From f3eef3861693b68a42cbbde553aa4131f20534f4 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 5 Apr 2024 10:46:26 +0200 Subject: [PATCH 01/97] Clean up the README file and license/copyright notices - Amend `README.rst` - Add metadata to `__init__.py` - Remove license/copyright notices from other places as we are now using `README.rst` for this purpose. - Remove `version.py` --- README.rst | 37 ++++++++++++++++++++++++++++++++++--- bin/pytroll-mongo.py | 30 ++++++------------------------ trolldb/__init__.py | 8 ++++++++ trolldb/version.py | 26 -------------------------- 4 files changed, 48 insertions(+), 53 deletions(-) delete mode 100644 trolldb/version.py diff --git a/README.rst b/README.rst index 707024b..6fa0b57 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,35 @@ -doobie -====== -Database interface for pytroll +The database interface of `Pytroll `_ + + +Copyright (C) + 2012, 2014, 2015, 2024 + + Martin Raspaud, Pouria Khalaj, Esben S. Nielsen, Adam Dybbroe, Kristian Rune Larsen + + +Authors + - Martin Raspaud + - Pouria Khalaj + - Esben S. Nielsen + - Adam Dybbroe + - Kristian Rune Larsen + + +License + This program, i.e. **pytroll-db**, is part of `Pytroll `_. + + **pytroll-db** is free software: you can redistribute it and/or modify + it under the terms of the `GNU General Public License `_ + as published by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + +Disclaimer + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. You should have + received a copy of the GNU General Public License + along with this program. If not, see . + diff --git a/bin/pytroll-mongo.py b/bin/pytroll-mongo.py index 0e6a10d..466e0b1 100644 --- a/bin/pytroll-mongo.py +++ b/bin/pytroll-mongo.py @@ -1,32 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2019 Martin Raspaud - -# Author(s): - -# Martin Raspaud - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from posttroll.subscriber import Subscribe import logging -from threading import Thread -import yaml import os +from threading import Thread +import yaml +from posttroll.subscriber import Subscribe from pymongo import MongoClient + logger = logging.getLogger(__name__) @@ -118,8 +100,8 @@ def setup_logging(cmd_args): help="Log config file to use instead of the standard logging.") parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, help="Verbosity (between 1 and 2 occurrences with more leading to more " - "verbose logging). WARN=0, INFO=1, " - "DEBUG=2. This is overridden by the log config file if specified.") + "verbose logging). WARN=0, INFO=1, " + "DEBUG=2. This is overridden by the log config file if specified.") cmd_args = parser.parse_args() logger = logging.getLogger("mongo_recorder") diff --git a/trolldb/__init__.py b/trolldb/__init__.py index e69de29..d7e5058 100644 --- a/trolldb/__init__.py +++ b/trolldb/__init__.py @@ -0,0 +1,8 @@ +__author__ = "Martin Raspaud, Pouria Khalaj, Esben S. Nielsen, Adam Dybbroe, Kristian Rune Larsen" +__copyright__ = "Copyright (C) 2012, 2014, 2015, 2024 Martin Raspaud, Pouria Khalaj, Esben S. Nielsen, Adam Dybbroe, Kristian Rune Larsen" + +__license__ = "GPLv3" +__version__ = "0.1.0" +__maintainer__ = "Martin Raspaud" +__email__ = "martin.raspaud@smhi.se" +__status__ = "Production" diff --git a/trolldb/version.py b/trolldb/version.py deleted file mode 100644 index 6ca7299..0000000 --- a/trolldb/version.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright (c) 2012, 2014, 2015 Martin Raspaud - -# Author(s): - -# Martin Raspaud - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""Version file. -""" - -__version__ = "0.2.0" From 559b20f7e006c756a1187a0548c42a644e7c1af8 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Sun, 5 May 2024 22:32:58 +0200 Subject: [PATCH 02/97] Preliminary implementation of the API and SDK. --- .gitignore | 11 +- .readthedocs.yaml | 23 +++ trolldb/__init__.py => __init__.py | 0 docs/Makefile | 20 ++ docs/source/conf.py | 64 +++++++ docs/source/index.rst | 17 ++ trolldb/api/__init__.py | 11 ++ trolldb/api/api.py | 101 ++++++++++ trolldb/api/errors/__init__.py | 0 trolldb/api/errors/errors.py | 107 +++++++++++ trolldb/api/routes/__init__.py | 3 + trolldb/api/routes/common.py | 128 +++++++++++++ trolldb/api/routes/databases.py | 69 +++++++ trolldb/api/routes/datetime_.py | 74 +++++++ trolldb/api/routes/platforms.py | 21 ++ trolldb/api/routes/queries.py | 47 +++++ trolldb/api/routes/root.py | 15 ++ trolldb/api/routes/router.py | 18 ++ trolldb/api/routes/sensors.py | 21 ++ trolldb/api/tests/conftest.py | 19 ++ trolldb/api/tests/pytest.ini | 2 + trolldb/api/tests/test_api.py | 71 +++++++ trolldb/config/__init__.py | 0 trolldb/config/config.py | 187 ++++++++++++++++++ trolldb/database/__init__.py | 1 + trolldb/database/mongodb.py | 201 ++++++++++++++++++++ trolldb/database/piplines.py | 50 +++++ trolldb/database/tests/conftest.py | 20 ++ trolldb/database/tests/pytest.ini | 2 + trolldb/database/tests/test_mongodb.py | 97 ++++++++++ trolldb/run_api.py | 11 ++ trolldb/template_config.yaml | 46 +++++ trolldb/test_utils/__init__.py | 0 trolldb/test_utils/common.py | 56 ++++++ trolldb/test_utils/mock_mongodb_database.py | 113 +++++++++++ trolldb/test_utils/mock_mongodb_instance.py | 97 ++++++++++ 36 files changed, 1722 insertions(+), 1 deletion(-) create mode 100644 .readthedocs.yaml rename trolldb/__init__.py => __init__.py (100%) create mode 100644 docs/Makefile create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 trolldb/api/__init__.py create mode 100644 trolldb/api/api.py create mode 100644 trolldb/api/errors/__init__.py create mode 100644 trolldb/api/errors/errors.py create mode 100644 trolldb/api/routes/__init__.py create mode 100644 trolldb/api/routes/common.py create mode 100644 trolldb/api/routes/databases.py create mode 100644 trolldb/api/routes/datetime_.py create mode 100644 trolldb/api/routes/platforms.py create mode 100644 trolldb/api/routes/queries.py create mode 100644 trolldb/api/routes/root.py create mode 100644 trolldb/api/routes/router.py create mode 100644 trolldb/api/routes/sensors.py create mode 100644 trolldb/api/tests/conftest.py create mode 100644 trolldb/api/tests/pytest.ini create mode 100644 trolldb/api/tests/test_api.py create mode 100644 trolldb/config/__init__.py create mode 100644 trolldb/config/config.py create mode 100644 trolldb/database/__init__.py create mode 100644 trolldb/database/mongodb.py create mode 100644 trolldb/database/piplines.py create mode 100644 trolldb/database/tests/conftest.py create mode 100644 trolldb/database/tests/pytest.ini create mode 100644 trolldb/database/tests/test_mongodb.py create mode 100644 trolldb/run_api.py create mode 100644 trolldb/template_config.yaml create mode 100644 trolldb/test_utils/__init__.py create mode 100644 trolldb/test_utils/common.py create mode 100644 trolldb/test_utils/mock_mongodb_database.py create mode 100644 trolldb/test_utils/mock_mongodb_instance.py diff --git a/.gitignore b/.gitignore index 219af40..a45834e 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,13 @@ coverage.xml *.pot # Sphinx documentation -docs/_build/ +docs/build/ +*.rst +!index.rst + +# the actual config file [HAS TO BE ALWAYS EXCLUDED!] +config.yaml +config.yml + +# temp log and storage for the test database +__temp* diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..16d1c72 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,23 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/trolldb/__init__.py b/__init__.py similarity index 100% rename from trolldb/__init__.py rename to __init__.py diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..d298115 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,64 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath('../../trolldb')) + +# -- Project information ----------------------------------------------------- + +project = 'Pytroll-db' +copyright = '2024, Pytroll' +author = 'Pouria Khalaj' + +# The full version, including alpha/beta/rc tags +release = '0.1' + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', +] +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["*test*"] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +autodoc_default_options = { + 'member-order': 'bysource', + 'special-members': '__init__', + 'undoc-members': True, + 'exclude-members': '__weakref__' +} + +root_doc = "index" diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..9e0c6e7 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,17 @@ +Welcome to Pytroll documentation! +=========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + modules + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/trolldb/api/__init__.py b/trolldb/api/__init__.py new file mode 100644 index 0000000..8604270 --- /dev/null +++ b/trolldb/api/__init__.py @@ -0,0 +1,11 @@ +""" +Note: + The following applies to the :package:`trolldb.api` and all its subpackages/modules. + + To avoid double documentations and inconsistencies, only non-FastAPI components are documented via the docstrings. + For the documentation related to the FastAPI components, check out the auto-generated documentation by FastAPI. + Assuming that the API server is running on ``_ (example) the auto-generated documentation can + be accessed via either ``_ or ``_. + + Read more at `FastAPI automatics docs `_. +""" diff --git a/trolldb/api/api.py b/trolldb/api/api.py new file mode 100644 index 0000000..dc95730 --- /dev/null +++ b/trolldb/api/api.py @@ -0,0 +1,101 @@ +""" +The module which includes the main functionalities of the API package. This is the main module which is supposed to be +imported by the users of the package. + +Note: + Functions in this module are decorated with + `pydantic.validate_call `_ + so that their arguments can be validated using the corresponding type hints, when calling the function at runtime. + +Note: + The following applies to the :obj:`trolldb.api` package and all its subpackages/modules. + + To avoid redundant documentation and inconsistencies, only non-FastAPI components are documented via the docstrings. + For the documentation related to the FastAPI components, check out the auto-generated documentation by FastAPI. + Assuming that the API server is running on ``_ (example) the auto-generated documentation can + be accessed via either ``_ or ``_. + + Read more at `FastAPI automatics docs `_. +""" + +import asyncio +import time +from contextlib import contextmanager +from multiprocessing import Process + +import uvicorn +from fastapi import FastAPI +from pydantic import FilePath, validate_call + +from api.routes import api_router +from config.config import AppConfig, parse, Timeout +from database.mongodb import mongodb_context + + +@validate_call +def run_server(config: AppConfig | FilePath, **kwargs) -> None: + """ + Runs the API server with all the routes and connection to the database. It first creates a FastAPI + application and runs it using `uvicorn `_ which is + ASGI (Asynchronous Server Gateway Interface) compliant. This function runs the event loop using + `asyncio `_ and does not yield! + + Args: + config: + The configuration of the application which includes both the server and database configurations. In case of + a :class:`FilePath`, it should be a valid path to an existing config file which will parsed as a ``.YAML`` + file. + + **kwargs: + The keyword arguments are the same as those accepted by the + `FastAPI class `_ and are directly passed + to it. These keyword arguments will be first concatenated with the configurations of the API server which + are read from the ``config`` argument. The keyword arguments which are passed + explicitly to the function take precedence over ``config``. + """ + + config = parse(config) + app = FastAPI(**(config.api_server._asdict() | kwargs)) + app.include_router(api_router) + + async def _serve(): + """ + An auxiliary coroutine to be used in the asynchronous execution of the FastAPI application. + """ + async with mongodb_context(config.database): + await uvicorn.Server( + config=uvicorn.Config( + host=config.api_server.url.host, + port=config.api_server.url.port, + app=app + ) + ).serve() + + asyncio.run(_serve()) + + +@contextmanager +@validate_call +def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2000): + """ + A synchronous context manager to run the API server in a separate process (non-blocking) using the + `multiprocessing `_ package. The main use case is envisaged + to be in testing environments. + + Args: + config: + Same as ``config`` argument for :func:`run_server`. + + startup_time: + The overall time that is expected for the server and the database connections to be established before + actual requests can be sent to the server. For testing purposes ensure that this is sufficiently large so + that the tests will not time out. + """ + config = parse(config) + process = Process(target=run_server, args=(config,)) + process.start() + try: + time.sleep(startup_time / 1000) # `time.sleep()` expects an argument in seconds, hence the division by 1000. + yield process + finally: + process.terminate() diff --git a/trolldb/api/errors/__init__.py b/trolldb/api/errors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/trolldb/api/errors/errors.py b/trolldb/api/errors/errors.py new file mode 100644 index 0000000..7a46d8e --- /dev/null +++ b/trolldb/api/errors/errors.py @@ -0,0 +1,107 @@ +""" +The module which defines error responses that will be returned by the API. +""" + +from collections import OrderedDict +from typing import Self + +from fastapi import status +from fastapi.responses import PlainTextResponse + + +class FailureResponse: + descriptor_delimiter = " |OR| " + defaultResponseClass = PlainTextResponse + + def __init__(self, args_dict: dict): + self.__dict = OrderedDict(args_dict) + + def __or__(self, other: Self): + buff = OrderedDict(self.__dict) + for key, value in other.__dict.items(): + buff[key] = FailureResponse.listify(buff.get(key, [])) + buff[key].extend(FailureResponse.listify(value)) + return FailureResponse(buff) + + def __str__(self): + return str(self.__dict) + + def fastapi_response(self, status_code: int | None = None): + if status_code is None and len(self.__dict) > 1: + raise ValueError("In case of multiple response status codes, please provide one.") + status_code, content = [(k, v) for k, v in self.__dict.items()][0] + try: + return FailureResponse.defaultResponseClass( + content=FailureResponse.stringify(content), + status_code=status_code) + except KeyError: + raise KeyError(f"No default response found for the given status code: {status_code}") + + @property + def fastapi_descriptor(self): + return {k: {"description": FailureResponse.stringify(v)} for k, v in self.__dict.items()} + + @staticmethod + def listify(item: str | list[str]) -> list[str]: + return item if isinstance(item, list) else [item] + + @staticmethod + def stringify(item: str | list[str]) -> str: + return FailureResponse.descriptor_delimiter.join(FailureResponse.listify(item)) + + +class BaseFailureResponses: + + @classmethod + def fields(cls): + return {k: v for k, v in cls.__dict__.items() if isinstance(v, FailureResponse)} + + @classmethod + def union(cls): + buff = FailureResponse({}) + for k, v in cls.fields().items(): + buff |= v + return buff + + +class CollectionFail(BaseFailureResponses): + NOT_FOUND = FailureResponse({ + status.HTTP_404_NOT_FOUND: + "Collection name does not exist." + }) + + WRONG_TYPE = FailureResponse({ + status.HTTP_422_UNPROCESSABLE_ENTITY: + "Collection name must be either a string or None; or both database name and collection name must be None." + }) + + +class DatabaseFail(BaseFailureResponses): + NOT_FOUND = FailureResponse({ + status.HTTP_404_NOT_FOUND: + "Database name does not exist." + }) + + WRONG_TYPE = FailureResponse({ + status.HTTP_422_UNPROCESSABLE_ENTITY: + "Database name must be either a string or None." + }) + + +class DocumentsFail(BaseFailureResponses): + NOT_FOUND = FailureResponse({ + status.HTTP_404_NOT_FOUND: + "Could not find any document with the given object id." + }) + + +Database_Collection_Fail = DatabaseFail | CollectionFail +Database_Collection_Document_Fail = DatabaseFail | CollectionFail | DocumentsFail + +database_collection_fail_descriptor = ( + DatabaseFail.union() | CollectionFail.union() +).fastapi_descriptor + +database_collection_document_fail_descriptor = ( + DatabaseFail.union() | CollectionFail.union() | DocumentsFail.union() +).fastapi_descriptor diff --git a/trolldb/api/routes/__init__.py b/trolldb/api/routes/__init__.py new file mode 100644 index 0000000..08584de --- /dev/null +++ b/trolldb/api/routes/__init__.py @@ -0,0 +1,3 @@ +from .router import api_router + +__all__ = ("api_router",) diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py new file mode 100644 index 0000000..92867bb --- /dev/null +++ b/trolldb/api/routes/common.py @@ -0,0 +1,128 @@ +""" +The module which defines common functions to be used in handling requests related to `databases` and `collections`. +""" + +from typing import Annotated + +from fastapi import Response, Query, Depends +from motor.motor_asyncio import AsyncIOMotorCollection, AsyncIOMotorDatabase + +from api.errors.errors import CollectionFail, DatabaseFail, Database_Collection_Fail, FailureResponse +from database.mongodb import MongoDB + +exclude_defaults_query = Query( + True, + title="Query string", + description="A boolean to exclude default databases from a MongoDB instance. Refer to " + "`trolldb.database.mongodb.MongoDB.default_database_names` for more information.") + + +async def check_database(database_name: str | None = None) -> Response | AsyncIOMotorDatabase: + """ + A dependency for route handlers to check for the existence of a database given + its name. + + Args: + database_name (Optional, default ``None``): + The name of the database to check. In case of ``None``, the main database will be picked. + + Returns: + -- The database object if it exists. + + -- :obj:`api.errors.errors.DatabaseFail.NOT_FOUND`, if the database does not exist. + + -- :obj:`api.errors.errors.DatabaseFail.WRONG_TYPE`, if the type of the database name is not ``str`` or + ``None``. + """ + + match database_name: + case None: + return MongoDB.main_database() + + case str(): + if database_name in await MongoDB.client().list_database_names(): + return MongoDB.client()[database_name] + return DatabaseFail.NOT_FOUND.fastapi_response() + + case _: + return DatabaseFail.WRONG_TYPE.fastapi_response() + + +async def check_collection( + database_name: str | None = None, + collection_name: str | None = None) -> Response | AsyncIOMotorCollection: + """ + A dependency for route handlers to check for the existence of a collection given + its name and the name of the database it resides in. It first checks for the existence of the database using + :func:`check_database`. + + Args: + database_name (Optional, default ``None``): + The name of the database to check. In case of ``None``, the main database will be picked. + collection_name (Optional, default ``None``): + The name of the collection to check. In case of ``None``, the main collection will be picked. + + Warning: + Both of ``database_name`` and ``collection_name`` must be ``None`` so that the main database and collection + will be picked. In case only one of them is ``None``, this is treated as an unacceptable request. + + Returns: + -- The collection object if it exists in the designated database. + + -- A response from :func:`check_database`, if the database does not exist or the type of ``database_name`` is + not valid. + + -- :obj:`api.errors.errors.CollectionFail.NOT_FOUND`, if the parent database exists but the collection does not. + + -- :obj:`api.errors.errors.CollectionFail.WRONG_TYPE`, if only one of ``database_name`` or ``collection_name`` + is ``None``; or if the type of ``collection_name`` is not ``str``. + """ + + res = await check_database(database_name) + if isinstance(res, Response): + return res + + match database_name, collection_name: + case None, None: + return MongoDB.main_collection() + + case str(), str(): + if collection_name in await MongoDB.client()[database_name].list_collection_names(): + return MongoDB.client()[database_name][collection_name] + return CollectionFail.NOT_FOUND.fastapi_response() + + case _: + return CollectionFail.WRONG_TYPE.fastapi_response() + + +async def get_distinct_items_in_collection( + res_coll: Response | AsyncIOMotorCollection, + field_name: str) -> Response | list[str]: + """ + An auxiliary function to either return (verbatim echo) the given response; or return a list of distinct (unique) + values for the given ``field_name`` via a search which is conducted in all documents of the given collection. The + latter behaviour is equivalent to the ``distinct`` function from MongoDB. The former is the behaviour of an + identity function + + Args: + res_coll: + Either a response object, or a collection in which documents will be queried for the ``field_name``. + + field_name: + The name of the target field in the documents + + Returns: + -- In case of a response as input, the same response will be returned. + + -- In case of a collection as input, all the documents of the collection will be searched for ``field_name``, + and the corresponding values will be retrieved. Finally, a list of all the distinct values is returned. + """ + + if isinstance(res_coll, Response): + return res_coll + + return await res_coll.distinct(field_name) + + +CheckCollectionDependency = Annotated[FailureResponse | AsyncIOMotorCollection, Depends(check_collection)] +CheckDataBaseDependency = Annotated[FailureResponse | AsyncIOMotorDatabase, Depends(check_database)] diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py new file mode 100644 index 0000000..c368e40 --- /dev/null +++ b/trolldb/api/routes/databases.py @@ -0,0 +1,69 @@ +""" +The module which handles all requests related to getting the list of `databases` and `collections`. + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +from fastapi import APIRouter, Response +from pymongo.collection import _DocumentType + +from api.errors.errors import ( + DatabaseFail, + DocumentsFail, + database_collection_fail_descriptor, + database_collection_document_fail_descriptor) +from api.routes.common import ( + exclude_defaults_query, CheckCollectionDependency, CheckDataBaseDependency) +from config.config import MongoObjectId +from database.mongodb import MongoDB + +router = APIRouter() + + +@router.get("/", + response_model=list[str], + summary="Gets the list of all database names") +async def database_names(exclude_defaults: bool = exclude_defaults_query) -> list[str]: + db_names = await MongoDB.client().list_database_names() + + if not exclude_defaults: + return db_names + + return [db for db in db_names if db not in MongoDB.default_database_names] + + +@router.get("/{database_name}", + response_model=list[str], + responses=DatabaseFail.union().fastapi_descriptor, + summary="Gets the list of all collection names for the given database name") +async def collection_names(res_db: CheckDataBaseDependency) -> Response | list[str]: + if isinstance(res_db, Response): + return res_db + + return await res_db.list_collection_names() + + +@router.get("/{database_name}/{collection_name}", + response_model=list[str], + responses=database_collection_fail_descriptor, + summary="Gets the object ids of all documents for the given database and collection name") +async def documents(res_coll: CheckCollectionDependency) -> Response | list[str]: + if isinstance(res_coll, Response): + return res_coll + + return await MongoDB.get_ids(res_coll.find({})) + + +@router.get("/{database_name}/{collection_name}/{_id}", + response_model=_DocumentType, + responses=database_collection_document_fail_descriptor, + summary="Gets the document content in json format given its object id, database, and collection name") +async def document_by_id(res_coll: CheckCollectionDependency, _id: MongoObjectId) -> Response | _DocumentType: + if isinstance(res_coll, Response): + return res_coll + + if document := await res_coll.find_one({"_id": _id}): + return dict(document) | {"_id": str(_id)} + + return DocumentsFail.NOT_FOUND.fastapi_response() diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py new file mode 100644 index 0000000..13768a0 --- /dev/null +++ b/trolldb/api/routes/datetime_.py @@ -0,0 +1,74 @@ +""" +The module which handles all requests related to `datetime`. + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +from datetime import datetime +from typing import TypedDict + +from fastapi import APIRouter, Response +from pydantic import BaseModel + +from api.errors.errors import database_collection_fail_descriptor +from api.routes.common import CheckCollectionDependency +from database.mongodb import MongoDB + + +class TimeModel(TypedDict): + _id: str + _time: datetime + + +class TimeEntry(TypedDict): + _min: TimeModel + _max: TimeModel + + +class ResponseModel(BaseModel): + start_time: TimeEntry + end_time: TimeEntry + + +router = APIRouter() + + +@router.get("", + response_model=ResponseModel, + responses=database_collection_fail_descriptor, + summary="Gets the the minimum and maximum values for the start and end times") +async def datetime(res_coll: CheckCollectionDependency) -> Response | ResponseModel: + if isinstance(res_coll, Response): + return res_coll + + agg_result = await res_coll.aggregate([{ + "$group": { + "_id": None, + "min_start_time": {"$min": "$start_time"}, + "max_start_time": {"$max": "$start_time"}, + "min_end_time": {"$min": "$end_time"}, + "max_end_time": {"$max": "$end_time"} + }}]).next() + + def _aux(query): + return MongoDB.get_id(res_coll.find_one(query)) + + return ResponseModel( + start_time=TimeEntry( + _min=TimeModel( + _id=await _aux({"start_time": agg_result["min_start_time"]}), + _time=agg_result["min_start_time"]), + _max=TimeModel( + _id=await _aux({"start_time": agg_result["max_start_time"]}), + _time=agg_result["max_start_time"]) + ), + end_time=TimeEntry( + _min=TimeModel( + _id=await _aux({"end_time": agg_result["min_end_time"]}), + _time=agg_result["min_end_time"]), + _max=TimeModel( + _id=await _aux({"end_time": agg_result["max_end_time"]}), + _time=agg_result["max_end_time"]) + ) + ) diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py new file mode 100644 index 0000000..47c62e4 --- /dev/null +++ b/trolldb/api/routes/platforms.py @@ -0,0 +1,21 @@ +""" +The module which handles all requests regarding `platforms`. + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +from fastapi import APIRouter, Response + +from api.errors.errors import database_collection_fail_descriptor +from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency + +router = APIRouter() + + +@router.get("", + response_model=list[str], + responses=database_collection_fail_descriptor, + summary="Gets the list of all platform names") +async def platform_names(res_coll: CheckCollectionDependency) -> Response | list[str]: + return await get_distinct_items_in_collection(res_coll, "platform_name") diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py new file mode 100644 index 0000000..bad9a21 --- /dev/null +++ b/trolldb/api/routes/queries.py @@ -0,0 +1,47 @@ +""" +The module which handles all requests to the queries route. + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +import datetime + +from fastapi import APIRouter, Query, Response + +from api.errors.errors import database_collection_fail_descriptor +from api.routes.common import CheckCollectionDependency +from database.mongodb import MongoDB +from database.piplines import PipelineAttribute, Pipelines + +router = APIRouter() + + +@router.get("", + response_model=list[str], + responses=database_collection_fail_descriptor, + summary="Gets the database UUIDs of the documents that match specifications determined by the query string") +async def queries( + res_coll: CheckCollectionDependency, + platform: list[str] = Query(None), + sensor: list[str] = Query(None), + time_min: datetime.datetime = Query(None), + time_max: datetime.datetime = Query(None)) -> Response | list[str]: + if isinstance(res_coll, Response): + return res_coll + + pipelines = Pipelines() + + if platform: + pipelines += PipelineAttribute("platform_name") == platform + + if sensor: + pipelines += PipelineAttribute("sensor") == sensor + + if [time_min, time_max] != [None, None]: + start_time = PipelineAttribute("start_time") + end_time = PipelineAttribute("end_time") + pipelines += ((start_time >= time_min) | (start_time <= time_max) | + (end_time >= time_min) | (end_time <= time_max)) + + return await MongoDB.get_ids(res_coll.aggregate(pipelines)) diff --git a/trolldb/api/routes/root.py b/trolldb/api/routes/root.py new file mode 100644 index 0000000..785be36 --- /dev/null +++ b/trolldb/api/routes/root.py @@ -0,0 +1,15 @@ +""" +The module which handles all requests to the root route, i.e. "/". + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +from fastapi import APIRouter, Response, status + +router = APIRouter() + + +@router.get("/", summary="The root route which is mainly used to check the status of connection") +async def root() -> Response: + return Response(status_code=status.HTTP_200_OK) diff --git a/trolldb/api/routes/router.py b/trolldb/api/routes/router.py new file mode 100644 index 0000000..2c75288 --- /dev/null +++ b/trolldb/api/routes/router.py @@ -0,0 +1,18 @@ +""" +The module which defines all the routes with their corresponding tags. + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +from fastapi import APIRouter + +from . import databases, datetime_, platforms, queries, root, sensors + +api_router = APIRouter() +api_router.include_router(root.router, tags=["root"]) +api_router.include_router(databases.router, tags=["databases"], prefix="/databases") +api_router.include_router(datetime_.router, tags=["datetime"], prefix="/datetime") +api_router.include_router(platforms.router, tags=["platforms"], prefix="/platforms") +api_router.include_router(queries.router, tags=["queries"], prefix="/queries") +api_router.include_router(sensors.router, tags=["sensors"], prefix="/sensors") diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py new file mode 100644 index 0000000..e73268c --- /dev/null +++ b/trolldb/api/routes/sensors.py @@ -0,0 +1,21 @@ +""" +The module which handles all requests regarding `sensors`. + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +from fastapi import APIRouter, Response + +from api.errors.errors import database_collection_fail_descriptor +from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency + +router = APIRouter() + + +@router.get("", + response_model=list[str], + responses=database_collection_fail_descriptor, + summary="Gets the list of all sensor names") +async def sensor_names(res_coll: CheckCollectionDependency) -> Response | list[str]: + return await get_distinct_items_in_collection(res_coll, "sensor") diff --git a/trolldb/api/tests/conftest.py b/trolldb/api/tests/conftest.py new file mode 100644 index 0000000..496e777 --- /dev/null +++ b/trolldb/api/tests/conftest.py @@ -0,0 +1,19 @@ +import pytest + +from api.api import server_process_context +from test_utils.mock_mongodb_instance import mongodb_instance_server_process_context +from test_utils.common import test_app_config +from test_utils.mock_mongodb_database import TestDatabase + + +@pytest.fixture(scope="session") +def run_mongodb_server_instance(): + with mongodb_instance_server_process_context(): + yield + + +@pytest.fixture(scope="session", autouse=True) +def test_server_fixture(run_mongodb_server_instance): + TestDatabase.prepare() + with server_process_context(test_app_config, startup_time=2000): + yield diff --git a/trolldb/api/tests/pytest.ini b/trolldb/api/tests/pytest.ini new file mode 100644 index 0000000..4088045 --- /dev/null +++ b/trolldb/api/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode=auto diff --git a/trolldb/api/tests/test_api.py b/trolldb/api/tests/test_api.py new file mode 100644 index 0000000..1796a35 --- /dev/null +++ b/trolldb/api/tests/test_api.py @@ -0,0 +1,71 @@ +from fastapi import status + +from test_utils.common import assert_equal, http_get +from test_utils.mock_mongodb_database import test_mongodb_context, TestDatabase + + +def test_root(): + """ + Checks that the server is up and running, i.e. the root routes responds with 200. + """ + assert_equal(http_get().status, status.HTTP_200_OK) + + +def test_platforms(): + """ + Checks that the retrieved platform names match the expected names. + """ + assert_equal(http_get("platforms").json(), TestDatabase.platform_names) + + +def test_sensors(): + """ + Checks that the retrieved sensor names match the expected names. + """ + assert_equal(http_get("sensors").json(), TestDatabase.sensors) + + +def test_database_names(): + """ + Checks that the retrieved database names match the expected names. + """ + assert_equal(http_get("databases").json(), TestDatabase.database_names) + assert_equal(http_get("databases?exclude_defaults=True").json(), TestDatabase.database_names) + assert_equal(http_get("databases?exclude_defaults=False").json(), TestDatabase.all_database_names) + + +def test_database_names_negative(): + """ + Checks that the non-existing databases cannot be found. + """ + assert_equal(http_get(f"databases/non_existing_database").status, status.HTTP_404_NOT_FOUND) + + +def test_collections(): + """ + Check the presence of existing collections and that the ids of documents therein can be correctly retrieved. + """ + with test_mongodb_context() as client: + for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names): + # Collections exist + assert_equal( + http_get(f"databases/{database_name}").json(), + [collection_name] + ) + + # Document ids are correct + assert_equal( + http_get(f"databases/{database_name}/{collection_name}").json(), + {str(doc["_id"]) for doc in client[database_name][collection_name].find({})} + ) + + +def test_collections_negative(): + """ + Checks that the non-existing collections cannot be found. + """ + for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names): + assert_equal( + http_get(f"databases/{database_name}/non_existing_collection").status, + status.HTTP_404_NOT_FOUND + ) diff --git a/trolldb/config/__init__.py b/trolldb/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/trolldb/config/config.py b/trolldb/config/config.py new file mode 100644 index 0000000..ad65e2d --- /dev/null +++ b/trolldb/config/config.py @@ -0,0 +1,187 @@ +""" +The module which handles parsing and validating the config (YAML) file. +The validation is performed using `Pydantic `_. + +Note: + Functions in this module are decorated with + `pydantic.validate_call `_ + so that their arguments can be validated using the corresponding type hints, when calling the function at runtime. +""" + +import errno +import sys +from typing import Optional, NamedTuple, TypedDict + +from bson import ObjectId +from bson.errors import InvalidId +from loguru import logger +from pydantic import AnyUrl, BaseModel, Field, FilePath, MongoDsn, ValidationError, validate_call +from pydantic.functional_validators import AfterValidator +from typing_extensions import Annotated +from yaml import safe_load + +Timeout = Annotated[int, Field(gt=0)] + + +def id_must_be_valid(v: str) -> ObjectId: + try: + return ObjectId(v) + except InvalidId as e: + raise ValueError(str(e)) + + +MongoObjectId = Annotated[str, AfterValidator(id_must_be_valid)] + + +class MongoDocument(BaseModel): + _id: MongoObjectId + + +class LicenseInfo(TypedDict): + """ + A dictionary type to hold the summary of the license information. One has to always consult the included `LICENSE` + file for more information. + """ + + name: str + """ + The full name of the license including the exact variant and the version (if any), e.g. + ``"The GNU General Public License v3.0"`` + """ + + url: AnyUrl + """ + The URL to access the license, e.g. ``"https://www.gnu.org/licenses/gpl-3.0.en.html"`` + """ + + +class APIServerConfig(NamedTuple): + """ + A named tuple to hold all the configurations of the API server (excluding the database). + + Note: + Except for the ``url``, the attributes herein are a subset of the keyword arguments accepted by + `FastAPI class `_ and are directly passed + to the FastAPI class. + """ + + url: AnyUrl + """ + The URL of the API server including the port, e.g. ``mongodb://localhost:8000``. This will not be passed to the + FastAPI class. Instead, it will be used by the `uvicorn` to determine the URL of the server. + """ + + title: str + """ + The title of the API server, as appears in the automatically generated documentation by the FastAPI. + """ + + version: str + """ + The version of the API server as appears in the automatically generated documentation by the FastAPI. + """ + + summary: Optional[str] = None + """ + The summary of the API server, as appears in the automatically generated documentation by the FastAPI. + """ + + description: Optional[str] = None + """ + The more comprehensive description (extended summary) of the API server, as appears in the automatically generated + documentation by the FastAPI. + """ + + license_info: Optional[LicenseInfo] = None + """ + The license information of the API server, as appears in the automatically generated documentation by the FastAPI. + """ + + +class DatabaseConfig(NamedTuple): + """ + A named tuple to hold all the configurations of the Database which will be used by the MongoDB instance. + """ + + main_database_name: str + """ + The name of the main database which includes the ``main_collection``, e.g. ``"satellite_database"``. + """ + + main_collection_name: str + """ + The name of the main collection which resides inside the ``main_database`` and includes the actual data for the + files, e.g. ``"files"`` + """ + + url: MongoDsn + """ + The URL of the MongoDB server excluding the port part, e.g. ``"mongodb://localhost:27017"`` + """ + + timeout: Annotated[int, Field(gt=-1)] + """ + The timeout in milliseconds (non-negative integer), after which an exception is raised if a connection with the + MongoDB instance is not established successfully, e.g. ``1000``. + """ + + +class AppConfig(BaseModel): + """ + A model to hold all the configurations of the application including both the API server and the database. This will + be used by Pydantic to validate the parsed YAML file. + """ + api_server: APIServerConfig + database: DatabaseConfig + + +@validate_call +def from_yaml(filename: FilePath) -> AppConfig: + """ + Parses and validates the configurations from a YAML file. + + Args: + filename: + The filename of a valid YAML file which holds the configurations. + + Raises: + -- ParserError: + If the file cannot be properly parsed. + + -- ValidationError: + If the successfully parsed file fails the validation, i.e. its schema or the content does not conform to + :class:`AppConfig`. + + Returns: + An instance of :class:`AppConfig`. + """ + + with open(filename, "r") as file: + config = safe_load(file) + try: + return AppConfig(**config) + except ValidationError as e: + logger.error(e) + sys.exit(errno.EIO) + + +@validate_call +def parse(config: AppConfig | FilePath) -> AppConfig: + """ + Tries to return a valid object of type :class:`AppConfig` + + Args: + config: + Either an object of type :class:`AppConfig` or :class:`FilePath`. + + Returns: + -- In case of an object of type :class:`AppConfig` as input, the same object will be returned. + + -- An input object of type ``str`` will be interpreted as a YAML filename, in which case the function returns + the result of parsing the file. + """ + match config: + case AppConfig(): + return config + case _: + return from_yaml(config) diff --git a/trolldb/database/__init__.py b/trolldb/database/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/trolldb/database/__init__.py @@ -0,0 +1 @@ + diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py new file mode 100644 index 0000000..1484821 --- /dev/null +++ b/trolldb/database/mongodb.py @@ -0,0 +1,201 @@ +""" +The module which handles database CRUD operations for MongoDB. It is based on +`PyMongo `_ and `motor `_. +""" + +import errno +import sys +from contextlib import asynccontextmanager +from typing import Any, Coroutine + +from loguru import logger +from motor.motor_asyncio import ( + AsyncIOMotorClient, + AsyncIOMotorDatabase, + AsyncIOMotorCollection, + AsyncIOMotorCommandCursor, + AsyncIOMotorCursor +) +from pydantic import validate_call +from pymongo.collection import _DocumentType +from pymongo.errors import ( + ConnectionFailure, + ServerSelectionTimeoutError) + +from config.config import DatabaseConfig + + +class MongoDB: + """ + A wrapper class around the `motor async driver `_ for Mongo DB with + convenience methods tailored to our specific needs. As such, most of the methods return coroutines whose results + need to be awaited. + + Note: + This class is not meant to be instantiated! That's why all the methods in this class are decorated with + ``@classmethods``. This choice has been made to guarantee optimal performance, i.e. for each running process + there must be only a single motor client to handle all database operations. Having different clients which are + constantly opened/closed degrades the performance. The expected usage is that we open a client in the beginning + of the program and keep it open until the program finishes. It is okay to reopen/close the client for testing + purposes when isolation is needed. + + Note: + The main difference between this wrapper class and the original motor driver class is that we attempt to access + the database and collections during the initialization to see if we succeed or fail. This is contrary to the + behaviour of the motor driver which simply creates a client object and does not attempt to access the database + until some time later when an actual operation is performed on the database. This behaviour is not desired for + us, we would like to fail early! + """ + + __client: AsyncIOMotorClient | None = None + __main_collection: AsyncIOMotorCollection = None + __main_database: AsyncIOMotorDatabase = None + + default_database_names = ["admin", "config", "local"] + """ + MongoDB creates these databases by default for self usage. + """ + + @classmethod + async def initialize(cls, database_config: DatabaseConfig): + """ + Initializes the motor client. Note that this method has to be awaited! + + Args: + database_config: + A named tuple which includes the database configurations. + + Raises :obj:`~SystemExit(errno.EIO)`: + If connection is not established (``ConnectionFailure``) or if the attempt times out + (``ServerSelectionTimeoutError``) + + Raises :obj:`~SystemExit(errno.ENODATA)`: + If either ``database_config.main_database`` or ``database_config.main_collection`` does not exist. + + Returns: + On success ``None``. + """ + + # This only makes the reference and does not establish an actual connection until the first attempt is made + # to access the database. + cls.__client = AsyncIOMotorClient( + database_config.url.unicode_string(), + serverSelectionTimeoutMS=database_config.timeout) + + try: + # Here we attempt to access the database + __database_names = await cls.__client.list_database_names() + except (ConnectionFailure, ServerSelectionTimeoutError): + logger.error(f"Could not connect to the database with URL: {database_config.url.unicode_string()}") + sys.exit(errno.EIO) + + if database_config.main_database_name not in __database_names: + logger.error(f"Could not find any database with the given name: {database_config.main_database_name}") + sys.exit(errno.ENODATA) + cls.__main_database = cls.__client.get_database(database_config.main_database_name) + + if database_config.main_collection_name not in await cls.__main_database.list_collection_names(): + logger.error(f"Could not find any collection in database `{database_config.main_database_name}` with the " + f"given name: {database_config.main_database_name}") + sys.exit(errno.ENODATA) + cls.__main_collection = cls.__main_database.get_collection(database_config.main_collection_name) + + @classmethod + def client(cls) -> AsyncIOMotorClient: + """ + Returns: + The actual motor client so that it can be used to perform database CRUD operations. + """ + return cls.__client + + @classmethod + def main_collection(cls) -> AsyncIOMotorCollection: + """ + A convenience method to get the main collection. + + Returns: + The main collection which resides inside the main database. + Equivalent to ``MongoDB.client()[][]``. + """ + return cls.__main_collection + + @classmethod + def main_database(cls) -> AsyncIOMotorDatabase: + """ + A convenience method to get the main database. + + Returns: + The main database which includes the main collection, which in turn includes the desired documents. + Equivalent to ``MongoDB.client()[]``. + """ + return cls.__main_database + + @staticmethod + async def get_id(doc: Coroutine[Any, Any, _DocumentType | None] | _DocumentType) -> str: + """ + Retrieves the ID of a document as a simple flat string. + + Note: + The rationale behind this method is as follows. In MongoDB, each document has a unique ID which is of type + :class:`~bson.objectid.ObjectId`. This is not suitable for purposes when a simple string is needed, hence + the need for this method. + + Args: + doc: + A MongoDB document as a :class:`_DocumentType` object or in the coroutine form. The latter could be e.g. + the result of applying the standard ``find_one`` method from MongoDB on a collection given a ``filter``. + + Returns: + The ID of a document as a simple string. For example, when applied on a document with + ``_id: ObjectId('000000000000000000000000')``, the method returns ``'000000000000000000000000'``. + """ + match doc: + case _DocumentType(): + return str(doc["_id"]) + case Coroutine(): + return str((await doc)["_id"]) + case _: + raise TypeError("The type of `doc` must be either `_DocumentType` or " + "`Coroutine[Any, Any, _DocumentType | None] `.") + + @staticmethod + async def get_ids(docs: AsyncIOMotorCommandCursor | AsyncIOMotorCursor | list[_DocumentType]) -> list[str]: + """ + Similar to :func:`~MongoDB.get_id` but for a list of documents. + + Args: + docs: + A list of MongoDB documents each as a :class:`DocumentType`, or all as an + :obj:`~AsyncIOMotorCommandCursor`. The latter could be e.g. the result of applying the + standard ``aggregate`` method from MongoDB on a collection given a ``pipeline``. + + Returns: + The list of all IDs, each as a simple string. + """ + match docs: + case list(): + return [str(doc["_id"]) for doc in docs] + case AsyncIOMotorCommandCursor() | AsyncIOMotorCursor(): + return [str(doc["_id"]) async for doc in docs] + case _: + raise TypeError("The type of `docs` must be either `list[_DocumentType]` or " + "`AsyncIOMotorCommandCursor`.") + + +@asynccontextmanager +@validate_call +async def mongodb_context(database_config: DatabaseConfig): + """ + An asynchronous context manager to connect to the MongoDB client. + It can be either used in production or in testing environments. + + Args: + database_config: + The configuration of the database. + """ + try: + await MongoDB.initialize(database_config) + yield + finally: + if MongoDB.client() is not None: + MongoDB.client().close() diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py new file mode 100644 index 0000000..9d39d4b --- /dev/null +++ b/trolldb/database/piplines.py @@ -0,0 +1,50 @@ +from typing import Any, Self + + +class PipelineDict(dict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def __or__(self, other: Self): + return PipelineDict(**{"$or": [self, other]}) + + def __and__(self, other: Self): + return PipelineDict(**{"$and": [self, other]}) + + +class PipelineAttribute: + def __init__(self, key: str): + self.__key = key + + def __eq__(self, other: Any) -> PipelineDict: + if isinstance(other, list): + return PipelineDict(**{"$or": [{self.__key: v} for v in other]}) + return PipelineDict(**{self.__key: other}) + + def __aux_operators(self, other: Any, operator: str) -> PipelineDict: + return PipelineDict(**{self.__key: {operator: other}} if other else {}) + + def __ge__(self, other: Any) -> PipelineDict: + return self.__aux_operators(other, "$gte") + + def __gt__(self, other: Any) -> PipelineDict: + return self.__aux_operators(other, "$gt") + + def __le__(self, other: Any) -> PipelineDict: + return self.__aux_operators(other, "$lte") + + def __lt__(self, other: Any) -> PipelineDict: + return self.__aux_operators(other, "$le") + + +class Pipelines(list): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def __iadd__(self, other): + self.extend([{"$match": other}]) + return self + + def __add__(self, other): + self.append({"$match": other}) + return self diff --git a/trolldb/database/tests/conftest.py b/trolldb/database/tests/conftest.py new file mode 100644 index 0000000..ab10d98 --- /dev/null +++ b/trolldb/database/tests/conftest.py @@ -0,0 +1,20 @@ +import pytest +import pytest_asyncio + +from test_utils.mock_mongodb_instance import mongodb_instance_server_process_context +from test_utils.common import test_app_config +from test_utils.mock_mongodb_database import TestDatabase +from trolldb.database.mongodb import mongodb_context + + +@pytest.fixture(scope="session") +def run_mongodb_server_instance(): + with mongodb_instance_server_process_context(): + yield + + +@pytest_asyncio.fixture(autouse=True) +async def mongodb_fixture(run_mongodb_server_instance): + TestDatabase.prepare() + async with mongodb_context(test_app_config.database): + yield diff --git a/trolldb/database/tests/pytest.ini b/trolldb/database/tests/pytest.ini new file mode 100644 index 0000000..4088045 --- /dev/null +++ b/trolldb/database/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode=auto diff --git a/trolldb/database/tests/test_mongodb.py b/trolldb/database/tests/test_mongodb.py new file mode 100644 index 0000000..60f8234 --- /dev/null +++ b/trolldb/database/tests/test_mongodb.py @@ -0,0 +1,97 @@ +import errno +import time + +import pytest +from pydantic import AnyUrl +from pymongo.errors import InvalidOperation + +from test_utils.common import test_app_config +from trolldb.database.mongodb import DatabaseConfig, MongoDB, mongodb_context + + +async def test_connection_timeout_negative(): + """ + Expect to see the connection attempt times out since the MongoDB URL is invalid. + """ + timeout = 3000 + with pytest.raises(SystemExit) as pytest_wrapped_e: + t1 = time.time() + async with mongodb_context( + DatabaseConfig(url=AnyUrl("mongodb://invalid_url_that_does_not_exist:8000"), + timeout=timeout, main_database_name=" ", main_collection_name=" ")): + pass + t2 = time.time() + assert pytest_wrapped_e.value.code == errno.EIO + assert t2 - t1 >= timeout / 1000 + + +async def test_main_database_negative(): + """ + Expect to fail when giving an invalid name for the main database, given a valid collection name. + """ + with pytest.raises(SystemExit) as pytest_wrapped_e: + async with mongodb_context(DatabaseConfig( + timeout=1000, + url=test_app_config.database.url, + main_database_name=" ", + main_collection_name=test_app_config.database.main_collection_name)): + pass + assert pytest_wrapped_e.value.code == errno.ENODATA + + +async def test_main_collection_negative(): + """ + Expect to fail when giving an invalid name for the main collection, given a valid database name. + """ + with pytest.raises(SystemExit) as pytest_wrapped_e: + async with mongodb_context(DatabaseConfig( + timeout=1000, + url=test_app_config.database.url, + main_database_name=test_app_config.database.main_database_name, + main_collection_name=" ")): + pass + assert pytest_wrapped_e.value.code == errno.ENODATA + + +async def test_connection_success(): + """ + Expect to establish a connection with the MongoDB instance successfully (with default args). + """ + pass + + +async def test_get_client(): + """ + This is our way of testing that MongoDB.client() returns the valid client object. + + Expect: + - Have a MongoDB client which is not `None` + - The `close` method can be called on the client and leads to the closure of the client + - Further attempts to access the database after closing the client fails. + """ + assert MongoDB.client() is not None + MongoDB.client().close() + with pytest.raises(InvalidOperation): + await MongoDB.client().list_database_names() + + +async def test_main_collection(): + """ + Expect: + - The retrieved main collection is not `None` + - It has the correct name + - It is the same object that can be accessed via the `client` object of the MongoDB. + """ + assert MongoDB.main_collection() is not None + assert MongoDB.main_collection().name == test_app_config.database.main_collection_name + assert MongoDB.main_collection() == \ + MongoDB.client()[test_app_config.database.main_database_name][test_app_config.database.main_collection_name] + + +async def test_main_database(): + """ + Same as test_main_collection but for the main database. + """ + assert MongoDB.main_database() is not None + assert MongoDB.main_database().name == test_app_config.database.main_database_name + assert MongoDB.main_database() == MongoDB.client()[test_app_config.database.main_database_name] diff --git a/trolldb/run_api.py b/trolldb/run_api.py new file mode 100644 index 0000000..f5a6cbb --- /dev/null +++ b/trolldb/run_api.py @@ -0,0 +1,11 @@ +""" +The main entry point to run the API server according to the configurations given in `config.yaml` + +Note: + For more information on the API server, see the automatically generated documentation by FastAPI. +""" + +from api.api import run_server + +if __name__ == "__main__": + run_server("config.yaml") diff --git a/trolldb/template_config.yaml b/trolldb/template_config.yaml new file mode 100644 index 0000000..8afd05a --- /dev/null +++ b/trolldb/template_config.yaml @@ -0,0 +1,46 @@ +# This is just a template. +# For use in production, rename this file to `config.yaml` so that the actual configurations can be correctly picked up. +# In addition, adapt the values accordingly. +# Finally, to improve readability please remove the comments! + +# Required +database: + #Required + main_database_name: "satellite_database" + + #Required + main_collection_name: "files" + + #Required + url: "mongodb://localhost:27017" + + #Required + timeout: 1000 # milliseconds + + +# Required +api_server: + # Required + url: "http://localhost:8000" + + # Required + "title": "Pytroll-db API" + + # Required + "version": "0.1" + + # Optional + "summary": "The awesome API of Pytroll-db" + + # Optional + "description": " + The API allows you to perform CRUD operations as well as querying the database. + At the moment only MongoDB is supported. It is based on the following Python packages + \n * **PyMongo** (https://github.com/mongodb/mongo-python-driver) + \n * **motor** (https://github.com/mongodb/motor) + " + + # Optional + "license_info": + "name": "The GNU General Public License v3.0" + "url": "https://www.gnu.org/licenses/gpl-3.0.en.html" diff --git a/trolldb/test_utils/__init__.py b/trolldb/test_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py new file mode 100644 index 0000000..568de5c --- /dev/null +++ b/trolldb/test_utils/common.py @@ -0,0 +1,56 @@ +from typing import Any +from urllib.parse import urljoin + + +from pydantic import AnyUrl +from urllib3 import request, BaseHTTPResponse + +from config.config import APIServerConfig, DatabaseConfig, AppConfig + +test_app_config = AppConfig( + api_server=APIServerConfig(url=AnyUrl("http://localhost:8080"), title="Test API Server", version="0.1"), + database=DatabaseConfig( + main_database_name="mock_database", + main_collection_name="mock_collection", + url=AnyUrl("mongodb://localhost:28017"), + timeout=1000) +) + + +def http_get(route: str = "") -> BaseHTTPResponse: + """ + An auxiliary function to make a GET request using :func:`urllib.request`. + + Args: + route: + The desired route (excluding the root URL) which can include a query string as well. + + Returns: + The response from the GET request. + """ + return request("GET", urljoin(test_app_config.api_server.url.unicode_string(), route)) + + +def assert_equal(test, expected) -> None: + """ + An auxiliary function to assert the equality of two objects using the ``==`` operator. In case an input is a list or + a tuple, it will be first converted to a set so that the order of items there in does not affect the assertion + outcome. + + Warning: + In case of a list or tuple of items as inputs, do not use this function if the order of items matters. + + Args: + test: + The object to be tested. + expected: + The object to test against. + """ + + def _setify(obj: Any) -> Any: + """ + An auxiliary function to convert an object to a set if it is a tuple or a list. + """ + return set(obj) if isinstance(obj, list | tuple) else obj + + assert _setify(test) == _setify(expected) diff --git a/trolldb/test_utils/mock_mongodb_database.py b/trolldb/test_utils/mock_mongodb_database.py new file mode 100644 index 0000000..316de8c --- /dev/null +++ b/trolldb/test_utils/mock_mongodb_database.py @@ -0,0 +1,113 @@ +from contextlib import contextmanager +from datetime import datetime, timedelta +from random import randint, shuffle +from typing import Any + +from pymongo import MongoClient + +from config.config import DatabaseConfig +from test_utils.common import test_app_config + + +@contextmanager +def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database): + client = None + try: + client = MongoClient(database_config.url.unicode_string(), connectTimeoutMS=database_config.timeout) + yield client + finally: + if client is not None: + client.close() + + +def random_sample(items: list[Any], size=10): + last_index = len(items) - 1 + indices = [randint(0, last_index) for _ in range(size)] + return [items[i] for i in indices] + + +class Time: + min_start_time = datetime(2019, 1, 1, 0, 0, 0) + max_end_time = datetime(2024, 1, 1, 0, 0, 0) + delta_time = int((max_end_time - min_start_time).total_seconds()) + + @staticmethod + def random_interval_secs(max_interval_secs): + return timedelta(seconds=randint(0, max_interval_secs)) + + @staticmethod + def random_start_time(): + return Time.min_start_time + Time.random_interval_secs(Time.delta_time) + + @staticmethod + def random_end_time(start_time: datetime, max_interval_secs: int = 300): + return start_time + Time.random_interval_secs(max_interval_secs) + + +class Document: + def __init__(self, platform_name: str, sensor: str): + self.platform_name = platform_name + self.sensor = sensor + self.start_time = Time.random_start_time() + self.end_time = Time.random_end_time(self.start_time) + + def generate_dataset(self, max_count: int): + dataset = [] + n = randint(1, max_count) + for i in range(n): + txt = f"{self.platform_name}_{self.sensor}_{self.start_time}_{self.end_time}_{i}" + dataset.append({ + "uri": f"/pytroll/{txt}", + "uid": f"{txt}.EXT1", + "path": f"{txt}.EXT1.EXT2" + }) + return dataset + + def like_mongodb_document(self): + return { + "platform_name": self.platform_name, + "sensor": self.sensor, + "start_time": self.start_time, + "end_time": self.end_time, + "dataset": self.generate_dataset(30) + } + + +class TestDatabase: + platform_names = random_sample(["PA", "PB", "PC"]) + sensors = random_sample(["SA", "SB", "SC"]) + + database_names = [test_app_config.database.main_database_name, "another_mock_database"] + collection_names = [test_app_config.database.main_collection_name, "another_mock_collection"] + all_database_names = ["admin", "config", "local", *database_names] + + documents = [] + + @classmethod + def generate_documents(cls, random_shuffle=True) -> list: + documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors)] + if random_shuffle: + shuffle(documents) + return documents + + @classmethod + def reset(cls): + with test_mongodb_context() as client: + for db_name, coll_name in zip(cls.database_names, cls.collection_names): + db = client[db_name] + collection = db[coll_name] + collection.delete_many({}) + collection.insert_one({}) + + @classmethod + def write_mock_date(cls): + with test_mongodb_context() as client: + cls.documents = cls.generate_documents() + collection = client[test_app_config.database.main_database_name][ + test_app_config.database.main_collection_name] + collection.insert_many(cls.documents) + + @classmethod + def prepare(cls): + cls.reset() + cls.write_mock_date() diff --git a/trolldb/test_utils/mock_mongodb_instance.py b/trolldb/test_utils/mock_mongodb_instance.py new file mode 100644 index 0000000..0587056 --- /dev/null +++ b/trolldb/test_utils/mock_mongodb_instance.py @@ -0,0 +1,97 @@ +""" +The module which defines functionalities to run a MongoDB instance which is to be used in the testing environment. +""" +import errno +import subprocess +import sys +import tempfile +import time +from contextlib import contextmanager +from os import path, mkdir +from shutil import rmtree + +from loguru import logger + +from config.config import DatabaseConfig +from test_utils.common import test_app_config + + +class TestMongoInstance: + log_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_log") + storage_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_storage") + port: int = 28017 + process: subprocess.Popen | None = None + + @classmethod + def prepare_dir(cls, directory: str): + cls.remove_dir(directory) + mkdir(directory) + + @classmethod + def remove_dir(cls, directory: str): + if path.exists(directory) and path.isdir(directory): + rmtree(directory) + + @classmethod + def run_subprocess(cls, args: list[str], wait=True): + cls.process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if wait: + outs, errs = cls.process.communicate() + return outs, errs + return None + + @classmethod + def mongodb_exists(cls) -> bool: + outs, errs = cls.run_subprocess(["which", "mongod"]) + if outs and not errs: + return True + return False + + @classmethod + def prepare_dirs(cls) -> None: + cls.prepare_dir(cls.log_dir) + cls.prepare_dir(cls.storage_dir) + + @classmethod + def run_instance(cls): + cls.run_subprocess( + ["mongod", "--dbpath", cls.storage_dir, "--logpath", f"{cls.log_dir}/mongod.log", "--port", f"{cls.port}"] + , wait=False) + + @classmethod + def shutdown_instance(cls): + cls.process.kill() + for d in [cls.log_dir, cls.storage_dir]: + cls.remove_dir(d) + + +@contextmanager +def mongodb_instance_server_process_context( + database_config: DatabaseConfig = test_app_config.database, + startup_time=2000): + """ + A synchronous context manager to run the MongoDB instance in a separate process (non-blocking) using the + `subprocess `_ package. The main use case is envisaged to be in + testing environments. + + Args: + database_config: + The configuration of the database. + + startup_time: + The overall time that is expected for the MongoDB server instance to run before the database content can be + accessed. + """ + TestMongoInstance.port = database_config.url.hosts()[0]["port"] + TestMongoInstance.prepare_dirs() + + if not TestMongoInstance.mongodb_exists(): + logger.error("`mongod` is not available!") + sys.exit(errno.EIO) + + try: + TestMongoInstance.run_instance() + time.sleep(startup_time / 1000) + yield + finally: + TestMongoInstance.shutdown_instance() From f92b25ceb8b1abecfaf3125722fd1797d914e813 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 10:17:43 +0200 Subject: [PATCH 03/97] Commit --- docs/source/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 9e0c6e7..7091703 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,10 +2,10 @@ Welcome to Pytroll documentation! =========================================== .. toctree:: - :maxdepth: 2 - :caption: Contents: + :maxdepth: 2 + :caption: Contents: - modules + modules From b4e97842b111f7004e219d8f5406031842506aac Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 11:13:07 +0200 Subject: [PATCH 04/97] Commit --- docs/Makefile | 20 -------------------- docs/requirements.txt | 2 ++ docs/source/conf.py | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 26 deletions(-) delete mode 100644 docs/Makefile create mode 100644 docs/requirements.txt diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d0c3cbf..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..d5476d8 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx==7.2.6 +sphinx-rtd-theme==2.0.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index d298115..1725acb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,6 +14,7 @@ import sys sys.path.insert(0, os.path.abspath('../../trolldb')) +sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- @@ -33,6 +34,10 @@ 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', + 'sphinx.ext.duration', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -47,6 +52,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # +html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, @@ -54,11 +60,11 @@ # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] -autodoc_default_options = { - 'member-order': 'bysource', - 'special-members': '__init__', - 'undoc-members': True, - 'exclude-members': '__weakref__' -} +# autodoc_default_options = { +# 'member-order': 'bysource', +# 'special-members': '__init__', +# 'undoc-members': True, +# 'exclude-members': '__weakref__' +# } root_doc = "index" From 3d81e5de067a3b2a6a10aa2e8e646dd13adffaaa Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 11:21:45 +0200 Subject: [PATCH 05/97] Fix Sphinx --- .readthedocs.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 16d1c72..0e20fea 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -18,6 +18,6 @@ sphinx: # Optional but recommended, declare the Python requirements required # to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt +python: + install: + - requirements: docs/requirements.txt From 5622ad5858b29f1679a77cc28c2b88d6a593172d Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 12:11:57 +0200 Subject: [PATCH 06/97] Fix Sphinx --- .gitignore | 2 ++ __init__.py | 8 -------- docs/source/conf.py | 17 ++++++++--------- trolldb/api/__init__.py | 11 ----------- 4 files changed, 10 insertions(+), 28 deletions(-) delete mode 100644 __init__.py diff --git a/.gitignore b/.gitignore index a45834e..5a0f0d9 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,8 @@ coverage.xml docs/build/ *.rst !index.rst +*.doctree +*.pickle # the actual config file [HAS TO BE ALWAYS EXCLUDED!] config.yaml diff --git a/__init__.py b/__init__.py deleted file mode 100644 index d7e5058..0000000 --- a/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -__author__ = "Martin Raspaud, Pouria Khalaj, Esben S. Nielsen, Adam Dybbroe, Kristian Rune Larsen" -__copyright__ = "Copyright (C) 2012, 2014, 2015, 2024 Martin Raspaud, Pouria Khalaj, Esben S. Nielsen, Adam Dybbroe, Kristian Rune Larsen" - -__license__ = "GPLv3" -__version__ = "0.1.0" -__maintainer__ = "Martin Raspaud" -__email__ = "martin.raspaud@smhi.se" -__status__ = "Production" diff --git a/docs/source/conf.py b/docs/source/conf.py index 1725acb..cfdb580 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,8 +13,9 @@ import os import sys +from sphinx.ext import apidoc + sys.path.insert(0, os.path.abspath('../../trolldb')) -sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- @@ -45,7 +46,8 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["*test*"] +exclude_patterns = ["*tests/*"] +include_patterns = ["**"] # -- Options for HTML output ------------------------------------------------- @@ -54,17 +56,14 @@ # html_theme = 'sphinx_rtd_theme' - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] -# autodoc_default_options = { -# 'member-order': 'bysource', -# 'special-members': '__init__', -# 'undoc-members': True, -# 'exclude-members': '__weakref__' -# } root_doc = "index" + +output_dir = os.path.join('.') +module_dir = os.path.abspath('../../trolldb') +apidoc.main(['-f', '-o', output_dir, module_dir]) diff --git a/trolldb/api/__init__.py b/trolldb/api/__init__.py index 8604270..e69de29 100644 --- a/trolldb/api/__init__.py +++ b/trolldb/api/__init__.py @@ -1,11 +0,0 @@ -""" -Note: - The following applies to the :package:`trolldb.api` and all its subpackages/modules. - - To avoid double documentations and inconsistencies, only non-FastAPI components are documented via the docstrings. - For the documentation related to the FastAPI components, check out the auto-generated documentation by FastAPI. - Assuming that the API server is running on ``_ (example) the auto-generated documentation can - be accessed via either ``_ or ``_. - - Read more at `FastAPI automatics docs `_. -""" From 5109c96e7592a884a059c1ff1863fb63ee6f0b68 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 12:19:45 +0200 Subject: [PATCH 07/97] Fix Sphinx --- docs/requirements.txt | 4 ++++ trolldb/__init__.py | 0 2 files changed, 4 insertions(+) create mode 100644 trolldb/__init__.py diff --git a/docs/requirements.txt b/docs/requirements.txt index d5476d8..9c504dc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,6 @@ sphinx==7.2.6 sphinx-rtd-theme==2.0.0 +motor==3.4.0 +pydantic==2.6.4 +fastapi==0.110.1 +uvicorn==0.29.0 diff --git a/trolldb/__init__.py b/trolldb/__init__.py new file mode 100644 index 0000000..e69de29 From 621cb6b9414ba90b86aa278b2cf3ccfc8ca55860 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 13:03:54 +0200 Subject: [PATCH 08/97] Fix Sphinx --- docs/source/conf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index cfdb580..794f85f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,7 +15,10 @@ from sphinx.ext import apidoc -sys.path.insert(0, os.path.abspath('../../trolldb')) +sys.path.insert(0, os.path.abspath('../../')) +sys.path.append(os.path.abspath(os.path.dirname(__file__))) +for x in os.walk('../../trolldb'): + sys.path.append(x[0]) # -- Project information ----------------------------------------------------- @@ -66,4 +69,4 @@ output_dir = os.path.join('.') module_dir = os.path.abspath('../../trolldb') -apidoc.main(['-f', '-o', output_dir, module_dir]) +apidoc.main(['-q', '-f', '-o', output_dir, module_dir, *include_patterns]) From ae934350097e6111f02e77f2d4fd8e6dd13c4c3e Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 13:07:31 +0200 Subject: [PATCH 09/97] Fix Sphinx --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 9c504dc..8e858dc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,3 +4,4 @@ motor==3.4.0 pydantic==2.6.4 fastapi==0.110.1 uvicorn==0.29.0 +loguru==0.7.2 From d15b0e80fe5f477173096f927dbe0c2b7ca9d760 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 13:10:35 +0200 Subject: [PATCH 10/97] Fix Sphinx --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 8e858dc..d808c6d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,3 +5,4 @@ pydantic==2.6.4 fastapi==0.110.1 uvicorn==0.29.0 loguru==0.7.2 +yaml==0.2.5 From 4baba93531d4a99aeafbc370377ea8f622b75c2b Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 6 May 2024 13:13:32 +0200 Subject: [PATCH 11/97] Fix Sphinx --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index d808c6d..a89bb8e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,4 +5,4 @@ pydantic==2.6.4 fastapi==0.110.1 uvicorn==0.29.0 loguru==0.7.2 -yaml==0.2.5 +pyyaml==6.0.1 From 04b9fa12a8368e206220c8d4b23e37ece3b32b18 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 13 May 2024 11:34:51 +0200 Subject: [PATCH 12/97] Refactor error handling. New functionality has been added to the MongoDB class. As a result, the errors are now raised there instead of the API routes. TODO: Documentation and more tests. --- trolldb/api/api.py | 12 +- trolldb/api/errors/errors.py | 107 --------- trolldb/api/routes/common.py | 51 ++--- trolldb/api/routes/databases.py | 37 ++- trolldb/api/routes/datetime_.py | 15 +- trolldb/api/routes/platforms.py | 8 +- trolldb/api/routes/queries.py | 15 +- trolldb/api/routes/sensors.py | 8 +- trolldb/api/tests/conftest.py | 4 +- trolldb/api/tests/test_api.py | 2 +- trolldb/database/errors.py | 73 ++++++ trolldb/database/mongodb.py | 212 +++++++++++++----- trolldb/database/piplines.py | 24 +- trolldb/database/tests/conftest.py | 6 +- trolldb/database/tests/test_mongodb.py | 28 +-- trolldb/{api => }/errors/__init__.py | 0 trolldb/errors/errors.py | 108 +++++++++ ...ongodb_database.py => mongodb_database.py} | 0 ...ongodb_instance.py => mongodb_instance.py} | 0 19 files changed, 430 insertions(+), 280 deletions(-) delete mode 100644 trolldb/api/errors/errors.py create mode 100644 trolldb/database/errors.py rename trolldb/{api => }/errors/__init__.py (100%) create mode 100644 trolldb/errors/errors.py rename trolldb/test_utils/{mock_mongodb_database.py => mongodb_database.py} (100%) rename trolldb/test_utils/{mock_mongodb_instance.py => mongodb_instance.py} (100%) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index dc95730..87fcd5a 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -24,12 +24,14 @@ from multiprocessing import Process import uvicorn -from fastapi import FastAPI +from fastapi import FastAPI, status +from fastapi.responses import PlainTextResponse from pydantic import FilePath, validate_call from api.routes import api_router from config.config import AppConfig, parse, Timeout from database.mongodb import mongodb_context +from errors.errors import ResponseError @validate_call @@ -58,6 +60,14 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: app = FastAPI(**(config.api_server._asdict() | kwargs)) app.include_router(api_router) + @app.exception_handler(ResponseError) + async def unicorn_exception_handler(_, exc: ResponseError): + status_code, message = exc.get_error_info() + return PlainTextResponse( + status_code=status_code if status_code else status.HTTP_500_INTERNAL_SERVER_ERROR, + content=message if message else "Generic Error [This is not okay, check why we have the generic error!]", + ) + async def _serve(): """ An auxiliary coroutine to be used in the asynchronous execution of the FastAPI application. diff --git a/trolldb/api/errors/errors.py b/trolldb/api/errors/errors.py deleted file mode 100644 index 7a46d8e..0000000 --- a/trolldb/api/errors/errors.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -The module which defines error responses that will be returned by the API. -""" - -from collections import OrderedDict -from typing import Self - -from fastapi import status -from fastapi.responses import PlainTextResponse - - -class FailureResponse: - descriptor_delimiter = " |OR| " - defaultResponseClass = PlainTextResponse - - def __init__(self, args_dict: dict): - self.__dict = OrderedDict(args_dict) - - def __or__(self, other: Self): - buff = OrderedDict(self.__dict) - for key, value in other.__dict.items(): - buff[key] = FailureResponse.listify(buff.get(key, [])) - buff[key].extend(FailureResponse.listify(value)) - return FailureResponse(buff) - - def __str__(self): - return str(self.__dict) - - def fastapi_response(self, status_code: int | None = None): - if status_code is None and len(self.__dict) > 1: - raise ValueError("In case of multiple response status codes, please provide one.") - status_code, content = [(k, v) for k, v in self.__dict.items()][0] - try: - return FailureResponse.defaultResponseClass( - content=FailureResponse.stringify(content), - status_code=status_code) - except KeyError: - raise KeyError(f"No default response found for the given status code: {status_code}") - - @property - def fastapi_descriptor(self): - return {k: {"description": FailureResponse.stringify(v)} for k, v in self.__dict.items()} - - @staticmethod - def listify(item: str | list[str]) -> list[str]: - return item if isinstance(item, list) else [item] - - @staticmethod - def stringify(item: str | list[str]) -> str: - return FailureResponse.descriptor_delimiter.join(FailureResponse.listify(item)) - - -class BaseFailureResponses: - - @classmethod - def fields(cls): - return {k: v for k, v in cls.__dict__.items() if isinstance(v, FailureResponse)} - - @classmethod - def union(cls): - buff = FailureResponse({}) - for k, v in cls.fields().items(): - buff |= v - return buff - - -class CollectionFail(BaseFailureResponses): - NOT_FOUND = FailureResponse({ - status.HTTP_404_NOT_FOUND: - "Collection name does not exist." - }) - - WRONG_TYPE = FailureResponse({ - status.HTTP_422_UNPROCESSABLE_ENTITY: - "Collection name must be either a string or None; or both database name and collection name must be None." - }) - - -class DatabaseFail(BaseFailureResponses): - NOT_FOUND = FailureResponse({ - status.HTTP_404_NOT_FOUND: - "Database name does not exist." - }) - - WRONG_TYPE = FailureResponse({ - status.HTTP_422_UNPROCESSABLE_ENTITY: - "Database name must be either a string or None." - }) - - -class DocumentsFail(BaseFailureResponses): - NOT_FOUND = FailureResponse({ - status.HTTP_404_NOT_FOUND: - "Could not find any document with the given object id." - }) - - -Database_Collection_Fail = DatabaseFail | CollectionFail -Database_Collection_Document_Fail = DatabaseFail | CollectionFail | DocumentsFail - -database_collection_fail_descriptor = ( - DatabaseFail.union() | CollectionFail.union() -).fastapi_descriptor - -database_collection_document_fail_descriptor = ( - DatabaseFail.union() | CollectionFail.union() | DocumentsFail.union() -).fastapi_descriptor diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py index 92867bb..a7fe0ff 100644 --- a/trolldb/api/routes/common.py +++ b/trolldb/api/routes/common.py @@ -7,7 +7,6 @@ from fastapi import Response, Query, Depends from motor.motor_asyncio import AsyncIOMotorCollection, AsyncIOMotorDatabase -from api.errors.errors import CollectionFail, DatabaseFail, Database_Collection_Fail, FailureResponse from database.mongodb import MongoDB exclude_defaults_query = Query( @@ -17,7 +16,7 @@ "`trolldb.database.mongodb.MongoDB.default_database_names` for more information.") -async def check_database(database_name: str | None = None) -> Response | AsyncIOMotorDatabase: +async def check_database(database_name: str | None = None) -> AsyncIOMotorDatabase: """ A dependency for route handlers to check for the existence of a database given its name. @@ -29,28 +28,15 @@ async def check_database(database_name: str | None = None) -> Response | AsyncIO Returns: -- The database object if it exists. - -- :obj:`api.errors.errors.DatabaseFail.NOT_FOUND`, if the database does not exist. - - -- :obj:`api.errors.errors.DatabaseFail.WRONG_TYPE`, if the type of the database name is not ``str`` or - ``None``. + -- Raises a :class:`~trolldb.errors.errors.ResponseError` otherwise. Check + :func:`~trolldb.database.mongodb.MongoDB.get_database` for more information. """ - - match database_name: - case None: - return MongoDB.main_database() - - case str(): - if database_name in await MongoDB.client().list_database_names(): - return MongoDB.client()[database_name] - return DatabaseFail.NOT_FOUND.fastapi_response() - - case _: - return DatabaseFail.WRONG_TYPE.fastapi_response() + return await MongoDB.get_database(database_name) async def check_collection( database_name: str | None = None, - collection_name: str | None = None) -> Response | AsyncIOMotorCollection: + collection_name: str | None = None) -> AsyncIOMotorCollection: """ A dependency for route handlers to check for the existence of a collection given its name and the name of the database it resides in. It first checks for the existence of the database using @@ -78,21 +64,7 @@ async def check_collection( is ``None``; or if the type of ``collection_name`` is not ``str``. """ - res = await check_database(database_name) - if isinstance(res, Response): - return res - - match database_name, collection_name: - case None, None: - return MongoDB.main_collection() - - case str(), str(): - if collection_name in await MongoDB.client()[database_name].list_collection_names(): - return MongoDB.client()[database_name][collection_name] - return CollectionFail.NOT_FOUND.fastapi_response() - - case _: - return CollectionFail.WRONG_TYPE.fastapi_response() + return await MongoDB.get_collection(database_name, collection_name) async def get_distinct_items_in_collection( @@ -124,5 +96,12 @@ async def get_distinct_items_in_collection( return await res_coll.distinct(field_name) -CheckCollectionDependency = Annotated[FailureResponse | AsyncIOMotorCollection, Depends(check_collection)] -CheckDataBaseDependency = Annotated[FailureResponse | AsyncIOMotorDatabase, Depends(check_database)] +CheckCollectionDependency = Annotated[AsyncIOMotorCollection, Depends(check_collection)] +""" +Type annotation for the FastAPI dependency injection of checking a collection (function). +""" + +CheckDataBaseDependency = Annotated[AsyncIOMotorDatabase, Depends(check_database)] +""" +Type annotation for the FastAPI dependency injection of checking a database (function). +""" diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py index c368e40..09015ac 100644 --- a/trolldb/api/routes/databases.py +++ b/trolldb/api/routes/databases.py @@ -5,18 +5,18 @@ For more information on the API server, see the automatically generated documentation by FastAPI. """ -from fastapi import APIRouter, Response +from fastapi import APIRouter from pymongo.collection import _DocumentType -from api.errors.errors import ( +from api.routes.common import ( + exclude_defaults_query, CheckCollectionDependency, CheckDataBaseDependency) +from config.config import MongoObjectId +from database.errors import ( DatabaseFail, DocumentsFail, database_collection_fail_descriptor, database_collection_document_fail_descriptor) -from api.routes.common import ( - exclude_defaults_query, CheckCollectionDependency, CheckDataBaseDependency) -from config.config import MongoObjectId -from database.mongodb import MongoDB +from database.mongodb import MongoDB, get_ids router = APIRouter() @@ -25,7 +25,7 @@ response_model=list[str], summary="Gets the list of all database names") async def database_names(exclude_defaults: bool = exclude_defaults_query) -> list[str]: - db_names = await MongoDB.client().list_database_names() + db_names = await MongoDB.list_database_names() if not exclude_defaults: return db_names @@ -37,33 +37,24 @@ async def database_names(exclude_defaults: bool = exclude_defaults_query) -> lis response_model=list[str], responses=DatabaseFail.union().fastapi_descriptor, summary="Gets the list of all collection names for the given database name") -async def collection_names(res_db: CheckDataBaseDependency) -> Response | list[str]: - if isinstance(res_db, Response): - return res_db - - return await res_db.list_collection_names() +async def collection_names(db: CheckDataBaseDependency) -> list[str]: + return await db.list_collection_names() @router.get("/{database_name}/{collection_name}", response_model=list[str], responses=database_collection_fail_descriptor, summary="Gets the object ids of all documents for the given database and collection name") -async def documents(res_coll: CheckCollectionDependency) -> Response | list[str]: - if isinstance(res_coll, Response): - return res_coll - - return await MongoDB.get_ids(res_coll.find({})) +async def documents(collection: CheckCollectionDependency) -> list[str]: + return await get_ids(collection.find({})) @router.get("/{database_name}/{collection_name}/{_id}", response_model=_DocumentType, responses=database_collection_document_fail_descriptor, summary="Gets the document content in json format given its object id, database, and collection name") -async def document_by_id(res_coll: CheckCollectionDependency, _id: MongoObjectId) -> Response | _DocumentType: - if isinstance(res_coll, Response): - return res_coll - - if document := await res_coll.find_one({"_id": _id}): +async def document_by_id(collection: CheckCollectionDependency, _id: MongoObjectId) -> _DocumentType: + if document := await collection.find_one({"_id": _id}): return dict(document) | {"_id": str(_id)} - return DocumentsFail.NOT_FOUND.fastapi_response() + raise DocumentsFail.NotFound diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index 13768a0..995d8ea 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -8,12 +8,12 @@ from datetime import datetime from typing import TypedDict -from fastapi import APIRouter, Response +from fastapi import APIRouter from pydantic import BaseModel -from api.errors.errors import database_collection_fail_descriptor from api.routes.common import CheckCollectionDependency -from database.mongodb import MongoDB +from database.errors import database_collection_fail_descriptor +from database.mongodb import get_id class TimeModel(TypedDict): @@ -38,11 +38,8 @@ class ResponseModel(BaseModel): response_model=ResponseModel, responses=database_collection_fail_descriptor, summary="Gets the the minimum and maximum values for the start and end times") -async def datetime(res_coll: CheckCollectionDependency) -> Response | ResponseModel: - if isinstance(res_coll, Response): - return res_coll - - agg_result = await res_coll.aggregate([{ +async def datetime(collection: CheckCollectionDependency) -> ResponseModel: + agg_result = await collection.aggregate([{ "$group": { "_id": None, "min_start_time": {"$min": "$start_time"}, @@ -52,7 +49,7 @@ async def datetime(res_coll: CheckCollectionDependency) -> Response | ResponseMo }}]).next() def _aux(query): - return MongoDB.get_id(res_coll.find_one(query)) + return get_id(collection.find_one(query)) return ResponseModel( start_time=TimeEntry( diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py index 47c62e4..0026214 100644 --- a/trolldb/api/routes/platforms.py +++ b/trolldb/api/routes/platforms.py @@ -5,10 +5,10 @@ For more information on the API server, see the automatically generated documentation by FastAPI. """ -from fastapi import APIRouter, Response +from fastapi import APIRouter -from api.errors.errors import database_collection_fail_descriptor from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency +from database.errors import database_collection_fail_descriptor router = APIRouter() @@ -17,5 +17,5 @@ response_model=list[str], responses=database_collection_fail_descriptor, summary="Gets the list of all platform names") -async def platform_names(res_coll: CheckCollectionDependency) -> Response | list[str]: - return await get_distinct_items_in_collection(res_coll, "platform_name") +async def platform_names(collection: CheckCollectionDependency) -> list[str]: + return await get_distinct_items_in_collection(collection, "platform_name") diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index bad9a21..96afe55 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -7,11 +7,11 @@ import datetime -from fastapi import APIRouter, Query, Response +from fastapi import APIRouter, Query -from api.errors.errors import database_collection_fail_descriptor from api.routes.common import CheckCollectionDependency -from database.mongodb import MongoDB +from database.errors import database_collection_fail_descriptor +from database.mongodb import get_ids from database.piplines import PipelineAttribute, Pipelines router = APIRouter() @@ -22,14 +22,11 @@ responses=database_collection_fail_descriptor, summary="Gets the database UUIDs of the documents that match specifications determined by the query string") async def queries( - res_coll: CheckCollectionDependency, + collection: CheckCollectionDependency, platform: list[str] = Query(None), sensor: list[str] = Query(None), time_min: datetime.datetime = Query(None), - time_max: datetime.datetime = Query(None)) -> Response | list[str]: - if isinstance(res_coll, Response): - return res_coll - + time_max: datetime.datetime = Query(None)) -> list[str]: pipelines = Pipelines() if platform: @@ -44,4 +41,4 @@ async def queries( pipelines += ((start_time >= time_min) | (start_time <= time_max) | (end_time >= time_min) | (end_time <= time_max)) - return await MongoDB.get_ids(res_coll.aggregate(pipelines)) + return await get_ids(collection.aggregate(pipelines)) diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py index e73268c..7dbe466 100644 --- a/trolldb/api/routes/sensors.py +++ b/trolldb/api/routes/sensors.py @@ -5,10 +5,10 @@ For more information on the API server, see the automatically generated documentation by FastAPI. """ -from fastapi import APIRouter, Response +from fastapi import APIRouter -from api.errors.errors import database_collection_fail_descriptor from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency +from database.errors import database_collection_fail_descriptor router = APIRouter() @@ -17,5 +17,5 @@ response_model=list[str], responses=database_collection_fail_descriptor, summary="Gets the list of all sensor names") -async def sensor_names(res_coll: CheckCollectionDependency) -> Response | list[str]: - return await get_distinct_items_in_collection(res_coll, "sensor") +async def sensor_names(collection: CheckCollectionDependency) -> list[str]: + return await get_distinct_items_in_collection(collection, "sensor") diff --git a/trolldb/api/tests/conftest.py b/trolldb/api/tests/conftest.py index 496e777..916cb75 100644 --- a/trolldb/api/tests/conftest.py +++ b/trolldb/api/tests/conftest.py @@ -1,9 +1,9 @@ import pytest from api.api import server_process_context -from test_utils.mock_mongodb_instance import mongodb_instance_server_process_context +from test_utils.mongodb_instance import mongodb_instance_server_process_context from test_utils.common import test_app_config -from test_utils.mock_mongodb_database import TestDatabase +from test_utils.mongodb_database import TestDatabase @pytest.fixture(scope="session") diff --git a/trolldb/api/tests/test_api.py b/trolldb/api/tests/test_api.py index 1796a35..fa76fde 100644 --- a/trolldb/api/tests/test_api.py +++ b/trolldb/api/tests/test_api.py @@ -1,7 +1,7 @@ from fastapi import status from test_utils.common import assert_equal, http_get -from test_utils.mock_mongodb_database import test_mongodb_context, TestDatabase +from test_utils.mongodb_database import test_mongodb_context, TestDatabase def test_root(): diff --git a/trolldb/database/errors.py b/trolldb/database/errors.py new file mode 100644 index 0000000..d02750c --- /dev/null +++ b/trolldb/database/errors.py @@ -0,0 +1,73 @@ +from fastapi import status + +from errors.errors import ResponsesErrorGroup, ResponseError + + +class ClientFail(ResponsesErrorGroup): + CloseNotAllowedError = ResponseError({ + status.HTTP_405_METHOD_NOT_ALLOWED: + "Calling `close()` on a client which has not been initialized is not allowed!" + }) + + ReinitializeConfigError = ResponseError({ + status.HTTP_405_METHOD_NOT_ALLOWED: + "The client is already initialized with a different database configuration!" + }) + + AlreadyOpenError = ResponseError({ + status.HTTP_100_CONTINUE: + "The client has been already initialized with the same configuration." + }) + + InconsistencyError = ResponseError({ + status.HTTP_405_METHOD_NOT_ALLOWED: + "Something must have been wrong as we are in an inconsistent state. " + "The internal database configuration is not empty and is the same as what we just " + "received but the client is `None` or has been already closed!" + }) + + ConnectionError = ResponseError({ + status.HTTP_400_BAD_REQUEST: + "Could not connect to the database with URL." + }) + + +class CollectionFail(ResponsesErrorGroup): + NotFoundError = ResponseError({ + status.HTTP_404_NOT_FOUND: + "Could not find the given collection name inside the specified database." + }) + + WrongTypeError = ResponseError({ + status.HTTP_422_UNPROCESSABLE_ENTITY: + "Both the Database and collection name must be `None` if one of them is `None`." + }) + + +class DatabaseFail(ResponsesErrorGroup): + NotFoundError = ResponseError({ + status.HTTP_404_NOT_FOUND: + "Could not find the given database name." + }) + + WrongTypeError = ResponseError({ + status.HTTP_422_UNPROCESSABLE_ENTITY: + "Database name must be either of type `str` or `None.`" + }) + + +class DocumentsFail(ResponsesErrorGroup): + NotFound = ResponseError({ + status.HTTP_404_NOT_FOUND: + "Could not find any document with the given object id." + }) + + +Database_Collection_Fail = DatabaseFail | CollectionFail +Database_Collection_Document_Fail = DatabaseFail | CollectionFail | DocumentsFail +database_collection_fail_descriptor = ( + DatabaseFail.union() | CollectionFail.union() +).fastapi_descriptor +database_collection_document_fail_descriptor = ( + DatabaseFail.union() | CollectionFail.union() | DocumentsFail.union() +).fastapi_descriptor diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 1484821..0eac928 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -4,11 +4,9 @@ """ import errno -import sys from contextlib import asynccontextmanager -from typing import Any, Coroutine +from typing import Any, AsyncGenerator, Coroutine, TypeVar -from loguru import logger from motor.motor_asyncio import ( AsyncIOMotorClient, AsyncIOMotorDatabase, @@ -16,20 +14,72 @@ AsyncIOMotorCommandCursor, AsyncIOMotorCursor ) -from pydantic import validate_call +from pydantic import validate_call, BaseModel from pymongo.collection import _DocumentType from pymongo.errors import ( ConnectionFailure, ServerSelectionTimeoutError) from config.config import DatabaseConfig +from database.errors import CollectionFail, DatabaseFail, ClientFail +from errors.errors import ResponseError + +T = TypeVar("T") +CoroutineLike = Coroutine[Any, Any, T] +CoroutineDocument = CoroutineLike[_DocumentType | None] +CoroutineStrList = CoroutineLike[list[str]] + + +class DatabaseName(BaseModel): + name: str | None + + +class CollectionName(BaseModel): + name: str | None + + +async def get_id(doc: CoroutineDocument) -> str: + """ + Retrieves the ID of a document as a simple flat string. + + Note: + The rationale behind this method is as follows. In MongoDB, each document has a unique ID which is of type + :class:`~bson.objectid.ObjectId`. This is not suitable for purposes when a simple string is needed, hence + the need for this method. + + Args: + doc: + A MongoDB document in the coroutine form. This could be e.g. the result of applying the standard + ``find_one`` method from MongoDB on a collection given a ``filter``. + + Returns: + The ID of a document as a simple string. For example, when applied on a document with + ``_id: ObjectId('000000000000000000000000')``, the method returns ``'000000000000000000000000'``. + """ + return str((await doc)["_id"]) + + +async def get_ids(docs: AsyncIOMotorCommandCursor | AsyncIOMotorCursor) -> list[str]: + """ + Similar to :func:`~MongoDB.get_id` but for a list of documents. + + Args: + docs: + A list of MongoDB documents as :obj:`~AsyncIOMotorCommandCursor` or :obj:`~AsyncIOMotorCursor`. + This could be e.g. the result of applying the standard ``aggregate`` method from MongoDB on a + collection given a ``pipeline``. + + Returns: + The list of all IDs, each as a simple string. + """ + return [str(doc["_id"]) async for doc in docs] class MongoDB: """ A wrapper class around the `motor async driver `_ for Mongo DB with - convenience methods tailored to our specific needs. As such, most of the methods return coroutines whose results - need to be awaited. + convenience methods tailored to our specific needs. As such, the :func:`~MongoDB.initialize()`` method returns a + coroutine which needs to be awaited. Note: This class is not meant to be instantiated! That's why all the methods in this class are decorated with @@ -48,6 +98,7 @@ class MongoDB: """ __client: AsyncIOMotorClient | None = None + __database_config: DatabaseConfig | None = None __main_collection: AsyncIOMotorCollection = None __main_database: AsyncIOMotorDatabase = None @@ -65,48 +116,70 @@ async def initialize(cls, database_config: DatabaseConfig): database_config: A named tuple which includes the database configurations. - Raises :obj:`~SystemExit(errno.EIO)`: - If connection is not established (``ConnectionFailure``) or if the attempt times out - (``ServerSelectionTimeoutError``) + Raises ``SystemExit(errno.EIO)``: + - If connection is not established (``ConnectionFailure``) + - If the attempt times out (``ServerSelectionTimeoutError``) + - If one attempts reinitializing the class with new (different) database configurations without calling + :func:`~close()` first. + - If the state is not consistent, i.e. the client is closed or ``None`` but the internal database + configurations still exist and are different from the new ones which have been just provided. - Raises :obj:`~SystemExit(errno.ENODATA)`: + Raises ``SystemExit(errno.ENODATA)``: If either ``database_config.main_database`` or ``database_config.main_collection`` does not exist. Returns: On success ``None``. """ + if cls.__database_config: + if database_config == cls.__database_config: + if cls.__client: + return ClientFail.AlreadyOpenError.log_warning() + ClientFail.InconsistencyError.raise_error_log_and_exit(errno.EIO) + else: + ClientFail.ReinitializeConfigError.raise_error_log_and_exit(errno.EIO) + # This only makes the reference and does not establish an actual connection until the first attempt is made # to access the database. cls.__client = AsyncIOMotorClient( database_config.url.unicode_string(), serverSelectionTimeoutMS=database_config.timeout) + __database_names = [] try: # Here we attempt to access the database __database_names = await cls.__client.list_database_names() except (ConnectionFailure, ServerSelectionTimeoutError): - logger.error(f"Could not connect to the database with URL: {database_config.url.unicode_string()}") - sys.exit(errno.EIO) + ClientFail.ConnectionError.raise_error_log_and_exit( + errno.EIO, {"url": database_config.url.unicode_string()} + ) + + err_extra_information = {"database_name": database_config.main_database_name} if database_config.main_database_name not in __database_names: - logger.error(f"Could not find any database with the given name: {database_config.main_database_name}") - sys.exit(errno.ENODATA) + DatabaseFail.NotFoundError.raise_error_log_and_exit(errno.ENODATA, err_extra_information) cls.__main_database = cls.__client.get_database(database_config.main_database_name) + err_extra_information |= {"collection_name": database_config.main_collection_name} + if database_config.main_collection_name not in await cls.__main_database.list_collection_names(): - logger.error(f"Could not find any collection in database `{database_config.main_database_name}` with the " - f"given name: {database_config.main_database_name}") - sys.exit(errno.ENODATA) + CollectionFail.NotFoundError.raise_error_log_and_exit(errno.ENODATA, err_extra_information) + cls.__main_collection = cls.__main_database.get_collection(database_config.main_collection_name) @classmethod - def client(cls) -> AsyncIOMotorClient: + def close(cls) -> None: """ - Returns: - The actual motor client so that it can be used to perform database CRUD operations. + Closes the motor client. """ - return cls.__client + if cls.__client: + cls.__database_config = None + return cls.__client.close() + ClientFail.CloseNotAllowedError.raise_error_log_and_exit(errno.EIO) + + @classmethod + def list_database_names(cls) -> CoroutineStrList: + return cls.__client.list_database_names() @classmethod def main_collection(cls) -> AsyncIOMotorCollection: @@ -130,61 +203,83 @@ def main_database(cls) -> AsyncIOMotorDatabase: """ return cls.__main_database - @staticmethod - async def get_id(doc: Coroutine[Any, Any, _DocumentType | None] | _DocumentType) -> str: + @classmethod + async def get_collection( + cls, + database_name: str, + collection_name: str) -> AsyncIOMotorCollection | ResponseError: """ - Retrieves the ID of a document as a simple flat string. - - Note: - The rationale behind this method is as follows. In MongoDB, each document has a unique ID which is of type - :class:`~bson.objectid.ObjectId`. This is not suitable for purposes when a simple string is needed, hence - the need for this method. + Gets the collection object given its name and the database name in which it resides. Args: - doc: - A MongoDB document as a :class:`_DocumentType` object or in the coroutine form. The latter could be e.g. - the result of applying the standard ``find_one`` method from MongoDB on a collection given a ``filter``. + database_name: + The name of the parent database which includes the collection. + collection_name: + The name of the collection which resides inside the parent database labelled by ``database_name``. + + Raises: + ``ValidationError``: + If input args are invalid according to the pydantic. + + ``KeyError``: + If the database name exists, but it does not include any collection with the given name. + + ``TypeError``: + If only one of the database or collection names are ``None``. + + ``_``: + This method relies on :func:`get_database` to check for the existence of the database which can raise + exceptions. Check its documentation for more information. Returns: - The ID of a document as a simple string. For example, when applied on a document with - ``_id: ObjectId('000000000000000000000000')``, the method returns ``'000000000000000000000000'``. + The database object. In case of ``None`` for both the database name and collection name, the main collection + will be returned. """ - match doc: - case _DocumentType(): - return str(doc["_id"]) - case Coroutine(): - return str((await doc)["_id"]) + + database_name = DatabaseName(name=database_name).name + collection_name = CollectionName(name=collection_name).name + + match database_name, collection_name: + case None, None: + return cls.main_collection() + + case str(), str(): + db = await cls.get_database(database_name) + if collection_name in await db.list_collection_names(): + return db[collection_name] + raise CollectionFail.NotFoundError case _: - raise TypeError("The type of `doc` must be either `_DocumentType` or " - "`Coroutine[Any, Any, _DocumentType | None] `.") + raise CollectionFail.WrongTypeError - @staticmethod - async def get_ids(docs: AsyncIOMotorCommandCursor | AsyncIOMotorCursor | list[_DocumentType]) -> list[str]: + @classmethod + async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | ResponseError: """ - Similar to :func:`~MongoDB.get_id` but for a list of documents. + Gets the database object given its name. Args: - docs: - A list of MongoDB documents each as a :class:`DocumentType`, or all as an - :obj:`~AsyncIOMotorCommandCursor`. The latter could be e.g. the result of applying the - standard ``aggregate`` method from MongoDB on a collection given a ``pipeline``. + database_name: + The name of the database to retrieve. + Raises: + ``KeyError``: + If the database name does not exist in the list of database names. Returns: - The list of all IDs, each as a simple string. + The database object. """ - match docs: - case list(): - return [str(doc["_id"]) for doc in docs] - case AsyncIOMotorCommandCursor() | AsyncIOMotorCursor(): - return [str(doc["_id"]) async for doc in docs] + database_name = DatabaseName(name=database_name).name + + match database_name: + case None: + return cls.main_database() + case _ if database_name in await cls.list_database_names(): + return cls.__client[database_name] case _: - raise TypeError("The type of `docs` must be either `list[_DocumentType]` or " - "`AsyncIOMotorCommandCursor`.") + raise DatabaseFail.NotFoundError @asynccontextmanager @validate_call -async def mongodb_context(database_config: DatabaseConfig): +async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: """ An asynchronous context manager to connect to the MongoDB client. It can be either used in production or in testing environments. @@ -197,5 +292,4 @@ async def mongodb_context(database_config: DatabaseConfig): await MongoDB.initialize(database_config) yield finally: - if MongoDB.client() is not None: - MongoDB.client().close() + MongoDB.close() diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 9d39d4b..1c23f53 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -1,15 +1,31 @@ +""" +The module which defines some convenience classes to facilitate the use of aggregation pipelines. +""" + from typing import Any, Self class PipelineDict(dict): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + """ + A subclass of dict which overrides the behaviour of bitwise or ``|`` and bitwise and ``&``. The operators are only + defined for operands of type :class:`PipelineDict`. For each of the aforementioned operators, the result will be a + dictionary with a single key/value pair. The key is either ``$or`` or ``$and`` depending on the operator being used. + The corresponding value is a list with two elements only. The first element of the list is the left operand and the + second element is the right operand. + + Example: + ``` + pd1 = PipelineDict({"number": 2}) + pd2 = PipelineDict({"kind": 1}) + pd3 = pd1 & pd2 + ``` + """ def __or__(self, other: Self): - return PipelineDict(**{"$or": [self, other]}) + return PipelineDict({"$or": [self, other]}) def __and__(self, other: Self): - return PipelineDict(**{"$and": [self, other]}) + return PipelineDict({"$and": [self, other]}) class PipelineAttribute: diff --git a/trolldb/database/tests/conftest.py b/trolldb/database/tests/conftest.py index ab10d98..a646f4b 100644 --- a/trolldb/database/tests/conftest.py +++ b/trolldb/database/tests/conftest.py @@ -1,9 +1,9 @@ import pytest import pytest_asyncio -from test_utils.mock_mongodb_instance import mongodb_instance_server_process_context +from test_utils.mongodb_instance import mongodb_instance_server_process_context from test_utils.common import test_app_config -from test_utils.mock_mongodb_database import TestDatabase +from test_utils.mongodb_database import TestDatabase from trolldb.database.mongodb import mongodb_context @@ -13,7 +13,7 @@ def run_mongodb_server_instance(): yield -@pytest_asyncio.fixture(autouse=True) +@pytest_asyncio.fixture() async def mongodb_fixture(run_mongodb_server_instance): TestDatabase.prepare() async with mongodb_context(test_app_config.database): diff --git a/trolldb/database/tests/test_mongodb.py b/trolldb/database/tests/test_mongodb.py index 60f8234..b74d370 100644 --- a/trolldb/database/tests/test_mongodb.py +++ b/trolldb/database/tests/test_mongodb.py @@ -25,7 +25,7 @@ async def test_connection_timeout_negative(): assert t2 - t1 >= timeout / 1000 -async def test_main_database_negative(): +async def test_main_database_negative(run_mongodb_server_instance): """ Expect to fail when giving an invalid name for the main database, given a valid collection name. """ @@ -39,7 +39,7 @@ async def test_main_database_negative(): assert pytest_wrapped_e.value.code == errno.ENODATA -async def test_main_collection_negative(): +async def test_main_collection_negative(run_mongodb_server_instance): """ Expect to fail when giving an invalid name for the main collection, given a valid database name. """ @@ -53,29 +53,20 @@ async def test_main_collection_negative(): assert pytest_wrapped_e.value.code == errno.ENODATA -async def test_connection_success(): - """ - Expect to establish a connection with the MongoDB instance successfully (with default args). - """ - pass - - -async def test_get_client(): +async def test_get_client(mongodb_fixture): """ This is our way of testing that MongoDB.client() returns the valid client object. Expect: - - Have a MongoDB client which is not `None` - The `close` method can be called on the client and leads to the closure of the client - Further attempts to access the database after closing the client fails. """ - assert MongoDB.client() is not None - MongoDB.client().close() + MongoDB.close() with pytest.raises(InvalidOperation): - await MongoDB.client().list_database_names() + await MongoDB.list_database_names() -async def test_main_collection(): +async def test_main_collection(mongodb_fixture): """ Expect: - The retrieved main collection is not `None` @@ -85,13 +76,14 @@ async def test_main_collection(): assert MongoDB.main_collection() is not None assert MongoDB.main_collection().name == test_app_config.database.main_collection_name assert MongoDB.main_collection() == \ - MongoDB.client()[test_app_config.database.main_database_name][test_app_config.database.main_collection_name] + (await MongoDB.get_database(test_app_config.database.main_database_name))[ + test_app_config.database.main_collection_name] -async def test_main_database(): +async def test_main_database(mongodb_fixture): """ Same as test_main_collection but for the main database. """ assert MongoDB.main_database() is not None assert MongoDB.main_database().name == test_app_config.database.main_database_name - assert MongoDB.main_database() == MongoDB.client()[test_app_config.database.main_database_name] + assert MongoDB.main_database() == await MongoDB.get_database(test_app_config.database.main_database_name) diff --git a/trolldb/api/errors/__init__.py b/trolldb/errors/__init__.py similarity index 100% rename from trolldb/api/errors/__init__.py rename to trolldb/errors/__init__.py diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py new file mode 100644 index 0000000..d351a10 --- /dev/null +++ b/trolldb/errors/errors.py @@ -0,0 +1,108 @@ +""" +The module which defines error responses that will be returned by the API. +""" + +from collections import OrderedDict +from sys import exit +from typing import Literal, Self + +from fastapi import Response +from fastapi.responses import PlainTextResponse +from loguru import logger + +StatusCode = int + + +class ResponseError(Exception): + descriptor_delimiter: str = " |OR| " + defaultResponseClass: Response = PlainTextResponse + + def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) -> None: + self.__dict: OrderedDict = OrderedDict(args_dict) + self.extra_information: dict | None = None + + def __or__(self, other: Self): + buff = OrderedDict(self.__dict) + for key, msg in other.__dict.items(): + self_msg = buff.get(key, None) + buff[key] = ResponseError.__listify(self_msg) if self_msg else [] + buff[key].extend(ResponseError.__listify(msg)) + return ResponseError(buff) + + def __assert_existence_multiple_response_codes( + self, + status_code: StatusCode | None = None) -> (StatusCode, str): + match status_code, len(self.__dict): + case None, n if n > 1: + raise ValueError("In case of multiple response status codes, the status code must be specified.") + case StatusCode(), n if n > 1: + if status_code in self.__dict.keys(): + return status_code, self.__dict[status_code] + raise KeyError(f"Status code {status_code} cannot be found.") + case _, 1: + return [(k, v) for k, v in self.__dict.items()][0] + + def get_error_info( + self, + extra_information: dict | None = None, + status_code: int | None = None) -> (StatusCode, str): + status_code, msg = self.__assert_existence_multiple_response_codes(status_code) + return ( + status_code, + ResponseError.__stringify(msg) + (f" :=> {extra_information}" if extra_information else "") + ) + + def fastapi_response( + self, + extra_information: dict | None = None, + status_code: StatusCode | None = None) -> defaultResponseClass: + try: + msg, _ = self.get_error_info(extra_information, status_code) + return ResponseError.defaultResponseClass(content=msg, status_code=status_code) + except KeyError: + raise KeyError(f"No default response found for the given status code: {status_code}") + + def raise_error_log_and_exit( + self, + exit_code: int = -1, + extra_information: dict | None = None, + status_code: int | None = None) -> None: + msg, _ = self.get_error_info(extra_information, status_code) + logger.error(msg) + exit(exit_code) + + def log_warning( + self, + extra_information: dict | None = None, + status_code: int | None = None): + msg, _ = self.get_error_info(extra_information, status_code) + logger.warning(msg) + + @property + def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], str]]: + return {status: {Literal["description"]: ResponseError.__stringify(msg)} for status, msg in self.__dict.items()} + + @staticmethod + def __listify(item: str | list[str]) -> list[str]: + return item if isinstance(item, list) else [item] + + @staticmethod + def __stringify(item: str | list[str]) -> str: + return ResponseError.descriptor_delimiter.join(ResponseError.__listify(item)) + + +class ResponsesErrorGroup: + + @classmethod + def fields(cls): + return {k: v for k, v in cls.__dict__.items() if isinstance(v, ResponseError)} + + @classmethod + def union(cls): + buff = None + for k, v in cls.fields().items(): + if buff is None: + buff = v + else: + buff |= v + return buff diff --git a/trolldb/test_utils/mock_mongodb_database.py b/trolldb/test_utils/mongodb_database.py similarity index 100% rename from trolldb/test_utils/mock_mongodb_database.py rename to trolldb/test_utils/mongodb_database.py diff --git a/trolldb/test_utils/mock_mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py similarity index 100% rename from trolldb/test_utils/mock_mongodb_instance.py rename to trolldb/test_utils/mongodb_instance.py From 29a3b4556567c71d24a3e231f01bbf6753235730 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 09:48:16 +0200 Subject: [PATCH 13/97] Better errors handling --- docs/source/conf.py | 3 ++ trolldb/api/api.py | 4 +-- trolldb/api/routes/common.py | 4 +-- trolldb/api/routes/databases.py | 16 ++++----- trolldb/api/routes/datetime_.py | 4 +-- trolldb/api/routes/platforms.py | 4 +-- trolldb/api/routes/queries.py | 4 +-- trolldb/api/routes/sensors.py | 4 +-- trolldb/database/errors.py | 49 +++++++++++++++++++++------ trolldb/database/mongodb.py | 29 +++++++++------- trolldb/database/piplines.py | 2 ++ trolldb/errors/errors.py | 59 ++++++++++++++++++++++++--------- 12 files changed, 124 insertions(+), 58 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 794f85f..c68eb3d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,6 +20,9 @@ for x in os.walk('../../trolldb'): sys.path.append(x[0]) +# autodoc_mock_imports = ["motor", "pydantic", "fastapi", "uvicorn", "loguru", "pyyaml"] + + # -- Project information ----------------------------------------------------- project = 'Pytroll-db' diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 87fcd5a..73561b7 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -8,7 +8,7 @@ so that their arguments can be validated using the corresponding type hints, when calling the function at runtime. Note: - The following applies to the :obj:`trolldb.api` package and all its subpackages/modules. + The following applies to the :obj:`api` package and all its subpackages/modules. To avoid redundant documentation and inconsistencies, only non-FastAPI components are documented via the docstrings. For the documentation related to the FastAPI components, check out the auto-generated documentation by FastAPI. @@ -62,7 +62,7 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: @app.exception_handler(ResponseError) async def unicorn_exception_handler(_, exc: ResponseError): - status_code, message = exc.get_error_info() + status_code, message = exc.get_error_details() return PlainTextResponse( status_code=status_code if status_code else status.HTTP_500_INTERNAL_SERVER_ERROR, content=message if message else "Generic Error [This is not okay, check why we have the generic error!]", diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py index a7fe0ff..2a63cd4 100644 --- a/trolldb/api/routes/common.py +++ b/trolldb/api/routes/common.py @@ -58,9 +58,9 @@ async def check_collection( -- A response from :func:`check_database`, if the database does not exist or the type of ``database_name`` is not valid. - -- :obj:`api.errors.errors.CollectionFail.NOT_FOUND`, if the parent database exists but the collection does not. + -- if the parent database exists but the collection does not. - -- :obj:`api.errors.errors.CollectionFail.WRONG_TYPE`, if only one of ``database_name`` or ``collection_name`` + -- if only one of ``database_name`` or ``collection_name`` is ``None``; or if the type of ``collection_name`` is not ``str``. """ diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py index 09015ac..515697b 100644 --- a/trolldb/api/routes/databases.py +++ b/trolldb/api/routes/databases.py @@ -12,10 +12,10 @@ exclude_defaults_query, CheckCollectionDependency, CheckDataBaseDependency) from config.config import MongoObjectId from database.errors import ( - DatabaseFail, - DocumentsFail, - database_collection_fail_descriptor, - database_collection_document_fail_descriptor) + Databases, + Documents, + database_collection_error_descriptor, + database_collection_document_error_descriptor) from database.mongodb import MongoDB, get_ids router = APIRouter() @@ -35,7 +35,7 @@ async def database_names(exclude_defaults: bool = exclude_defaults_query) -> lis @router.get("/{database_name}", response_model=list[str], - responses=DatabaseFail.union().fastapi_descriptor, + responses=Databases.union().fastapi_descriptor, summary="Gets the list of all collection names for the given database name") async def collection_names(db: CheckDataBaseDependency) -> list[str]: return await db.list_collection_names() @@ -43,7 +43,7 @@ async def collection_names(db: CheckDataBaseDependency) -> list[str]: @router.get("/{database_name}/{collection_name}", response_model=list[str], - responses=database_collection_fail_descriptor, + responses=database_collection_error_descriptor, summary="Gets the object ids of all documents for the given database and collection name") async def documents(collection: CheckCollectionDependency) -> list[str]: return await get_ids(collection.find({})) @@ -51,10 +51,10 @@ async def documents(collection: CheckCollectionDependency) -> list[str]: @router.get("/{database_name}/{collection_name}/{_id}", response_model=_DocumentType, - responses=database_collection_document_fail_descriptor, + responses=database_collection_document_error_descriptor, summary="Gets the document content in json format given its object id, database, and collection name") async def document_by_id(collection: CheckCollectionDependency, _id: MongoObjectId) -> _DocumentType: if document := await collection.find_one({"_id": _id}): return dict(document) | {"_id": str(_id)} - raise DocumentsFail.NotFound + raise Documents.NotFound diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index 995d8ea..2aeeeaf 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -12,7 +12,7 @@ from pydantic import BaseModel from api.routes.common import CheckCollectionDependency -from database.errors import database_collection_fail_descriptor +from database.errors import database_collection_error_descriptor from database.mongodb import get_id @@ -36,7 +36,7 @@ class ResponseModel(BaseModel): @router.get("", response_model=ResponseModel, - responses=database_collection_fail_descriptor, + responses=database_collection_error_descriptor, summary="Gets the the minimum and maximum values for the start and end times") async def datetime(collection: CheckCollectionDependency) -> ResponseModel: agg_result = await collection.aggregate([{ diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py index 0026214..f19ed7f 100644 --- a/trolldb/api/routes/platforms.py +++ b/trolldb/api/routes/platforms.py @@ -8,14 +8,14 @@ from fastapi import APIRouter from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency -from database.errors import database_collection_fail_descriptor +from database.errors import database_collection_error_descriptor router = APIRouter() @router.get("", response_model=list[str], - responses=database_collection_fail_descriptor, + responses=database_collection_error_descriptor, summary="Gets the list of all platform names") async def platform_names(collection: CheckCollectionDependency) -> list[str]: return await get_distinct_items_in_collection(collection, "platform_name") diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index 96afe55..95ebb5c 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -10,7 +10,7 @@ from fastapi import APIRouter, Query from api.routes.common import CheckCollectionDependency -from database.errors import database_collection_fail_descriptor +from database.errors import database_collection_error_descriptor from database.mongodb import get_ids from database.piplines import PipelineAttribute, Pipelines @@ -19,7 +19,7 @@ @router.get("", response_model=list[str], - responses=database_collection_fail_descriptor, + responses=database_collection_error_descriptor, summary="Gets the database UUIDs of the documents that match specifications determined by the query string") async def queries( collection: CheckCollectionDependency, diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py index 7dbe466..53c867e 100644 --- a/trolldb/api/routes/sensors.py +++ b/trolldb/api/routes/sensors.py @@ -8,14 +8,14 @@ from fastapi import APIRouter from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency -from database.errors import database_collection_fail_descriptor +from database.errors import database_collection_error_descriptor router = APIRouter() @router.get("", response_model=list[str], - responses=database_collection_fail_descriptor, + responses=database_collection_error_descriptor, summary="Gets the list of all sensor names") async def sensor_names(collection: CheckCollectionDependency) -> list[str]: return await get_distinct_items_in_collection(collection, "sensor") diff --git a/trolldb/database/errors.py b/trolldb/database/errors.py index d02750c..1b88a6a 100644 --- a/trolldb/database/errors.py +++ b/trolldb/database/errors.py @@ -1,9 +1,23 @@ +""" +The modules which defines the error responses that might occur while working with the +MongoDB database. + +Note: + The error responses are grouped into classes, with each class representing the major + category (context) in which the errors occur. As such, the attributes of the top classes + are (expected to be) self-explanatory and require no additional documentation. +""" + from fastapi import status from errors.errors import ResponsesErrorGroup, ResponseError -class ClientFail(ResponsesErrorGroup): +class Client(ResponsesErrorGroup): + """ + Client error responses, e.g. if something goes wrong with initialization or closing the + client. + """ CloseNotAllowedError = ResponseError({ status.HTTP_405_METHOD_NOT_ALLOWED: "Calling `close()` on a client which has not been initialized is not allowed!" @@ -32,7 +46,10 @@ class ClientFail(ResponsesErrorGroup): }) -class CollectionFail(ResponsesErrorGroup): +class Collections(ResponsesErrorGroup): + """ + Collections error responses, e.g. if a requested collection cannot be found. + """ NotFoundError = ResponseError({ status.HTTP_404_NOT_FOUND: "Could not find the given collection name inside the specified database." @@ -44,7 +61,10 @@ class CollectionFail(ResponsesErrorGroup): }) -class DatabaseFail(ResponsesErrorGroup): +class Databases(ResponsesErrorGroup): + """ + Databases error responses, e.g. if a requested database cannot be found. + """ NotFoundError = ResponseError({ status.HTTP_404_NOT_FOUND: "Could not find the given database name." @@ -56,18 +76,27 @@ class DatabaseFail(ResponsesErrorGroup): }) -class DocumentsFail(ResponsesErrorGroup): +class Documents(ResponsesErrorGroup): + """ + Documents error responses, e.g. if a requested document cannot be found. + """ NotFound = ResponseError({ status.HTTP_404_NOT_FOUND: "Could not find any document with the given object id." }) -Database_Collection_Fail = DatabaseFail | CollectionFail -Database_Collection_Document_Fail = DatabaseFail | CollectionFail | DocumentsFail -database_collection_fail_descriptor = ( - DatabaseFail.union() | CollectionFail.union() +database_collection_error_descriptor = ( + Databases.union() | Collections.union() ).fastapi_descriptor -database_collection_document_fail_descriptor = ( - DatabaseFail.union() | CollectionFail.union() | DocumentsFail.union() +""" +A response descriptor for the Fast API routes. This combines all the error messages that might +occur as result of working with databases and collections. See the fast api documentation for TODO. +""" + +database_collection_document_error_descriptor = ( + Databases.union() | Collections.union() | Documents.union() ).fastapi_descriptor +""" +Same as :obj:`database_collection_error_descriptor` but including documents as well. +""" diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 0eac928..0d8c92c 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -21,7 +21,7 @@ ServerSelectionTimeoutError) from config.config import DatabaseConfig -from database.errors import CollectionFail, DatabaseFail, ClientFail +from database.errors import Collections, Databases, Client from errors.errors import ResponseError T = TypeVar("T") @@ -117,13 +117,18 @@ async def initialize(cls, database_config: DatabaseConfig): A named tuple which includes the database configurations. Raises ``SystemExit(errno.EIO)``: + - If connection is not established (``ConnectionFailure``) + - If the attempt times out (``ServerSelectionTimeoutError``) + - If one attempts reinitializing the class with new (different) database configurations without calling - :func:`~close()` first. + :func:`~close()` first. + - If the state is not consistent, i.e. the client is closed or ``None`` but the internal database configurations still exist and are different from the new ones which have been just provided. + Raises ``SystemExit(errno.ENODATA)``: If either ``database_config.main_database`` or ``database_config.main_collection`` does not exist. @@ -134,10 +139,10 @@ async def initialize(cls, database_config: DatabaseConfig): if cls.__database_config: if database_config == cls.__database_config: if cls.__client: - return ClientFail.AlreadyOpenError.log_warning() - ClientFail.InconsistencyError.raise_error_log_and_exit(errno.EIO) + return Client.AlreadyOpenError.log_as_warning() + Client.InconsistencyError.sys_exit_log(errno.EIO) else: - ClientFail.ReinitializeConfigError.raise_error_log_and_exit(errno.EIO) + Client.ReinitializeConfigError.sys_exit_log(errno.EIO) # This only makes the reference and does not establish an actual connection until the first attempt is made # to access the database. @@ -150,20 +155,20 @@ async def initialize(cls, database_config: DatabaseConfig): # Here we attempt to access the database __database_names = await cls.__client.list_database_names() except (ConnectionFailure, ServerSelectionTimeoutError): - ClientFail.ConnectionError.raise_error_log_and_exit( + Client.ConnectionError.sys_exit_log( errno.EIO, {"url": database_config.url.unicode_string()} ) err_extra_information = {"database_name": database_config.main_database_name} if database_config.main_database_name not in __database_names: - DatabaseFail.NotFoundError.raise_error_log_and_exit(errno.ENODATA, err_extra_information) + Databases.NotFoundError.sys_exit_log(errno.ENODATA, err_extra_information) cls.__main_database = cls.__client.get_database(database_config.main_database_name) err_extra_information |= {"collection_name": database_config.main_collection_name} if database_config.main_collection_name not in await cls.__main_database.list_collection_names(): - CollectionFail.NotFoundError.raise_error_log_and_exit(errno.ENODATA, err_extra_information) + Collections.NotFoundError.sys_exit_log(errno.ENODATA, err_extra_information) cls.__main_collection = cls.__main_database.get_collection(database_config.main_collection_name) @@ -175,7 +180,7 @@ def close(cls) -> None: if cls.__client: cls.__database_config = None return cls.__client.close() - ClientFail.CloseNotAllowedError.raise_error_log_and_exit(errno.EIO) + Client.CloseNotAllowedError.sys_exit_log(errno.EIO) @classmethod def list_database_names(cls) -> CoroutineStrList: @@ -247,9 +252,9 @@ async def get_collection( db = await cls.get_database(database_name) if collection_name in await db.list_collection_names(): return db[collection_name] - raise CollectionFail.NotFoundError + raise Collections.NotFoundError case _: - raise CollectionFail.WrongTypeError + raise Collections.WrongTypeError @classmethod async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | ResponseError: @@ -274,7 +279,7 @@ async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | Respon case _ if database_name in await cls.list_database_names(): return cls.__client[database_name] case _: - raise DatabaseFail.NotFoundError + raise Databases.NotFoundError @asynccontextmanager diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 1c23f53..59a9774 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -14,10 +14,12 @@ class PipelineDict(dict): second element is the right operand. Example: + ``` pd1 = PipelineDict({"number": 2}) pd2 = PipelineDict({"kind": 1}) pd3 = pd1 & pd2 + ``` """ diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index d351a10..59a397e 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -1,5 +1,7 @@ """ -The module which defines error responses that will be returned by the API. +The module which defines the base functionality for error responses that will be returned by the API. +This module only includes the generic utilities using which each module should define its own error responses +specifically. See :obj:`trolldb.database.errors` as an example on how this module is used. """ from collections import OrderedDict @@ -14,14 +16,47 @@ class ResponseError(Exception): + """ + The base class for all error responses. This is derivative of the ``Exception`` class. + """ + descriptor_delimiter: str = " |OR| " + """ + A delimiter to combine the message part of several error responses into a single one. This will be shown in textual + format for the response descriptors of the Fast API routes. For example: + + ``ErrorA |OR| ErrorB`` + """ + defaultResponseClass: Response = PlainTextResponse + """ + The default type of the response which will be returned when an error occurs. + """ def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) -> None: self.__dict: OrderedDict = OrderedDict(args_dict) self.extra_information: dict | None = None def __or__(self, other: Self): + """ + Combines the error responses into a single error response. + Args: + other: + Another error response of the same base type to combine with. + + Returns: + A new error response which includes the combined error response. In case of different http status codes, + the returned response includes the `{status-code: message}` pairs for both ``self`` and the ``other``. + In case of the same status codes, the messages will be appended to a list and saved as a list. + + Example: + ErrorA = ResponseError({200: "OK"}) + ErrorB = ResponseError({400: "Bad Request"}) + ErrorC = ResponseError({200: "Still Okay"}) + + ErrorCombined = ErrorA | ErrorB | ErrorC + + """ buff = OrderedDict(self.__dict) for key, msg in other.__dict.items(): self_msg = buff.get(key, None) @@ -41,8 +76,10 @@ def __assert_existence_multiple_response_codes( raise KeyError(f"Status code {status_code} cannot be found.") case _, 1: return [(k, v) for k, v in self.__dict.items()][0] + case _: + return 500, "Generic Response Error" - def get_error_info( + def get_error_details( self, extra_information: dict | None = None, status_code: int | None = None) -> (StatusCode, str): @@ -52,30 +89,20 @@ def get_error_info( ResponseError.__stringify(msg) + (f" :=> {extra_information}" if extra_information else "") ) - def fastapi_response( - self, - extra_information: dict | None = None, - status_code: StatusCode | None = None) -> defaultResponseClass: - try: - msg, _ = self.get_error_info(extra_information, status_code) - return ResponseError.defaultResponseClass(content=msg, status_code=status_code) - except KeyError: - raise KeyError(f"No default response found for the given status code: {status_code}") - - def raise_error_log_and_exit( + def sys_exit_log( self, exit_code: int = -1, extra_information: dict | None = None, status_code: int | None = None) -> None: - msg, _ = self.get_error_info(extra_information, status_code) + msg, _ = self.get_error_details(extra_information, status_code) logger.error(msg) exit(exit_code) - def log_warning( + def log_as_warning( self, extra_information: dict | None = None, status_code: int | None = None): - msg, _ = self.get_error_info(extra_information, status_code) + msg, _ = self.get_error_details(extra_information, status_code) logger.warning(msg) @property From 8522453f34992caf83be74a5ba797a08455c4ce5 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 10:37:15 +0200 Subject: [PATCH 14/97] Change imports to absolute --- trolldb/api/api.py | 8 ++++---- trolldb/api/routes/common.py | 2 +- trolldb/api/routes/databases.py | 8 ++++---- trolldb/api/routes/datetime_.py | 6 +++--- trolldb/api/routes/platforms.py | 4 ++-- trolldb/api/routes/queries.py | 8 ++++---- trolldb/api/routes/router.py | 2 +- trolldb/api/routes/sensors.py | 4 ++-- trolldb/api/tests/conftest.py | 8 ++++---- trolldb/api/tests/test_api.py | 4 ++-- trolldb/database/errors.py | 2 +- trolldb/database/mongodb.py | 8 ++++---- trolldb/database/tests/conftest.py | 6 +++--- trolldb/database/tests/test_mongodb.py | 2 +- trolldb/test_utils/common.py | 2 +- trolldb/test_utils/mongodb_database.py | 4 ++-- trolldb/test_utils/mongodb_instance.py | 4 ++-- 17 files changed, 41 insertions(+), 41 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 73561b7..62502a4 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -28,10 +28,10 @@ from fastapi.responses import PlainTextResponse from pydantic import FilePath, validate_call -from api.routes import api_router -from config.config import AppConfig, parse, Timeout -from database.mongodb import mongodb_context -from errors.errors import ResponseError +from trolldb.api.routes import api_router +from trolldb.config.config import AppConfig, parse, Timeout +from trolldb.database.mongodb import mongodb_context +from trolldb.errors.errors import ResponseError @validate_call diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py index 2a63cd4..8d5b983 100644 --- a/trolldb/api/routes/common.py +++ b/trolldb/api/routes/common.py @@ -7,7 +7,7 @@ from fastapi import Response, Query, Depends from motor.motor_asyncio import AsyncIOMotorCollection, AsyncIOMotorDatabase -from database.mongodb import MongoDB +from trolldb.database.mongodb import MongoDB exclude_defaults_query = Query( True, diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py index 515697b..6513d95 100644 --- a/trolldb/api/routes/databases.py +++ b/trolldb/api/routes/databases.py @@ -8,15 +8,15 @@ from fastapi import APIRouter from pymongo.collection import _DocumentType -from api.routes.common import ( +from trolldb.api.routes.common import ( exclude_defaults_query, CheckCollectionDependency, CheckDataBaseDependency) -from config.config import MongoObjectId -from database.errors import ( +from trolldb.config.config import MongoObjectId +from trolldb.database.errors import ( Databases, Documents, database_collection_error_descriptor, database_collection_document_error_descriptor) -from database.mongodb import MongoDB, get_ids +from trolldb.database.mongodb import MongoDB, get_ids router = APIRouter() diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index 2aeeeaf..552df04 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -11,9 +11,9 @@ from fastapi import APIRouter from pydantic import BaseModel -from api.routes.common import CheckCollectionDependency -from database.errors import database_collection_error_descriptor -from database.mongodb import get_id +from trolldb.api.routes.common import CheckCollectionDependency +from trolldb.database.errors import database_collection_error_descriptor +from trolldb.database.mongodb import get_id class TimeModel(TypedDict): diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py index f19ed7f..f116947 100644 --- a/trolldb/api/routes/platforms.py +++ b/trolldb/api/routes/platforms.py @@ -7,8 +7,8 @@ from fastapi import APIRouter -from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency -from database.errors import database_collection_error_descriptor +from trolldb.api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency +from trolldb.database.errors import database_collection_error_descriptor router = APIRouter() diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index 95ebb5c..24c89e9 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -9,10 +9,10 @@ from fastapi import APIRouter, Query -from api.routes.common import CheckCollectionDependency -from database.errors import database_collection_error_descriptor -from database.mongodb import get_ids -from database.piplines import PipelineAttribute, Pipelines +from trolldb.api.routes.common import CheckCollectionDependency +from trolldb.database.errors import database_collection_error_descriptor +from trolldb.database.mongodb import get_ids +from trolldb.database.piplines import PipelineAttribute, Pipelines router = APIRouter() diff --git a/trolldb/api/routes/router.py b/trolldb/api/routes/router.py index 2c75288..7ba4d93 100644 --- a/trolldb/api/routes/router.py +++ b/trolldb/api/routes/router.py @@ -7,7 +7,7 @@ from fastapi import APIRouter -from . import databases, datetime_, platforms, queries, root, sensors +from trolldb.api.routes import databases, datetime_, platforms, queries, root, sensors api_router = APIRouter() api_router.include_router(root.router, tags=["root"]) diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py index 53c867e..52832e6 100644 --- a/trolldb/api/routes/sensors.py +++ b/trolldb/api/routes/sensors.py @@ -7,8 +7,8 @@ from fastapi import APIRouter -from api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency -from database.errors import database_collection_error_descriptor +from trolldb.api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency +from trolldb.database.errors import database_collection_error_descriptor router = APIRouter() diff --git a/trolldb/api/tests/conftest.py b/trolldb/api/tests/conftest.py index 916cb75..e6a81da 100644 --- a/trolldb/api/tests/conftest.py +++ b/trolldb/api/tests/conftest.py @@ -1,9 +1,9 @@ import pytest -from api.api import server_process_context -from test_utils.mongodb_instance import mongodb_instance_server_process_context -from test_utils.common import test_app_config -from test_utils.mongodb_database import TestDatabase +from trolldb.api.api import server_process_context +from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context +from trolldb.test_utils.common import test_app_config +from trolldb.test_utils.mongodb_database import TestDatabase @pytest.fixture(scope="session") diff --git a/trolldb/api/tests/test_api.py b/trolldb/api/tests/test_api.py index fa76fde..4d6f386 100644 --- a/trolldb/api/tests/test_api.py +++ b/trolldb/api/tests/test_api.py @@ -1,7 +1,7 @@ from fastapi import status -from test_utils.common import assert_equal, http_get -from test_utils.mongodb_database import test_mongodb_context, TestDatabase +from trolldb.test_utils.common import assert_equal, http_get +from trolldb.test_utils.mongodb_database import test_mongodb_context, TestDatabase def test_root(): diff --git a/trolldb/database/errors.py b/trolldb/database/errors.py index 1b88a6a..8874819 100644 --- a/trolldb/database/errors.py +++ b/trolldb/database/errors.py @@ -10,7 +10,7 @@ from fastapi import status -from errors.errors import ResponsesErrorGroup, ResponseError +from trolldb.errors.errors import ResponsesErrorGroup, ResponseError class Client(ResponsesErrorGroup): diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 0d8c92c..7309110 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -20,9 +20,9 @@ ConnectionFailure, ServerSelectionTimeoutError) -from config.config import DatabaseConfig -from database.errors import Collections, Databases, Client -from errors.errors import ResponseError +from trolldb.config.config import DatabaseConfig +from trolldb.database.errors import Collections, Databases, Client +from trolldb.errors.errors import ResponseError T = TypeVar("T") CoroutineLike = Coroutine[Any, Any, T] @@ -104,7 +104,7 @@ class MongoDB: default_database_names = ["admin", "config", "local"] """ - MongoDB creates these databases by default for self usage. + MongoDB creates these databases by default for self usage. """ @classmethod diff --git a/trolldb/database/tests/conftest.py b/trolldb/database/tests/conftest.py index a646f4b..01948de 100644 --- a/trolldb/database/tests/conftest.py +++ b/trolldb/database/tests/conftest.py @@ -1,9 +1,9 @@ import pytest import pytest_asyncio -from test_utils.mongodb_instance import mongodb_instance_server_process_context -from test_utils.common import test_app_config -from test_utils.mongodb_database import TestDatabase +from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context +from trolldb.test_utils.common import test_app_config +from trolldb.test_utils.mongodb_database import TestDatabase from trolldb.database.mongodb import mongodb_context diff --git a/trolldb/database/tests/test_mongodb.py b/trolldb/database/tests/test_mongodb.py index b74d370..1ee2a76 100644 --- a/trolldb/database/tests/test_mongodb.py +++ b/trolldb/database/tests/test_mongodb.py @@ -5,7 +5,7 @@ from pydantic import AnyUrl from pymongo.errors import InvalidOperation -from test_utils.common import test_app_config +from trolldb.test_utils.common import test_app_config from trolldb.database.mongodb import DatabaseConfig, MongoDB, mongodb_context diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 568de5c..8d8db3d 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -5,7 +5,7 @@ from pydantic import AnyUrl from urllib3 import request, BaseHTTPResponse -from config.config import APIServerConfig, DatabaseConfig, AppConfig +from trolldb.config.config import APIServerConfig, DatabaseConfig, AppConfig test_app_config = AppConfig( api_server=APIServerConfig(url=AnyUrl("http://localhost:8080"), title="Test API Server", version="0.1"), diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index 316de8c..f23949d 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -5,8 +5,8 @@ from pymongo import MongoClient -from config.config import DatabaseConfig -from test_utils.common import test_app_config +from trolldb.config.config import DatabaseConfig +from trolldb.test_utils.common import test_app_config @contextmanager diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 0587056..a38ae29 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -12,8 +12,8 @@ from loguru import logger -from config.config import DatabaseConfig -from test_utils.common import test_app_config +from trolldb.config.config import DatabaseConfig +from trolldb.test_utils.common import test_app_config class TestMongoInstance: From 55edaa4c860637b8c7e7c328424b0dbce84a75d0 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 10:37:37 +0200 Subject: [PATCH 15/97] Switch to pyproject.toml --- pyproject.toml | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 58 -------------------------------------------------- 2 files changed, 58 insertions(+), 58 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..45b7c44 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,58 @@ +[project] +name = "pytroll-db" +dynamic = ["version"] +description = "The database API of Pytroll." +authors = [ + { name = "Pouria Khalaj", email = "pouria.khalaj@smhi.se" } +] +dependencies = ['pymongo', 'posttroll', 'motor', 'pydantic', 'fastapi', 'uvicorn', "loguru", "pyyaml", "urllib3"] +readme = "README.rst" +requires-python = ">= 3.12" +license = {file = "LICENSE"} +classifiers = [ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python', + 'Operating System :: OS Independent', + 'Intended Audience :: Science/Research', + 'Topic :: Scientific/Engineering', + 'Topic :: Database' +] + +[project.urls] +"Documentation" = "https://pytroll-db.readthedocs.io/en/latest/" + +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["trolldb"] + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "trolldb/version.py" + + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +# See https://docs.astral.sh/ruff/rules/ +select = ["A", "B", "D", "E", "W", "F", "I", "N", "PT", "S", "TID", "C90", "Q", "T10", "T20"] + +[tool.ruff.lint.per-file-ignores] +"*/tests/*" = ["S101"] # assert allowed in tests +"docs/source/conf.py" = ["D100", "A001"] # sphinx misbihaving +"src/trolldb/version.py" = ["D100", "Q000"] # automatically generated by hatch-vcs + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.mccabe] +max-complexity = 10 diff --git a/setup.py b/setup.py deleted file mode 100644 index e50ad92..0000000 --- a/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2011, 2012, 2014, 2015. - -# Author(s): - -# The pytroll team: -# Martin Raspaud - -# This file is part of pytroll. - -# This is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. - -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -# details. - -# You should have received a copy of the GNU General Public License along with -# this program. If not, see . - -from setuptools import setup -import imp - -version = imp.load_source('trolldb.version', 'trolldb/version.py') - - -requirements = ['pymongo', 'posttroll'] - -setup(name="pytroll-db", - version=version.__version__, - description='Messaging system for pytroll', - author='The pytroll team', - author_email='martin.raspaud@smhi.se', - url="http://github.com/pytroll/pytroll-db", - packages=['trolldb'], - zip_safe=False, - license="GPLv3", - install_requires=requirements, - classifiers=[ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Programming Language :: Python', - 'Operating System :: OS Independent', - 'Intended Audience :: Science/Research', - 'Topic :: Scientific/Engineering', - 'Topic :: Database' - ], - entry_points={ - 'console_scripts': [ - 'db_cleanup = trolldb.pytroll_cleanup:threaded_check_all', - ], - }, - scripts=['bin/pytroll-mongo.py'], - ) From 83aa075e571e027b9a3df142c9d221cb0c2ce183 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 10:47:42 +0200 Subject: [PATCH 16/97] Add github actions --- .github/workflows/ci.yml | 47 +++++++++++++++++++++++++++++ .github/workflows/deploy-sdist.yaml | 27 +++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/deploy-sdist.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..84c5a16 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + env: + PYTHON_VERSION: ${{ matrix.python-version }} + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt install -y mongodb + python -m pip install --upgrade pip + python -m pip install ruff pytest pytest-asyncio + python -m pip install -e . + - name: Lint with ruff + run: | + ruff check . + - name: Test with pytest + run: | + pytest --cov=pytroll_db tests --cov-report=xml + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: pytroll/pytroll-db + file: ./coverage.xml + env_vars: PYTHON_VERSION diff --git a/.github/workflows/deploy-sdist.yaml b/.github/workflows/deploy-sdist.yaml new file mode 100644 index 0000000..1499397 --- /dev/null +++ b/.github/workflows/deploy-sdist.yaml @@ -0,0 +1,27 @@ +name: Deploy sdist + +on: + release: + types: + - published + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Create sdist + shell: bash -l {0} + run: | + python -m pip install -q build + python -m build -s + + - name: Publish package to PyPI + if: github.event.action == 'published' + uses: pypa/gh-action-pypi-publish@v1.8.14 + with: + user: __token__ + password: ${{ secrets.pypi_password }} From cfdaa82be75cc08f55e933b59120559be4e17141 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 11:12:49 +0200 Subject: [PATCH 17/97] Fix ruff stuff (WIP). --- .gitignore | 3 ++ docs/source/conf.py | 40 ++++++++--------- trolldb/api/api.py | 27 ++++++------ trolldb/api/routes/common.py | 28 +++++------- trolldb/api/routes/databases.py | 9 ++-- trolldb/api/routes/datetime_.py | 3 +- trolldb/api/routes/platforms.py | 5 +-- trolldb/api/routes/queries.py | 3 +- trolldb/api/routes/root.py | 3 +- trolldb/api/routes/router.py | 3 +- trolldb/api/routes/sensors.py | 5 +-- trolldb/api/tests/conftest.py | 2 +- trolldb/api/tests/test_api.py | 36 ++++++---------- trolldb/config/config.py | 31 ++++++-------- trolldb/database/errors.py | 22 +++------- trolldb/database/mongodb.py | 59 +++++++++++--------------- trolldb/database/piplines.py | 14 +++--- trolldb/database/tests/conftest.py | 4 +- trolldb/database/tests/test_mongodb.py | 24 +++-------- trolldb/errors/errors.py | 11 ++--- trolldb/run_api.py | 3 +- trolldb/test_utils/common.py | 14 +++--- trolldb/test_utils/mongodb_database.py | 4 +- trolldb/test_utils/mongodb_instance.py | 13 +++--- 24 files changed, 147 insertions(+), 219 deletions(-) diff --git a/.gitignore b/.gitignore index 5a0f0d9..835f613 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ config.yml # temp log and storage for the test database __temp* + +# version file +version.py diff --git a/docs/source/conf.py b/docs/source/conf.py index c68eb3d..0d0bfb3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,9 +15,9 @@ from sphinx.ext import apidoc -sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath("../../")) sys.path.append(os.path.abspath(os.path.dirname(__file__))) -for x in os.walk('../../trolldb'): +for x in os.walk("../../trolldb"): sys.path.append(x[0]) # autodoc_mock_imports = ["motor", "pydantic", "fastapi", "uvicorn", "loguru", "pyyaml"] @@ -25,29 +25,29 @@ # -- Project information ----------------------------------------------------- -project = 'Pytroll-db' -copyright = '2024, Pytroll' -author = 'Pouria Khalaj' +project = "Pytroll-db" +copyright = "2024, Pytroll" +author = "Pouria Khalaj" # The full version, including alpha/beta/rc tags -release = '0.1' +release = "0.1" # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# extensions coming with Sphinx (named "sphinx.ext.*") or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - 'sphinx.ext.napoleon', - 'sphinx.ext.duration', - 'sphinx.ext.doctest', - 'sphinx.ext.autosummary', - 'sphinx.ext.intersphinx', + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", + "sphinx.ext.duration", + "sphinx.ext.doctest", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -60,16 +60,16 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] +# html_static_path = ["_static"] root_doc = "index" -output_dir = os.path.join('.') -module_dir = os.path.abspath('../../trolldb') -apidoc.main(['-q', '-f', '-o', output_dir, module_dir, *include_patterns]) +output_dir = os.path.join(".") +module_dir = os.path.abspath("../../trolldb") +apidoc.main(["-q", "-f", "-o", output_dir, module_dir, *include_patterns]) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 62502a4..7a1b1b6 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -1,6 +1,6 @@ -""" -The module which includes the main functionalities of the API package. This is the main module which is supposed to be -imported by the users of the package. +"""The module which includes the main functionalities of the API package. + +This is the main module which is supposed to be imported by the users of the package. Note: Functions in this module are decorated with @@ -29,16 +29,16 @@ from pydantic import FilePath, validate_call from trolldb.api.routes import api_router -from trolldb.config.config import AppConfig, parse, Timeout +from trolldb.config.config import AppConfig, Timeout, parse from trolldb.database.mongodb import mongodb_context from trolldb.errors.errors import ResponseError @validate_call def run_server(config: AppConfig | FilePath, **kwargs) -> None: - """ - Runs the API server with all the routes and connection to the database. It first creates a FastAPI - application and runs it using `uvicorn `_ which is + """Runs the API server with all the routes and connection to the database. + + It first creates a FastAPI application and runs it using `uvicorn `_ which is ASGI (Asynchronous Server Gateway Interface) compliant. This function runs the event loop using `asyncio `_ and does not yield! @@ -55,7 +55,6 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: are read from the ``config`` argument. The keyword arguments which are passed explicitly to the function take precedence over ``config``. """ - config = parse(config) app = FastAPI(**(config.api_server._asdict() | kwargs)) app.include_router(api_router) @@ -69,9 +68,7 @@ async def unicorn_exception_handler(_, exc: ResponseError): ) async def _serve(): - """ - An auxiliary coroutine to be used in the asynchronous execution of the FastAPI application. - """ + """An auxiliary coroutine to be used in the asynchronous execution of the FastAPI application.""" async with mongodb_context(config.database): await uvicorn.Server( config=uvicorn.Config( @@ -87,10 +84,10 @@ async def _serve(): @contextmanager @validate_call def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2000): - """ - A synchronous context manager to run the API server in a separate process (non-blocking) using the - `multiprocessing `_ package. The main use case is envisaged - to be in testing environments. + """A synchronous context manager to run the API server in a separate process (non-blocking). + + It uses the `multiprocessing `_ package. The main use case + is envisaged to be in testing environments. Args: config: diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py index 8d5b983..7acada5 100644 --- a/trolldb/api/routes/common.py +++ b/trolldb/api/routes/common.py @@ -1,10 +1,9 @@ -""" -The module which defines common functions to be used in handling requests related to `databases` and `collections`. +"""The module which defines common functions to be used in handling requests related to `databases` and `collections`. """ from typing import Annotated -from fastapi import Response, Query, Depends +from fastapi import Depends, Query, Response from motor.motor_asyncio import AsyncIOMotorCollection, AsyncIOMotorDatabase from trolldb.database.mongodb import MongoDB @@ -17,9 +16,7 @@ async def check_database(database_name: str | None = None) -> AsyncIOMotorDatabase: - """ - A dependency for route handlers to check for the existence of a database given - its name. + """A dependency for route handlers to check for the existence of a database given its name. Args: database_name (Optional, default ``None``): @@ -37,10 +34,10 @@ async def check_database(database_name: str | None = None) -> AsyncIOMotorDataba async def check_collection( database_name: str | None = None, collection_name: str | None = None) -> AsyncIOMotorCollection: - """ - A dependency for route handlers to check for the existence of a collection given - its name and the name of the database it resides in. It first checks for the existence of the database using - :func:`check_database`. + """A dependency for route handlers to check for the existence of a collection. + + It performs the check given the collection name and the name of the database it resides in. It first checks for the + existence of the database using :func:`check_database`. Args: database_name (Optional, default ``None``): @@ -63,18 +60,16 @@ async def check_collection( -- if only one of ``database_name`` or ``collection_name`` is ``None``; or if the type of ``collection_name`` is not ``str``. """ - return await MongoDB.get_collection(database_name, collection_name) async def get_distinct_items_in_collection( res_coll: Response | AsyncIOMotorCollection, field_name: str) -> Response | list[str]: - """ - An auxiliary function to either return (verbatim echo) the given response; or return a list of distinct (unique) - values for the given ``field_name`` via a search which is conducted in all documents of the given collection. The - latter behaviour is equivalent to the ``distinct`` function from MongoDB. The former is the behaviour of an - identity function + """An auxiliary function to either return the given response; or return a list of distinct (unique) values + + Given the ``field_name`` it conducts a search in all documents of the given collection. The latter behaviour is + equivalent to the ``distinct`` function from MongoDB. The former is the behaviour of an identity function. Args: res_coll: @@ -89,7 +84,6 @@ async def get_distinct_items_in_collection( -- In case of a collection as input, all the documents of the collection will be searched for ``field_name``, and the corresponding values will be retrieved. Finally, a list of all the distinct values is returned. """ - if isinstance(res_coll, Response): return res_coll diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py index 6513d95..798e798 100644 --- a/trolldb/api/routes/databases.py +++ b/trolldb/api/routes/databases.py @@ -1,5 +1,4 @@ -""" -The module which handles all requests related to getting the list of `databases` and `collections`. +"""The module which handles all requests related to getting the list of `databases` and `collections`. Note: For more information on the API server, see the automatically generated documentation by FastAPI. @@ -8,14 +7,14 @@ from fastapi import APIRouter from pymongo.collection import _DocumentType -from trolldb.api.routes.common import ( - exclude_defaults_query, CheckCollectionDependency, CheckDataBaseDependency) +from trolldb.api.routes.common import CheckCollectionDependency, CheckDataBaseDependency, exclude_defaults_query from trolldb.config.config import MongoObjectId from trolldb.database.errors import ( Databases, Documents, + database_collection_document_error_descriptor, database_collection_error_descriptor, - database_collection_document_error_descriptor) +) from trolldb.database.mongodb import MongoDB, get_ids router = APIRouter() diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index 552df04..f5d4e48 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -1,5 +1,4 @@ -""" -The module which handles all requests related to `datetime`. +"""The module which handles all requests related to `datetime`. Note: For more information on the API server, see the automatically generated documentation by FastAPI. diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py index f116947..4e6e868 100644 --- a/trolldb/api/routes/platforms.py +++ b/trolldb/api/routes/platforms.py @@ -1,5 +1,4 @@ -""" -The module which handles all requests regarding `platforms`. +"""The module which handles all requests regarding `platforms`. Note: For more information on the API server, see the automatically generated documentation by FastAPI. @@ -7,7 +6,7 @@ from fastapi import APIRouter -from trolldb.api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency +from trolldb.api.routes.common import CheckCollectionDependency, get_distinct_items_in_collection from trolldb.database.errors import database_collection_error_descriptor router = APIRouter() diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index 24c89e9..20d3984 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -1,5 +1,4 @@ -""" -The module which handles all requests to the queries route. +"""The module which handles all requests to the queries route. Note: For more information on the API server, see the automatically generated documentation by FastAPI. diff --git a/trolldb/api/routes/root.py b/trolldb/api/routes/root.py index 785be36..5099fe9 100644 --- a/trolldb/api/routes/root.py +++ b/trolldb/api/routes/root.py @@ -1,5 +1,4 @@ -""" -The module which handles all requests to the root route, i.e. "/". +"""The module which handles all requests to the root route, i.e. "/". Note: For more information on the API server, see the automatically generated documentation by FastAPI. diff --git a/trolldb/api/routes/router.py b/trolldb/api/routes/router.py index 7ba4d93..7f0125c 100644 --- a/trolldb/api/routes/router.py +++ b/trolldb/api/routes/router.py @@ -1,5 +1,4 @@ -""" -The module which defines all the routes with their corresponding tags. +"""The module which defines all the routes with their corresponding tags. Note: For more information on the API server, see the automatically generated documentation by FastAPI. diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py index 52832e6..2fa5d0b 100644 --- a/trolldb/api/routes/sensors.py +++ b/trolldb/api/routes/sensors.py @@ -1,5 +1,4 @@ -""" -The module which handles all requests regarding `sensors`. +"""The module which handles all requests regarding `sensors`. Note: For more information on the API server, see the automatically generated documentation by FastAPI. @@ -7,7 +6,7 @@ from fastapi import APIRouter -from trolldb.api.routes.common import get_distinct_items_in_collection, CheckCollectionDependency +from trolldb.api.routes.common import CheckCollectionDependency, get_distinct_items_in_collection from trolldb.database.errors import database_collection_error_descriptor router = APIRouter() diff --git a/trolldb/api/tests/conftest.py b/trolldb/api/tests/conftest.py index e6a81da..5a8d7c1 100644 --- a/trolldb/api/tests/conftest.py +++ b/trolldb/api/tests/conftest.py @@ -1,9 +1,9 @@ import pytest from trolldb.api.api import server_process_context -from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context from trolldb.test_utils.common import test_app_config from trolldb.test_utils.mongodb_database import TestDatabase +from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context @pytest.fixture(scope="session") diff --git a/trolldb/api/tests/test_api.py b/trolldb/api/tests/test_api.py index 4d6f386..445b56a 100644 --- a/trolldb/api/tests/test_api.py +++ b/trolldb/api/tests/test_api.py @@ -1,52 +1,42 @@ from fastapi import status from trolldb.test_utils.common import assert_equal, http_get -from trolldb.test_utils.mongodb_database import test_mongodb_context, TestDatabase +from trolldb.test_utils.mongodb_database import TestDatabase, test_mongodb_context def test_root(): - """ - Checks that the server is up and running, i.e. the root routes responds with 200. - """ + """Checks that the server is up and running, i.e. the root routes responds with 200.""" assert_equal(http_get().status, status.HTTP_200_OK) def test_platforms(): - """ - Checks that the retrieved platform names match the expected names. - """ + """Checks that the retrieved platform names match the expected names.""" assert_equal(http_get("platforms").json(), TestDatabase.platform_names) def test_sensors(): - """ - Checks that the retrieved sensor names match the expected names. + """Checks that the retrieved sensor names match the expected names. """ assert_equal(http_get("sensors").json(), TestDatabase.sensors) def test_database_names(): - """ - Checks that the retrieved database names match the expected names. - """ + """Checks that the retrieved database names match the expected names.""" assert_equal(http_get("databases").json(), TestDatabase.database_names) assert_equal(http_get("databases?exclude_defaults=True").json(), TestDatabase.database_names) assert_equal(http_get("databases?exclude_defaults=False").json(), TestDatabase.all_database_names) def test_database_names_negative(): - """ - Checks that the non-existing databases cannot be found. - """ - assert_equal(http_get(f"databases/non_existing_database").status, status.HTTP_404_NOT_FOUND) + """Checks that the non-existing databases cannot be found.""" + assert_equal(http_get("databases/non_existing_database").status, status.HTTP_404_NOT_FOUND) def test_collections(): - """ - Check the presence of existing collections and that the ids of documents therein can be correctly retrieved. - """ + """Check the presence of existing collections and that the ids of documents therein can be correctly retrieved.""" with test_mongodb_context() as client: - for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names): + for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names, + strict=False): # Collections exist assert_equal( http_get(f"databases/{database_name}").json(), @@ -61,10 +51,8 @@ def test_collections(): def test_collections_negative(): - """ - Checks that the non-existing collections cannot be found. - """ - for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names): + """Checks that the non-existing collections cannot be found.""" + for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names, strict=False): assert_equal( http_get(f"databases/{database_name}/non_existing_collection").status, status.HTTP_404_NOT_FOUND diff --git a/trolldb/config/config.py b/trolldb/config/config.py index ad65e2d..67be871 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -1,5 +1,4 @@ -""" -The module which handles parsing and validating the config (YAML) file. +"""The module which handles parsing and validating the config (YAML) file. The validation is performed using `Pydantic `_. Note: @@ -10,7 +9,7 @@ import errno import sys -from typing import Optional, NamedTuple, TypedDict +from typing import NamedTuple, Optional, TypedDict from bson import ObjectId from bson.errors import InvalidId @@ -38,9 +37,10 @@ class MongoDocument(BaseModel): class LicenseInfo(TypedDict): - """ - A dictionary type to hold the summary of the license information. One has to always consult the included `LICENSE` - file for more information. + """A dictionary type to hold the summary of the license information. + + Warning: + One has to always consult the included `LICENSE` file for more information. """ name: str @@ -56,8 +56,7 @@ class LicenseInfo(TypedDict): class APIServerConfig(NamedTuple): - """ - A named tuple to hold all the configurations of the API server (excluding the database). + """A named tuple to hold all the configurations of the API server (excluding the database). Note: Except for the ``url``, the attributes herein are a subset of the keyword arguments accepted by @@ -99,8 +98,7 @@ class APIServerConfig(NamedTuple): class DatabaseConfig(NamedTuple): - """ - A named tuple to hold all the configurations of the Database which will be used by the MongoDB instance. + """A named tuple to hold all the configurations of the Database which will be used by the MongoDB instance. """ main_database_name: str @@ -127,9 +125,9 @@ class DatabaseConfig(NamedTuple): class AppConfig(BaseModel): - """ - A model to hold all the configurations of the application including both the API server and the database. This will - be used by Pydantic to validate the parsed YAML file. + """A model to hold all the configurations of the application including both the API server and the database. + + This will be used by Pydantic to validate the parsed YAML file. """ api_server: APIServerConfig database: DatabaseConfig @@ -137,8 +135,7 @@ class AppConfig(BaseModel): @validate_call def from_yaml(filename: FilePath) -> AppConfig: - """ - Parses and validates the configurations from a YAML file. + """Parses and validates the configurations from a YAML file. Args: filename: @@ -155,7 +152,6 @@ def from_yaml(filename: FilePath) -> AppConfig: Returns: An instance of :class:`AppConfig`. """ - with open(filename, "r") as file: config = safe_load(file) try: @@ -167,8 +163,7 @@ def from_yaml(filename: FilePath) -> AppConfig: @validate_call def parse(config: AppConfig | FilePath) -> AppConfig: - """ - Tries to return a valid object of type :class:`AppConfig` + """Tries to return a valid object of type :class:`AppConfig` Args: config: diff --git a/trolldb/database/errors.py b/trolldb/database/errors.py index 8874819..a15ec43 100644 --- a/trolldb/database/errors.py +++ b/trolldb/database/errors.py @@ -1,5 +1,4 @@ -""" -The modules which defines the error responses that might occur while working with the +"""The modules which defines the error responses that might occur while working with the MongoDB database. Note: @@ -10,14 +9,11 @@ from fastapi import status -from trolldb.errors.errors import ResponsesErrorGroup, ResponseError +from trolldb.errors.errors import ResponseError, ResponsesErrorGroup class Client(ResponsesErrorGroup): - """ - Client error responses, e.g. if something goes wrong with initialization or closing the - client. - """ + """Client error responses, e.g. if something goes wrong with initialization or closing the client.""" CloseNotAllowedError = ResponseError({ status.HTTP_405_METHOD_NOT_ALLOWED: "Calling `close()` on a client which has not been initialized is not allowed!" @@ -47,9 +43,7 @@ class Client(ResponsesErrorGroup): class Collections(ResponsesErrorGroup): - """ - Collections error responses, e.g. if a requested collection cannot be found. - """ + """Collections error responses, e.g. if a requested collection cannot be found.""" NotFoundError = ResponseError({ status.HTTP_404_NOT_FOUND: "Could not find the given collection name inside the specified database." @@ -62,9 +56,7 @@ class Collections(ResponsesErrorGroup): class Databases(ResponsesErrorGroup): - """ - Databases error responses, e.g. if a requested database cannot be found. - """ + """Databases error responses, e.g. if a requested database cannot be found.""" NotFoundError = ResponseError({ status.HTTP_404_NOT_FOUND: "Could not find the given database name." @@ -77,9 +69,7 @@ class Databases(ResponsesErrorGroup): class Documents(ResponsesErrorGroup): - """ - Documents error responses, e.g. if a requested document cannot be found. - """ + """Documents error responses, e.g. if a requested document cannot be found.""" NotFound = ResponseError({ status.HTTP_404_NOT_FOUND: "Could not find any document with the given object id." diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 7309110..04a3c73 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -1,6 +1,7 @@ -""" -The module which handles database CRUD operations for MongoDB. It is based on -`PyMongo `_ and `motor `_. +"""The module which handles database CRUD operations for MongoDB. + +It is based on `PyMongo `_ and +`motor `_. """ import errno @@ -9,19 +10,17 @@ from motor.motor_asyncio import ( AsyncIOMotorClient, - AsyncIOMotorDatabase, AsyncIOMotorCollection, AsyncIOMotorCommandCursor, - AsyncIOMotorCursor + AsyncIOMotorCursor, + AsyncIOMotorDatabase, ) -from pydantic import validate_call, BaseModel +from pydantic import BaseModel, validate_call from pymongo.collection import _DocumentType -from pymongo.errors import ( - ConnectionFailure, - ServerSelectionTimeoutError) +from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError from trolldb.config.config import DatabaseConfig -from trolldb.database.errors import Collections, Databases, Client +from trolldb.database.errors import Client, Collections, Databases from trolldb.errors.errors import ResponseError T = TypeVar("T") @@ -39,8 +38,7 @@ class CollectionName(BaseModel): async def get_id(doc: CoroutineDocument) -> str: - """ - Retrieves the ID of a document as a simple flat string. + """Retrieves the ID of a document as a simple flat string. Note: The rationale behind this method is as follows. In MongoDB, each document has a unique ID which is of type @@ -60,8 +58,7 @@ async def get_id(doc: CoroutineDocument) -> str: async def get_ids(docs: AsyncIOMotorCommandCursor | AsyncIOMotorCursor) -> list[str]: - """ - Similar to :func:`~MongoDB.get_id` but for a list of documents. + """Similar to :func:`~MongoDB.get_id` but for a list of documents. Args: docs: @@ -76,10 +73,10 @@ async def get_ids(docs: AsyncIOMotorCommandCursor | AsyncIOMotorCursor) -> list[ class MongoDB: - """ - A wrapper class around the `motor async driver `_ for Mongo DB with - convenience methods tailored to our specific needs. As such, the :func:`~MongoDB.initialize()`` method returns a - coroutine which needs to be awaited. + """A wrapper class around the `motor async driver `_ for Mongo DB. + + It includes convenience methods tailored to our specific needs. As such, the :func:`~MongoDB.initialize()`` method + returns a coroutine which needs to be awaited. Note: This class is not meant to be instantiated! That's why all the methods in this class are decorated with @@ -109,8 +106,7 @@ class MongoDB: @classmethod async def initialize(cls, database_config: DatabaseConfig): - """ - Initializes the motor client. Note that this method has to be awaited! + """Initializes the motor client. Note that this method has to be awaited! Args: database_config: @@ -135,7 +131,6 @@ async def initialize(cls, database_config: DatabaseConfig): Returns: On success ``None``. """ - if cls.__database_config: if database_config == cls.__database_config: if cls.__client: @@ -174,9 +169,7 @@ async def initialize(cls, database_config: DatabaseConfig): @classmethod def close(cls) -> None: - """ - Closes the motor client. - """ + """Closes the motor client.""" if cls.__client: cls.__database_config = None return cls.__client.close() @@ -188,8 +181,7 @@ def list_database_names(cls) -> CoroutineStrList: @classmethod def main_collection(cls) -> AsyncIOMotorCollection: - """ - A convenience method to get the main collection. + """A convenience method to get the main collection. Returns: The main collection which resides inside the main database. @@ -199,8 +191,7 @@ def main_collection(cls) -> AsyncIOMotorCollection: @classmethod def main_database(cls) -> AsyncIOMotorDatabase: - """ - A convenience method to get the main database. + """A convenience method to get the main database. Returns: The main database which includes the main collection, which in turn includes the desired documents. @@ -213,8 +204,7 @@ async def get_collection( cls, database_name: str, collection_name: str) -> AsyncIOMotorCollection | ResponseError: - """ - Gets the collection object given its name and the database name in which it resides. + """Gets the collection object given its name and the database name in which it resides. Args: database_name: @@ -240,7 +230,6 @@ async def get_collection( The database object. In case of ``None`` for both the database name and collection name, the main collection will be returned. """ - database_name = DatabaseName(name=database_name).name collection_name = CollectionName(name=collection_name).name @@ -258,12 +247,12 @@ async def get_collection( @classmethod async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | ResponseError: - """ - Gets the database object given its name. + """Gets the database object given its name. Args: database_name: The name of the database to retrieve. + Raises: ``KeyError``: If the database name does not exist in the list of database names. @@ -285,8 +274,8 @@ async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | Respon @asynccontextmanager @validate_call async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: - """ - An asynchronous context manager to connect to the MongoDB client. + """An asynchronous context manager to connect to the MongoDB client. + It can be either used in production or in testing environments. Args: diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 59a9774..9a43dad 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -1,20 +1,18 @@ -""" -The module which defines some convenience classes to facilitate the use of aggregation pipelines. +"""The module which defines some convenience classes to facilitate the use of aggregation pipelines. """ from typing import Any, Self class PipelineDict(dict): - """ - A subclass of dict which overrides the behaviour of bitwise or ``|`` and bitwise and ``&``. The operators are only - defined for operands of type :class:`PipelineDict`. For each of the aforementioned operators, the result will be a - dictionary with a single key/value pair. The key is either ``$or`` or ``$and`` depending on the operator being used. - The corresponding value is a list with two elements only. The first element of the list is the left operand and the + """A subclass of dict which overrides the behaviour of bitwise or ``|`` and bitwise and ``&``. + + The operators are only defined for operands of type :class:`PipelineDict`. For each of the aforementioned operators, + the result will be a dictionary with a single key/value pair. The key is either ``$or`` or ``$and`` depending on the + operator being used. The corresponding value is a list with two elements only. The first element of the list is the left operand and the second element is the right operand. Example: - ``` pd1 = PipelineDict({"number": 2}) pd2 = PipelineDict({"kind": 1}) diff --git a/trolldb/database/tests/conftest.py b/trolldb/database/tests/conftest.py index 01948de..247683f 100644 --- a/trolldb/database/tests/conftest.py +++ b/trolldb/database/tests/conftest.py @@ -1,10 +1,10 @@ import pytest import pytest_asyncio -from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context +from trolldb.database.mongodb import mongodb_context from trolldb.test_utils.common import test_app_config from trolldb.test_utils.mongodb_database import TestDatabase -from trolldb.database.mongodb import mongodb_context +from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context @pytest.fixture(scope="session") diff --git a/trolldb/database/tests/test_mongodb.py b/trolldb/database/tests/test_mongodb.py index 1ee2a76..b24ec13 100644 --- a/trolldb/database/tests/test_mongodb.py +++ b/trolldb/database/tests/test_mongodb.py @@ -5,14 +5,12 @@ from pydantic import AnyUrl from pymongo.errors import InvalidOperation -from trolldb.test_utils.common import test_app_config from trolldb.database.mongodb import DatabaseConfig, MongoDB, mongodb_context +from trolldb.test_utils.common import test_app_config async def test_connection_timeout_negative(): - """ - Expect to see the connection attempt times out since the MongoDB URL is invalid. - """ + """Expect to see the connection attempt times out since the MongoDB URL is invalid.""" timeout = 3000 with pytest.raises(SystemExit) as pytest_wrapped_e: t1 = time.time() @@ -26,9 +24,7 @@ async def test_connection_timeout_negative(): async def test_main_database_negative(run_mongodb_server_instance): - """ - Expect to fail when giving an invalid name for the main database, given a valid collection name. - """ + """Expect to fail when giving an invalid name for the main database, given a valid collection name.""" with pytest.raises(SystemExit) as pytest_wrapped_e: async with mongodb_context(DatabaseConfig( timeout=1000, @@ -40,9 +36,7 @@ async def test_main_database_negative(run_mongodb_server_instance): async def test_main_collection_negative(run_mongodb_server_instance): - """ - Expect to fail when giving an invalid name for the main collection, given a valid database name. - """ + """Expect to fail when giving an invalid name for the main collection, given a valid database name.""" with pytest.raises(SystemExit) as pytest_wrapped_e: async with mongodb_context(DatabaseConfig( timeout=1000, @@ -54,8 +48,7 @@ async def test_main_collection_negative(run_mongodb_server_instance): async def test_get_client(mongodb_fixture): - """ - This is our way of testing that MongoDB.client() returns the valid client object. + """This is our way of testing that MongoDB.client() returns the valid client object. Expect: - The `close` method can be called on the client and leads to the closure of the client @@ -67,8 +60,7 @@ async def test_get_client(mongodb_fixture): async def test_main_collection(mongodb_fixture): - """ - Expect: + """Expect: - The retrieved main collection is not `None` - It has the correct name - It is the same object that can be accessed via the `client` object of the MongoDB. @@ -81,9 +73,7 @@ async def test_main_collection(mongodb_fixture): async def test_main_database(mongodb_fixture): - """ - Same as test_main_collection but for the main database. - """ + """Same as test_main_collection but for the main database.""" assert MongoDB.main_database() is not None assert MongoDB.main_database().name == test_app_config.database.main_database_name assert MongoDB.main_database() == await MongoDB.get_database(test_app_config.database.main_database_name) diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index 59a397e..723c4cb 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -1,5 +1,4 @@ -""" -The module which defines the base functionality for error responses that will be returned by the API. +"""The module which defines the base functionality for error responses that will be returned by the API. This module only includes the generic utilities using which each module should define its own error responses specifically. See :obj:`trolldb.database.errors` as an example on how this module is used. """ @@ -16,9 +15,7 @@ class ResponseError(Exception): - """ - The base class for all error responses. This is derivative of the ``Exception`` class. - """ + """The base class for all error responses. This is derivative of the ``Exception`` class.""" descriptor_delimiter: str = " |OR| " """ @@ -38,8 +35,8 @@ def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) - self.extra_information: dict | None = None def __or__(self, other: Self): - """ - Combines the error responses into a single error response. + """Combines the error responses into a single error response. + Args: other: Another error response of the same base type to combine with. diff --git a/trolldb/run_api.py b/trolldb/run_api.py index f5a6cbb..d1b0ea2 100644 --- a/trolldb/run_api.py +++ b/trolldb/run_api.py @@ -1,5 +1,4 @@ -""" -The main entry point to run the API server according to the configurations given in `config.yaml` +"""The main entry point to run the API server according to the configurations given in `config.yaml` Note: For more information on the API server, see the automatically generated documentation by FastAPI. diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 8d8db3d..7994308 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,11 +1,10 @@ from typing import Any from urllib.parse import urljoin - from pydantic import AnyUrl -from urllib3 import request, BaseHTTPResponse +from urllib3 import BaseHTTPResponse, request -from trolldb.config.config import APIServerConfig, DatabaseConfig, AppConfig +from trolldb.config.config import APIServerConfig, AppConfig, DatabaseConfig test_app_config = AppConfig( api_server=APIServerConfig(url=AnyUrl("http://localhost:8080"), title="Test API Server", version="0.1"), @@ -18,8 +17,7 @@ def http_get(route: str = "") -> BaseHTTPResponse: - """ - An auxiliary function to make a GET request using :func:`urllib.request`. + """An auxiliary function to make a GET request using :func:`urllib.request`. Args: route: @@ -32,8 +30,7 @@ def http_get(route: str = "") -> BaseHTTPResponse: def assert_equal(test, expected) -> None: - """ - An auxiliary function to assert the equality of two objects using the ``==`` operator. In case an input is a list or + """An auxiliary function to assert the equality of two objects using the ``==`` operator. In case an input is a list or a tuple, it will be first converted to a set so that the order of items there in does not affect the assertion outcome. @@ -48,8 +45,7 @@ def assert_equal(test, expected) -> None: """ def _setify(obj: Any) -> Any: - """ - An auxiliary function to convert an object to a set if it is a tuple or a list. + """An auxiliary function to convert an object to a set if it is a tuple or a list. """ return set(obj) if isinstance(obj, list | tuple) else obj diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index f23949d..9eeb4d2 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -85,7 +85,7 @@ class TestDatabase: @classmethod def generate_documents(cls, random_shuffle=True) -> list: - documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors)] + documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors, strict=False)] if random_shuffle: shuffle(documents) return documents @@ -93,7 +93,7 @@ def generate_documents(cls, random_shuffle=True) -> list: @classmethod def reset(cls): with test_mongodb_context() as client: - for db_name, coll_name in zip(cls.database_names, cls.collection_names): + for db_name, coll_name in zip(cls.database_names, cls.collection_names, strict=False): db = client[db_name] collection = db[coll_name] collection.delete_many({}) diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index a38ae29..7102b05 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -1,5 +1,4 @@ -""" -The module which defines functionalities to run a MongoDB instance which is to be used in the testing environment. +"""The module which defines functionalities to run a MongoDB instance which is to be used in the testing environment. """ import errno import subprocess @@ -7,7 +6,7 @@ import tempfile import time from contextlib import contextmanager -from os import path, mkdir +from os import mkdir, path from shutil import rmtree from loguru import logger @@ -69,10 +68,10 @@ def shutdown_instance(cls): def mongodb_instance_server_process_context( database_config: DatabaseConfig = test_app_config.database, startup_time=2000): - """ - A synchronous context manager to run the MongoDB instance in a separate process (non-blocking) using the - `subprocess `_ package. The main use case is envisaged to be in - testing environments. + """A synchronous context manager to run the MongoDB instance in a separate process (non-blocking). + + It uses the `subprocess `_ package. The main use case is + envisaged to be in testing environments. Args: database_config: From 5482c2937ebaf0a13811523f18a8e7f107b9cc81 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 11:18:55 +0200 Subject: [PATCH 18/97] Try fixing ci --- .github/workflows/ci.yml | 2 +- pyproject.toml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84c5a16..8ad5585 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12"] + python-version: ["3.11", "3.12"] env: PYTHON_VERSION: ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index 45b7c44..b145efd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,18 +5,18 @@ description = "The database API of Pytroll." authors = [ { name = "Pouria Khalaj", email = "pouria.khalaj@smhi.se" } ] -dependencies = ['pymongo', 'posttroll', 'motor', 'pydantic', 'fastapi', 'uvicorn', "loguru", "pyyaml", "urllib3"] +dependencies = ["pymongo", "posttroll", "motor", "pydantic", "fastapi", "uvicorn", "loguru", "pyyaml", "urllib3"] readme = "README.rst" requires-python = ">= 3.12" license = {file = "LICENSE"} classifiers = [ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Programming Language :: Python', - 'Operating System :: OS Independent', - 'Intended Audience :: Science/Research', - 'Topic :: Scientific/Engineering', - 'Topic :: Database' + "Development Status :: 4 - Beta", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python", + "Operating System :: OS Independent", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "Topic :: Database" ] [project.urls] From 4591f648cf68a5074b2cab8e052ca361cf461e28 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 11:24:25 +0200 Subject: [PATCH 19/97] Fix ci --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ad5585..af837d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,13 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: Python package +name: Continuous Integration on: push: - branches: [ "master" ] + branches: [ "master", "main" ] pull_request: - branches: [ "master" ] + branches: [ "master", "main" ] jobs: build: From 959ff5d2a66eeb45fdd209acaa8c679307165b3c Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 11:29:37 +0200 Subject: [PATCH 20/97] Add ci.yml --- .github/workflows/ci.yml | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..af837d4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Continuous Integration + +on: + push: + branches: [ "master", "main" ] + pull_request: + branches: [ "master", "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12"] + env: + PYTHON_VERSION: ${{ matrix.python-version }} + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt install -y mongodb + python -m pip install --upgrade pip + python -m pip install ruff pytest pytest-asyncio + python -m pip install -e . + - name: Lint with ruff + run: | + ruff check . + - name: Test with pytest + run: | + pytest --cov=pytroll_db tests --cov-report=xml + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: pytroll/pytroll-db + file: ./coverage.xml + env_vars: PYTHON_VERSION From 1b5c61a68eca4470ed78f14337512b8fe94e59e4 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 11:39:52 +0200 Subject: [PATCH 21/97] Fix package --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af837d4..a95d781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt install -y mongodb + sudo apt install -y mongodb-org python -m pip install --upgrade pip python -m pip install ruff pytest pytest-asyncio python -m pip install -e . From 6f36e15d277ed871a44e1a801af9595d1948851f Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 11:49:10 +0200 Subject: [PATCH 22/97] Fix mongodb installation --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a95d781..59fce5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,15 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Install mongodb + run: | + sudo apt install -y gnupg curl + curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor + echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list + sudo apt-get update + sudo apt install -y mongodb-org - name: Install dependencies run: | - sudo apt install -y mongodb-org python -m pip install --upgrade pip python -m pip install ruff pytest pytest-asyncio python -m pip install -e . From e3af5bf07b5cbffac5673bd79b89c0e77a9a6d22 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 11:51:03 +0200 Subject: [PATCH 23/97] Add pytest cov to ci installation --- .github/workflows/ci.yml | 2 +- .github/workflows/pylint.yml | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 .github/workflows/pylint.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59fce5f..18f43df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install ruff pytest pytest-asyncio + python -m pip install ruff pytest pytest-asyncio pytest-cov python -m pip install -e . - name: Lint with ruff run: | diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index db12d51..0000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Pylint - -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - - name: Analysing the code with pylint - run: | - pylint $(git ls-files '*.py') From 93dd84d4f7d5192aeb4dc44e76a87a2bc5e3ffa1 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 11:54:36 +0200 Subject: [PATCH 24/97] Fix more ruff stuff (WIP). --- trolldb/__init__.py | 1 + trolldb/api/__init__.py | 1 + trolldb/api/routes/__init__.py | 2 + trolldb/api/routes/common.py | 13 ++---- trolldb/config/__init__.py | 1 + trolldb/config/config.py | 57 ++++++++++---------------- trolldb/database/__init__.py | 2 +- trolldb/database/mongodb.py | 7 ++-- trolldb/database/piplines.py | 7 ++-- trolldb/database/tests/conftest.py | 14 ++++++- trolldb/database/tests/test_mongodb.py | 14 +++++-- trolldb/errors/__init__.py | 1 + trolldb/test_utils/__init__.py | 1 + 13 files changed, 62 insertions(+), 59 deletions(-) diff --git a/trolldb/__init__.py b/trolldb/__init__.py index e69de29..f054e81 100644 --- a/trolldb/__init__.py +++ b/trolldb/__init__.py @@ -0,0 +1 @@ +"""trolldb package.""" diff --git a/trolldb/api/__init__.py b/trolldb/api/__init__.py index e69de29..2780698 100644 --- a/trolldb/api/__init__.py +++ b/trolldb/api/__init__.py @@ -0,0 +1 @@ +"""api package.""" diff --git a/trolldb/api/routes/__init__.py b/trolldb/api/routes/__init__.py index 08584de..4c69061 100644 --- a/trolldb/api/routes/__init__.py +++ b/trolldb/api/routes/__init__.py @@ -1,3 +1,5 @@ +"""routes package.""" + from .router import api_router __all__ = ("api_router",) diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py index 7acada5..e7d9092 100644 --- a/trolldb/api/routes/common.py +++ b/trolldb/api/routes/common.py @@ -1,5 +1,4 @@ -"""The module which defines common functions to be used in handling requests related to `databases` and `collections`. -""" +"""The module with common functions to be used in handling requests related to `databases` and `collections`.""" from typing import Annotated @@ -66,7 +65,7 @@ async def check_collection( async def get_distinct_items_in_collection( res_coll: Response | AsyncIOMotorCollection, field_name: str) -> Response | list[str]: - """An auxiliary function to either return the given response; or return a list of distinct (unique) values + """An auxiliary function to either return the given response; or return a list of distinct (unique) values. Given the ``field_name`` it conducts a search in all documents of the given collection. The latter behaviour is equivalent to the ``distinct`` function from MongoDB. The former is the behaviour of an identity function. @@ -91,11 +90,7 @@ async def get_distinct_items_in_collection( CheckCollectionDependency = Annotated[AsyncIOMotorCollection, Depends(check_collection)] -""" -Type annotation for the FastAPI dependency injection of checking a collection (function). -""" +"""Type annotation for the FastAPI dependency injection of checking a collection (function).""" CheckDataBaseDependency = Annotated[AsyncIOMotorDatabase, Depends(check_database)] -""" -Type annotation for the FastAPI dependency injection of checking a database (function). -""" +"""Type annotation for the FastAPI dependency injection of checking a database (function).""" diff --git a/trolldb/config/__init__.py b/trolldb/config/__init__.py index e69de29..28c6e45 100644 --- a/trolldb/config/__init__.py +++ b/trolldb/config/__init__.py @@ -0,0 +1 @@ +"""config package.""" diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 67be871..bf48b90 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -1,4 +1,5 @@ """The module which handles parsing and validating the config (YAML) file. + The validation is performed using `Pydantic `_. Note: @@ -23,16 +24,18 @@ def id_must_be_valid(v: str) -> ObjectId: + """Checks that the given string can be converted to a valid MongoDB ObjectId.""" try: return ObjectId(v) except InvalidId as e: - raise ValueError(str(e)) + raise ValueError from e MongoObjectId = Annotated[str, AfterValidator(id_must_be_valid)] class MongoDocument(BaseModel): + """Pydantic model for a MongoDB document.""" _id: MongoObjectId @@ -44,15 +47,12 @@ class LicenseInfo(TypedDict): """ name: str - """ - The full name of the license including the exact variant and the version (if any), e.g. + """The full name of the license including the exact variant and the version (if any), e.g. ``"The GNU General Public License v3.0"`` """ url: AnyUrl - """ - The URL to access the license, e.g. ``"https://www.gnu.org/licenses/gpl-3.0.en.html"`` - """ + """The URL to access the license, e.g. ``"https://www.gnu.org/licenses/gpl-3.0.en.html"``""" class APIServerConfig(NamedTuple): @@ -65,61 +65,46 @@ class APIServerConfig(NamedTuple): """ url: AnyUrl - """ - The URL of the API server including the port, e.g. ``mongodb://localhost:8000``. This will not be passed to the + """The URL of the API server including the port, e.g. ``mongodb://localhost:8000``. This will not be passed to the FastAPI class. Instead, it will be used by the `uvicorn` to determine the URL of the server. """ title: str - """ - The title of the API server, as appears in the automatically generated documentation by the FastAPI. - """ + """The title of the API server, as appears in the automatically generated documentation by the FastAPI.""" version: str - """ - The version of the API server as appears in the automatically generated documentation by the FastAPI. - """ + """The version of the API server as appears in the automatically generated documentation by the FastAPI.""" summary: Optional[str] = None - """ - The summary of the API server, as appears in the automatically generated documentation by the FastAPI. - """ + """The summary of the API server, as appears in the automatically generated documentation by the FastAPI.""" description: Optional[str] = None - """ - The more comprehensive description (extended summary) of the API server, as appears in the automatically generated - documentation by the FastAPI. + """The more comprehensive description (extended summary) of the API server, as appears in the automatically + generated documentation by the FastAPI. """ license_info: Optional[LicenseInfo] = None - """ - The license information of the API server, as appears in the automatically generated documentation by the FastAPI. + """The license information of the API server, as appears in the automatically generated documentation by the + FastAPI. """ class DatabaseConfig(NamedTuple): - """A named tuple to hold all the configurations of the Database which will be used by the MongoDB instance. - """ + """A named tuple to hold all the configurations of the Database which will be used by the MongoDB instance.""" main_database_name: str - """ - The name of the main database which includes the ``main_collection``, e.g. ``"satellite_database"``. - """ + """The name of the main database which includes the ``main_collection``, e.g. ``"satellite_database"``.""" main_collection_name: str - """ - The name of the main collection which resides inside the ``main_database`` and includes the actual data for the - files, e.g. ``"files"`` + """The name of the main collection which resides inside the ``main_database`` and includes the actual data for the + files, e.g. ``"files"`` """ url: MongoDsn - """ - The URL of the MongoDB server excluding the port part, e.g. ``"mongodb://localhost:27017"`` - """ + """The URL of the MongoDB server excluding the port part, e.g. ``"mongodb://localhost:27017"``""" timeout: Annotated[int, Field(gt=-1)] - """ - The timeout in milliseconds (non-negative integer), after which an exception is raised if a connection with the + """The timeout in milliseconds (non-negative integer), after which an exception is raised if a connection with the MongoDB instance is not established successfully, e.g. ``1000``. """ @@ -163,7 +148,7 @@ def from_yaml(filename: FilePath) -> AppConfig: @validate_call def parse(config: AppConfig | FilePath) -> AppConfig: - """Tries to return a valid object of type :class:`AppConfig` + """Tries to return a valid object of type :class:`AppConfig`. Args: config: diff --git a/trolldb/database/__init__.py b/trolldb/database/__init__.py index 8b13789..f8444ef 100644 --- a/trolldb/database/__init__.py +++ b/trolldb/database/__init__.py @@ -1 +1 @@ - +"""database package.""" diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 04a3c73..d018c93 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -30,10 +30,12 @@ class DatabaseName(BaseModel): + """Pydantic model for a database name.""" name: str | None class CollectionName(BaseModel): + """Pydantic model for a collection name.""" name: str | None @@ -100,9 +102,7 @@ class MongoDB: __main_database: AsyncIOMotorDatabase = None default_database_names = ["admin", "config", "local"] - """ - MongoDB creates these databases by default for self usage. - """ + """MongoDB creates these databases by default for self usage.""" @classmethod async def initialize(cls, database_config: DatabaseConfig): @@ -177,6 +177,7 @@ def close(cls) -> None: @classmethod def list_database_names(cls) -> CoroutineStrList: + """Lists all the database names.""" return cls.__client.list_database_names() @classmethod diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 9a43dad..8fb95d5 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -1,5 +1,4 @@ -"""The module which defines some convenience classes to facilitate the use of aggregation pipelines. -""" +"""The module which defines some convenience classes to facilitate the use of aggregation pipelines.""" from typing import Any, Self @@ -9,8 +8,8 @@ class PipelineDict(dict): The operators are only defined for operands of type :class:`PipelineDict`. For each of the aforementioned operators, the result will be a dictionary with a single key/value pair. The key is either ``$or`` or ``$and`` depending on the - operator being used. The corresponding value is a list with two elements only. The first element of the list is the left operand and the - second element is the right operand. + operator being used. The corresponding value is a list with two elements only. The first element of the list is the + left operand and the second element is the right operand. Example: ``` diff --git a/trolldb/database/tests/conftest.py b/trolldb/database/tests/conftest.py index 247683f..6a73628 100644 --- a/trolldb/database/tests/conftest.py +++ b/trolldb/database/tests/conftest.py @@ -1,3 +1,8 @@ +"""Pytest config for database tests. + +This module provides fixtures for running a Mongo DB instance in test mode and filling the database with test data. +""" + import pytest import pytest_asyncio @@ -8,13 +13,18 @@ @pytest.fixture(scope="session") -def run_mongodb_server_instance(): +def _run_mongodb_server_instance(): + """Runs the MongoDB instance in test mode using a context manager. + + It is run once for all tests in this test suite. + """ with mongodb_instance_server_process_context(): yield @pytest_asyncio.fixture() -async def mongodb_fixture(run_mongodb_server_instance): +async def mongodb_fixture(_run_mongodb_server_instance): + """Fills the database with test data and then runs the mongodb client using a context manager.""" TestDatabase.prepare() async with mongodb_context(test_app_config.database): yield diff --git a/trolldb/database/tests/test_mongodb.py b/trolldb/database/tests/test_mongodb.py index b24ec13..b25f930 100644 --- a/trolldb/database/tests/test_mongodb.py +++ b/trolldb/database/tests/test_mongodb.py @@ -1,3 +1,5 @@ +"""Direct tests for `mongodb` module without an API server connection.""" + import errno import time @@ -12,8 +14,8 @@ async def test_connection_timeout_negative(): """Expect to see the connection attempt times out since the MongoDB URL is invalid.""" timeout = 3000 + t1 = time.time() with pytest.raises(SystemExit) as pytest_wrapped_e: - t1 = time.time() async with mongodb_context( DatabaseConfig(url=AnyUrl("mongodb://invalid_url_that_does_not_exist:8000"), timeout=timeout, main_database_name=" ", main_collection_name=" ")): @@ -23,7 +25,8 @@ async def test_connection_timeout_negative(): assert t2 - t1 >= timeout / 1000 -async def test_main_database_negative(run_mongodb_server_instance): +@pytest.mark.usefixtures("_run_mongodb_server_instance") +async def test_main_database_negative(): """Expect to fail when giving an invalid name for the main database, given a valid collection name.""" with pytest.raises(SystemExit) as pytest_wrapped_e: async with mongodb_context(DatabaseConfig( @@ -35,7 +38,8 @@ async def test_main_database_negative(run_mongodb_server_instance): assert pytest_wrapped_e.value.code == errno.ENODATA -async def test_main_collection_negative(run_mongodb_server_instance): +@pytest.mark.usefixtures("_run_mongodb_server_instance") +async def test_main_collection_negative(): """Expect to fail when giving an invalid name for the main collection, given a valid database name.""" with pytest.raises(SystemExit) as pytest_wrapped_e: async with mongodb_context(DatabaseConfig( @@ -60,7 +64,9 @@ async def test_get_client(mongodb_fixture): async def test_main_collection(mongodb_fixture): - """Expect: + """Tests the properties of the main collection. + + Expect: - The retrieved main collection is not `None` - It has the correct name - It is the same object that can be accessed via the `client` object of the MongoDB. diff --git a/trolldb/errors/__init__.py b/trolldb/errors/__init__.py index e69de29..da3944b 100644 --- a/trolldb/errors/__init__.py +++ b/trolldb/errors/__init__.py @@ -0,0 +1 @@ +"""errors package.""" diff --git a/trolldb/test_utils/__init__.py b/trolldb/test_utils/__init__.py index e69de29..e18f5c9 100644 --- a/trolldb/test_utils/__init__.py +++ b/trolldb/test_utils/__init__.py @@ -0,0 +1 @@ +"""test_utils package.""" From a48f9632252f086446e93ede35bf8f493422cc8a Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 12:00:12 +0200 Subject: [PATCH 25/97] Just run ci on 3.12 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18f43df..e5c851e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12"] + python-version: ["3.12"] env: PYTHON_VERSION: ${{ matrix.python-version }} From 351926d252d98058f72ce1ad6934be1c3dcb3707 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 12:01:12 +0200 Subject: [PATCH 26/97] Add pre-commit-ci config --- .pre-commit-config.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..867cafe --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +exclude: '^$' +fail_fast: false + +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: 'v0.3.7' + hooks: + - id: ruff + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: no-commit-to-branch From 9bc3f85b5b4f00f17eeabaf80de03dd84cf981c7 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 12:05:19 +0200 Subject: [PATCH 27/97] Add pre-commit-ci --- README.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 6fa0b57..dff5e31 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,11 @@ The database interface of `Pytroll `_ +.. image:: https://results.pre-commit.ci/badge/github/pytroll/pytroll-db/master.svg + :target: https://results.pre-commit.ci/latest/github/pytroll/pytroll-db/master + :alt: pre-commit.ci status + + Copyright (C) 2012, 2014, 2015, 2024 @@ -32,4 +37,3 @@ Disclaimer GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . - From 86d20f5971c1ce67be14c9bf673c76ae6d2dce10 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 10:05:28 +0000 Subject: [PATCH 28/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- LICENSE | 2 +- changelog.rst | 2 -- trolldb/errors/errors.py | 4 ++-- trolldb/template_config.yaml | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/LICENSE b/LICENSE index 70566f2..ef7e7ef 100644 --- a/LICENSE +++ b/LICENSE @@ -671,4 +671,4 @@ into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read -. \ No newline at end of file +. diff --git a/changelog.rst b/changelog.rst index 7c230d0..5b87e54 100644 --- a/changelog.rst +++ b/changelog.rst @@ -215,5 +215,3 @@ Other postgreSQL/postGIS database. [Adam Dybbroe] - Initial commit. [Martin Raspaud] - - diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index 723c4cb..c6c1c54 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -19,9 +19,9 @@ class ResponseError(Exception): descriptor_delimiter: str = " |OR| " """ - A delimiter to combine the message part of several error responses into a single one. This will be shown in textual + A delimiter to combine the message part of several error responses into a single one. This will be shown in textual format for the response descriptors of the Fast API routes. For example: - + ``ErrorA |OR| ErrorB`` """ diff --git a/trolldb/template_config.yaml b/trolldb/template_config.yaml index 8afd05a..b16a166 100644 --- a/trolldb/template_config.yaml +++ b/trolldb/template_config.yaml @@ -34,9 +34,9 @@ api_server: # Optional "description": " - The API allows you to perform CRUD operations as well as querying the database. + The API allows you to perform CRUD operations as well as querying the database. At the moment only MongoDB is supported. It is based on the following Python packages - \n * **PyMongo** (https://github.com/mongodb/mongo-python-driver) + \n * **PyMongo** (https://github.com/mongodb/mongo-python-driver) \n * **motor** (https://github.com/mongodb/motor) " From 9d74b51ea8be3421fcb9e7ca908eba869027880d Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 13:03:43 +0200 Subject: [PATCH 29/97] Remove ruff from ci in favour of pre-commit --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5c851e..44a3560 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,9 +38,6 @@ jobs: python -m pip install --upgrade pip python -m pip install ruff pytest pytest-asyncio pytest-cov python -m pip install -e . - - name: Lint with ruff - run: | - ruff check . - name: Test with pytest run: | pytest --cov=pytroll_db tests --cov-report=xml From 16fefaf30dc0d61b5cac6039d1449a676aaf78d7 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 13:06:33 +0200 Subject: [PATCH 30/97] Run pytest on all test files --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44a3560..d4a7bde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: python -m pip install -e . - name: Test with pytest run: | - pytest --cov=pytroll_db tests --cov-report=xml + pytest --cov=pytroll_db --cov-report=xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 with: From 99b78006a0eefc982ddc8b527265bac87ef19673 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 13:39:57 +0200 Subject: [PATCH 31/97] Make ruff happy. --- trolldb/api/routes/databases.py | 4 ++++ trolldb/api/routes/datetime_.py | 5 +++++ trolldb/api/routes/platforms.py | 1 + trolldb/api/routes/queries.py | 9 +++++---- trolldb/api/routes/root.py | 1 + trolldb/api/routes/sensors.py | 1 + trolldb/api/tests/conftest.py | 10 +++++++--- trolldb/api/tests/test_api.py | 14 ++++++++++--- trolldb/database/errors.py | 14 ++++++------- trolldb/database/piplines.py | 15 ++++++++++++++ trolldb/errors/errors.py | 26 +++++++++++++++++-------- trolldb/run_api.py | 2 +- trolldb/test_utils/common.py | 15 ++++++++------ trolldb/test_utils/mongodb_database.py | 27 ++++++++++++++++++++++---- trolldb/test_utils/mongodb_instance.py | 13 ++++++++++--- 15 files changed, 117 insertions(+), 40 deletions(-) diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py index 798e798..fa92264 100644 --- a/trolldb/api/routes/databases.py +++ b/trolldb/api/routes/databases.py @@ -24,6 +24,7 @@ response_model=list[str], summary="Gets the list of all database names") async def database_names(exclude_defaults: bool = exclude_defaults_query) -> list[str]: + """TODO.""" db_names = await MongoDB.list_database_names() if not exclude_defaults: @@ -37,6 +38,7 @@ async def database_names(exclude_defaults: bool = exclude_defaults_query) -> lis responses=Databases.union().fastapi_descriptor, summary="Gets the list of all collection names for the given database name") async def collection_names(db: CheckDataBaseDependency) -> list[str]: + """TODO.""" return await db.list_collection_names() @@ -45,6 +47,7 @@ async def collection_names(db: CheckDataBaseDependency) -> list[str]: responses=database_collection_error_descriptor, summary="Gets the object ids of all documents for the given database and collection name") async def documents(collection: CheckCollectionDependency) -> list[str]: + """TODO.""" return await get_ids(collection.find({})) @@ -53,6 +56,7 @@ async def documents(collection: CheckCollectionDependency) -> list[str]: responses=database_collection_document_error_descriptor, summary="Gets the document content in json format given its object id, database, and collection name") async def document_by_id(collection: CheckCollectionDependency, _id: MongoObjectId) -> _DocumentType: + """TODO.""" if document := await collection.find_one({"_id": _id}): return dict(document) | {"_id": str(_id)} diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index f5d4e48..78d412d 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -16,16 +16,19 @@ class TimeModel(TypedDict): + """TODO.""" _id: str _time: datetime class TimeEntry(TypedDict): + """TODO.""" _min: TimeModel _max: TimeModel class ResponseModel(BaseModel): + """TODO.""" start_time: TimeEntry end_time: TimeEntry @@ -38,6 +41,7 @@ class ResponseModel(BaseModel): responses=database_collection_error_descriptor, summary="Gets the the minimum and maximum values for the start and end times") async def datetime(collection: CheckCollectionDependency) -> ResponseModel: + """TODO.""" agg_result = await collection.aggregate([{ "$group": { "_id": None, @@ -48,6 +52,7 @@ async def datetime(collection: CheckCollectionDependency) -> ResponseModel: }}]).next() def _aux(query): + """TODO.""" return get_id(collection.find_one(query)) return ResponseModel( diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py index 4e6e868..53708a7 100644 --- a/trolldb/api/routes/platforms.py +++ b/trolldb/api/routes/platforms.py @@ -17,4 +17,5 @@ responses=database_collection_error_descriptor, summary="Gets the list of all platform names") async def platform_names(collection: CheckCollectionDependency) -> list[str]: + """TODO.""" return await get_distinct_items_in_collection(collection, "platform_name") diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index 20d3984..78284dd 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -22,10 +22,11 @@ summary="Gets the database UUIDs of the documents that match specifications determined by the query string") async def queries( collection: CheckCollectionDependency, - platform: list[str] = Query(None), - sensor: list[str] = Query(None), - time_min: datetime.datetime = Query(None), - time_max: datetime.datetime = Query(None)) -> list[str]: + platform: list[str] = Query(default=None), # noqa: B008 + sensor: list[str] = Query(default=None), # noqa: B008 + time_min: datetime.datetime = Query(default=None), # noqa: B008 + time_max: datetime.datetime = Query(default=None)) -> list[str]: # noqa: B008 + """TODO.""" pipelines = Pipelines() if platform: diff --git a/trolldb/api/routes/root.py b/trolldb/api/routes/root.py index 5099fe9..3fa975f 100644 --- a/trolldb/api/routes/root.py +++ b/trolldb/api/routes/root.py @@ -11,4 +11,5 @@ @router.get("/", summary="The root route which is mainly used to check the status of connection") async def root() -> Response: + """TODO.""" return Response(status_code=status.HTTP_200_OK) diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py index 2fa5d0b..826b163 100644 --- a/trolldb/api/routes/sensors.py +++ b/trolldb/api/routes/sensors.py @@ -17,4 +17,5 @@ responses=database_collection_error_descriptor, summary="Gets the list of all sensor names") async def sensor_names(collection: CheckCollectionDependency) -> list[str]: + """TODO.""" return await get_distinct_items_in_collection(collection, "sensor") diff --git a/trolldb/api/tests/conftest.py b/trolldb/api/tests/conftest.py index 5a8d7c1..b9a3a6d 100644 --- a/trolldb/api/tests/conftest.py +++ b/trolldb/api/tests/conftest.py @@ -1,3 +1,5 @@ +"""TODO.""" + import pytest from trolldb.api.api import server_process_context @@ -7,13 +9,15 @@ @pytest.fixture(scope="session") -def run_mongodb_server_instance(): +def _run_mongodb_server_instance(): + """TODO.""" with mongodb_instance_server_process_context(): yield -@pytest.fixture(scope="session", autouse=True) -def test_server_fixture(run_mongodb_server_instance): +@pytest.fixture(scope="session") +def _test_server_fixture(_run_mongodb_server_instance): + """TODO.""" TestDatabase.prepare() with server_process_context(test_app_config, startup_time=2000): yield diff --git a/trolldb/api/tests/test_api.py b/trolldb/api/tests/test_api.py index 445b56a..97743ee 100644 --- a/trolldb/api/tests/test_api.py +++ b/trolldb/api/tests/test_api.py @@ -1,25 +1,30 @@ +"""TODO.""" +import pytest from fastapi import status from trolldb.test_utils.common import assert_equal, http_get from trolldb.test_utils.mongodb_database import TestDatabase, test_mongodb_context +@pytest.mark.usefixtures("_test_server_fixture") def test_root(): """Checks that the server is up and running, i.e. the root routes responds with 200.""" assert_equal(http_get().status, status.HTTP_200_OK) +@pytest.mark.usefixtures("_test_server_fixture") def test_platforms(): """Checks that the retrieved platform names match the expected names.""" assert_equal(http_get("platforms").json(), TestDatabase.platform_names) +@pytest.mark.usefixtures("_test_server_fixture") def test_sensors(): - """Checks that the retrieved sensor names match the expected names. - """ + """Checks that the retrieved sensor names match the expected names.""" assert_equal(http_get("sensors").json(), TestDatabase.sensors) +@pytest.mark.usefixtures("_test_server_fixture") def test_database_names(): """Checks that the retrieved database names match the expected names.""" assert_equal(http_get("databases").json(), TestDatabase.database_names) @@ -27,11 +32,13 @@ def test_database_names(): assert_equal(http_get("databases?exclude_defaults=False").json(), TestDatabase.all_database_names) +@pytest.mark.usefixtures("_test_server_fixture") def test_database_names_negative(): """Checks that the non-existing databases cannot be found.""" assert_equal(http_get("databases/non_existing_database").status, status.HTTP_404_NOT_FOUND) +@pytest.mark.usefixtures("_test_server_fixture") def test_collections(): """Check the presence of existing collections and that the ids of documents therein can be correctly retrieved.""" with test_mongodb_context() as client: @@ -50,9 +57,10 @@ def test_collections(): ) +@pytest.mark.usefixtures("_test_server_fixture") def test_collections_negative(): """Checks that the non-existing collections cannot be found.""" - for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names, strict=False): + for database_name in TestDatabase.database_names: assert_equal( http_get(f"databases/{database_name}/non_existing_collection").status, status.HTTP_404_NOT_FOUND diff --git a/trolldb/database/errors.py b/trolldb/database/errors.py index a15ec43..044181f 100644 --- a/trolldb/database/errors.py +++ b/trolldb/database/errors.py @@ -1,5 +1,4 @@ -"""The modules which defines the error responses that might occur while working with the -MongoDB database. +"""The modules which defines the error responses that might occur while working with the MongoDB database. Note: The error responses are grouped into classes, with each class representing the major @@ -79,14 +78,13 @@ class Documents(ResponsesErrorGroup): database_collection_error_descriptor = ( Databases.union() | Collections.union() ).fastapi_descriptor -""" -A response descriptor for the Fast API routes. This combines all the error messages that might -occur as result of working with databases and collections. See the fast api documentation for TODO. +"""A response descriptor for the Fast API routes. + +This combines all the error messages that might occur as result of working with databases and collections. See the +fast api documentation for TODO. """ database_collection_document_error_descriptor = ( Databases.union() | Collections.union() | Documents.union() ).fastapi_descriptor -""" -Same as :obj:`database_collection_error_descriptor` but including documents as well. -""" +"""Same as :obj:`database_collection_error_descriptor` but including documents as well.""" diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 8fb95d5..eeb0cac 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -21,45 +21,60 @@ class PipelineDict(dict): """ def __or__(self, other: Self): + """TODO.""" return PipelineDict({"$or": [self, other]}) def __and__(self, other: Self): + """TODO.""" return PipelineDict({"$and": [self, other]}) class PipelineAttribute: + """TODO.""" + def __init__(self, key: str): + """TODO.""" self.__key = key def __eq__(self, other: Any) -> PipelineDict: + """TODO.""" if isinstance(other, list): return PipelineDict(**{"$or": [{self.__key: v} for v in other]}) return PipelineDict(**{self.__key: other}) def __aux_operators(self, other: Any, operator: str) -> PipelineDict: + """TODO.""" return PipelineDict(**{self.__key: {operator: other}} if other else {}) def __ge__(self, other: Any) -> PipelineDict: + """TODO.""" return self.__aux_operators(other, "$gte") def __gt__(self, other: Any) -> PipelineDict: + """TODO.""" return self.__aux_operators(other, "$gt") def __le__(self, other: Any) -> PipelineDict: + """TODO.""" return self.__aux_operators(other, "$lte") def __lt__(self, other: Any) -> PipelineDict: + """TODO.""" return self.__aux_operators(other, "$le") class Pipelines(list): + """TODO.""" def __init__(self, *args, **kwargs): + """TODO.""" super().__init__(*args, **kwargs) def __iadd__(self, other): + """TODO.""" self.extend([{"$match": other}]) return self def __add__(self, other): + """TODO.""" self.append({"$match": other}) return self diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index c6c1c54..eeb7551 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -1,4 +1,5 @@ """The module which defines the base functionality for error responses that will be returned by the API. + This module only includes the generic utilities using which each module should define its own error responses specifically. See :obj:`trolldb.database.errors` as an example on how this module is used. """ @@ -18,19 +19,18 @@ class ResponseError(Exception): """The base class for all error responses. This is derivative of the ``Exception`` class.""" descriptor_delimiter: str = " |OR| " - """ - A delimiter to combine the message part of several error responses into a single one. This will be shown in textual - format for the response descriptors of the Fast API routes. For example: + """A delimiter to combine the message part of several error responses into a single one. + + This will be shown in textual format for the response descriptors of the Fast API routes. For example: ``ErrorA |OR| ErrorB`` """ - defaultResponseClass: Response = PlainTextResponse - """ - The default type of the response which will be returned when an error occurs. - """ + DefaultResponseClass: Response = PlainTextResponse + """The default type of the response which will be returned when an error occurs.""" def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) -> None: + """TODO.""" self.__dict: OrderedDict = OrderedDict(args_dict) self.extra_information: dict | None = None @@ -64,6 +64,7 @@ def __or__(self, other: Self): def __assert_existence_multiple_response_codes( self, status_code: StatusCode | None = None) -> (StatusCode, str): + """TODO.""" match status_code, len(self.__dict): case None, n if n > 1: raise ValueError("In case of multiple response status codes, the status code must be specified.") @@ -80,6 +81,7 @@ def get_error_details( self, extra_information: dict | None = None, status_code: int | None = None) -> (StatusCode, str): + """TODO.""" status_code, msg = self.__assert_existence_multiple_response_codes(status_code) return ( status_code, @@ -91,6 +93,7 @@ def sys_exit_log( exit_code: int = -1, extra_information: dict | None = None, status_code: int | None = None) -> None: + """TODO.""" msg, _ = self.get_error_details(extra_information, status_code) logger.error(msg) exit(exit_code) @@ -99,32 +102,39 @@ def log_as_warning( self, extra_information: dict | None = None, status_code: int | None = None): + """TODO.""" msg, _ = self.get_error_details(extra_information, status_code) logger.warning(msg) @property def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], str]]: + """TODO.""" return {status: {Literal["description"]: ResponseError.__stringify(msg)} for status, msg in self.__dict.items()} @staticmethod def __listify(item: str | list[str]) -> list[str]: + """TODO.""" return item if isinstance(item, list) else [item] @staticmethod def __stringify(item: str | list[str]) -> str: + """TODO.""" return ResponseError.descriptor_delimiter.join(ResponseError.__listify(item)) class ResponsesErrorGroup: + """TODO.""" @classmethod def fields(cls): + """TODO.""" return {k: v for k, v in cls.__dict__.items() if isinstance(v, ResponseError)} @classmethod def union(cls): + """TODO.""" buff = None - for k, v in cls.fields().items(): + for v in cls.fields().values(): if buff is None: buff = v else: diff --git a/trolldb/run_api.py b/trolldb/run_api.py index d1b0ea2..b5bc6cf 100644 --- a/trolldb/run_api.py +++ b/trolldb/run_api.py @@ -1,4 +1,4 @@ -"""The main entry point to run the API server according to the configurations given in `config.yaml` +"""The main entry point to run the API server according to the configurations given in `config.yaml`. Note: For more information on the API server, see the automatically generated documentation by FastAPI. diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 7994308..f4bb333 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,3 +1,5 @@ +"""TODO.""" + from typing import Any from urllib.parse import urljoin @@ -30,9 +32,10 @@ def http_get(route: str = "") -> BaseHTTPResponse: def assert_equal(test, expected) -> None: - """An auxiliary function to assert the equality of two objects using the ``==`` operator. In case an input is a list or - a tuple, it will be first converted to a set so that the order of items there in does not affect the assertion - outcome. + """An auxiliary function to assert the equality of two objects using the ``==`` operator. + + In case an input is a list or a tuple, it will be first converted to a set so that the order of items there in does + not affect the assertion outcome. Warning: In case of a list or tuple of items as inputs, do not use this function if the order of items matters. @@ -45,8 +48,8 @@ def assert_equal(test, expected) -> None: """ def _setify(obj: Any) -> Any: - """An auxiliary function to convert an object to a set if it is a tuple or a list. - """ + """An auxiliary function to convert an object to a set if it is a tuple or a list.""" return set(obj) if isinstance(obj, list | tuple) else obj - assert _setify(test) == _setify(expected) + if not _setify(test) == _setify(expected): + raise AssertionError(f"{test} and {expected} are not equal.") diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index 9eeb4d2..c79ef18 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -1,3 +1,5 @@ +"""TODO.""" + from contextlib import contextmanager from datetime import datetime, timedelta from random import randint, shuffle @@ -11,6 +13,7 @@ @contextmanager def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database): + """TODO.""" client = None try: client = MongoClient(database_config.url.unicode_string(), connectTimeoutMS=database_config.timeout) @@ -21,39 +24,48 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab def random_sample(items: list[Any], size=10): + """TODO.""" last_index = len(items) - 1 - indices = [randint(0, last_index) for _ in range(size)] + indices = [randint(0, last_index) for _ in range(size)] # noqa: S311 return [items[i] for i in indices] class Time: + """TODO.""" min_start_time = datetime(2019, 1, 1, 0, 0, 0) max_end_time = datetime(2024, 1, 1, 0, 0, 0) delta_time = int((max_end_time - min_start_time).total_seconds()) @staticmethod def random_interval_secs(max_interval_secs): - return timedelta(seconds=randint(0, max_interval_secs)) + """TODO.""" + return timedelta(seconds=randint(0, max_interval_secs)) # noqa: S311 @staticmethod def random_start_time(): + """TODO.""" return Time.min_start_time + Time.random_interval_secs(Time.delta_time) @staticmethod def random_end_time(start_time: datetime, max_interval_secs: int = 300): + """TODO.""" return start_time + Time.random_interval_secs(max_interval_secs) class Document: + """TODO.""" + def __init__(self, platform_name: str, sensor: str): + """TODO.""" self.platform_name = platform_name self.sensor = sensor self.start_time = Time.random_start_time() self.end_time = Time.random_end_time(self.start_time) def generate_dataset(self, max_count: int): + """TODO.""" dataset = [] - n = randint(1, max_count) + n = randint(1, max_count) # noqa: S311 for i in range(n): txt = f"{self.platform_name}_{self.sensor}_{self.start_time}_{self.end_time}_{i}" dataset.append({ @@ -64,6 +76,7 @@ def generate_dataset(self, max_count: int): return dataset def like_mongodb_document(self): + """TODO.""" return { "platform_name": self.platform_name, "sensor": self.sensor, @@ -74,6 +87,7 @@ def like_mongodb_document(self): class TestDatabase: + """TODO.""" platform_names = random_sample(["PA", "PB", "PC"]) sensors = random_sample(["SA", "SB", "SC"]) @@ -85,13 +99,16 @@ class TestDatabase: @classmethod def generate_documents(cls, random_shuffle=True) -> list: - documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors, strict=False)] + """TODO.""" + documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors, + strict=False)] if random_shuffle: shuffle(documents) return documents @classmethod def reset(cls): + """TODO.""" with test_mongodb_context() as client: for db_name, coll_name in zip(cls.database_names, cls.collection_names, strict=False): db = client[db_name] @@ -101,6 +118,7 @@ def reset(cls): @classmethod def write_mock_date(cls): + """TODO.""" with test_mongodb_context() as client: cls.documents = cls.generate_documents() collection = client[test_app_config.database.main_database_name][ @@ -109,5 +127,6 @@ def write_mock_date(cls): @classmethod def prepare(cls): + """TODO.""" cls.reset() cls.write_mock_date() diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 7102b05..04fe553 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -1,5 +1,4 @@ -"""The module which defines functionalities to run a MongoDB instance which is to be used in the testing environment. -""" +"""The module which defines functionalities to run a MongoDB instance which is to be used in the testing environment.""" import errno import subprocess import sys @@ -16,6 +15,7 @@ class TestMongoInstance: + """TODO.""" log_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_log") storage_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_storage") port: int = 28017 @@ -23,17 +23,20 @@ class TestMongoInstance: @classmethod def prepare_dir(cls, directory: str): + """TODO.""" cls.remove_dir(directory) mkdir(directory) @classmethod def remove_dir(cls, directory: str): + """TODO.""" if path.exists(directory) and path.isdir(directory): rmtree(directory) @classmethod def run_subprocess(cls, args: list[str], wait=True): - cls.process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + """TODO.""" + cls.process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # noqa: S603 if wait: outs, errs = cls.process.communicate() return outs, errs @@ -41,6 +44,7 @@ def run_subprocess(cls, args: list[str], wait=True): @classmethod def mongodb_exists(cls) -> bool: + """TODO.""" outs, errs = cls.run_subprocess(["which", "mongod"]) if outs and not errs: return True @@ -48,17 +52,20 @@ def mongodb_exists(cls) -> bool: @classmethod def prepare_dirs(cls) -> None: + """TODO.""" cls.prepare_dir(cls.log_dir) cls.prepare_dir(cls.storage_dir) @classmethod def run_instance(cls): + """TODO.""" cls.run_subprocess( ["mongod", "--dbpath", cls.storage_dir, "--logpath", f"{cls.log_dir}/mongod.log", "--port", f"{cls.port}"] , wait=False) @classmethod def shutdown_instance(cls): + """TODO.""" cls.process.kill() for d in [cls.log_dir, cls.storage_dir]: cls.remove_dir(d) From e209a29c33a6a470ef1165c4003e5eb3529d1128 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 13:49:20 +0200 Subject: [PATCH 32/97] Make ruff even happier. --- bin/pytroll-mongo.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/pytroll-mongo.py b/bin/pytroll-mongo.py index 466e0b1..d8203b2 100644 --- a/bin/pytroll-mongo.py +++ b/bin/pytroll-mongo.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +"""TODO.""" import logging import os @@ -17,7 +18,7 @@ class MongoRecorder: def __init__(self, mongo_uri="mongodb://localhost:27017", - db_name='sat_db'): + db_name="sat_db"): """Init the recorder.""" self.db = MongoClient(mongo_uri)[db_name] self.loop = True @@ -38,7 +39,7 @@ def record(self): for msg in sub.recv(timeout=1): if msg: logger.debug("got msg %s", str(msg)) - if msg.type in ['collection', 'file', 'dataset']: + if msg.type in ["collection", "file", "dataset"]: self.insert_files(msg) if not self.loop: logger.info("Stop recording") @@ -67,7 +68,7 @@ def setup_logging(cmd_args): logging.config.dictConfig(log_dict) return - root = logging.getLogger('') + root = logging.getLogger("") root.setLevel(log_levels[cmd_args.verbosity]) if cmd_args.log: @@ -86,9 +87,9 @@ def setup_logging(cmd_args): LOG_FORMAT = "[%(asctime)s %(name)s %(levelname)s] %(message)s" -if __name__ == '__main__': - import time +if __name__ == "__main__": import argparse + import time parser = argparse.ArgumentParser() parser.add_argument("-d", "--database", @@ -116,4 +117,4 @@ def setup_logging(cmd_args): time.sleep(1) except KeyboardInterrupt: recorder.stop() - print("Thanks for using pytroll/mongo_recorder. See you soon on www.pytroll.org!") + # print("Thanks for using pytroll/mongo_recorder. See you soon on www.pytroll.org!") From 2b369063074de499478376123e53e7be69afe96e Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 13:53:04 +0200 Subject: [PATCH 33/97] Commit. --- bin/pytroll-mongo.py | 2 +- trolldb/api/routes/databases.py | 8 +++---- trolldb/api/routes/datetime_.py | 10 ++++---- trolldb/api/routes/platforms.py | 2 +- trolldb/api/routes/queries.py | 2 +- trolldb/api/routes/root.py | 2 +- trolldb/api/routes/sensors.py | 2 +- trolldb/api/tests/conftest.py | 6 ++--- trolldb/api/tests/test_api.py | 2 +- trolldb/database/errors.py | 2 +- trolldb/database/piplines.py | 28 +++++++++++----------- trolldb/errors/errors.py | 22 +++++++++--------- trolldb/test_utils/common.py | 2 +- trolldb/test_utils/mongodb_database.py | 32 +++++++++++++------------- trolldb/test_utils/mongodb_instance.py | 16 ++++++------- 15 files changed, 69 insertions(+), 69 deletions(-) diff --git a/bin/pytroll-mongo.py b/bin/pytroll-mongo.py index d8203b2..be1324d 100644 --- a/bin/pytroll-mongo.py +++ b/bin/pytroll-mongo.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""TODO.""" +"""Documentation to be added!""" import logging import os diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py index fa92264..0d444fa 100644 --- a/trolldb/api/routes/databases.py +++ b/trolldb/api/routes/databases.py @@ -24,7 +24,7 @@ response_model=list[str], summary="Gets the list of all database names") async def database_names(exclude_defaults: bool = exclude_defaults_query) -> list[str]: - """TODO.""" + """Documentation to be added!""" db_names = await MongoDB.list_database_names() if not exclude_defaults: @@ -38,7 +38,7 @@ async def database_names(exclude_defaults: bool = exclude_defaults_query) -> lis responses=Databases.union().fastapi_descriptor, summary="Gets the list of all collection names for the given database name") async def collection_names(db: CheckDataBaseDependency) -> list[str]: - """TODO.""" + """Documentation to be added!""" return await db.list_collection_names() @@ -47,7 +47,7 @@ async def collection_names(db: CheckDataBaseDependency) -> list[str]: responses=database_collection_error_descriptor, summary="Gets the object ids of all documents for the given database and collection name") async def documents(collection: CheckCollectionDependency) -> list[str]: - """TODO.""" + """Documentation to be added!""" return await get_ids(collection.find({})) @@ -56,7 +56,7 @@ async def documents(collection: CheckCollectionDependency) -> list[str]: responses=database_collection_document_error_descriptor, summary="Gets the document content in json format given its object id, database, and collection name") async def document_by_id(collection: CheckCollectionDependency, _id: MongoObjectId) -> _DocumentType: - """TODO.""" + """Documentation to be added!""" if document := await collection.find_one({"_id": _id}): return dict(document) | {"_id": str(_id)} diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index 78d412d..bff8c5c 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -16,19 +16,19 @@ class TimeModel(TypedDict): - """TODO.""" + """Documentation to be added!""" _id: str _time: datetime class TimeEntry(TypedDict): - """TODO.""" + """Documentation to be added!""" _min: TimeModel _max: TimeModel class ResponseModel(BaseModel): - """TODO.""" + """Documentation to be added!""" start_time: TimeEntry end_time: TimeEntry @@ -41,7 +41,7 @@ class ResponseModel(BaseModel): responses=database_collection_error_descriptor, summary="Gets the the minimum and maximum values for the start and end times") async def datetime(collection: CheckCollectionDependency) -> ResponseModel: - """TODO.""" + """Documentation to be added!""" agg_result = await collection.aggregate([{ "$group": { "_id": None, @@ -52,7 +52,7 @@ async def datetime(collection: CheckCollectionDependency) -> ResponseModel: }}]).next() def _aux(query): - """TODO.""" + """Documentation to be added!""" return get_id(collection.find_one(query)) return ResponseModel( diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py index 53708a7..9c7354b 100644 --- a/trolldb/api/routes/platforms.py +++ b/trolldb/api/routes/platforms.py @@ -17,5 +17,5 @@ responses=database_collection_error_descriptor, summary="Gets the list of all platform names") async def platform_names(collection: CheckCollectionDependency) -> list[str]: - """TODO.""" + """Documentation to be added!""" return await get_distinct_items_in_collection(collection, "platform_name") diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index 78284dd..4fd4c80 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -26,7 +26,7 @@ async def queries( sensor: list[str] = Query(default=None), # noqa: B008 time_min: datetime.datetime = Query(default=None), # noqa: B008 time_max: datetime.datetime = Query(default=None)) -> list[str]: # noqa: B008 - """TODO.""" + """Documentation to be added!""" pipelines = Pipelines() if platform: diff --git a/trolldb/api/routes/root.py b/trolldb/api/routes/root.py index 3fa975f..0c543c8 100644 --- a/trolldb/api/routes/root.py +++ b/trolldb/api/routes/root.py @@ -11,5 +11,5 @@ @router.get("/", summary="The root route which is mainly used to check the status of connection") async def root() -> Response: - """TODO.""" + """Documentation to be added!""" return Response(status_code=status.HTTP_200_OK) diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py index 826b163..9852822 100644 --- a/trolldb/api/routes/sensors.py +++ b/trolldb/api/routes/sensors.py @@ -17,5 +17,5 @@ responses=database_collection_error_descriptor, summary="Gets the list of all sensor names") async def sensor_names(collection: CheckCollectionDependency) -> list[str]: - """TODO.""" + """Documentation to be added!""" return await get_distinct_items_in_collection(collection, "sensor") diff --git a/trolldb/api/tests/conftest.py b/trolldb/api/tests/conftest.py index b9a3a6d..bc32a39 100644 --- a/trolldb/api/tests/conftest.py +++ b/trolldb/api/tests/conftest.py @@ -1,4 +1,4 @@ -"""TODO.""" +"""Documentation to be added!""" import pytest @@ -10,14 +10,14 @@ @pytest.fixture(scope="session") def _run_mongodb_server_instance(): - """TODO.""" + """Documentation to be added!""" with mongodb_instance_server_process_context(): yield @pytest.fixture(scope="session") def _test_server_fixture(_run_mongodb_server_instance): - """TODO.""" + """Documentation to be added!""" TestDatabase.prepare() with server_process_context(test_app_config, startup_time=2000): yield diff --git a/trolldb/api/tests/test_api.py b/trolldb/api/tests/test_api.py index 97743ee..220d993 100644 --- a/trolldb/api/tests/test_api.py +++ b/trolldb/api/tests/test_api.py @@ -1,4 +1,4 @@ -"""TODO.""" +"""Documentation to be added!""" import pytest from fastapi import status diff --git a/trolldb/database/errors.py b/trolldb/database/errors.py index 044181f..1063e68 100644 --- a/trolldb/database/errors.py +++ b/trolldb/database/errors.py @@ -81,7 +81,7 @@ class Documents(ResponsesErrorGroup): """A response descriptor for the Fast API routes. This combines all the error messages that might occur as result of working with databases and collections. See the -fast api documentation for TODO. +fast api documentation for Documentation to be added! """ database_collection_document_error_descriptor = ( diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index eeb0cac..00b645c 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -21,60 +21,60 @@ class PipelineDict(dict): """ def __or__(self, other: Self): - """TODO.""" + """Documentation to be added!""" return PipelineDict({"$or": [self, other]}) def __and__(self, other: Self): - """TODO.""" + """Documentation to be added!""" return PipelineDict({"$and": [self, other]}) class PipelineAttribute: - """TODO.""" + """Documentation to be added!""" def __init__(self, key: str): - """TODO.""" + """Documentation to be added!""" self.__key = key def __eq__(self, other: Any) -> PipelineDict: - """TODO.""" + """Documentation to be added!""" if isinstance(other, list): return PipelineDict(**{"$or": [{self.__key: v} for v in other]}) return PipelineDict(**{self.__key: other}) def __aux_operators(self, other: Any, operator: str) -> PipelineDict: - """TODO.""" + """Documentation to be added!""" return PipelineDict(**{self.__key: {operator: other}} if other else {}) def __ge__(self, other: Any) -> PipelineDict: - """TODO.""" + """Documentation to be added!""" return self.__aux_operators(other, "$gte") def __gt__(self, other: Any) -> PipelineDict: - """TODO.""" + """Documentation to be added!""" return self.__aux_operators(other, "$gt") def __le__(self, other: Any) -> PipelineDict: - """TODO.""" + """Documentation to be added!""" return self.__aux_operators(other, "$lte") def __lt__(self, other: Any) -> PipelineDict: - """TODO.""" + """Documentation to be added!""" return self.__aux_operators(other, "$le") class Pipelines(list): - """TODO.""" + """Documentation to be added!""" def __init__(self, *args, **kwargs): - """TODO.""" + """Documentation to be added!""" super().__init__(*args, **kwargs) def __iadd__(self, other): - """TODO.""" + """Documentation to be added!""" self.extend([{"$match": other}]) return self def __add__(self, other): - """TODO.""" + """Documentation to be added!""" self.append({"$match": other}) return self diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index eeb7551..3204bd5 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -30,7 +30,7 @@ class ResponseError(Exception): """The default type of the response which will be returned when an error occurs.""" def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) -> None: - """TODO.""" + """Documentation to be added!""" self.__dict: OrderedDict = OrderedDict(args_dict) self.extra_information: dict | None = None @@ -64,7 +64,7 @@ def __or__(self, other: Self): def __assert_existence_multiple_response_codes( self, status_code: StatusCode | None = None) -> (StatusCode, str): - """TODO.""" + """Documentation to be added!""" match status_code, len(self.__dict): case None, n if n > 1: raise ValueError("In case of multiple response status codes, the status code must be specified.") @@ -81,7 +81,7 @@ def get_error_details( self, extra_information: dict | None = None, status_code: int | None = None) -> (StatusCode, str): - """TODO.""" + """Documentation to be added!""" status_code, msg = self.__assert_existence_multiple_response_codes(status_code) return ( status_code, @@ -93,7 +93,7 @@ def sys_exit_log( exit_code: int = -1, extra_information: dict | None = None, status_code: int | None = None) -> None: - """TODO.""" + """Documentation to be added!""" msg, _ = self.get_error_details(extra_information, status_code) logger.error(msg) exit(exit_code) @@ -102,37 +102,37 @@ def log_as_warning( self, extra_information: dict | None = None, status_code: int | None = None): - """TODO.""" + """Documentation to be added!""" msg, _ = self.get_error_details(extra_information, status_code) logger.warning(msg) @property def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], str]]: - """TODO.""" + """Documentation to be added!""" return {status: {Literal["description"]: ResponseError.__stringify(msg)} for status, msg in self.__dict.items()} @staticmethod def __listify(item: str | list[str]) -> list[str]: - """TODO.""" + """Documentation to be added!""" return item if isinstance(item, list) else [item] @staticmethod def __stringify(item: str | list[str]) -> str: - """TODO.""" + """Documentation to be added!""" return ResponseError.descriptor_delimiter.join(ResponseError.__listify(item)) class ResponsesErrorGroup: - """TODO.""" + """Documentation to be added!""" @classmethod def fields(cls): - """TODO.""" + """Documentation to be added!""" return {k: v for k, v in cls.__dict__.items() if isinstance(v, ResponseError)} @classmethod def union(cls): - """TODO.""" + """Documentation to be added!""" buff = None for v in cls.fields().values(): if buff is None: diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index f4bb333..93445d9 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,4 +1,4 @@ -"""TODO.""" +"""Documentation to be added!""" from typing import Any from urllib.parse import urljoin diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index c79ef18..d6f58c3 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -1,4 +1,4 @@ -"""TODO.""" +"""Documentation to be added!""" from contextlib import contextmanager from datetime import datetime, timedelta @@ -13,7 +13,7 @@ @contextmanager def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database): - """TODO.""" + """Documentation to be added!""" client = None try: client = MongoClient(database_config.url.unicode_string(), connectTimeoutMS=database_config.timeout) @@ -24,46 +24,46 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab def random_sample(items: list[Any], size=10): - """TODO.""" + """Documentation to be added!""" last_index = len(items) - 1 indices = [randint(0, last_index) for _ in range(size)] # noqa: S311 return [items[i] for i in indices] class Time: - """TODO.""" + """Documentation to be added!""" min_start_time = datetime(2019, 1, 1, 0, 0, 0) max_end_time = datetime(2024, 1, 1, 0, 0, 0) delta_time = int((max_end_time - min_start_time).total_seconds()) @staticmethod def random_interval_secs(max_interval_secs): - """TODO.""" + """Documentation to be added!""" return timedelta(seconds=randint(0, max_interval_secs)) # noqa: S311 @staticmethod def random_start_time(): - """TODO.""" + """Documentation to be added!""" return Time.min_start_time + Time.random_interval_secs(Time.delta_time) @staticmethod def random_end_time(start_time: datetime, max_interval_secs: int = 300): - """TODO.""" + """Documentation to be added!""" return start_time + Time.random_interval_secs(max_interval_secs) class Document: - """TODO.""" + """Documentation to be added!""" def __init__(self, platform_name: str, sensor: str): - """TODO.""" + """Documentation to be added!""" self.platform_name = platform_name self.sensor = sensor self.start_time = Time.random_start_time() self.end_time = Time.random_end_time(self.start_time) def generate_dataset(self, max_count: int): - """TODO.""" + """Documentation to be added!""" dataset = [] n = randint(1, max_count) # noqa: S311 for i in range(n): @@ -76,7 +76,7 @@ def generate_dataset(self, max_count: int): return dataset def like_mongodb_document(self): - """TODO.""" + """Documentation to be added!""" return { "platform_name": self.platform_name, "sensor": self.sensor, @@ -87,7 +87,7 @@ def like_mongodb_document(self): class TestDatabase: - """TODO.""" + """Documentation to be added!""" platform_names = random_sample(["PA", "PB", "PC"]) sensors = random_sample(["SA", "SB", "SC"]) @@ -99,7 +99,7 @@ class TestDatabase: @classmethod def generate_documents(cls, random_shuffle=True) -> list: - """TODO.""" + """Documentation to be added!""" documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors, strict=False)] if random_shuffle: @@ -108,7 +108,7 @@ def generate_documents(cls, random_shuffle=True) -> list: @classmethod def reset(cls): - """TODO.""" + """Documentation to be added!""" with test_mongodb_context() as client: for db_name, coll_name in zip(cls.database_names, cls.collection_names, strict=False): db = client[db_name] @@ -118,7 +118,7 @@ def reset(cls): @classmethod def write_mock_date(cls): - """TODO.""" + """Documentation to be added!""" with test_mongodb_context() as client: cls.documents = cls.generate_documents() collection = client[test_app_config.database.main_database_name][ @@ -127,6 +127,6 @@ def write_mock_date(cls): @classmethod def prepare(cls): - """TODO.""" + """Documentation to be added!""" cls.reset() cls.write_mock_date() diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 04fe553..43b0154 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -15,7 +15,7 @@ class TestMongoInstance: - """TODO.""" + """Documentation to be added!""" log_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_log") storage_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_storage") port: int = 28017 @@ -23,19 +23,19 @@ class TestMongoInstance: @classmethod def prepare_dir(cls, directory: str): - """TODO.""" + """Documentation to be added!""" cls.remove_dir(directory) mkdir(directory) @classmethod def remove_dir(cls, directory: str): - """TODO.""" + """Documentation to be added!""" if path.exists(directory) and path.isdir(directory): rmtree(directory) @classmethod def run_subprocess(cls, args: list[str], wait=True): - """TODO.""" + """Documentation to be added!""" cls.process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # noqa: S603 if wait: outs, errs = cls.process.communicate() @@ -44,7 +44,7 @@ def run_subprocess(cls, args: list[str], wait=True): @classmethod def mongodb_exists(cls) -> bool: - """TODO.""" + """Documentation to be added!""" outs, errs = cls.run_subprocess(["which", "mongod"]) if outs and not errs: return True @@ -52,20 +52,20 @@ def mongodb_exists(cls) -> bool: @classmethod def prepare_dirs(cls) -> None: - """TODO.""" + """Documentation to be added!""" cls.prepare_dir(cls.log_dir) cls.prepare_dir(cls.storage_dir) @classmethod def run_instance(cls): - """TODO.""" + """Documentation to be added!""" cls.run_subprocess( ["mongod", "--dbpath", cls.storage_dir, "--logpath", f"{cls.log_dir}/mongod.log", "--port", f"{cls.port}"] , wait=False) @classmethod def shutdown_instance(cls): - """TODO.""" + """Documentation to be added!""" cls.process.kill() for d in [cls.log_dir, cls.storage_dir]: cls.remove_dir(d) From 3513776a3bfd21c992c0e07808503742175270e0 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 15:07:58 +0200 Subject: [PATCH 34/97] Terminate mongodb instead of killing it --- trolldb/test_utils/mongodb_instance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 43b0154..34a9895 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -66,7 +66,8 @@ def run_instance(cls): @classmethod def shutdown_instance(cls): """Documentation to be added!""" - cls.process.kill() + cls.process.terminate() + cls.process.wait() for d in [cls.log_dir, cls.storage_dir]: cls.remove_dir(d) From 7336f0d3b5eb8ce63294fc250904a275966cbcb4 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 15:14:45 +0200 Subject: [PATCH 35/97] Refactor tests into a separate directory. --- trolldb/database/tests/pytest.ini | 2 -- trolldb/{api => }/tests/pytest.ini | 0 trolldb/{api/tests => tests/tests_api}/conftest.py | 0 trolldb/{api/tests => tests/tests_api}/test_api.py | 0 trolldb/{database/tests => tests/tests_database}/conftest.py | 0 .../{database/tests => tests/tests_database}/test_mongodb.py | 0 6 files changed, 2 deletions(-) delete mode 100644 trolldb/database/tests/pytest.ini rename trolldb/{api => }/tests/pytest.ini (100%) rename trolldb/{api/tests => tests/tests_api}/conftest.py (100%) rename trolldb/{api/tests => tests/tests_api}/test_api.py (100%) rename trolldb/{database/tests => tests/tests_database}/conftest.py (100%) rename trolldb/{database/tests => tests/tests_database}/test_mongodb.py (100%) diff --git a/trolldb/database/tests/pytest.ini b/trolldb/database/tests/pytest.ini deleted file mode 100644 index 4088045..0000000 --- a/trolldb/database/tests/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -asyncio_mode=auto diff --git a/trolldb/api/tests/pytest.ini b/trolldb/tests/pytest.ini similarity index 100% rename from trolldb/api/tests/pytest.ini rename to trolldb/tests/pytest.ini diff --git a/trolldb/api/tests/conftest.py b/trolldb/tests/tests_api/conftest.py similarity index 100% rename from trolldb/api/tests/conftest.py rename to trolldb/tests/tests_api/conftest.py diff --git a/trolldb/api/tests/test_api.py b/trolldb/tests/tests_api/test_api.py similarity index 100% rename from trolldb/api/tests/test_api.py rename to trolldb/tests/tests_api/test_api.py diff --git a/trolldb/database/tests/conftest.py b/trolldb/tests/tests_database/conftest.py similarity index 100% rename from trolldb/database/tests/conftest.py rename to trolldb/tests/tests_database/conftest.py diff --git a/trolldb/database/tests/test_mongodb.py b/trolldb/tests/tests_database/test_mongodb.py similarity index 100% rename from trolldb/database/tests/test_mongodb.py rename to trolldb/tests/tests_database/test_mongodb.py From f6c5a72db66d2b1a87988f70c8bf9e2d751ff9f8 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 15 May 2024 16:49:50 +0200 Subject: [PATCH 36/97] Add a cli db test --- trolldb/cli.py | 25 ++++++ .../tests/{tests_database => }/conftest.py | 17 ++++ trolldb/tests/test_db.py | 80 +++++++++++++++++++ trolldb/tests/tests_api/conftest.py | 23 ------ 4 files changed, 122 insertions(+), 23 deletions(-) create mode 100644 trolldb/cli.py rename trolldb/tests/{tests_database => }/conftest.py (68%) create mode 100644 trolldb/tests/test_db.py delete mode 100644 trolldb/tests/tests_api/conftest.py diff --git a/trolldb/cli.py b/trolldb/cli.py new file mode 100644 index 0000000..bc5634a --- /dev/null +++ b/trolldb/cli.py @@ -0,0 +1,25 @@ +"""Main interface.""" + +from posttroll.message import Message +from posttroll.subscriber import create_subscriber_from_dict_config + +from trolldb.database.mongodb import MongoDB, mongodb_context +from trolldb.test_utils.common import test_app_config + + +async def record_messages(subscriber_config): + """Record the metadata of messages into the database.""" + async with mongodb_context(test_app_config.database): + sub = create_subscriber_from_dict_config(subscriber_config) + collection = await MongoDB.get_collection("mock_database", "mock_collection") + for m in sub.recv(): + msg = Message.decode(m) + match msg.type: + case "file": + collection.insert_one(msg.data) + case "delete": + deletion_result = await collection.delete_many({"uri": msg.data["uri"]}) + if deletion_result.deleted_count != 1: + raise ValueError("Multiple deletions!") # Replace with logging + case _: + raise KeyError(f"Don't know what to do with {msg.type} message.") # Replace with logging diff --git a/trolldb/tests/tests_database/conftest.py b/trolldb/tests/conftest.py similarity index 68% rename from trolldb/tests/tests_database/conftest.py rename to trolldb/tests/conftest.py index 6a73628..edb8ee7 100644 --- a/trolldb/tests/tests_database/conftest.py +++ b/trolldb/tests/conftest.py @@ -3,15 +3,32 @@ This module provides fixtures for running a Mongo DB instance in test mode and filling the database with test data. """ + import pytest import pytest_asyncio +from trolldb.api.api import server_process_context from trolldb.database.mongodb import mongodb_context from trolldb.test_utils.common import test_app_config from trolldb.test_utils.mongodb_database import TestDatabase from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context +@pytest.fixture(scope="session") +def _run_mongodb_server_instance(): + """Documentation to be added!""" + with mongodb_instance_server_process_context(): + yield + + +@pytest.fixture(scope="session") +def _test_server_fixture(_run_mongodb_server_instance): + """Documentation to be added!""" + TestDatabase.prepare() + with server_process_context(test_app_config, startup_time=2000): + yield + + @pytest.fixture(scope="session") def _run_mongodb_server_instance(): """Runs the MongoDB instance in test mode using a context manager. diff --git a/trolldb/tests/test_db.py b/trolldb/tests/test_db.py new file mode 100644 index 0000000..8b865e4 --- /dev/null +++ b/trolldb/tests/test_db.py @@ -0,0 +1,80 @@ +"""Tests for the message recording into database.""" + +import pytest +from posttroll.message import Message +from posttroll.testing import patched_subscriber_recv + +from trolldb.cli import record_messages +from trolldb.database.mongodb import MongoDB, mongodb_context +from trolldb.test_utils.common import test_app_config +from trolldb.test_utils.mongodb_database import TestDatabase +from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context + +FILENAME = "20191103_153936-s1b-ew-hh.tiff" + +@pytest.fixture() +def tmp_filename(tmp_path): + """Create a filename for the messages.""" + return tmp_path / FILENAME + +@pytest.fixture() +def file_message(tmp_filename): + """Create a string for a file message.""" + return ('pytroll://segment/raster/L2/SAR file a001673@c20969.ad.smhi.se 2019-11-05T13:00:10.366023 v1.01 ' + 'application/json {"platform_name": "S1B", "scan_mode": "EW", "type": "GRDM", "data_source": "1SDH", ' + '"start_time": "2019-11-03T15:39:36.543000", "end_time": "2019-11-03T15:40:40.821000", "orbit_number": ' + '18765, "random_string1": "0235EA", "random_string2": "747D", "uri": ' + f'"{str(tmp_filename)}", "uid": "20191103_153936-s1b-ew-hh.tiff", ' + '"polarization": "hh", "sensor": "sar-c", "format": "GeoTIFF", "pass_direction": "ASCENDING"}') + + +@pytest.fixture() +def del_message(tmp_filename): + """Create a string for a delete message.""" + return ('pytroll://segment/raster/L2/SAR delete a001673@c20969.ad.smhi.se 2019-11-05T13:00:10.366023 v1.01 ' + 'application/json {"platform_name": "S1B", "scan_mode": "EW", "type": "GRDM", "data_source": "1SDH", ' + '"start_time": "2019-11-03T15:39:36.543000", "end_time": "2019-11-03T15:40:40.821000", "orbit_number": ' + '18765, "random_string1": "0235EA", "random_string2": "747D", "uri": ' + f'"{str(tmp_filename)}", "uid": "20191103_153936-s1b-ew-hh.tiff", ' + '"polarization": "hh", "sensor": "sar-c", "format": "GeoTIFF", "pass_direction": "ASCENDING"}') + + +async def test_record_adds_message(tmp_path, file_message, tmp_filename): + """Test that message recording adds a message to the database.""" + msg = Message.decode(file_message) + + subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) + + with mongodb_instance_server_process_context(): + TestDatabase.prepare() + with patched_subscriber_recv([file_message]): + + await record_messages(subscriber_config) + + async with mongodb_context(test_app_config.database): + collection = await MongoDB.get_collection("mock_database", "mock_collection") + + result = await collection.find_one(dict(scan_mode="EW")) + result.pop("_id") + assert result == msg.data + + deletion_result = await collection.delete_many({"uri": str(tmp_filename)}) + + assert deletion_result.deleted_count == 1 + + +async def test_record_deletes_message(tmp_path, file_message, del_message): + """Test that message recording can delete a record in the database.""" + subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) + + with mongodb_instance_server_process_context(): + TestDatabase.prepare() + + with patched_subscriber_recv([file_message, del_message]): + + await record_messages(subscriber_config) + + async with mongodb_context(test_app_config.database): + collection = await MongoDB.get_collection("mock_database", "mock_collection") + result = await collection.find_one(dict(scan_mode="EW")) + assert result is None diff --git a/trolldb/tests/tests_api/conftest.py b/trolldb/tests/tests_api/conftest.py deleted file mode 100644 index bc32a39..0000000 --- a/trolldb/tests/tests_api/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Documentation to be added!""" - -import pytest - -from trolldb.api.api import server_process_context -from trolldb.test_utils.common import test_app_config -from trolldb.test_utils.mongodb_database import TestDatabase -from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context - - -@pytest.fixture(scope="session") -def _run_mongodb_server_instance(): - """Documentation to be added!""" - with mongodb_instance_server_process_context(): - yield - - -@pytest.fixture(scope="session") -def _test_server_fixture(_run_mongodb_server_instance): - """Documentation to be added!""" - TestDatabase.prepare() - with server_process_context(test_app_config, startup_time=2000): - yield From 4453f645f326b7f7108b6ad9d6e62b3daa494310 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 15 May 2024 16:57:37 +0200 Subject: [PATCH 37/97] Sphinx --- docs/make.sh | 2 ++ trolldb/database/mongodb.py | 8 ++++---- trolldb/database/piplines.py | 11 +++++------ 3 files changed, 11 insertions(+), 10 deletions(-) create mode 100755 docs/make.sh diff --git a/docs/make.sh b/docs/make.sh new file mode 100755 index 0000000..e4c8ff0 --- /dev/null +++ b/docs/make.sh @@ -0,0 +1,2 @@ +cd source +python -m sphinx -T -b html -d _build/doctrees -D language=en . build/html diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index d018c93..fdc968b 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -114,14 +114,14 @@ async def initialize(cls, database_config: DatabaseConfig): Raises ``SystemExit(errno.EIO)``: - - If connection is not established (``ConnectionFailure``) + If connection is not established (``ConnectionFailure``) - - If the attempt times out (``ServerSelectionTimeoutError``) + If the attempt times out (``ServerSelectionTimeoutError``) - - If one attempts reinitializing the class with new (different) database configurations without calling + If one attempts reinitializing the class with new (different) database configurations without calling :func:`~close()` first. - - If the state is not consistent, i.e. the client is closed or ``None`` but the internal database + If the state is not consistent, i.e. the client is closed or ``None`` but the internal database configurations still exist and are different from the new ones which have been just provided. diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 00b645c..6a314a9 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -12,12 +12,11 @@ class PipelineDict(dict): left operand and the second element is the right operand. Example: - ``` - pd1 = PipelineDict({"number": 2}) - pd2 = PipelineDict({"kind": 1}) - pd3 = pd1 & pd2 - - ``` + ``` + pd1 = PipelineDict({"number": 2}) + pd2 = PipelineDict({"kind": 1}) + pd3 = pd1 & pd2 + ``` """ def __or__(self, other: Self): From 42a9eba1501c586e6ff005761b71f00265e9ff63 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 10:52:10 +0200 Subject: [PATCH 38/97] Docstrings! --- trolldb/api/routes/databases.py | 8 ++++---- trolldb/api/routes/datetime_.py | 10 +++++----- trolldb/api/routes/platforms.py | 2 +- trolldb/api/routes/queries.py | 2 +- trolldb/api/routes/root.py | 2 +- trolldb/api/routes/sensors.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/trolldb/api/routes/databases.py b/trolldb/api/routes/databases.py index 0d444fa..e114ef4 100644 --- a/trolldb/api/routes/databases.py +++ b/trolldb/api/routes/databases.py @@ -24,7 +24,7 @@ response_model=list[str], summary="Gets the list of all database names") async def database_names(exclude_defaults: bool = exclude_defaults_query) -> list[str]: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" db_names = await MongoDB.list_database_names() if not exclude_defaults: @@ -38,7 +38,7 @@ async def database_names(exclude_defaults: bool = exclude_defaults_query) -> lis responses=Databases.union().fastapi_descriptor, summary="Gets the list of all collection names for the given database name") async def collection_names(db: CheckDataBaseDependency) -> list[str]: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" return await db.list_collection_names() @@ -47,7 +47,7 @@ async def collection_names(db: CheckDataBaseDependency) -> list[str]: responses=database_collection_error_descriptor, summary="Gets the object ids of all documents for the given database and collection name") async def documents(collection: CheckCollectionDependency) -> list[str]: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" return await get_ids(collection.find({})) @@ -56,7 +56,7 @@ async def documents(collection: CheckCollectionDependency) -> list[str]: responses=database_collection_document_error_descriptor, summary="Gets the document content in json format given its object id, database, and collection name") async def document_by_id(collection: CheckCollectionDependency, _id: MongoObjectId) -> _DocumentType: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" if document := await collection.find_one({"_id": _id}): return dict(document) | {"_id": str(_id)} diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index bff8c5c..6a0d4bc 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -16,19 +16,19 @@ class TimeModel(TypedDict): - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" _id: str _time: datetime class TimeEntry(TypedDict): - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" _min: TimeModel _max: TimeModel class ResponseModel(BaseModel): - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" start_time: TimeEntry end_time: TimeEntry @@ -41,7 +41,7 @@ class ResponseModel(BaseModel): responses=database_collection_error_descriptor, summary="Gets the the minimum and maximum values for the start and end times") async def datetime(collection: CheckCollectionDependency) -> ResponseModel: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" agg_result = await collection.aggregate([{ "$group": { "_id": None, @@ -52,7 +52,7 @@ async def datetime(collection: CheckCollectionDependency) -> ResponseModel: }}]).next() def _aux(query): - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" return get_id(collection.find_one(query)) return ResponseModel( diff --git a/trolldb/api/routes/platforms.py b/trolldb/api/routes/platforms.py index 9c7354b..7d0c756 100644 --- a/trolldb/api/routes/platforms.py +++ b/trolldb/api/routes/platforms.py @@ -17,5 +17,5 @@ responses=database_collection_error_descriptor, summary="Gets the list of all platform names") async def platform_names(collection: CheckCollectionDependency) -> list[str]: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" return await get_distinct_items_in_collection(collection, "platform_name") diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index 4fd4c80..5d0a399 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -26,7 +26,7 @@ async def queries( sensor: list[str] = Query(default=None), # noqa: B008 time_min: datetime.datetime = Query(default=None), # noqa: B008 time_max: datetime.datetime = Query(default=None)) -> list[str]: # noqa: B008 - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" pipelines = Pipelines() if platform: diff --git a/trolldb/api/routes/root.py b/trolldb/api/routes/root.py index 0c543c8..1aaaccc 100644 --- a/trolldb/api/routes/root.py +++ b/trolldb/api/routes/root.py @@ -11,5 +11,5 @@ @router.get("/", summary="The root route which is mainly used to check the status of connection") async def root() -> Response: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" return Response(status_code=status.HTTP_200_OK) diff --git a/trolldb/api/routes/sensors.py b/trolldb/api/routes/sensors.py index 9852822..c3883a1 100644 --- a/trolldb/api/routes/sensors.py +++ b/trolldb/api/routes/sensors.py @@ -17,5 +17,5 @@ responses=database_collection_error_descriptor, summary="Gets the list of all sensor names") async def sensor_names(collection: CheckCollectionDependency) -> list[str]: - """Documentation to be added!""" + """Please consult the auto-generated documentation by FastAPI.""" return await get_distinct_items_in_collection(collection, "sensor") From 43f9b076b2f4071fb2cff8d46d2ad8abfe0a97b0 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 11:13:01 +0200 Subject: [PATCH 39/97] Hard code some API configs. --- trolldb/api/api.py | 16 ++++++++++++++- trolldb/config/config.py | 39 ++---------------------------------- trolldb/database/piplines.py | 2 ++ trolldb/test_utils/common.py | 2 +- 4 files changed, 20 insertions(+), 39 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 7a1b1b6..4e7e93a 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -33,6 +33,20 @@ from trolldb.database.mongodb import mongodb_context from trolldb.errors.errors import ResponseError +API_INFO = { + "title": "pytroll-db", + "version": "0.1", + "summary": "The database API of Pytroll", + "description": "The API allows you to perform CRUD operations as well as querying the database" + "At the moment only MongoDB is supported. It is based on the following Python packages" + "\n * **PyMongo** (https://github.com/mongodb/mongo-python-driver)" + "\n * **motor** (https://github.com/mongodb/motor)", + "license_info": { + "name": "The GNU General Public License v3.0", + "url": "https://www.gnu.org/licenses/gpl-3.0.en.html" + } +} + @validate_call def run_server(config: AppConfig | FilePath, **kwargs) -> None: @@ -56,7 +70,7 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: explicitly to the function take precedence over ``config``. """ config = parse(config) - app = FastAPI(**(config.api_server._asdict() | kwargs)) + app = FastAPI(**(config.api_server._asdict() | kwargs | API_INFO)) app.include_router(api_router) @app.exception_handler(ResponseError) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index bf48b90..acd89fc 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -10,7 +10,7 @@ import errno import sys -from typing import NamedTuple, Optional, TypedDict +from typing import NamedTuple from bson import ObjectId from bson.errors import InvalidId @@ -39,27 +39,11 @@ class MongoDocument(BaseModel): _id: MongoObjectId -class LicenseInfo(TypedDict): - """A dictionary type to hold the summary of the license information. - - Warning: - One has to always consult the included `LICENSE` file for more information. - """ - - name: str - """The full name of the license including the exact variant and the version (if any), e.g. - ``"The GNU General Public License v3.0"`` - """ - - url: AnyUrl - """The URL to access the license, e.g. ``"https://www.gnu.org/licenses/gpl-3.0.en.html"``""" - - class APIServerConfig(NamedTuple): """A named tuple to hold all the configurations of the API server (excluding the database). Note: - Except for the ``url``, the attributes herein are a subset of the keyword arguments accepted by + Except for the ``url``, the attributes herein (if any!) are a subset of the keyword arguments accepted by `FastAPI class `_ and are directly passed to the FastAPI class. """ @@ -69,25 +53,6 @@ class APIServerConfig(NamedTuple): FastAPI class. Instead, it will be used by the `uvicorn` to determine the URL of the server. """ - title: str - """The title of the API server, as appears in the automatically generated documentation by the FastAPI.""" - - version: str - """The version of the API server as appears in the automatically generated documentation by the FastAPI.""" - - summary: Optional[str] = None - """The summary of the API server, as appears in the automatically generated documentation by the FastAPI.""" - - description: Optional[str] = None - """The more comprehensive description (extended summary) of the API server, as appears in the automatically - generated documentation by the FastAPI. - """ - - license_info: Optional[LicenseInfo] = None - """The license information of the API server, as appears in the automatically generated documentation by the - FastAPI. - """ - class DatabaseConfig(NamedTuple): """A named tuple to hold all the configurations of the Database which will be used by the MongoDB instance.""" diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 6a314a9..ae91f4f 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -14,7 +14,9 @@ class PipelineDict(dict): Example: ``` pd1 = PipelineDict({"number": 2}) + pd2 = PipelineDict({"kind": 1}) + pd3 = pd1 & pd2 ``` """ diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 93445d9..c58616f 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -9,7 +9,7 @@ from trolldb.config.config import APIServerConfig, AppConfig, DatabaseConfig test_app_config = AppConfig( - api_server=APIServerConfig(url=AnyUrl("http://localhost:8080"), title="Test API Server", version="0.1"), + api_server=APIServerConfig(url=AnyUrl("http://localhost:8080")), database=DatabaseConfig( main_database_name="mock_database", main_collection_name="mock_collection", From ab855bfb720fbbea56c1299466677e7666109552 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 11:21:59 +0200 Subject: [PATCH 40/97] Hard code some API configs. --- trolldb/api/api.py | 11 +++++++++-- trolldb/template_config.yaml | 22 ---------------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 4e7e93a..b0a22ce 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -67,10 +67,17 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: `FastAPI class `_ and are directly passed to it. These keyword arguments will be first concatenated with the configurations of the API server which are read from the ``config`` argument. The keyword arguments which are passed - explicitly to the function take precedence over ``config``. + explicitly to the function take precedence over ``config``. Finally, ``API_INFO``, which are hard-coded + information for the API server, will be concatenated and takes precedence over all. """ config = parse(config) - app = FastAPI(**(config.api_server._asdict() | kwargs | API_INFO)) + + # Keep all except for the "url" key, as that one will be handled by the `uvicorn` + config_fast_api = {k: v for k, v in config.api_server._asdict() if k != "url"} + + # concatenate the keyword arguments for the API server in the order of precedence (lower to higher). + app = FastAPI(**(config_fast_api | kwargs | API_INFO)) + app.include_router(api_router) @app.exception_handler(ResponseError) diff --git a/trolldb/template_config.yaml b/trolldb/template_config.yaml index b16a166..01c78f7 100644 --- a/trolldb/template_config.yaml +++ b/trolldb/template_config.yaml @@ -22,25 +22,3 @@ database: api_server: # Required url: "http://localhost:8000" - - # Required - "title": "Pytroll-db API" - - # Required - "version": "0.1" - - # Optional - "summary": "The awesome API of Pytroll-db" - - # Optional - "description": " - The API allows you to perform CRUD operations as well as querying the database. - At the moment only MongoDB is supported. It is based on the following Python packages - \n * **PyMongo** (https://github.com/mongodb/mongo-python-driver) - \n * **motor** (https://github.com/mongodb/motor) - " - - # Optional - "license_info": - "name": "The GNU General Public License v3.0" - "url": "https://www.gnu.org/licenses/gpl-3.0.en.html" From 386cf7b5b0fb173a702bcd6c5fa57d43fb8a9be1 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 11:27:25 +0200 Subject: [PATCH 41/97] Hard code some API configs. --- trolldb/api/api.py | 5 +---- trolldb/config/config.py | 6 ++---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index b0a22ce..6a57303 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -72,11 +72,8 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: """ config = parse(config) - # Keep all except for the "url" key, as that one will be handled by the `uvicorn` - config_fast_api = {k: v for k, v in config.api_server._asdict() if k != "url"} - # concatenate the keyword arguments for the API server in the order of precedence (lower to higher). - app = FastAPI(**(config_fast_api | kwargs | API_INFO)) + app = FastAPI(**(config.api_server._asdict() | kwargs | API_INFO)) app.include_router(api_router) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index acd89fc..872bcf9 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -43,15 +43,13 @@ class APIServerConfig(NamedTuple): """A named tuple to hold all the configurations of the API server (excluding the database). Note: - Except for the ``url``, the attributes herein (if any!) are a subset of the keyword arguments accepted by + The attributes herein are a subset of the keyword arguments accepted by `FastAPI class `_ and are directly passed to the FastAPI class. """ url: AnyUrl - """The URL of the API server including the port, e.g. ``mongodb://localhost:8000``. This will not be passed to the - FastAPI class. Instead, it will be used by the `uvicorn` to determine the URL of the server. - """ + """The URL of the API server including the port, e.g. ``mongodb://localhost:8000``.""" class DatabaseConfig(NamedTuple): From abf0354c098b553577ee29b17f159f32dba7f63e Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 12:23:08 +0200 Subject: [PATCH 42/97] More docstrings! --- trolldb/api/api.py | 10 ++-- trolldb/database/piplines.py | 97 ++++++++++++++++++++++-------------- 2 files changed, 65 insertions(+), 42 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 6a57303..9ac5d27 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -37,15 +37,17 @@ "title": "pytroll-db", "version": "0.1", "summary": "The database API of Pytroll", - "description": "The API allows you to perform CRUD operations as well as querying the database" - "At the moment only MongoDB is supported. It is based on the following Python packages" - "\n * **PyMongo** (https://github.com/mongodb/mongo-python-driver)" - "\n * **motor** (https://github.com/mongodb/motor)", + "description": + "The API allows you to perform CRUD operations as well as querying the database" + "At the moment only MongoDB is supported. It is based on the following Python packages" + "\n * **PyMongo** (https://github.com/mongodb/mongo-python-driver)" + "\n * **motor** (https://github.com/mongodb/motor)", "license_info": { "name": "The GNU General Public License v3.0", "url": "https://www.gnu.org/licenses/gpl-3.0.en.html" } } +"""These will appear the auto-generated documentation and are passed to the ``FastAPI`` class.""" @validate_call diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index ae91f4f..f10f2e8 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -1,10 +1,12 @@ -"""The module which defines some convenience classes to facilitate the use of aggregation pipelines.""" +"""The module which defines some convenience classes to facilitate the use of aggregation pipelines in MongoDB.""" from typing import Any, Self -class PipelineDict(dict): - """A subclass of dict which overrides the behaviour of bitwise or ``|`` and bitwise and ``&``. +class PipelineBooleanDict(dict): + """A subclass of dict which overrides the behaviour of bitwise `OR` (``|``) and bitwise `AND` (``&``). + + This class makes it easier to chain and nest `And/Or` operations. The operators are only defined for operands of type :class:`PipelineDict`. For each of the aforementioned operators, the result will be a dictionary with a single key/value pair. The key is either ``$or`` or ``$and`` depending on the @@ -12,70 +14,89 @@ class PipelineDict(dict): left operand and the second element is the right operand. Example: - ``` - pd1 = PipelineDict({"number": 2}) + .. code-block:: python + + pd1 = PipelineDict({"number": 2}) + pd2 = PipelineDict({"kind": 1}) - pd2 = PipelineDict({"kind": 1}) + pd_and = pd1 & pd2 # is equivalent to the following + pd_and_literal = PipelineDict({"$and": [{"number": 2}, {"kind": 1}]}) - pd3 = pd1 & pd2 - ``` + pd_or = pd1 | pd2 # is equivalent to the following + pd_or_literal = PipelineDict({"$or": [{"number": 2}, {"kind": 1}]}) """ def __or__(self, other: Self): - """Documentation to be added!""" - return PipelineDict({"$or": [self, other]}) + """Implements the bitwise or operator, i.e. ``|``.""" + return PipelineBooleanDict({"$or": [self, other]}) def __and__(self, other: Self): - """Documentation to be added!""" - return PipelineDict({"$and": [self, other]}) + """Implements the bitwise and operator, i.e. ``&``.""" + return PipelineBooleanDict({"$and": [self, other]}) class PipelineAttribute: - """Documentation to be added!""" + """A class which defines a single pipeline attribute on which boolean operations will be performed. - def __init__(self, key: str): - """Documentation to be added!""" + The boolean operations are in the form of boolean dicts of type :class:`PipelineBooleanDict`. + """ + + def __init__(self, key: str) -> None: + """The constructor which specifies the pipeline attribute to work with.""" self.__key = key - def __eq__(self, other: Any) -> PipelineDict: - """Documentation to be added!""" + def __eq__(self, other: Any) -> PipelineBooleanDict: + """Implements the equality operator, i.e. ``==``. + + This makes a boolean filter in which the attribute can match any of the items in ``other`` if it is a list, or + the ``other`` itself, otherwise. + """ if isinstance(other, list): - return PipelineDict(**{"$or": [{self.__key: v} for v in other]}) - return PipelineDict(**{self.__key: other}) + return PipelineBooleanDict(**{"$or": [{self.__key: v} for v in other]}) + return PipelineBooleanDict(**{self.__key: other}) - def __aux_operators(self, other: Any, operator: str) -> PipelineDict: - """Documentation to be added!""" - return PipelineDict(**{self.__key: {operator: other}} if other else {}) + def __aux_operators(self, other: Any, operator: str) -> PipelineBooleanDict: + """An auxiliary function to perform comparison operations.""" + return PipelineBooleanDict(**{self.__key: {operator: other}} if other else {}) - def __ge__(self, other: Any) -> PipelineDict: - """Documentation to be added!""" + def __ge__(self, other: Any) -> PipelineBooleanDict: + """Implements the `greater than or equal to` operator, i.e. ``>=``.""" return self.__aux_operators(other, "$gte") - def __gt__(self, other: Any) -> PipelineDict: - """Documentation to be added!""" + def __gt__(self, other: Any) -> PipelineBooleanDict: + """Implements the `greater than` operator, i.e. ``>``.""" return self.__aux_operators(other, "$gt") - def __le__(self, other: Any) -> PipelineDict: - """Documentation to be added!""" + def __le__(self, other: Any) -> PipelineBooleanDict: + """Implements the `less than or equal to` operator, i.e. ``<=``.""" return self.__aux_operators(other, "$lte") - def __lt__(self, other: Any) -> PipelineDict: - """Documentation to be added!""" + def __lt__(self, other: Any) -> PipelineBooleanDict: + """Implements the `less than` operator, i.e. ``<``.""" return self.__aux_operators(other, "$le") class Pipelines(list): - """Documentation to be added!""" - def __init__(self, *args, **kwargs): - """Documentation to be added!""" - super().__init__(*args, **kwargs) + """A class which defines a list of pipelines. - def __iadd__(self, other): - """Documentation to be added!""" + Each item in the list is a dictionary with its key being the literal string ``"$match"`` and its corresponding value + being of type :class:`PipelineBooleanDict`. The``"$match"`` key is what actually triggers the matching operation in + the MongoDB aggregation pipeline. The condition against which the matching will be performed is given by the value + which is a simply a boolean pipeline dictionary which has a hierarchical structure. + """ + + def __iadd__(self, other: PipelineBooleanDict) -> Self: + """Implements the augmented (aka in-place) addition operator, i.e. ``+=``. + + This is similar to :func:`extend` function of a list. + """ self.extend([{"$match": other}]) return self - def __add__(self, other): - """Documentation to be added!""" + def __add__(self, other: PipelineBooleanDict) -> Self: + """Implements the addition operator, i.e. ``+``. + + This is similar to :func:`append` function of a list. + """ self.append({"$match": other}) return self From 9ef3340fce3b33d6511c8af2302d4f6fd16f7545 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 12:28:37 +0200 Subject: [PATCH 43/97] More docstrings! --- trolldb/tests/conftest.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/trolldb/tests/conftest.py b/trolldb/tests/conftest.py index edb8ee7..ffb7312 100644 --- a/trolldb/tests/conftest.py +++ b/trolldb/tests/conftest.py @@ -3,7 +3,6 @@ This module provides fixtures for running a Mongo DB instance in test mode and filling the database with test data. """ - import pytest import pytest_asyncio @@ -16,32 +15,22 @@ @pytest.fixture(scope="session") def _run_mongodb_server_instance(): - """Documentation to be added!""" + """Encloses all tests (session scope) in a context manager of a running MongoDB instance (in a separate process).""" with mongodb_instance_server_process_context(): yield @pytest.fixture(scope="session") def _test_server_fixture(_run_mongodb_server_instance): - """Documentation to be added!""" + """Encloses all tests (session scope) in a context manager of a running API server (in a separate process).""" TestDatabase.prepare() with server_process_context(test_app_config, startup_time=2000): yield -@pytest.fixture(scope="session") -def _run_mongodb_server_instance(): - """Runs the MongoDB instance in test mode using a context manager. - - It is run once for all tests in this test suite. - """ - with mongodb_instance_server_process_context(): - yield - - @pytest_asyncio.fixture() async def mongodb_fixture(_run_mongodb_server_instance): - """Fills the database with test data and then runs the mongodb client using a context manager.""" + """Fills the database with test data and then enclose each test in a mongodb context manager.""" TestDatabase.prepare() async with mongodb_context(test_app_config.database): yield From c75d270d8c67356ee8c957eb5da50c477eb34739 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 12:38:23 +0200 Subject: [PATCH 44/97] More docstrings! --- trolldb/tests/tests_api/test_api.py | 10 +++++++++- trolldb/tests/tests_database/test_mongodb.py | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/trolldb/tests/tests_api/test_api.py b/trolldb/tests/tests_api/test_api.py index 220d993..78150eb 100644 --- a/trolldb/tests/tests_api/test_api.py +++ b/trolldb/tests/tests_api/test_api.py @@ -1,4 +1,12 @@ -"""Documentation to be added!""" +"""Tests for the API server. + +Note: + The functionalities of the API server is not mocked! For the tests herein an actual API server will be running in a + separate process. Moreover, a MongoDB instance is run with databases which are pre-filled with random data having + similar characteristics to the real data. Actual requests will be sent to the API and the results will be asserted + against expectations. +""" + import pytest from fastapi import status diff --git a/trolldb/tests/tests_database/test_mongodb.py b/trolldb/tests/tests_database/test_mongodb.py index b25f930..a03b3bd 100644 --- a/trolldb/tests/tests_database/test_mongodb.py +++ b/trolldb/tests/tests_database/test_mongodb.py @@ -1,4 +1,10 @@ -"""Direct tests for `mongodb` module without an API server connection.""" +"""Direct tests for :obj:`trolldb.database.mongodb` module without an API server connection. + +Note: + The functionalities of the MongoDB client is not mocked! For the tests herein an actual MongoDB instance will be + run. It includes databases which are pre-filled with random data having similar characteristics to the real data. + Actual calls will be made to the running MongoDB instance via the client. +""" import errno import time From c5008a5f34a2605d9a8aae33978d66f52306d089 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 16 May 2024 13:06:28 +0200 Subject: [PATCH 45/97] Add a command line interface function --- trolldb/cli.py | 22 +++++++++- trolldb/config/config.py | 4 +- trolldb/test_utils/common.py | 3 +- trolldb/tests/test_db.py | 84 +++++++++++++++++++++++++++++++++--- 4 files changed, 104 insertions(+), 9 deletions(-) diff --git a/trolldb/cli.py b/trolldb/cli.py index bc5634a..0d42da6 100644 --- a/trolldb/cli.py +++ b/trolldb/cli.py @@ -1,8 +1,11 @@ """Main interface.""" +import argparse + from posttroll.message import Message from posttroll.subscriber import create_subscriber_from_dict_config +from trolldb.config import config from trolldb.database.mongodb import MongoDB, mongodb_context from trolldb.test_utils.common import test_app_config @@ -16,10 +19,27 @@ async def record_messages(subscriber_config): msg = Message.decode(m) match msg.type: case "file": - collection.insert_one(msg.data) + await collection.insert_one(msg.data) case "delete": deletion_result = await collection.delete_many({"uri": msg.data["uri"]}) if deletion_result.deleted_count != 1: raise ValueError("Multiple deletions!") # Replace with logging case _: raise KeyError(f"Don't know what to do with {msg.type} message.") # Replace with logging + + +async def record_messages_from_config(config_file): + """Record messages into the database, getting the configuration from a file.""" + config_obj = config.parse(config_file) + await record_messages(config_obj.subscriber_config) + + +async def record_messages_from_command_line(args=None): + """Record messages into the database, command-line interface.""" + parser = argparse.ArgumentParser() + parser.add_argument("configuration_file", + help="Path to the configuration file") + cmd_args = parser.parse_args(args) + + config_file = cmd_args.configuration_file + await record_messages_from_config(config_file) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 872bcf9..b5a2a32 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -10,7 +10,7 @@ import errno import sys -from typing import NamedTuple +from typing import Any, Dict, NamedTuple from bson import ObjectId from bson.errors import InvalidId @@ -79,7 +79,7 @@ class AppConfig(BaseModel): """ api_server: APIServerConfig database: DatabaseConfig - + subscriber_config: Dict[Any, Any] @validate_call def from_yaml(filename: FilePath) -> AppConfig: diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index c58616f..ab8c243 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -14,7 +14,8 @@ main_database_name="mock_database", main_collection_name="mock_collection", url=AnyUrl("mongodb://localhost:28017"), - timeout=1000) + timeout=1000), + subscriber_config=dict() ) diff --git a/trolldb/tests/test_db.py b/trolldb/tests/test_db.py index 8b865e4..2b9d3b2 100644 --- a/trolldb/tests/test_db.py +++ b/trolldb/tests/test_db.py @@ -1,10 +1,13 @@ """Tests for the message recording into database.""" +from contextlib import contextmanager + import pytest +import yaml from posttroll.message import Message from posttroll.testing import patched_subscriber_recv -from trolldb.cli import record_messages +from trolldb.cli import record_messages, record_messages_from_command_line, record_messages_from_config from trolldb.database.mongodb import MongoDB, mongodb_context from trolldb.test_utils.common import test_app_config from trolldb.test_utils.mongodb_database import TestDatabase @@ -39,14 +42,21 @@ def del_message(tmp_filename): '"polarization": "hh", "sensor": "sar-c", "format": "GeoTIFF", "pass_direction": "ASCENDING"}') +@contextmanager +def running_prepared_database(): + """Starts and prepares a database instance for tests.""" + with mongodb_instance_server_process_context(): + TestDatabase.prepare() + yield + + async def test_record_adds_message(tmp_path, file_message, tmp_filename): """Test that message recording adds a message to the database.""" msg = Message.decode(file_message) subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) - with mongodb_instance_server_process_context(): - TestDatabase.prepare() + with running_prepared_database(): with patched_subscriber_recv([file_message]): await record_messages(subscriber_config) @@ -67,8 +77,7 @@ async def test_record_deletes_message(tmp_path, file_message, del_message): """Test that message recording can delete a record in the database.""" subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) - with mongodb_instance_server_process_context(): - TestDatabase.prepare() + with running_prepared_database(): with patched_subscriber_recv([file_message, del_message]): @@ -78,3 +87,68 @@ async def test_record_deletes_message(tmp_path, file_message, del_message): collection = await MongoDB.get_collection("mock_database", "mock_collection") result = await collection.find_one(dict(scan_mode="EW")) assert result is None + + +async def test_record_from_config(tmp_path, file_message, tmp_filename): + """Test that we can record when passed a config file.""" + config_file = create_config_file(tmp_path) + + msg = Message.decode(file_message) + + with running_prepared_database(): + + with patched_subscriber_recv([file_message]): + + await record_messages_from_config(config_file) + + async with mongodb_context(test_app_config.database): + collection = await MongoDB.get_collection("mock_database", "mock_collection") + + result = await collection.find_one(dict(scan_mode="EW")) + result.pop("_id") + assert result == msg.data + + deletion_result = await collection.delete_many({"uri": str(tmp_filename)}) + + assert deletion_result.deleted_count == 1 + + +def create_config_file(tmp_path): + """Create a config file for tests.""" + config_file = tmp_path / "config.yaml" + subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) + db_config = {"main_database_name": "sat_db", + "main_collection_name": "files", + "url": "mongodb://localhost:27017", + "timeout": 1000} + api_server_config = {"url": "http://localhost:8000"} + + config_dict = dict(subscriber_config=subscriber_config, + database = db_config, + api_server = api_server_config) + with open(config_file, "w") as fd: + fd.write(yaml.dump(config_dict)) + + return config_file + +async def test_record_cli(tmp_path, file_message, tmp_filename): + """Test that we can record when passed a config file.""" + config_file = create_config_file(tmp_path) + + msg = Message.decode(file_message) + + with running_prepared_database(): + + with patched_subscriber_recv([file_message]): + await record_messages_from_command_line([str(config_file)]) + + async with mongodb_context(test_app_config.database): + collection = await MongoDB.get_collection("mock_database", "mock_collection") + + result = await collection.find_one(dict(scan_mode="EW")) + result.pop("_id") + assert result == msg.data + + deletion_result = await collection.delete_many({"uri": str(tmp_filename)}) + + assert deletion_result.deleted_count == 1 From 56a5ab6fa94a6219ec4f0707524d5d94a14cf8d8 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 13:15:32 +0200 Subject: [PATCH 46/97] More docstrings! --- trolldb/database/errors.py | 2 +- trolldb/test_utils/mongodb_database.py | 97 +++++++++++++++++++------- 2 files changed, 71 insertions(+), 28 deletions(-) diff --git a/trolldb/database/errors.py b/trolldb/database/errors.py index 1063e68..b3830e3 100644 --- a/trolldb/database/errors.py +++ b/trolldb/database/errors.py @@ -81,7 +81,7 @@ class Documents(ResponsesErrorGroup): """A response descriptor for the Fast API routes. This combines all the error messages that might occur as result of working with databases and collections. See the -fast api documentation for Documentation to be added! +FastAPI documentation for `additional responses `_. """ database_collection_document_error_descriptor = ( diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index d6f58c3..27a588c 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -1,9 +1,9 @@ -"""Documentation to be added!""" +"""The module which provides testing utilities to make MongoDB databases/collections and fill them with test data.""" from contextlib import contextmanager from datetime import datetime, timedelta from random import randint, shuffle -from typing import Any +from typing import Any, Iterator from pymongo import MongoClient @@ -12,8 +12,13 @@ @contextmanager -def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database): - """Documentation to be added!""" +def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database) -> Iterator[MongoClient]: + """A context manager for the MongoDB client given test configurations. + + Note: + This is based on `pymongo` and not the `motor` async driver. For testing purposes this is sufficient and we + do not need async capabilities. + """ client = None try: client = MongoClient(database_config.url.unicode_string(), connectTimeoutMS=database_config.timeout) @@ -23,48 +28,68 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab client.close() -def random_sample(items: list[Any], size=10): - """Documentation to be added!""" +def random_sample(items: list[Any], size: int = 10) -> list[Any]: + """Generates a random sample of ``size`` elements, using the given list of items.""" last_index = len(items) - 1 + # We suppress ruff here as we are not generating anything cryptographic here! indices = [randint(0, last_index) for _ in range(size)] # noqa: S311 return [items[i] for i in indices] class Time: - """Documentation to be added!""" + """A static class to enclose functionalities for generating random time stamps.""" + min_start_time = datetime(2019, 1, 1, 0, 0, 0) + """The minimum timestamp.""" + max_end_time = datetime(2024, 1, 1, 0, 0, 0) + """The maximum timestamp.""" + delta_time = int((max_end_time - min_start_time).total_seconds()) + """The difference between the maximum and minimum timestamps in seconds.""" @staticmethod - def random_interval_secs(max_interval_secs): - """Documentation to be added!""" - return timedelta(seconds=randint(0, max_interval_secs)) # noqa: S311 + def random_interval_secs(max_interval_secs: int) -> timedelta: + """Generates a random time interval between zero and the given max interval.""" + # We suppress ruff here as we are not generating anything cryptographic here! + return timedelta(seconds=randint(0, max_interval_secs)) # noqa: S311 @staticmethod - def random_start_time(): - """Documentation to be added!""" + def random_start_time() -> datetime: + """Generates a random start time. + + The start time has a lower bound which is specified by :obj:`~Time.min_start_time` and an upper bound given by + :obj:`~Time.max_end_time`. + """ return Time.min_start_time + Time.random_interval_secs(Time.delta_time) @staticmethod - def random_end_time(start_time: datetime, max_interval_secs: int = 300): - """Documentation to be added!""" + def random_end_time(start_time: datetime, max_interval_secs: int = 300) -> datetime: + """Generates a random end time. + + The end time is within ``max_interval_secs`` seconds from the given ``start_time``. + """ return start_time + Time.random_interval_secs(max_interval_secs) class Document: - """Documentation to be added!""" + """A class which defines functionalities to generate documents data which are similar to real data.""" - def __init__(self, platform_name: str, sensor: str): - """Documentation to be added!""" + def __init__(self, platform_name: str, sensor: str) -> None: + """Initializes the document given its platform and sensor names.""" self.platform_name = platform_name self.sensor = sensor self.start_time = Time.random_start_time() self.end_time = Time.random_end_time(self.start_time) - def generate_dataset(self, max_count: int): - """Documentation to be added!""" + def generate_dataset(self, max_count: int) -> list[dict]: + """Generates the dataset for a given document. + + This corresponds to the list of files which are stored in each document. The number of datasets is randomly + chosen from 1 to ``max_count`` for each document. + """ dataset = [] + # We suppress ruff here as we are not generating anything cryptographic here! n = randint(1, max_count) # noqa: S311 for i in range(n): txt = f"{self.platform_name}_{self.sensor}_{self.start_time}_{self.end_time}_{i}" @@ -75,8 +100,8 @@ def generate_dataset(self, max_count: int): }) return dataset - def like_mongodb_document(self): - """Documentation to be added!""" + def like_mongodb_document(self) -> dict: + """Returns a dictionary which resembles the format we have for our real data when saving them to MongoDB.""" return { "platform_name": self.platform_name, "sensor": self.sensor, @@ -87,19 +112,37 @@ def like_mongodb_document(self): class TestDatabase: - """Documentation to be added!""" + """The class which encloses functionalities to prepare and fill the test database with mock data.""" + platform_names = random_sample(["PA", "PB", "PC"]) + """Example platform names.""" + sensors = random_sample(["SA", "SB", "SC"]) + """Example sensor names.""" database_names = [test_app_config.database.main_database_name, "another_mock_database"] + """List of all database names. + + The first element is the main database that will be queried by the API and includes the mock data. The second + database is for testing scenarios when one attempts to access another existing database or collection. + """ + collection_names = [test_app_config.database.main_collection_name, "another_mock_collection"] + """List of all collection names. + + The first element is the main collection that will be queried by the API and includes the mock data. The second + collection is for testing scenarios when one attempts to access another existing collection. + """ + all_database_names = ["admin", "config", "local", *database_names] + """All database names including the default ones which are automatically created by MongoDB.""" documents = [] + """The list of documents which include mock data.""" @classmethod - def generate_documents(cls, random_shuffle=True) -> list: - """Documentation to be added!""" + def generate_documents(cls, random_shuffle: bool = True) -> list: + """Generates test documents which for practical purposes resemble real data.""" documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors, strict=False)] if random_shuffle: @@ -108,7 +151,7 @@ def generate_documents(cls, random_shuffle=True) -> list: @classmethod def reset(cls): - """Documentation to be added!""" + """Resets all the databases/collections.""" with test_mongodb_context() as client: for db_name, coll_name in zip(cls.database_names, cls.collection_names, strict=False): db = client[db_name] @@ -118,7 +161,7 @@ def reset(cls): @classmethod def write_mock_date(cls): - """Documentation to be added!""" + """Fills databases/collections with mock data.""" with test_mongodb_context() as client: cls.documents = cls.generate_documents() collection = client[test_app_config.database.main_database_name][ @@ -127,6 +170,6 @@ def write_mock_date(cls): @classmethod def prepare(cls): - """Documentation to be added!""" + """Prepares the instance by first resetting all databases/collections and filling them with mock data.""" cls.reset() cls.write_mock_date() From ddd5e09478f570a964029f4661f2f3650eb76488 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 13:39:31 +0200 Subject: [PATCH 47/97] More docstrings! --- trolldb/errors/errors.py | 13 ++++---- trolldb/test_utils/common.py | 2 +- trolldb/test_utils/mongodb_instance.py | 41 +++++++++++++++++--------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index 3204bd5..0ff42ae 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -30,7 +30,7 @@ class ResponseError(Exception): """The default type of the response which will be returned when an error occurs.""" def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) -> None: - """Documentation to be added!""" + """Initializes the response error object given a dictionary of error (HTTP) codes and messages.""" self.__dict: OrderedDict = OrderedDict(args_dict) self.extra_information: dict | None = None @@ -47,12 +47,13 @@ def __or__(self, other: Self): In case of the same status codes, the messages will be appended to a list and saved as a list. Example: - ErrorA = ResponseError({200: "OK"}) - ErrorB = ResponseError({400: "Bad Request"}) - ErrorC = ResponseError({200: "Still Okay"}) + .. code-block:: python - ErrorCombined = ErrorA | ErrorB | ErrorC + ErrorA = ResponseError({200: "OK"}) + ErrorB = ResponseError({400: "Bad Request"}) + ErrorC = ResponseError({200: "Still Okay"}) + ErrorCombined = ErrorA | ErrorB | ErrorC """ buff = OrderedDict(self.__dict) for key, msg in other.__dict.items(): @@ -64,7 +65,7 @@ def __or__(self, other: Self): def __assert_existence_multiple_response_codes( self, status_code: StatusCode | None = None) -> (StatusCode, str): - """Documentation to be added!""" + """Assert whether the response error includes multiple items.""" match status_code, len(self.__dict): case None, n if n > 1: raise ValueError("In case of multiple response status codes, the status code must be specified.") diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index ab8c243..929aa52 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,4 +1,4 @@ -"""Documentation to be added!""" +"""Common functionalities for testing, shared between tests and other test utility modules.""" from typing import Any from urllib.parse import urljoin diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 34a9895..c45bec4 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -15,27 +15,40 @@ class TestMongoInstance: - """Documentation to be added!""" + """A static class to enclose functionalities for running a MongoDB instance.""" + log_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_log") + """Temp directory for logging messages by the MongoDB instance.""" + storage_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_storage") + """Temp directory for storing database files by the MongoDB instance.""" + port: int = 28017 + """The port on which the instance will run.""" + process: subprocess.Popen | None = None + """The (sub-)process which will be used to run the MongoDB instance.""" @classmethod - def prepare_dir(cls, directory: str): - """Documentation to be added!""" - cls.remove_dir(directory) + def __prepare_dir(cls, directory: str): + """Auxiliary function to prepare a single directory. + + That is making a directory if it does not exist, or removing it if it does and then remaking it. + """ + cls.__remove_dir(directory) mkdir(directory) @classmethod - def remove_dir(cls, directory: str): - """Documentation to be added!""" + def __remove_dir(cls, directory: str): + """Auxiliary function to remove temporary directories.""" if path.exists(directory) and path.isdir(directory): rmtree(directory) @classmethod def run_subprocess(cls, args: list[str], wait=True): - """Documentation to be added!""" + """Runs the subprocess in shell given its arguments.""" + # We suppress ruff here as we are not receiving any args from outside, e.g. port is hard-coded. Therefore, + # sanitization of ``args`` is not required. cls.process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # noqa: S603 if wait: outs, errs = cls.process.communicate() @@ -44,7 +57,7 @@ def run_subprocess(cls, args: list[str], wait=True): @classmethod def mongodb_exists(cls) -> bool: - """Documentation to be added!""" + """Checks if ``mongod`` command exists.""" outs, errs = cls.run_subprocess(["which", "mongod"]) if outs and not errs: return True @@ -52,24 +65,24 @@ def mongodb_exists(cls) -> bool: @classmethod def prepare_dirs(cls) -> None: - """Documentation to be added!""" - cls.prepare_dir(cls.log_dir) - cls.prepare_dir(cls.storage_dir) + """Prepares the temp directories.""" + cls.__prepare_dir(cls.log_dir) + cls.__prepare_dir(cls.storage_dir) @classmethod def run_instance(cls): - """Documentation to be added!""" + """Runs the MongoDB instance and does not wait for it, i.e. the process runs in the background.""" cls.run_subprocess( ["mongod", "--dbpath", cls.storage_dir, "--logpath", f"{cls.log_dir}/mongod.log", "--port", f"{cls.port}"] , wait=False) @classmethod def shutdown_instance(cls): - """Documentation to be added!""" + """Shuts down the MongoDB instance by terminating its process.""" cls.process.terminate() cls.process.wait() for d in [cls.log_dir, cls.storage_dir]: - cls.remove_dir(d) + cls.__remove_dir(d) @contextmanager From d50824e3218c494b3de6dbbc0689f3afcfaf0153 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 19:17:40 +0200 Subject: [PATCH 48/97] More docstrings! --- docs/source/conf.py | 17 +++-- trolldb/errors/errors.py | 153 ++++++++++++++++++++++++++++++--------- 2 files changed, 131 insertions(+), 39 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 0d0bfb3..f424a52 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,7 +9,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# + import os import sys @@ -20,9 +20,6 @@ for x in os.walk("../../trolldb"): sys.path.append(x[0]) -# autodoc_mock_imports = ["motor", "pydantic", "fastapi", "uvicorn", "loguru", "pyyaml"] - - # -- Project information ----------------------------------------------------- project = "Pytroll-db" @@ -44,8 +41,8 @@ "sphinx.ext.duration", "sphinx.ext.doctest", "sphinx.ext.autosummary", - "sphinx.ext.intersphinx", ] + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -67,9 +64,17 @@ # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ["_static"] +autodoc_default_options = { + "members": True, + "member-order": "bysource", + "private-members": True, + "special-members": "__init__, __or__", + "undoc-members": True, + "exclude-members": "__weakref__" +} root_doc = "index" output_dir = os.path.join(".") module_dir = os.path.abspath("../../trolldb") -apidoc.main(["-q", "-f", "-o", output_dir, module_dir, *include_patterns]) +apidoc.main(["-e", "-M", "-q", "-f", "-o", output_dir, module_dir, *include_patterns]) diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index 0ff42ae..dcdf00a 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -1,7 +1,7 @@ -"""The module which defines the base functionality for error responses that will be returned by the API. +"""The module which defines the base functionalities for errors that will be risen when using the package or the API. This module only includes the generic utilities using which each module should define its own error responses -specifically. See :obj:`trolldb.database.errors` as an example on how this module is used. +specifically. See :obj:`trolldb.database.errors` as an example on how to achieve this. """ from collections import OrderedDict @@ -13,59 +13,139 @@ from loguru import logger StatusCode = int +"""An alias for the built-in ``int`` type, which is used for HTTP status codes.""" + + +def _listify(item: str | list[str]) -> list[str]: + """Encloses the given (single) string in a list or returns the same input as-is in case of a list of strings. + + Args: + item: + The item that needs to be converted to a list. + + Returns: + If the input is itself a list of strings the same list is returned as-is, otherwise, the given input + string is enclosed in ``[]`` and returned. + + Example: + .. code-block:: python + + # The following evaluate to ``True`` + __listify("test") == ["test"] + __listify(["a", "b"]) = ["a", "b"] + __listify([]) == [] + """ + return item if isinstance(item, list) else [item] + + +def _stringify(item: str | list[str], delimiter: str) -> str: + """Makes a single string out of the item(s) by delimiting them with ``delimiter``. + + Args: + item: + A string or list of strings to be delimited. + delimiter: + A string as delimiter. + + Returns: + The same input string, or in case of a list of items, a single string delimited by ``delimiter``. + """ + return delimiter.join(_listify(item)) class ResponseError(Exception): - """The base class for all error responses. This is derivative of the ``Exception`` class.""" + """The base class for all error responses. + + This is a derivative of the ``Exception`` class and therefore can be used directly in ``raise`` statements. + + Attributes: + __dict (:obj:`OrderedDict[StatusCode, str]`): + An ordered dictionary in which the keys are (HTTP) status codes and the values are the corresponding + messages. + """ descriptor_delimiter: str = " |OR| " - """A delimiter to combine the message part of several error responses into a single one. + """A delimiter to divide the message part of several error responses which have been combined into a single one. + + This will be shown in textual format for the response descriptors of the Fast API routes. - This will be shown in textual format for the response descriptors of the Fast API routes. For example: + Example: + .. code-block:: python - ``ErrorA |OR| ErrorB`` + error_a = ResponseError({400: "Bad Request"}) + error_b = ResponseError({404: "Not Found"}) + errors = error_a | error_b + + # When used in a FastAPI response descriptor, the following string will be generated for ``errors`` + "Bad Request |OR| Not Found" """ DefaultResponseClass: Response = PlainTextResponse - """The default type of the response which will be returned when an error occurs.""" + """The default type of the response which will be returned when an error occurs. + + This must be a valid member (class) of ``fastapi.responses``. + """ def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) -> None: - """Initializes the response error object given a dictionary of error (HTTP) codes and messages.""" + """Initializes the error object given a dictionary of error (HTTP) codes (keys) and messages (values). + + Note: + The order of items will be preserved as we use an ordered dictionary to store the items internally. + + Example: + .. code-block:: python + + # The following are all valid error objects + error_a = ResponseError({400: "Bad Request"}) + error_b = ResponseError({404: "Not Found"}) + errors = error_a | error_b + errors_a_or_b = ResponseError({400: "Bad Request", 404: "Not Found"}) + """ self.__dict: OrderedDict = OrderedDict(args_dict) self.extra_information: dict | None = None def __or__(self, other: Self): - """Combines the error responses into a single error response. + """Implements the bitwise `or` (``|``) which combines the error objects into a single error response. Args: other: Another error response of the same base type to combine with. Returns: - A new error response which includes the combined error response. In case of different http status codes, - the returned response includes the `{status-code: message}` pairs for both ``self`` and the ``other``. - In case of the same status codes, the messages will be appended to a list and saved as a list. + A new error response which includes the combined error response. In case of different (HTTP) status codes, + the returned response includes the ``{: }`` pairs for both ``self`` and the ``other``. + In case of the same status codes, the messages will be stored in a list. Example: .. code-block:: python - ErrorA = ResponseError({200: "OK"}) - ErrorB = ResponseError({400: "Bad Request"}) - ErrorC = ResponseError({200: "Still Okay"}) + error_a = ResponseError({400: "Bad Request"}) + error_b = ResponseError({404: "Not Found"}) + error_c = ResponseError({400: "Still Bad Request"}) - ErrorCombined = ErrorA | ErrorB | ErrorC + errors_combined = error_a | error_b | error_c + + # which is equivalent to the following + errors_combined_literal = ResponseError({ + 400: ["Bad Request", "Still Bad Request"], + 404: "Not Found" + } """ buff = OrderedDict(self.__dict) for key, msg in other.__dict.items(): self_msg = buff.get(key, None) - buff[key] = ResponseError.__listify(self_msg) if self_msg else [] - buff[key].extend(ResponseError.__listify(msg)) + buff[key] = _listify(self_msg) if self_msg else [] + buff[key].extend(_listify(msg)) return ResponseError(buff) - def __assert_existence_multiple_response_codes( + def __retrieve_one_from_multiple_response_codes( self, status_code: StatusCode | None = None) -> (StatusCode, str): - """Assert whether the response error includes multiple items.""" + """Retrieves a single tuple of ``(, )`` from the internal dictionary ``self.__dict``. + + Asserts whether the response error includes multiple items. + + """ match status_code, len(self.__dict): case None, n if n > 1: raise ValueError("In case of multiple response status codes, the status code must be specified.") @@ -82,11 +162,11 @@ def get_error_details( self, extra_information: dict | None = None, status_code: int | None = None) -> (StatusCode, str): - """Documentation to be added!""" - status_code, msg = self.__assert_existence_multiple_response_codes(status_code) + """Gets the details of the error response.""" + status_code, msg = self.__retrieve_one_from_multiple_response_codes(status_code) return ( status_code, - ResponseError.__stringify(msg) + (f" :=> {extra_information}" if extra_information else "") + _stringify(msg, self.descriptor_delimiter) + (f" :=> {extra_information}" if extra_information else "") ) def sys_exit_log( @@ -109,18 +189,25 @@ def log_as_warning( @property def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], str]]: - """Documentation to be added!""" - return {status: {Literal["description"]: ResponseError.__stringify(msg)} for status, msg in self.__dict.items()} + """Gets the FastAPI descriptor (dictionary) of the error items stored in :obj:`~ResponseError.__dict`. - @staticmethod - def __listify(item: str | list[str]) -> list[str]: - """Documentation to be added!""" - return item if isinstance(item, list) else [item] + Example: + .. code-block:: python - @staticmethod - def __stringify(item: str | list[str]) -> str: - """Documentation to be added!""" - return ResponseError.descriptor_delimiter.join(ResponseError.__listify(item)) + error_a = ResponseError({400: "Bad Request"}) + error_b = ResponseError({404: "Not Found"}) + error_c = ResponseError({400: "Still Bad Request"}) + + errors_combined = error_a | error_b | error_c + errors_combined.fastapi_descriptor == { + 400: {"description": "Bad Request |OR| Still Bad Request"}, + 404: {"description": "Not Found"} + } + """ + return { + status: {Literal["description"]: _stringify(msg, self.descriptor_delimiter)} + for status, msg in self.__dict.items() + } class ResponsesErrorGroup: From bec9f6478d32b11970e63305333a677a04b514d3 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 19:38:39 +0200 Subject: [PATCH 49/97] More docstrings! --- trolldb/errors/errors.py | 75 ++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index dcdf00a..6ac3fbc 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -1,4 +1,4 @@ -"""The module which defines the base functionalities for errors that will be risen when using the package or the API. +"""The module which defines the base functionalities for errors that will be raised when using the package or the API. This module only includes the generic utilities using which each module should define its own error responses specifically. See :obj:`trolldb.database.errors` as an example on how to achieve this. @@ -138,23 +138,43 @@ def __or__(self, other: Self): buff[key].extend(_listify(msg)) return ResponseError(buff) - def __retrieve_one_from_multiple_response_codes( + def __retrieve_one_from_some( self, status_code: StatusCode | None = None) -> (StatusCode, str): """Retrieves a single tuple of ``(, )`` from the internal dictionary ``self.__dict``. - Asserts whether the response error includes multiple items. + Args: + status_code (Optional, default: ``None``): + The status code to retrieve from the internal dictionary. In case of ``None``, the internal dictionary + must include only a single entry which will be returned. + + Returns: + The tuple of ``(, )``. + Raises: + ValueError: + In case of ambiguity, i.e. there are multiple items in the internal dictionary and the + ``status_code`` is ``None``. + + KeyError: + When the given ``status_code`` cannot be found. """ match status_code, len(self.__dict): + # Ambiguity, several items in the dictionary but the status code has not been given case None, n if n > 1: raise ValueError("In case of multiple response status codes, the status code must be specified.") - case StatusCode(), n if n > 1: + + # The status code has been specified + case StatusCode(), n if n >= 1: if status_code in self.__dict.keys(): return status_code, self.__dict[status_code] raise KeyError(f"Status code {status_code} cannot be found.") + + # The status code has not been given and there is only a single item in the dictionary case _, 1: return [(k, v) for k, v in self.__dict.items()][0] + + # The internal dictionary is empty and the status code is None. case _: return 500, "Generic Response Error" @@ -162,31 +182,48 @@ def get_error_details( self, extra_information: dict | None = None, status_code: int | None = None) -> (StatusCode, str): - """Gets the details of the error response.""" - status_code, msg = self.__retrieve_one_from_multiple_response_codes(status_code) - return ( - status_code, - _stringify(msg, self.descriptor_delimiter) + (f" :=> {extra_information}" if extra_information else "") - ) + """Gets the details of the error response. + + Args: + extra_information (Optional, default ``None``): + Some more information to be added in the message string. + status_code (Optional, default ``None``): + The status to retrieve. This is useful when there are several error items in the internal dictionary. + In case of ``None``, the internal dictionary must include a single entry, otherwise an error is raised. + + Returns: + A tuple, in which the first element is the status code and the second element is a single string message. + """ + status_code, msg = self.__retrieve_one_from_some(status_code) + return status_code, msg + (f" :=> {extra_information}" if extra_information else "") + + def log_as_warning( + self, + extra_information: dict | None = None, + status_code: int | None = None): + """Same as :func:`~ResponseError.get_error_details` but logs the error as a warning and returns ``None``.""" + msg, _ = self.get_error_details(extra_information, status_code) + logger.warning(msg) def sys_exit_log( self, exit_code: int = -1, extra_information: dict | None = None, status_code: int | None = None) -> None: - """Documentation to be added!""" + """Same as :func:`~ResponseError.get_error_details` but logs the error and calls the ``sys.exit``. + + This is supposed to be done in case of non-recoverable errors, e.g. database issues. + + The arguments are the same as :func:`~ResponseError.get_error_details` with the addition of ``exit_code`` + which is optional and is set to ``-1`` by default. + + Returns: + Does not return anything, but logs the error and exits the program. + """ msg, _ = self.get_error_details(extra_information, status_code) logger.error(msg) exit(exit_code) - def log_as_warning( - self, - extra_information: dict | None = None, - status_code: int | None = None): - """Documentation to be added!""" - msg, _ = self.get_error_details(extra_information, status_code) - logger.warning(msg) - @property def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], str]]: """Gets the FastAPI descriptor (dictionary) of the error items stored in :obj:`~ResponseError.__dict`. From 7b5f6a5555e401211b08c08633dc328eb38f02d3 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Thu, 16 May 2024 19:48:13 +0200 Subject: [PATCH 50/97] More docstrings! --- trolldb/errors/errors.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index 6ac3fbc..d2b065e 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -248,16 +248,24 @@ def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], st class ResponsesErrorGroup: - """Documentation to be added!""" + """A class which groups related errors. + + This provides a base class from which actual error groups are derived. The attributes of this class are all static. + + See :obj:`trolldb.database.errors` as an example on how to achieve this. + """ @classmethod - def fields(cls): - """Documentation to be added!""" + def fields(cls) -> dict[str, ResponseError]: + """Retrieves a dictionary of all errors which are members of the class.""" return {k: v for k, v in cls.__dict__.items() if isinstance(v, ResponseError)} @classmethod - def union(cls): - """Documentation to be added!""" + def union(cls) -> ResponseError: + """Gets the union of all member errors in the group. + + This utilizes the bitwise `or` ``|`` functionality of :obj:`ResponseError`. + """ buff = None for v in cls.fields().values(): if buff is None: From 592e72a1003f92933228cc7946dff0eb52c5c680 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 14:01:12 +0200 Subject: [PATCH 51/97] More docstrings! --- docs/make.sh | 2 - trolldb/api/__init__.py | 10 +++- trolldb/api/api.py | 59 +++++++++++--------- trolldb/api/routes/common.py | 42 +++++++-------- trolldb/api/routes/datetime_.py | 14 +++-- trolldb/api/routes/queries.py | 11 +++- trolldb/api/routes/root.py | 3 +- trolldb/config/__init__.py | 2 +- trolldb/config/config.py | 23 ++++++-- trolldb/database/__init__.py | 2 +- trolldb/database/mongodb.py | 75 +++++++++++++++----------- trolldb/database/piplines.py | 2 +- trolldb/test_utils/mongodb_database.py | 32 +++++++++-- 13 files changed, 179 insertions(+), 98 deletions(-) delete mode 100755 docs/make.sh diff --git a/docs/make.sh b/docs/make.sh deleted file mode 100755 index e4c8ff0..0000000 --- a/docs/make.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd source -python -m sphinx -T -b html -d _build/doctrees -D language=en . build/html diff --git a/trolldb/api/__init__.py b/trolldb/api/__init__.py index 2780698..6590a31 100644 --- a/trolldb/api/__init__.py +++ b/trolldb/api/__init__.py @@ -1 +1,9 @@ -"""api package.""" +"""This package contains the API capabilities for the pytroll-db package. + +It provides functionality for interacting with the database via the `FastAPI `_ +framework. + +For more information and documentation, please refer to the following sub-packages and modules: + - :obj:`trolldb.api.routes`: The package which defines the API routes. + - :obj:`trollddb.api.api`: The module which defines the API server and how it is run via the given configuration. +""" diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 9ac5d27..ad6be2a 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -33,21 +33,21 @@ from trolldb.database.mongodb import mongodb_context from trolldb.errors.errors import ResponseError -API_INFO = { - "title": "pytroll-db", - "version": "0.1", - "summary": "The database API of Pytroll", - "description": - "The API allows you to perform CRUD operations as well as querying the database" - "At the moment only MongoDB is supported. It is based on the following Python packages" - "\n * **PyMongo** (https://github.com/mongodb/mongo-python-driver)" - "\n * **motor** (https://github.com/mongodb/motor)", - "license_info": { - "name": "The GNU General Public License v3.0", - "url": "https://www.gnu.org/licenses/gpl-3.0.en.html" - } -} -"""These will appear the auto-generated documentation and are passed to the ``FastAPI`` class.""" +API_INFO = dict( + title="pytroll-db", + version="0.1", + summary="The database API of Pytroll", + description= + "The API allows you to perform CRUD operations as well as querying the database" + "At the moment only MongoDB is supported. It is based on the following Python packages" + "\n * **PyMongo** (https://github.com/mongodb/mongo-python-driver)" + "\n * **motor** (https://github.com/mongodb/motor)", + license_info=dict( + name="The GNU General Public License v3.0", + url="https://www.gnu.org/licenses/gpl-3.0.en.html" + ) +) +"""These will appear int the auto-generated documentation and are passed to the ``FastAPI`` class as keyword args.""" @validate_call @@ -68,19 +68,27 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: The keyword arguments are the same as those accepted by the `FastAPI class `_ and are directly passed to it. These keyword arguments will be first concatenated with the configurations of the API server which - are read from the ``config`` argument. The keyword arguments which are passed - explicitly to the function take precedence over ``config``. Finally, ``API_INFO``, which are hard-coded - information for the API server, will be concatenated and takes precedence over all. + are read from the ``config`` argument. The keyword arguments which are passed explicitly to the function + take precedence over ``config``. Finally, ``API_INFO``, which are hard-coded information for the API server, + will be concatenated and takes precedence over all. + + Example: + .. code-block:: python + + from api.api import run_server + if __name__ == "__main__": + run_server("config.yaml") """ config = parse(config) - # concatenate the keyword arguments for the API server in the order of precedence (lower to higher). + # Concatenate the keyword arguments for the API server in the order of precedence (lower to higher). app = FastAPI(**(config.api_server._asdict() | kwargs | API_INFO)) app.include_router(api_router) @app.exception_handler(ResponseError) async def unicorn_exception_handler(_, exc: ResponseError): + """Catches all the exceptions raised as a ResponseError, e.g. accessing non-existing databases/collections.""" status_code, message = exc.get_error_details() return PlainTextResponse( status_code=status_code if status_code else status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -103,26 +111,27 @@ async def _serve(): @contextmanager @validate_call -def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2000): +def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2): """A synchronous context manager to run the API server in a separate process (non-blocking). It uses the `multiprocessing `_ package. The main use case - is envisaged to be in testing environments. + is envisaged to be in `TESTING` environments. Args: config: Same as ``config`` argument for :func:`run_server`. startup_time: - The overall time that is expected for the server and the database connections to be established before - actual requests can be sent to the server. For testing purposes ensure that this is sufficiently large so - that the tests will not time out. + The overall time in seconds that is expected for the server and the database connections to be established + before actual requests can be sent to the server. For testing purposes ensure that this is sufficiently + large so that the tests will not time out. """ config = parse(config) process = Process(target=run_server, args=(config,)) process.start() try: - time.sleep(startup_time / 1000) # `time.sleep()` expects an argument in seconds, hence the division by 1000. + time.sleep(startup_time) yield process finally: process.terminate() + process.join() diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py index e7d9092..f7a995a 100644 --- a/trolldb/api/routes/common.py +++ b/trolldb/api/routes/common.py @@ -10,8 +10,10 @@ exclude_defaults_query = Query( True, title="Query string", - description="A boolean to exclude default databases from a MongoDB instance. Refer to " - "`trolldb.database.mongodb.MongoDB.default_database_names` for more information.") + description= + "A boolean to exclude default databases from a MongoDB instance. Refer to " + "`trolldb.database.mongodb.MongoDB.default_database_names` for more information." +) async def check_database(database_name: str | None = None) -> AsyncIOMotorDatabase: @@ -22,10 +24,11 @@ async def check_database(database_name: str | None = None) -> AsyncIOMotorDataba The name of the database to check. In case of ``None``, the main database will be picked. Returns: - -- The database object if it exists. + The database object if it exists. - -- Raises a :class:`~trolldb.errors.errors.ResponseError` otherwise. Check - :func:`~trolldb.database.mongodb.MongoDB.get_database` for more information. + Raises: + :class:`~trolldb.errors.errors.ResponseError`: + Check :func:`~trolldb.database.mongodb.MongoDB.get_database` for more information. """ return await MongoDB.get_database(database_name) @@ -36,7 +39,7 @@ async def check_collection( """A dependency for route handlers to check for the existence of a collection. It performs the check given the collection name and the name of the database it resides in. It first checks for the - existence of the database using :func:`check_database`. + existence of the database. Args: database_name (Optional, default ``None``): @@ -49,21 +52,17 @@ async def check_collection( will be picked. In case only one of them is ``None``, this is treated as an unacceptable request. Returns: - -- The collection object if it exists in the designated database. + - The collection object if it exists in the designated database. - -- A response from :func:`check_database`, if the database does not exist or the type of ``database_name`` is - not valid. - - -- if the parent database exists but the collection does not. - - -- if only one of ``database_name`` or ``collection_name`` - is ``None``; or if the type of ``collection_name`` is not ``str``. + Raises: + :class:`~trolldb.errors.errors.ResponseError`: + Check :func:`~trolldb.database.mongodb.MongoDB.get_collection` for more information. """ return await MongoDB.get_collection(database_name, collection_name) async def get_distinct_items_in_collection( - res_coll: Response | AsyncIOMotorCollection, + response_or_collection: Response | AsyncIOMotorCollection, field_name: str) -> Response | list[str]: """An auxiliary function to either return the given response; or return a list of distinct (unique) values. @@ -71,22 +70,21 @@ async def get_distinct_items_in_collection( equivalent to the ``distinct`` function from MongoDB. The former is the behaviour of an identity function. Args: - res_coll: + response_or_collection: Either a response object, or a collection in which documents will be queried for the ``field_name``. field_name: The name of the target field in the documents Returns: - -- In case of a response as input, the same response will be returned. - - -- In case of a collection as input, all the documents of the collection will be searched for ``field_name``, + - In case of a response object as input, the same response will be returned as-is. + - In case of a collection as input, all the documents of the collection will be searched for ``field_name``, and the corresponding values will be retrieved. Finally, a list of all the distinct values is returned. """ - if isinstance(res_coll, Response): - return res_coll + if isinstance(response_or_collection, Response): + return response_or_collection - return await res_coll.distinct(field_name) + return await response_or_collection.distinct(field_name) CheckCollectionDependency = Annotated[AsyncIOMotorCollection, Depends(check_collection)] diff --git a/trolldb/api/routes/datetime_.py b/trolldb/api/routes/datetime_.py index 6a0d4bc..fbda3bf 100644 --- a/trolldb/api/routes/datetime_.py +++ b/trolldb/api/routes/datetime_.py @@ -5,7 +5,7 @@ """ from datetime import datetime -from typing import TypedDict +from typing import Any, Coroutine, TypedDict from fastapi import APIRouter from pydantic import BaseModel @@ -51,8 +51,16 @@ async def datetime(collection: CheckCollectionDependency) -> ResponseModel: "max_end_time": {"$max": "$end_time"} }}]).next() - def _aux(query): - """Please consult the auto-generated documentation by FastAPI.""" + def _aux(query: dict) -> Coroutine[Any, Any, str]: + """An auxiliary function that retrieves a single object UUID from the database based on the given query. + + Args: + query: + The query used to search for the desired document in the database. + + Returns: + The UUID of the document found in the database. + """ return get_id(collection.find_one(query)) return ResponseModel( diff --git a/trolldb/api/routes/queries.py b/trolldb/api/routes/queries.py index 5d0a399..e585101 100644 --- a/trolldb/api/routes/queries.py +++ b/trolldb/api/routes/queries.py @@ -22,11 +22,14 @@ summary="Gets the database UUIDs of the documents that match specifications determined by the query string") async def queries( collection: CheckCollectionDependency, + # We suppress ruff for the following four lines with `Query(default=None)`. + # Reason: This is the FastAPI way of defining optional queries and ruff is not happy about it! platform: list[str] = Query(default=None), # noqa: B008 sensor: list[str] = Query(default=None), # noqa: B008 time_min: datetime.datetime = Query(default=None), # noqa: B008 time_max: datetime.datetime = Query(default=None)) -> list[str]: # noqa: B008 """Please consult the auto-generated documentation by FastAPI.""" + # We pipelines = Pipelines() if platform: @@ -38,7 +41,11 @@ async def queries( if [time_min, time_max] != [None, None]: start_time = PipelineAttribute("start_time") end_time = PipelineAttribute("end_time") - pipelines += ((start_time >= time_min) | (start_time <= time_max) | - (end_time >= time_min) | (end_time <= time_max)) + pipelines += ( + (start_time >= time_min) | + (start_time <= time_max) | + (end_time >= time_min) | + (end_time <= time_max) + ) return await get_ids(collection.aggregate(pipelines)) diff --git a/trolldb/api/routes/root.py b/trolldb/api/routes/root.py index 1aaaccc..9a11da0 100644 --- a/trolldb/api/routes/root.py +++ b/trolldb/api/routes/root.py @@ -9,7 +9,8 @@ router = APIRouter() -@router.get("/", summary="The root route which is mainly used to check the status of connection") +@router.get("/", + summary="The root route which is mainly used to check the status of connection") async def root() -> Response: """Please consult the auto-generated documentation by FastAPI.""" return Response(status_code=status.HTTP_200_OK) diff --git a/trolldb/config/__init__.py b/trolldb/config/__init__.py index 28c6e45..ccb796b 100644 --- a/trolldb/config/__init__.py +++ b/trolldb/config/__init__.py @@ -1 +1 @@ -"""config package.""" +"""This package contains utilities (e.g. parser) to handle configuration files and settings for the pytroll-db.""" diff --git a/trolldb/config/config.py b/trolldb/config/config.py index b5a2a32..26c0e8b 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -20,13 +20,27 @@ from typing_extensions import Annotated from yaml import safe_load -Timeout = Annotated[int, Field(gt=0)] +Timeout = Annotated[float, Field(ge=0)] +"""A type hint for the timeout in seconds (non-negative float).""" -def id_must_be_valid(v: str) -> ObjectId: - """Checks that the given string can be converted to a valid MongoDB ObjectId.""" +def id_must_be_valid(id_like_string: str) -> ObjectId: + """Checks that the given string can be converted to a valid MongoDB ObjectId. + + Args: + id_like_string: + The string to be converted to an ObjectId. + + Returns: + The ObjectId object if successfully. + + Raises: + ValueError: + If the given string cannot be converted to a valid ObjectId. This will ultimately turn into a pydantic + validation error. + """ try: - return ObjectId(v) + return ObjectId(id_like_string) except InvalidId as e: raise ValueError from e @@ -81,6 +95,7 @@ class AppConfig(BaseModel): database: DatabaseConfig subscriber_config: Dict[Any, Any] + @validate_call def from_yaml(filename: FilePath) -> AppConfig: """Parses and validates the configurations from a YAML file. diff --git a/trolldb/database/__init__.py b/trolldb/database/__init__.py index f8444ef..0723ff5 100644 --- a/trolldb/database/__init__.py +++ b/trolldb/database/__init__.py @@ -1 +1 @@ -"""database package.""" +"""This package contains the main database functionalities of the pytroll-db.""" diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index fdc968b..7f685c6 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -1,7 +1,8 @@ """The module which handles database CRUD operations for MongoDB. -It is based on `PyMongo `_ and -`motor `_. +It is based on the following libraries: + - `PyMongo `_ + - `motor `_. """ import errno @@ -25,8 +26,13 @@ T = TypeVar("T") CoroutineLike = Coroutine[Any, Any, T] +"""A simple type hint for a coroutine of any type.""" + CoroutineDocument = CoroutineLike[_DocumentType | None] +"""Coroutine type hint for document like objects.""" + CoroutineStrList = CoroutineLike[list[str]] +"""Coroutine type hint for a list of strings.""" class DatabaseName(BaseModel): @@ -112,24 +118,23 @@ async def initialize(cls, database_config: DatabaseConfig): database_config: A named tuple which includes the database configurations. - Raises ``SystemExit(errno.EIO)``: - - If connection is not established (``ConnectionFailure``) - - If the attempt times out (``ServerSelectionTimeoutError``) - - If one attempts reinitializing the class with new (different) database configurations without calling - :func:`~close()` first. - - If the state is not consistent, i.e. the client is closed or ``None`` but the internal database - configurations still exist and are different from the new ones which have been just provided. - - - Raises ``SystemExit(errno.ENODATA)``: - If either ``database_config.main_database`` or ``database_config.main_collection`` does not exist. - Returns: On success ``None``. + + Raises: + SystemExit(errno.EIO): + If connection is not established (``ConnectionFailure``) + SystemExit(errno.EIO): + If the attempt times out (``ServerSelectionTimeoutError``) + SystemExit(errno.EIO): + If one attempts reinitializing the class with new (different) database configurations without calling + :func:`~close()` first. + SystemExit(errno.EIO): + If the state is not consistent, i.e. the client is closed or ``None`` but the internal database + configurations still exist and are different from the new ones which have been just provided. + + SystemExit(errno.ENODATA): + If either ``database_config.main_database`` or ``database_config.main_collection`` does not exist. """ if cls.__database_config: if database_config == cls.__database_config: @@ -196,7 +201,8 @@ def main_database(cls) -> AsyncIOMotorDatabase: Returns: The main database which includes the main collection, which in turn includes the desired documents. - Equivalent to ``MongoDB.client()[]``. + + This is equivalent to ``MongoDB.client()[]``. """ return cls.__main_database @@ -213,23 +219,23 @@ async def get_collection( collection_name: The name of the collection which resides inside the parent database labelled by ``database_name``. + Returns: + The database object. In case of ``None`` for both the database name and collection name, the main collection + will be returned. + Raises: - ``ValidationError``: + ValidationError: If input args are invalid according to the pydantic. - ``KeyError``: + KeyError: If the database name exists, but it does not include any collection with the given name. - ``TypeError``: + TypeError: If only one of the database or collection names are ``None``. - ``_``: + ...: This method relies on :func:`get_database` to check for the existence of the database which can raise exceptions. Check its documentation for more information. - - Returns: - The database object. In case of ``None`` for both the database name and collection name, the main collection - will be returned. """ database_name = DatabaseName(name=database_name).name collection_name = CollectionName(name=collection_name).name @@ -254,12 +260,12 @@ async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | Respon database_name: The name of the database to retrieve. - Raises: - ``KeyError``: - If the database name does not exist in the list of database names. - Returns: The database object. + + Raises: + KeyError: + If the database name does not exist in the list of database names. """ database_name = DatabaseName(name=database_name).name @@ -277,7 +283,12 @@ async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | Respon async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: """An asynchronous context manager to connect to the MongoDB client. - It can be either used in production or in testing environments. + It can be either used in `PRODUCTION` or in `TESTING` environments. + + + Note: + Since the :class:`MongoDB` is supposed to be used statically, this context manager does not yield anything! + One can simply use :class:`MongoDB` inside the context manager. Args: database_config: diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index f10f2e8..1b722d4 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -4,7 +4,7 @@ class PipelineBooleanDict(dict): - """A subclass of dict which overrides the behaviour of bitwise `OR` (``|``) and bitwise `AND` (``&``). + """A subclass of dict which overrides the behavior of bitwise `OR` (``|``) and bitwise `AND` (``&``). This class makes it easier to chain and nest `And/Or` operations. diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index 27a588c..a40ede3 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -3,7 +3,7 @@ from contextlib import contextmanager from datetime import datetime, timedelta from random import randint, shuffle -from typing import Any, Iterator +from typing import Iterator from pymongo import MongoClient @@ -18,6 +18,14 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab Note: This is based on `pymongo` and not the `motor` async driver. For testing purposes this is sufficient and we do not need async capabilities. + + Args: + database_config (Optional, default :obj:``~test_app_config.database``): + The configuration object for the database. Defaults to the test_app_config.database configuration. + + Yields: + MongoClient: + The MongoDB client object. """ client = None try: @@ -28,8 +36,26 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab client.close() -def random_sample(items: list[Any], size: int = 10) -> list[Any]: - """Generates a random sample of ``size`` elements, using the given list of items.""" +def random_sample(items, size=10): + """Generates a random sample of elements, using the given list of items. + + Args: + items: + The list of items from which the random sample will be generated. + size (Optional, default ``10``): + The number of elements in the random sample. Defaults to 10. + + Returns: + A list containing the random sample of elements. + + Raises: + None + + Example: + >>> items = [1, 2, 3, 4, 5] + >>> random_sample(items, 10) + [2, 4, 1, 5, 3, 4, 2, 1, 3, 5] + """ last_index = len(items) - 1 # We suppress ruff here as we are not generating anything cryptographic here! indices = [randint(0, last_index) for _ in range(size)] # noqa: S311 From 522139974cf30ae0430fefbbd1612500089ff848 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 14:01:42 +0200 Subject: [PATCH 52/97] Convert make.sh to an actual Makefile. --- docs/Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 docs/Makefile diff --git a/docs/Makefile b/docs/Makefile new file mode 100755 index 0000000..037c4de --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,11 @@ +SOURCE_DIR := source + +.DEFAULT_GOAL := html +.PHONY: clean html + +clean: + @cd $(SOURCE_DIR) && find . -type f ! -name 'index.rst' ! -name 'conf.py' -delete + @cd $(SOURCE_DIR) && find . -type d -delete + +html: + @cd $(SOURCE_DIR) && python -m sphinx -T -b html -d _build/doctrees -D language=en . build/html From 60f8145473e61ebaae62eec8f4cb98ebb793f4b9 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 14:02:00 +0200 Subject: [PATCH 53/97] Add tests for pipelines. --- trolldb/tests/tests_database/test_pipelines.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 trolldb/tests/tests_database/test_pipelines.py diff --git a/trolldb/tests/tests_database/test_pipelines.py b/trolldb/tests/tests_database/test_pipelines.py new file mode 100644 index 0000000..2e5fb34 --- /dev/null +++ b/trolldb/tests/tests_database/test_pipelines.py @@ -0,0 +1,16 @@ +"""Documentation to be added.""" +from trolldb.database.piplines import PipelineBooleanDict + + +def test_pipeline_boolean_dict(): + """Documentation to be added.""" + pd1 = PipelineBooleanDict({"number": 2}) + pd2 = PipelineBooleanDict({"kind": 1}) + + pd_and = pd1 & pd2 + pd_and_literal = PipelineBooleanDict({"$and": [{"number": 2}, {"kind": 1}]}) + assert pd_and == pd_and_literal + + pd_or = pd1 | pd2 + pd_or_literal = PipelineBooleanDict({"$or": [{"number": 2}, {"kind": 1}]}) + assert pd_or == pd_or_literal From 9c13ac21aa20670a03c3ed3a51c5d9d9545f029a Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 14:15:03 +0200 Subject: [PATCH 54/97] More docstrings! --- trolldb/config/config.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 26c0e8b..322b186 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -10,7 +10,7 @@ import errno import sys -from typing import Any, Dict, NamedTuple +from typing import Any, NamedTuple from bson import ObjectId from bson.errors import InvalidId @@ -46,6 +46,7 @@ def id_must_be_valid(id_like_string: str) -> ObjectId: MongoObjectId = Annotated[str, AfterValidator(id_must_be_valid)] +"""Type hint validator for object IDs.""" class MongoDocument(BaseModel): @@ -80,12 +81,19 @@ class DatabaseConfig(NamedTuple): url: MongoDsn """The URL of the MongoDB server excluding the port part, e.g. ``"mongodb://localhost:27017"``""" - timeout: Annotated[int, Field(gt=-1)] - """The timeout in milliseconds (non-negative integer), after which an exception is raised if a connection with the - MongoDB instance is not established successfully, e.g. ``1000``. + timeout: Timeout + """The timeout in seconds (non-negative float), after which an exception is raised if a connection with the + MongoDB instance is not established successfully, e.g. ``2.5``. """ +SubscriberConfig = dict[Any, Any] +"""A dictionary to hold all the configurations of the subscriber. + +TODO: This has to be moved to the `posttroll` package. +""" + + class AppConfig(BaseModel): """A model to hold all the configurations of the application including both the API server and the database. @@ -93,7 +101,7 @@ class AppConfig(BaseModel): """ api_server: APIServerConfig database: DatabaseConfig - subscriber_config: Dict[Any, Any] + subscriber_config: SubscriberConfig @validate_call @@ -104,16 +112,16 @@ def from_yaml(filename: FilePath) -> AppConfig: filename: The filename of a valid YAML file which holds the configurations. + Returns: + An instance of :class:`AppConfig`. + Raises: - -- ParserError: - If the file cannot be properly parsed. + ParserError: + If the file cannot be properly parsed - -- ValidationError: + ValidationError: If the successfully parsed file fails the validation, i.e. its schema or the content does not conform to :class:`AppConfig`. - - Returns: - An instance of :class:`AppConfig`. """ with open(filename, "r") as file: config = safe_load(file) @@ -133,9 +141,8 @@ def parse(config: AppConfig | FilePath) -> AppConfig: Either an object of type :class:`AppConfig` or :class:`FilePath`. Returns: - -- In case of an object of type :class:`AppConfig` as input, the same object will be returned. - - -- An input object of type ``str`` will be interpreted as a YAML filename, in which case the function returns + - In case of an object of type :class:`AppConfig` as input, the same object will be returned as-is. + - An input object of type ``str`` will be interpreted as a YAML filename, in which case the function returns the result of parsing the file. """ match config: From ceebf5ffbf17ea01d30adee8e06f6861b8a4e075 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 14:47:30 +0200 Subject: [PATCH 55/97] More docstrings! --- docs/source/conf.py | 4 +- trolldb/database/piplines.py | 74 +++++++++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index f424a52..ab40497 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -68,9 +68,9 @@ "members": True, "member-order": "bysource", "private-members": True, - "special-members": "__init__, __or__", + "special-members": True, "undoc-members": True, - "exclude-members": "__weakref__" + "exclude-members": "__weakref__, __dict__, __module__, __hash__" } root_doc = "index" diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 1b722d4..8f75ae1 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -4,26 +4,30 @@ class PipelineBooleanDict(dict): - """A subclass of dict which overrides the behavior of bitwise `OR` (``|``) and bitwise `AND` (``&``). + """A subclass of dict which overrides the behavior of bitwise `or` ``|`` and bitwise `and` ``&``. - This class makes it easier to chain and nest `And/Or` operations. + This class makes it easier to chain and nest `"and/or"` operations. - The operators are only defined for operands of type :class:`PipelineDict`. For each of the aforementioned operators, - the result will be a dictionary with a single key/value pair. The key is either ``$or`` or ``$and`` depending on the - operator being used. The corresponding value is a list with two elements only. The first element of the list is the - left operand and the second element is the right operand. + The operators are only defined for operands of type :class:`PipelineBooleanDict`. For each of the aforementioned + operators, the result will be a dictionary with a single key/value pair. The key is either ``$or`` or ``$and`` + depending on the operator being used. The corresponding value is a list with two elements only. The first element + of the list is the content of the left operand and the second element is the content of the right operand. Example: .. code-block:: python - pd1 = PipelineDict({"number": 2}) - pd2 = PipelineDict({"kind": 1}) + pd1 = PipelineBooleanDict({"number": 2}) + pd2 = PipelineBooleanDict({"kind": 1}) - pd_and = pd1 & pd2 # is equivalent to the following - pd_and_literal = PipelineDict({"$and": [{"number": 2}, {"kind": 1}]}) + pd_and = pd1 & pd2 + pd_and_literal = PipelineBooleanDict({"$and": [{"number": 2}, {"kind": 1}]}) + # The following evaluates to True + pd_and == pd_and_literal - pd_or = pd1 | pd2 # is equivalent to the following - pd_or_literal = PipelineDict({"$or": [{"number": 2}, {"kind": 1}]}) + pd_or = pd1 | pd2 + pd_or_literal = PipelineBooleanDict({"$or": [{"number": 2}, {"kind": 1}]}) + # The following evaluates to True + pd_or == pd_or_literal """ def __or__(self, other: Self): @@ -50,13 +54,36 @@ def __eq__(self, other: Any) -> PipelineBooleanDict: This makes a boolean filter in which the attribute can match any of the items in ``other`` if it is a list, or the ``other`` itself, otherwise. + + Warning: + Note how ``==`` behaves differently for :class:`PipelineBooleanDict` and :class:`PipelineAttribute`. + In the former, it asserts equality as per the standard behaviour of the operator in Python. However, in the + latter it acts as a filter and not an assertion of equality. + + Example: + .. code-block:: python + + pa_list = PipelineAttribute("letter") == ["A", "B"] + pd_list = PipelineBooleanDict({"$or": [{"letter": "A"}, {"letter": "B"}] + # The following evaluates to True + pa_list = pd_list + + pa_single = PipelineAttribute("letter") == "A" + pd_single = PipelineBooleanDict({"letter": "A"}) + # The following evaluates to True + pa_single = pd_single """ if isinstance(other, list): return PipelineBooleanDict(**{"$or": [{self.__key: v} for v in other]}) return PipelineBooleanDict(**{self.__key: other}) def __aux_operators(self, other: Any, operator: str) -> PipelineBooleanDict: - """An auxiliary function to perform comparison operations.""" + """An auxiliary function to perform comparison operations. + + Note: + The operators herein have similar behaviour to ``==`` in the sense that they make comparison filters and are + not to be interpreted as comparison assertions. + """ return PipelineBooleanDict(**{self.__key: {operator: other}} if other else {}) def __ge__(self, other: Any) -> PipelineBooleanDict: @@ -80,9 +107,28 @@ class Pipelines(list): """A class which defines a list of pipelines. Each item in the list is a dictionary with its key being the literal string ``"$match"`` and its corresponding value - being of type :class:`PipelineBooleanDict`. The``"$match"`` key is what actually triggers the matching operation in + being of type :class:`PipelineBooleanDict`. The ``"$match"`` key is what actually triggers the matching operation in the MongoDB aggregation pipeline. The condition against which the matching will be performed is given by the value which is a simply a boolean pipeline dictionary which has a hierarchical structure. + + Example: + .. code-block:: python + + pipelines = Pipelines() + pipelines += PipelineAttribute("platform_name") == "P" + pipelines += PipelineAttribute("sensor") == ["SA", "SB"] + + pipelines_literal = [ + {"$match": + {"platform_name": "P"} + }, + {"$match": + {"$or": [{"sensor_name": "SA"}, {"sensor_name": "SB"}]} + } + ] + + # The following evaluates to True + pipelines == pipelines_literal """ def __iadd__(self, other: PipelineBooleanDict) -> Self: From d7782d5ebafbb9be12ef29e755d3148c15c80394 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 16:22:49 +0200 Subject: [PATCH 56/97] More docstrings! --- docs/source/conf.py | 2 +- trolldb/errors/__init__.py | 2 +- trolldb/errors/errors.py | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ab40497..696e4cd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -70,7 +70,7 @@ "private-members": True, "special-members": True, "undoc-members": True, - "exclude-members": "__weakref__, __dict__, __module__, __hash__" + "exclude-members": "__weakref__, __dict__, __module__, __hash__, __annotations__" } root_doc = "index" diff --git a/trolldb/errors/__init__.py b/trolldb/errors/__init__.py index da3944b..4542061 100644 --- a/trolldb/errors/__init__.py +++ b/trolldb/errors/__init__.py @@ -1 +1 @@ -"""errors package.""" +"""This package provides custom error classes for the pytroll-db.""" diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index d2b065e..4f1fec7 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -30,7 +30,7 @@ def _listify(item: str | list[str]) -> list[str]: Example: .. code-block:: python - # The following evaluate to ``True`` + # The following evaluate to True __listify("test") == ["test"] __listify(["a", "b"]) = ["a", "b"] __listify([]) == [] @@ -59,7 +59,7 @@ class ResponseError(Exception): This is a derivative of the ``Exception`` class and therefore can be used directly in ``raise`` statements. Attributes: - __dict (:obj:`OrderedDict[StatusCode, str]`): + __dict (``OrderedDict[StatusCode, str]``): An ordered dictionary in which the keys are (HTTP) status codes and the values are the corresponding messages. """ @@ -76,7 +76,8 @@ class ResponseError(Exception): error_b = ResponseError({404: "Not Found"}) errors = error_a | error_b - # When used in a FastAPI response descriptor, the following string will be generated for ``errors`` + # When used in a FastAPI response descriptor, + # the following string will be generated for errors "Bad Request |OR| Not Found" """ @@ -100,12 +101,13 @@ def __init__(self, args_dict: OrderedDict[StatusCode, str | list[str]] | dict) - error_b = ResponseError({404: "Not Found"}) errors = error_a | error_b errors_a_or_b = ResponseError({400: "Bad Request", 404: "Not Found"}) + errors_list = ResponseError({404: ["Not Found", "Still Not Found"]}) """ self.__dict: OrderedDict = OrderedDict(args_dict) self.extra_information: dict | None = None def __or__(self, other: Self): - """Implements the bitwise `or` (``|``) which combines the error objects into a single error response. + """Implements the bitwise `or` ``|`` which combines the error objects into a single error response. Args: other: @@ -114,7 +116,7 @@ def __or__(self, other: Self): Returns: A new error response which includes the combined error response. In case of different (HTTP) status codes, the returned response includes the ``{: }`` pairs for both ``self`` and the ``other``. - In case of the same status codes, the messages will be stored in a list. + In case of the same status codes, the messages will be combined into a list. Example: .. code-block:: python @@ -141,10 +143,10 @@ def __or__(self, other: Self): def __retrieve_one_from_some( self, status_code: StatusCode | None = None) -> (StatusCode, str): - """Retrieves a single tuple of ``(, )`` from the internal dictionary ``self.__dict``. + """Retrieves a tuple ``(, )`` from the internal dictionary :obj:`ResponseError.__dict`. Args: - status_code (Optional, default: ``None``): + status_code (Optional, default ``None``): The status code to retrieve from the internal dictionary. In case of ``None``, the internal dictionary must include only a single entry which will be returned. @@ -226,7 +228,7 @@ def sys_exit_log( @property def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], str]]: - """Gets the FastAPI descriptor (dictionary) of the error items stored in :obj:`~ResponseError.__dict`. + """Gets the FastAPI descriptor (dictionary) of the error items stored in :obj:`ResponseError.__dict`. Example: .. code-block:: python From ac304b1ce65eee832f8d338979b7798990cf7b68 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 16:35:02 +0200 Subject: [PATCH 57/97] More docstrings! --- trolldb/errors/errors.py | 22 +++++++++++++--------- trolldb/test_utils/__init__.py | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index 4f1fec7..0d3ba88 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -174,7 +174,7 @@ def __retrieve_one_from_some( # The status code has not been given and there is only a single item in the dictionary case _, 1: - return [(k, v) for k, v in self.__dict.items()][0] + return next(iter(self.__dict.items())) # The internal dictionary is empty and the status code is None. case _: @@ -188,10 +188,11 @@ def get_error_details( Args: extra_information (Optional, default ``None``): - Some more information to be added in the message string. + More information (if any) that wants to be added to the message string. status_code (Optional, default ``None``): - The status to retrieve. This is useful when there are several error items in the internal dictionary. - In case of ``None``, the internal dictionary must include a single entry, otherwise an error is raised. + The status code to retrieve. This is useful when there are several error items in the internal + dictionary. In case of ``None``, the internal dictionary must include a single entry, otherwise an error + is raised. Returns: A tuple, in which the first element is the status code and the second element is a single string message. @@ -214,11 +215,13 @@ def sys_exit_log( status_code: int | None = None) -> None: """Same as :func:`~ResponseError.get_error_details` but logs the error and calls the ``sys.exit``. - This is supposed to be done in case of non-recoverable errors, e.g. database issues. - The arguments are the same as :func:`~ResponseError.get_error_details` with the addition of ``exit_code`` which is optional and is set to ``-1`` by default. + Warning: + This is supposed to be done in case of non-recoverable errors, e.g. database issues. For other cases, we try + to see if we can recover and continue. + Returns: Does not return anything, but logs the error and exits the program. """ @@ -258,7 +261,7 @@ class ResponsesErrorGroup: """ @classmethod - def fields(cls) -> dict[str, ResponseError]: + def members(cls) -> dict[str, ResponseError]: """Retrieves a dictionary of all errors which are members of the class.""" return {k: v for k, v in cls.__dict__.items() if isinstance(v, ResponseError)} @@ -266,10 +269,11 @@ def fields(cls) -> dict[str, ResponseError]: def union(cls) -> ResponseError: """Gets the union of all member errors in the group. - This utilizes the bitwise `or` ``|`` functionality of :obj:`ResponseError`. + This is useful when one wants to get the FastAPI response descriptor of all members. This function utilizes + the bitwise `or` ``|`` functionality of :obj:`ResponseError`. """ buff = None - for v in cls.fields().values(): + for v in cls.members().values(): if buff is None: buff = v else: diff --git a/trolldb/test_utils/__init__.py b/trolldb/test_utils/__init__.py index e18f5c9..e1fa351 100644 --- a/trolldb/test_utils/__init__.py +++ b/trolldb/test_utils/__init__.py @@ -1 +1 @@ -"""test_utils package.""" +"""This package provide tools to test the database and api packages.""" From 00aa2f5551166991c10c43dee58f35544aacf5b9 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 17:12:45 +0200 Subject: [PATCH 58/97] More docstrings! --- trolldb/test_utils/common.py | 42 ++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 929aa52..f6de7ab 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,5 +1,6 @@ """Common functionalities for testing, shared between tests and other test utility modules.""" +from collections import OrderedDict from typing import Any from urllib.parse import urljoin @@ -17,6 +18,7 @@ timeout=1000), subscriber_config=dict() ) +"""The app configuration when used in testing.""" def http_get(route: str = "") -> BaseHTTPResponse: @@ -32,25 +34,47 @@ def http_get(route: str = "") -> BaseHTTPResponse: return request("GET", urljoin(test_app_config.api_server.url.unicode_string(), route)) -def assert_equal(test, expected) -> None: +def assert_equal(test: Any, expected: Any, ordered: bool = False) -> None: """An auxiliary function to assert the equality of two objects using the ``==`` operator. - In case an input is a list or a tuple, it will be first converted to a set so that the order of items there in does - not affect the assertion outcome. + Examples: + - If ``ordered=False`` and the input is a list or a tuple, it will be first converted to a set + so that the order of items therein does not affect the assertion outcome. + - If ``ordered=True`` and the input is a dictionary, it will be first converted to an ``OrderedDict``. + + Note: + The rationale behind choosing ``ordered=False`` as the default behaviour is that this function is often used + in combination with API calls and/or querying the database. In such cases, the order of items which are returned + often does not matter. In addition, if the order really matters, one might as well simply use the built-in + ``assert`` statement. + + Note: + Dictionaries by default are unordered objects. Warning: - In case of a list or tuple of items as inputs, do not use this function if the order of items matters. + For the purpose of this function, the concept of ordered vs unordered only applies to lists, tuples, and + dictionaries. An object of any other type is assumed as-is, i.e. the default behaviour of Python applies. + For example, conceptually, two strings can be converted to two sets of characters and then be compared with + each other. However, this is not what we do for strings. Args: test: The object to be tested. expected: The object to test against. + ordered (Optional, default ``False``): + A flag to determine whether the order of items matters in case of a list, a tuple, or a dictionary. """ - def _setify(obj: Any) -> Any: - """An auxiliary function to convert an object to a set if it is a tuple or a list.""" - return set(obj) if isinstance(obj, list | tuple) else obj + def _ordered(obj: Any) -> Any: + """An auxiliary function to convert an object to ordered depending on its type and the ``ordered`` flag.""" + match obj: + case list() | tuple(): + return set(obj) if not ordered else obj + case dict(): + return OrderedDict(obj) if ordered else obj + case _: + return obj - if not _setify(test) == _setify(expected): - raise AssertionError(f"{test} and {expected} are not equal.") + if not _ordered(test) == _ordered(expected): + raise AssertionError(f"{test} and {expected} are not equal. The flag `ordered` is set to `{ordered}`.") From c4a03c3de53fae9cf95fc728d7fefd365c950d0b Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 17:58:42 +0200 Subject: [PATCH 59/97] More docstrings! --- docs/source/conf.py | 1 - trolldb/api/api.py | 5 +- trolldb/config/config.py | 8 +-- trolldb/test_utils/common.py | 6 ++- trolldb/test_utils/mongodb_database.py | 74 +++++++++++++++----------- 5 files changed, 54 insertions(+), 40 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 696e4cd..6f7ab16 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -70,7 +70,6 @@ "private-members": True, "special-members": True, "undoc-members": True, - "exclude-members": "__weakref__, __dict__, __module__, __hash__, __annotations__" } root_doc = "index" diff --git a/trolldb/api/api.py b/trolldb/api/api.py index ad6be2a..e139fd0 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -111,7 +111,7 @@ async def _serve(): @contextmanager @validate_call -def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2): +def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2000): """A synchronous context manager to run the API server in a separate process (non-blocking). It uses the `multiprocessing `_ package. The main use case @@ -130,7 +130,8 @@ def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = process = Process(target=run_server, args=(config,)) process.start() try: - time.sleep(startup_time) + # time.sleep() expects its argument to be in seconds, hence the division by 1000. + time.sleep(startup_time / 1000.) yield process finally: process.terminate() diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 322b186..6daf89f 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -20,8 +20,8 @@ from typing_extensions import Annotated from yaml import safe_load -Timeout = Annotated[float, Field(ge=0)] -"""A type hint for the timeout in seconds (non-negative float).""" +Timeout = Annotated[int, Field(ge=0)] +"""A type hint for the timeout in milliseconds (non-negative int).""" def id_must_be_valid(id_like_string: str) -> ObjectId: @@ -82,8 +82,8 @@ class DatabaseConfig(NamedTuple): """The URL of the MongoDB server excluding the port part, e.g. ``"mongodb://localhost:27017"``""" timeout: Timeout - """The timeout in seconds (non-negative float), after which an exception is raised if a connection with the - MongoDB instance is not established successfully, e.g. ``2.5``. + """The timeout in milliseconds (non-negative int), after which an exception is raised if a connection with the + MongoDB instance is not established successfully, e.g. ``1000``. """ diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index f6de7ab..fd481ef 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -21,17 +21,19 @@ """The app configuration when used in testing.""" -def http_get(route: str = "") -> BaseHTTPResponse: +def http_get(route: str = "", root: AnyUrl = test_app_config.api_server.url) -> BaseHTTPResponse: """An auxiliary function to make a GET request using :func:`urllib.request`. Args: route: The desired route (excluding the root URL) which can include a query string as well. + root (Optional, default :obj:`test_app_config.api_server.url`): + The root to which the given route will be added to make the complete URL. Returns: The response from the GET request. """ - return request("GET", urljoin(test_app_config.api_server.url.unicode_string(), route)) + return request("GET", urljoin(root.unicode_string(), route)) def assert_equal(test: Any, expected: Any, ordered: bool = False) -> None: diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index a40ede3..58c6a3d 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -16,16 +16,16 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab """A context manager for the MongoDB client given test configurations. Note: - This is based on `pymongo` and not the `motor` async driver. For testing purposes this is sufficient and we + This is based on `Pymongo` and not the `motor` async driver. For testing purposes this is sufficient and we do not need async capabilities. Args: - database_config (Optional, default :obj:``~test_app_config.database``): - The configuration object for the database. Defaults to the test_app_config.database configuration. + database_config (Optional, default :obj:`test_app_config.database`): + The configuration object for the database. Yields: MongoClient: - The MongoDB client object. + The MongoDB client object (from `Pymongo`) """ client = None try: @@ -36,28 +36,28 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab client.close() -def random_sample(items, size=10): - """Generates a random sample of elements, using the given list of items. +def random_sample(items: list, size: int = 10) -> list: + """Generates a random sample of items from the given list, with repetitions allowed. + + Note: + The length of the output can be larger than the lenght of the given list. See the example. Args: items: The list of items from which the random sample will be generated. size (Optional, default ``10``): - The number of elements in the random sample. Defaults to 10. + The number of elements in the random sample. Returns: A list containing the random sample of elements. - Raises: - None - Example: >>> items = [1, 2, 3, 4, 5] >>> random_sample(items, 10) [2, 4, 1, 5, 3, 4, 2, 1, 3, 5] """ last_index = len(items) - 1 - # We suppress ruff here as we are not generating anything cryptographic here! + # We suppress ruff (S311) here as we are not generating anything cryptographic here! indices = [randint(0, last_index) for _ in range(size)] # noqa: S311 return [items[i] for i in indices] @@ -66,18 +66,18 @@ class Time: """A static class to enclose functionalities for generating random time stamps.""" min_start_time = datetime(2019, 1, 1, 0, 0, 0) - """The minimum timestamp.""" + """The minimum timestamp which is allowed to appear in our data.""" max_end_time = datetime(2024, 1, 1, 0, 0, 0) - """The maximum timestamp.""" + """The maximum timestamp which is allowed to appear in our data.""" delta_time = int((max_end_time - min_start_time).total_seconds()) """The difference between the maximum and minimum timestamps in seconds.""" @staticmethod def random_interval_secs(max_interval_secs: int) -> timedelta: - """Generates a random time interval between zero and the given max interval.""" - # We suppress ruff here as we are not generating anything cryptographic here! + """Generates a random time interval between zero and the given max interval in seconds.""" + # We suppress ruff (S311) here as we are not generating anything cryptographic here! return timedelta(seconds=randint(0, max_interval_secs)) # noqa: S311 @staticmethod @@ -93,13 +93,14 @@ def random_start_time() -> datetime: def random_end_time(start_time: datetime, max_interval_secs: int = 300) -> datetime: """Generates a random end time. - The end time is within ``max_interval_secs`` seconds from the given ``start_time``. + The end time is within ``max_interval_secs`` seconds from the given ``start_time``. By default, the interval + is set to 300 seconds (5 minutes). """ return start_time + Time.random_interval_secs(max_interval_secs) class Document: - """A class which defines functionalities to generate documents data which are similar to real data.""" + """A class which defines functionalities to generate database documents/data which are similar to real data.""" def __init__(self, platform_name: str, sensor: str) -> None: """Initializes the document given its platform and sensor names.""" @@ -115,7 +116,7 @@ def generate_dataset(self, max_count: int) -> list[dict]: chosen from 1 to ``max_count`` for each document. """ dataset = [] - # We suppress ruff here as we are not generating anything cryptographic here! + # We suppress ruff (S311) here as we are not generating anything cryptographic here! n = randint(1, max_count) # noqa: S311 for i in range(n): txt = f"{self.platform_name}_{self.sensor}_{self.start_time}_{self.end_time}_{i}" @@ -138,7 +139,7 @@ def like_mongodb_document(self) -> dict: class TestDatabase: - """The class which encloses functionalities to prepare and fill the test database with mock data.""" + """A static class which encloses functionalities to prepare and fill the test database with mock data.""" platform_names = random_sample(["PA", "PB", "PC"]) """Example platform names.""" @@ -163,21 +164,28 @@ class TestDatabase: all_database_names = ["admin", "config", "local", *database_names] """All database names including the default ones which are automatically created by MongoDB.""" - documents = [] + documents: list[dict] = [] """The list of documents which include mock data.""" @classmethod - def generate_documents(cls, random_shuffle: bool = True) -> list: - """Generates test documents which for practical purposes resemble real data.""" - documents = [Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors, - strict=False)] + def generate_documents(cls, random_shuffle: bool = True) -> None: + """Generates test documents which for practical purposes resemble real data. + + Warning: + This method is not pure! The side effect is that the :obj:`TestDatabase.documents` is filled. + """ + cls.documents = [ + Document(p, s).like_mongodb_document() for p, s in zip(cls.platform_names, cls.sensors, strict=False)] if random_shuffle: - shuffle(documents) - return documents + shuffle(cls.documents) @classmethod def reset(cls): - """Resets all the databases/collections.""" + """Resets all the databases/collections. + + This is done by deleting all documents in the collections and then inserting a single empty ``{}`` document + in them. + """ with test_mongodb_context() as client: for db_name, coll_name in zip(cls.database_names, cls.collection_names, strict=False): db = client[db_name] @@ -189,13 +197,17 @@ def reset(cls): def write_mock_date(cls): """Fills databases/collections with mock data.""" with test_mongodb_context() as client: - cls.documents = cls.generate_documents() - collection = client[test_app_config.database.main_database_name][ - test_app_config.database.main_collection_name] + # The following function call has side effects! + cls.generate_documents() + collection = client[ + test_app_config.database.main_database_name + ][ + test_app_config.database.main_collection_name + ] collection.insert_many(cls.documents) @classmethod def prepare(cls): - """Prepares the instance by first resetting all databases/collections and filling them with mock data.""" + """Prepares the MongoDB instance by first resetting the database and then filling it with mock data.""" cls.reset() cls.write_mock_date() From c1508bb29f5c20f8e0330da1101aaf9cf7d091d0 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 19:15:42 +0200 Subject: [PATCH 60/97] More docstrings! --- trolldb/test_utils/mongodb_instance.py | 44 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index c45bec4..8da7dc2 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -9,8 +9,9 @@ from shutil import rmtree from loguru import logger +from pydantic import validate_call -from trolldb.config.config import DatabaseConfig +from trolldb.config.config import DatabaseConfig, Timeout from trolldb.test_utils.common import test_app_config @@ -18,37 +19,49 @@ class TestMongoInstance: """A static class to enclose functionalities for running a MongoDB instance.""" log_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_log") - """Temp directory for logging messages by the MongoDB instance.""" + """Temp directory for logging messages by the MongoDB instance. + + Warning: + The value of this attribute as shown above is just an example and will change in an unpredictable (secure) way! + """ storage_dir: str = tempfile.mkdtemp("__pytroll_db_temp_test_storage") - """Temp directory for storing database files by the MongoDB instance.""" + """Temp directory for storing database files by the MongoDB instance. + + Warning: + The value of this attribute as shown above is just an example and will change in an unpredictable (secure) way! + """ port: int = 28017 - """The port on which the instance will run.""" + """The port on which the instance will run. + + Warning: + This must be always hard-coded. + """ process: subprocess.Popen | None = None - """The (sub-)process which will be used to run the MongoDB instance.""" + """The process which is used to run the MongoDB instance.""" @classmethod def __prepare_dir(cls, directory: str): - """Auxiliary function to prepare a single directory. + """An auxiliary function to prepare a single directory. - That is making a directory if it does not exist, or removing it if it does and then remaking it. + It creates a directory if it does not exist, or removes it first if it exists and then recreates it. """ cls.__remove_dir(directory) mkdir(directory) @classmethod def __remove_dir(cls, directory: str): - """Auxiliary function to remove temporary directories.""" + """An auxiliary function to remove a directory and all its content recursively.""" if path.exists(directory) and path.isdir(directory): rmtree(directory) @classmethod def run_subprocess(cls, args: list[str], wait=True): """Runs the subprocess in shell given its arguments.""" - # We suppress ruff here as we are not receiving any args from outside, e.g. port is hard-coded. Therefore, - # sanitization of ``args`` is not required. + # We suppress ruff (S603) here as we are not receiving any args from outside, e.g. port is hard-coded. + # Therefore, sanitization of arguments is not required. cls.process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # noqa: S603 if wait: outs, errs = cls.process.communicate() @@ -66,8 +79,8 @@ def mongodb_exists(cls) -> bool: @classmethod def prepare_dirs(cls) -> None: """Prepares the temp directories.""" - cls.__prepare_dir(cls.log_dir) - cls.__prepare_dir(cls.storage_dir) + for d in [cls.log_dir, cls.storage_dir]: + cls.__prepare_dir(d) @classmethod def run_instance(cls): @@ -86,9 +99,10 @@ def shutdown_instance(cls): @contextmanager +@validate_call def mongodb_instance_server_process_context( database_config: DatabaseConfig = test_app_config.database, - startup_time=2000): + startup_time: Timeout = 2000): """A synchronous context manager to run the MongoDB instance in a separate process (non-blocking). It uses the `subprocess `_ package. The main use case is @@ -99,8 +113,8 @@ def mongodb_instance_server_process_context( The configuration of the database. startup_time: - The overall time that is expected for the MongoDB server instance to run before the database content can be - accessed. + The overall time in seconds that is expected for the MongoDB server instance to run before the database + content can be accessed. """ TestMongoInstance.port = database_config.url.hosts()[0]["port"] TestMongoInstance.prepare_dirs() From 80a985c975a29aa0565abbe4284b2c41901ca098 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 19:38:53 +0200 Subject: [PATCH 61/97] More docstrings! --- trolldb/api/api.py | 8 ++++++++ trolldb/config/config.py | 8 ++++++++ trolldb/database/mongodb.py | 4 ++++ trolldb/test_utils/mongodb_instance.py | 4 ++++ 4 files changed, 24 insertions(+) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index e139fd0..1d69c03 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -72,6 +72,10 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: take precedence over ``config``. Finally, ``API_INFO``, which are hard-coded information for the API server, will be concatenated and takes precedence over all. + Raises: + ValidationError: + If the function is not called with arguments of valid type. + Example: .. code-block:: python @@ -125,6 +129,10 @@ def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = The overall time in seconds that is expected for the server and the database connections to be established before actual requests can be sent to the server. For testing purposes ensure that this is sufficiently large so that the tests will not time out. + + Raises: + ValidationError: + If the function is not called with arguments of valid type. """ config = parse(config) process = Process(target=run_server, args=(config,)) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 6daf89f..b500779 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -122,6 +122,10 @@ def from_yaml(filename: FilePath) -> AppConfig: ValidationError: If the successfully parsed file fails the validation, i.e. its schema or the content does not conform to :class:`AppConfig`. + + Raises: + ValidationError: + If the function is not called with arguments of valid type. """ with open(filename, "r") as file: config = safe_load(file) @@ -144,6 +148,10 @@ def parse(config: AppConfig | FilePath) -> AppConfig: - In case of an object of type :class:`AppConfig` as input, the same object will be returned as-is. - An input object of type ``str`` will be interpreted as a YAML filename, in which case the function returns the result of parsing the file. + + Raises: + ValidationError: + If the function is not called with arguments of valid type. """ match config: case AppConfig(): diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 7f685c6..e6dd528 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -293,6 +293,10 @@ async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: Args: database_config: The configuration of the database. + + Raises: + ValidationError: + If the function is not called with arguments of valid type. """ try: await MongoDB.initialize(database_config) diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 8da7dc2..b54ee1d 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -115,6 +115,10 @@ def mongodb_instance_server_process_context( startup_time: The overall time in seconds that is expected for the MongoDB server instance to run before the database content can be accessed. + + Raises: + ValidationError: + If the function is not called with arguments of valid type. """ TestMongoInstance.port = database_config.url.hosts()[0]["port"] TestMongoInstance.prepare_dirs() From b1a7d7db99b48c293ec3fe4798312df068746c44 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 19:39:49 +0200 Subject: [PATCH 62/97] More docstrings! --- trolldb/config/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index b500779..4f881bf 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -123,7 +123,6 @@ def from_yaml(filename: FilePath) -> AppConfig: If the successfully parsed file fails the validation, i.e. its schema or the content does not conform to :class:`AppConfig`. - Raises: ValidationError: If the function is not called with arguments of valid type. """ From a06acabb37bc385736442d79b45160c873545dd9 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 20:35:10 +0200 Subject: [PATCH 63/97] More logging! --- trolldb/api/api.py | 17 ++++++++++++++--- trolldb/config/config.py | 9 ++++++++- trolldb/database/mongodb.py | 20 +++++++++++++++++--- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 1d69c03..0153141 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -26,6 +26,7 @@ import uvicorn from fastapi import FastAPI, status from fastapi.responses import PlainTextResponse +from loguru import logger from pydantic import FilePath, validate_call from trolldb.api.routes import api_router @@ -83,6 +84,7 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: if __name__ == "__main__": run_server("config.yaml") """ + logger.info("Attempt to run the API server ...") config = parse(config) # Concatenate the keyword arguments for the API server in the order of precedence (lower to higher). @@ -91,17 +93,20 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: app.include_router(api_router) @app.exception_handler(ResponseError) - async def unicorn_exception_handler(_, exc: ResponseError): + async def auto_exception_handler(_, exc: ResponseError): """Catches all the exceptions raised as a ResponseError, e.g. accessing non-existing databases/collections.""" status_code, message = exc.get_error_details() - return PlainTextResponse( + info = dict( status_code=status_code if status_code else status.HTTP_500_INTERNAL_SERVER_ERROR, content=message if message else "Generic Error [This is not okay, check why we have the generic error!]", ) + logger.error(f"Response error caught by the API auto exception handler: {info}") + return PlainTextResponse(**info) async def _serve(): """An auxiliary coroutine to be used in the asynchronous execution of the FastAPI application.""" async with mongodb_context(config.database): + logger.info("Attempt to start the uvicorn server ...") await uvicorn.Server( config=uvicorn.Config( host=config.api_server.url.host, @@ -110,6 +115,7 @@ async def _serve(): ) ).serve() + logger.info("Attempt to run the asyncio loop for the API server ...") asyncio.run(_serve()) @@ -134,13 +140,18 @@ def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = ValidationError: If the function is not called with arguments of valid type. """ + logger.info("Attempt to run the API server process in a context manager ...") + config = parse(config) process = Process(target=run_server, args=(config,)) - process.start() + try: + process.start() # time.sleep() expects its argument to be in seconds, hence the division by 1000. time.sleep(startup_time / 1000.) yield process finally: + logger.info("Attempt to terminate the API server process in the context manager ...") process.terminate() process.join() + logger.info("The API server process has terminated successfully.") diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 4f881bf..66d76ae 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -126,10 +126,15 @@ def from_yaml(filename: FilePath) -> AppConfig: ValidationError: If the function is not called with arguments of valid type. """ + logger.info("Attempt to parse the YAML file ...") with open(filename, "r") as file: config = safe_load(file) + logger.info("Parsing YAML file is successful.") try: - return AppConfig(**config) + logger.info("Attempt to validate the parsed YAML file ...") + config = AppConfig(**config) + logger.info("Validation of the parsed YAML file is successful.") + return config except ValidationError as e: logger.error(e) sys.exit(errno.EIO) @@ -152,8 +157,10 @@ def parse(config: AppConfig | FilePath) -> AppConfig: ValidationError: If the function is not called with arguments of valid type. """ + logger.info("Attempt to parse the config file or object ...") match config: case AppConfig(): + logger.info("Parsing config object successful.") return config case _: return from_yaml(config) diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index e6dd528..17fc5e2 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -9,6 +9,7 @@ from contextlib import asynccontextmanager from typing import Any, AsyncGenerator, Coroutine, TypeVar +from loguru import logger from motor.motor_asyncio import ( AsyncIOMotorClient, AsyncIOMotorCollection, @@ -136,6 +137,8 @@ async def initialize(cls, database_config: DatabaseConfig): SystemExit(errno.ENODATA): If either ``database_config.main_database`` or ``database_config.main_collection`` does not exist. """ + logger.info("Attempt to initialize the MongoDB client ...") + logger.info("Checking the database configs ...") if cls.__database_config: if database_config == cls.__database_config: if cls.__client: @@ -143,6 +146,7 @@ async def initialize(cls, database_config: DatabaseConfig): Client.InconsistencyError.sys_exit_log(errno.EIO) else: Client.ReinitializeConfigError.sys_exit_log(errno.EIO) + logger.info("Database configs are OK.") # This only makes the reference and does not establish an actual connection until the first attempt is made # to access the database. @@ -152,32 +156,41 @@ async def initialize(cls, database_config: DatabaseConfig): __database_names = [] try: - # Here we attempt to access the database + logger.info("Attempt to access list of databases ...") __database_names = await cls.__client.list_database_names() except (ConnectionFailure, ServerSelectionTimeoutError): Client.ConnectionError.sys_exit_log( errno.EIO, {"url": database_config.url.unicode_string()} ) + logger.info("Accessing the list of databases is successful.") err_extra_information = {"database_name": database_config.main_database_name} + logger.info("Checking if the main database name exists ...") if database_config.main_database_name not in __database_names: Databases.NotFoundError.sys_exit_log(errno.ENODATA, err_extra_information) cls.__main_database = cls.__client.get_database(database_config.main_database_name) + logger.info("The main database name exists.") err_extra_information |= {"collection_name": database_config.main_collection_name} + logger.info("Checking if the main collection name exists ...") if database_config.main_collection_name not in await cls.__main_database.list_collection_names(): Collections.NotFoundError.sys_exit_log(errno.ENODATA, err_extra_information) + logger.info("The main collection name exists.") cls.__main_collection = cls.__main_database.get_collection(database_config.main_collection_name) + logger.info("MongoDB is successfully initialized.") @classmethod def close(cls) -> None: """Closes the motor client.""" + logger.info("Attempt to close the MongoDB client ...") if cls.__client: cls.__database_config = None - return cls.__client.close() + cls.__client.close() + logger.info("Closes the MongoDB client successfully.") + return Client.CloseNotAllowedError.sys_exit_log(errno.EIO) @classmethod @@ -285,7 +298,6 @@ async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: It can be either used in `PRODUCTION` or in `TESTING` environments. - Note: Since the :class:`MongoDB` is supposed to be used statically, this context manager does not yield anything! One can simply use :class:`MongoDB` inside the context manager. @@ -298,8 +310,10 @@ async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: ValidationError: If the function is not called with arguments of valid type. """ + logger.info("Attempt to open the MongoDB context manager ...") try: await MongoDB.initialize(database_config) yield finally: MongoDB.close() + logger.info("The MongoDB context manager is successfully closed.") From 540a58ccaf3a0330497944f897e4c6eb945ded45 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 20:57:54 +0200 Subject: [PATCH 64/97] Update README.rst --- README.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.rst b/README.rst index dff5e31..f489eda 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,3 @@ - The database interface of `Pytroll `_ @@ -22,12 +21,7 @@ Authors License - This program, i.e. **pytroll-db**, is part of `Pytroll `_. - - **pytroll-db** is free software: you can redistribute it and/or modify - it under the terms of the `GNU General Public License `_ - as published by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. + Consult the `LICENSE` file which is included as a part of this package. Disclaimer From 332e9778c991ccf3fac7a9f13f44a3aeaea72964 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Fri, 17 May 2024 21:06:56 +0200 Subject: [PATCH 65/97] More logging --- trolldb/database/mongodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 17fc5e2..a052925 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -189,7 +189,7 @@ def close(cls) -> None: if cls.__client: cls.__database_config = None cls.__client.close() - logger.info("Closes the MongoDB client successfully.") + logger.info("The MongoDB client is closed successfully.") return Client.CloseNotAllowedError.sys_exit_log(errno.EIO) From e9fbb44eec9d30978ad1c7e8878f1e8aaf7fdb6a Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Sun, 19 May 2024 22:29:33 +0200 Subject: [PATCH 66/97] More docstrings. --- trolldb/database/piplines.py | 5 ++- trolldb/test_utils/common.py | 34 +++++++++++++++++++ trolldb/tests/tests_api/test_api.py | 2 +- .../tests/tests_database/test_pipelines.py | 23 +++++++++++-- 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index 8f75ae1..d84de95 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -84,6 +84,9 @@ def __aux_operators(self, other: Any, operator: str) -> PipelineBooleanDict: The operators herein have similar behaviour to ``==`` in the sense that they make comparison filters and are not to be interpreted as comparison assertions. """ + if isinstance(other, list): + return PipelineBooleanDict(**{"$or": [{self.__key: {operator: v}} for v in other]}) + return PipelineBooleanDict(**{self.__key: {operator: other}} if other else {}) def __ge__(self, other: Any) -> PipelineBooleanDict: @@ -100,7 +103,7 @@ def __le__(self, other: Any) -> PipelineBooleanDict: def __lt__(self, other: Any) -> PipelineBooleanDict: """Implements the `less than` operator, i.e. ``<``.""" - return self.__aux_operators(other, "$le") + return self.__aux_operators(other, "$lt") class Pipelines(list): diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index fd481ef..f076e29 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -80,3 +80,37 @@ def _ordered(obj: Any) -> Any: if not _ordered(test) == _ordered(expected): raise AssertionError(f"{test} and {expected} are not equal. The flag `ordered` is set to `{ordered}`.") + + +def compare_by_operator_name(operator: str, left: Any, right: Any) -> Any: + """Compares operands given the operator name in a string format. + + Args: + operator: + The name of the comparison operator. It can be any of the following: + ["$gte", "$gt", "$lte", "$lt", "$eq"] + left: + The left operand + right: + The right operand + + Returns: + The result of the comparison operation, i.e. . + + Raises: + ValueError: + If the operator name is not valid. + """ + match operator: + case "$gte": + return left >= right + case "$gt": + return left > right + case "$lte": + return left <= right + case "$lt": + return left < right + case "$eq": + return left == right + case _: + raise ValueError(f"Unknown operator: {operator}") diff --git a/trolldb/tests/tests_api/test_api.py b/trolldb/tests/tests_api/test_api.py index 78150eb..2d7ecd7 100644 --- a/trolldb/tests/tests_api/test_api.py +++ b/trolldb/tests/tests_api/test_api.py @@ -48,7 +48,7 @@ def test_database_names_negative(): @pytest.mark.usefixtures("_test_server_fixture") def test_collections(): - """Check the presence of existing collections and that the ids of documents therein can be correctly retrieved.""" + """Checks the presence of existing collections and that the ids of documents therein can be correctly retrieved.""" with test_mongodb_context() as client: for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names, strict=False): diff --git a/trolldb/tests/tests_database/test_pipelines.py b/trolldb/tests/tests_database/test_pipelines.py index 2e5fb34..e97fbdd 100644 --- a/trolldb/tests/tests_database/test_pipelines.py +++ b/trolldb/tests/tests_database/test_pipelines.py @@ -1,9 +1,10 @@ -"""Documentation to be added.""" -from trolldb.database.piplines import PipelineBooleanDict +"""Tests for the pipelines and applying comparison operations on them.""" +from trolldb.database.piplines import PipelineAttribute, PipelineBooleanDict +from trolldb.test_utils.common import assert_equal, compare_by_operator_name def test_pipeline_boolean_dict(): - """Documentation to be added.""" + """Checks the pipeline boolean dict for bitwise `and/or` operators.""" pd1 = PipelineBooleanDict({"number": 2}) pd2 = PipelineBooleanDict({"kind": 1}) @@ -14,3 +15,19 @@ def test_pipeline_boolean_dict(): pd_or = pd1 | pd2 pd_or_literal = PipelineBooleanDict({"$or": [{"number": 2}, {"kind": 1}]}) assert pd_or == pd_or_literal + + +def test_pipeline_attribute(): + """Tests different comparison operators for a pipeline attribute in a list and as a single item.""" + for op in ["$eq", "$gte", "$gt", "$lte", "$lt"]: + assert_equal( + compare_by_operator_name(op, PipelineAttribute("letter"), "A"), + PipelineBooleanDict({"letter": {op: "A"}} if op != "$eq" else {"letter": "A"}) + ) + assert_equal( + compare_by_operator_name(op, PipelineAttribute("letter"), ["A", "B"]), + PipelineBooleanDict({"$or": [ + {"letter": {op: "A"} if op != "$eq" else "A"}, + {"letter": {op: "B"} if op != "$eq" else "B"} + ]}) + ) From 2d166ab6e206b8c158686760527284fac9e08086 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Sun, 19 May 2024 22:43:43 +0200 Subject: [PATCH 67/97] More docstrings. --- trolldb/test_utils/common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index f076e29..4822d5d 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -83,19 +83,20 @@ def _ordered(obj: Any) -> Any: def compare_by_operator_name(operator: str, left: Any, right: Any) -> Any: - """Compares operands given the operator name in a string format. + """Compares two operands given the binary operator name in a string format. Args: operator: - The name of the comparison operator. It can be any of the following: - ["$gte", "$gt", "$lte", "$lt", "$eq"] + Any of ``["$gte", "$gt", "$lte", "$lt", "$eq"]``. + These match the MongoDB comparison operators described + `here `_. left: The left operand right: The right operand Returns: - The result of the comparison operation, i.e. . + The result of the comparison operation, i.e. `` ``. Raises: ValueError: From d1494a5e1ed5c74777115081693ab9cf1a597318 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Sun, 19 May 2024 22:49:44 +0200 Subject: [PATCH 68/97] More docstrings. --- trolldb/database/piplines.py | 6 +++--- .../tests/tests_database/test_pipelines.py | 21 ++++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/trolldb/database/piplines.py b/trolldb/database/piplines.py index d84de95..f85fa15 100644 --- a/trolldb/database/piplines.py +++ b/trolldb/database/piplines.py @@ -66,12 +66,12 @@ def __eq__(self, other: Any) -> PipelineBooleanDict: pa_list = PipelineAttribute("letter") == ["A", "B"] pd_list = PipelineBooleanDict({"$or": [{"letter": "A"}, {"letter": "B"}] # The following evaluates to True - pa_list = pd_list + pa_list == pd_list pa_single = PipelineAttribute("letter") == "A" pd_single = PipelineBooleanDict({"letter": "A"}) # The following evaluates to True - pa_single = pd_single + pa_single == pd_single """ if isinstance(other, list): return PipelineBooleanDict(**{"$or": [{self.__key: v} for v in other]}) @@ -126,7 +126,7 @@ class Pipelines(list): {"platform_name": "P"} }, {"$match": - {"$or": [{"sensor_name": "SA"}, {"sensor_name": "SB"}]} + {"$or": [{"sensor": "SA"}, {"sensor": "SB"}]} } ] diff --git a/trolldb/tests/tests_database/test_pipelines.py b/trolldb/tests/tests_database/test_pipelines.py index e97fbdd..19e4c97 100644 --- a/trolldb/tests/tests_database/test_pipelines.py +++ b/trolldb/tests/tests_database/test_pipelines.py @@ -1,5 +1,5 @@ """Tests for the pipelines and applying comparison operations on them.""" -from trolldb.database.piplines import PipelineAttribute, PipelineBooleanDict +from trolldb.database.piplines import PipelineAttribute, PipelineBooleanDict, Pipelines from trolldb.test_utils.common import assert_equal, compare_by_operator_name @@ -31,3 +31,22 @@ def test_pipeline_attribute(): {"letter": {op: "B"} if op != "$eq" else "B"} ]}) ) + + +def test_pipelines(): + """Tests the elements of Pipelines.""" + pipelines = Pipelines() + pipelines += PipelineAttribute("platform_name") == "P" + pipelines += PipelineAttribute("sensor") == ["SA", "SB"] + + pipelines_literal = [ + {"$match": + {"platform_name": "P"} + }, + {"$match": + {"$or": [{"sensor": "SA"}, {"sensor": "SB"}]} + } + ] + + for p1, p2 in zip(pipelines, pipelines_literal, strict=False): + assert_equal(p1, p2) From 8748ff92cb1824d05a645ac0fed6da79368c77e9 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 20 May 2024 09:33:37 +0200 Subject: [PATCH 69/97] Add script entrypoint --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b145efd..cfa0623 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,10 @@ classifiers = [ [project.urls] "Documentation" = "https://pytroll-db.readthedocs.io/en/latest/" +[project.scripts] +pytroll-db-recorder = "trolldb.cli:record_messages_from_command_line" + + [build-system] requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" From e206d658182d41cdece75d1e9ac7813110456c2a Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 20 May 2024 12:18:26 +0200 Subject: [PATCH 70/97] CI/CD. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4a7bde..664aec7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: python -m pip install -e . - name: Test with pytest run: | - pytest --cov=pytroll_db --cov-report=xml + pytest --cov=trolldb --cov-report=xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 with: From f84a76189f5c757c9028c6c83ba10c9a116b04df Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 20 May 2024 22:24:33 +0200 Subject: [PATCH 71/97] Refactor test_db tests. --- pyproject.toml | 2 +- trolldb/api/api.py | 15 +-- trolldb/cli.py | 32 +++--- trolldb/config/config.py | 48 +++----- trolldb/test_utils/common.py | 40 +++++-- trolldb/test_utils/mongodb_instance.py | 9 ++ trolldb/tests/conftest.py | 7 ++ trolldb/tests/test_db.py | 153 ++++++++----------------- 8 files changed, 137 insertions(+), 169 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cfa0623..182973b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ "Documentation" = "https://pytroll-db.readthedocs.io/en/latest/" [project.scripts] -pytroll-db-recorder = "trolldb.cli:record_messages_from_command_line" +pytroll-db-recorder = "trolldb.cli:run_sync" [build-system] diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 0153141..dbd3bca 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -30,7 +30,7 @@ from pydantic import FilePath, validate_call from trolldb.api.routes import api_router -from trolldb.config.config import AppConfig, Timeout, parse +from trolldb.config.config import AppConfig, Timeout, from_yaml from trolldb.database.mongodb import mongodb_context from trolldb.errors.errors import ResponseError @@ -61,9 +61,9 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: Args: config: - The configuration of the application which includes both the server and database configurations. In case of - a :class:`FilePath`, it should be a valid path to an existing config file which will parsed as a ``.YAML`` - file. + The configuration of the application which includes both the server and database configurations. Its type + should be a :class:`FilePath`, which is a valid path to an existing config file which will parsed as a + ``.YAML`` file. **kwargs: The keyword arguments are the same as those accepted by the @@ -85,7 +85,8 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: run_server("config.yaml") """ logger.info("Attempt to run the API server ...") - config = parse(config) + if not isinstance(config, AppConfig): + config = from_yaml(config) # Concatenate the keyword arguments for the API server in the order of precedence (lower to higher). app = FastAPI(**(config.api_server._asdict() | kwargs | API_INFO)) @@ -141,8 +142,8 @@ def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = If the function is not called with arguments of valid type. """ logger.info("Attempt to run the API server process in a context manager ...") - - config = parse(config) + if not isinstance(config, AppConfig): + config = from_yaml(config) process = Process(target=run_server, args=(config,)) try: diff --git a/trolldb/cli.py b/trolldb/cli.py index 0d42da6..7b33e9f 100644 --- a/trolldb/cli.py +++ b/trolldb/cli.py @@ -1,26 +1,27 @@ """Main interface.""" import argparse +import asyncio from posttroll.message import Message from posttroll.subscriber import create_subscriber_from_dict_config +from pydantic import FilePath -from trolldb.config import config +from trolldb.config.config import AppConfig, from_yaml from trolldb.database.mongodb import MongoDB, mongodb_context -from trolldb.test_utils.common import test_app_config -async def record_messages(subscriber_config): +async def record_messages(config: AppConfig): """Record the metadata of messages into the database.""" - async with mongodb_context(test_app_config.database): - sub = create_subscriber_from_dict_config(subscriber_config) + async with mongodb_context(config.database): + sub = create_subscriber_from_dict_config(config.subscriber) collection = await MongoDB.get_collection("mock_database", "mock_collection") for m in sub.recv(): msg = Message.decode(m) match msg.type: case "file": await collection.insert_one(msg.data) - case "delete": + case "del": deletion_result = await collection.delete_many({"uri": msg.data["uri"]}) if deletion_result.deleted_count != 1: raise ValueError("Multiple deletions!") # Replace with logging @@ -28,18 +29,23 @@ async def record_messages(subscriber_config): raise KeyError(f"Don't know what to do with {msg.type} message.") # Replace with logging -async def record_messages_from_config(config_file): +async def record_messages_from_config(config_file: FilePath): """Record messages into the database, getting the configuration from a file.""" - config_obj = config.parse(config_file) - await record_messages(config_obj.subscriber_config) + config = from_yaml(config_file) + await record_messages(config) async def record_messages_from_command_line(args=None): """Record messages into the database, command-line interface.""" parser = argparse.ArgumentParser() - parser.add_argument("configuration_file", - help="Path to the configuration file") + parser.add_argument( + "configuration_file", + help="Path to the configuration file") cmd_args = parser.parse_args(args) - config_file = cmd_args.configuration_file - await record_messages_from_config(config_file) + await record_messages_from_config(cmd_args.configuration_file) + + +def run_sync(): + """...""" + asyncio.run(record_messages_from_command_line()) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 66d76ae..eb2518f 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -15,7 +15,7 @@ from bson import ObjectId from bson.errors import InvalidId from loguru import logger -from pydantic import AnyUrl, BaseModel, Field, FilePath, MongoDsn, ValidationError, validate_call +from pydantic import AnyUrl, BaseModel, Field, FilePath, MongoDsn, ValidationError from pydantic.functional_validators import AfterValidator from typing_extensions import Annotated from yaml import safe_load @@ -101,10 +101,26 @@ class AppConfig(BaseModel): """ api_server: APIServerConfig database: DatabaseConfig - subscriber_config: SubscriberConfig + subscriber: SubscriberConfig + + def as_dict(self) -> dict: + """Converts the model to a dictionary, recursively.""" + def _aux(obj: Any) -> Any: + match obj: + case tuple(): + return {k: _aux(v) for k, v in obj._asdict().items()} + case BaseModel(): + return {k: _aux(v) for k, v in obj.__dict.items()} + case dict(): + return {k: _aux(v) for k, v in obj.items()} + case int() | str() | list() | bool(): + return obj + case _: + return str(obj) + + return _aux(self.__dict__) -@validate_call def from_yaml(filename: FilePath) -> AppConfig: """Parses and validates the configurations from a YAML file. @@ -138,29 +154,3 @@ def from_yaml(filename: FilePath) -> AppConfig: except ValidationError as e: logger.error(e) sys.exit(errno.EIO) - - -@validate_call -def parse(config: AppConfig | FilePath) -> AppConfig: - """Tries to return a valid object of type :class:`AppConfig`. - - Args: - config: - Either an object of type :class:`AppConfig` or :class:`FilePath`. - - Returns: - - In case of an object of type :class:`AppConfig` as input, the same object will be returned as-is. - - An input object of type ``str`` will be interpreted as a YAML filename, in which case the function returns - the result of parsing the file. - - Raises: - ValidationError: - If the function is not called with arguments of valid type. - """ - logger.info("Attempt to parse the config file or object ...") - match config: - case AppConfig(): - logger.info("Parsing config object successful.") - return config - case _: - return from_yaml(config) diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 4822d5d..79e8bdb 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -4,21 +4,39 @@ from typing import Any from urllib.parse import urljoin -from pydantic import AnyUrl +import yaml +from pydantic import AnyUrl, FilePath from urllib3 import BaseHTTPResponse, request from trolldb.config.config import APIServerConfig, AppConfig, DatabaseConfig -test_app_config = AppConfig( - api_server=APIServerConfig(url=AnyUrl("http://localhost:8080")), - database=DatabaseConfig( - main_database_name="mock_database", - main_collection_name="mock_collection", - url=AnyUrl("mongodb://localhost:28017"), - timeout=1000), - subscriber_config=dict() -) -"""The app configuration when used in testing.""" + +def make_test_app_config(subscriber_address: FilePath | None = None) -> AppConfig: + """The app configuration when used in testing.""" + return AppConfig( + api_server=APIServerConfig(url=AnyUrl("http://localhost:8080")), + database=DatabaseConfig( + main_database_name="mock_database", + main_collection_name="mock_collection", + url=AnyUrl("mongodb://localhost:28017"), + timeout=1000), + subscriber=dict() if subscriber_address is None else dict( + nameserver=False, + addresses=[f"ipc://{subscriber_address}/in.ipc"], + port=3000 + ) + ) + + +test_app_config = make_test_app_config() + + +def create_config_file(config_path: FilePath) -> FilePath: + """Create a config file for tests.""" + config_file = config_path / "config.yaml" + with open(config_file, "w") as f: + yaml.safe_dump(make_test_app_config(config_path).as_dict(), f) + return config_file def http_get(route: str = "", root: AnyUrl = test_app_config.api_server.url) -> BaseHTTPResponse: diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index b54ee1d..6f0486b 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -13,6 +13,7 @@ from trolldb.config.config import DatabaseConfig, Timeout from trolldb.test_utils.common import test_app_config +from trolldb.test_utils.mongodb_database import TestDatabase class TestMongoInstance: @@ -133,3 +134,11 @@ def mongodb_instance_server_process_context( yield finally: TestMongoInstance.shutdown_instance() + + +@contextmanager +def running_prepared_database_context(): + """A synchronous context manager to start and prepare a database instance for tests.""" + with mongodb_instance_server_process_context(): + TestDatabase.prepare() + yield diff --git a/trolldb/tests/conftest.py b/trolldb/tests/conftest.py index ffb7312..2ea5c70 100644 --- a/trolldb/tests/conftest.py +++ b/trolldb/tests/conftest.py @@ -34,3 +34,10 @@ async def mongodb_fixture(_run_mongodb_server_instance): TestDatabase.prepare() async with mongodb_context(test_app_config.database): yield + + +@pytest.fixture() +def tmp_data_filename(tmp_path): + """Create a filename for the messages.""" + filename = "20191103_153936-s1b-ew-hh.tiff" + return tmp_path / filename diff --git a/trolldb/tests/test_db.py b/trolldb/tests/test_db.py index 2b9d3b2..4158469 100644 --- a/trolldb/tests/test_db.py +++ b/trolldb/tests/test_db.py @@ -1,154 +1,91 @@ """Tests for the message recording into database.""" -from contextlib import contextmanager +from typing import Any import pytest -import yaml from posttroll.message import Message from posttroll.testing import patched_subscriber_recv +from pydantic import FilePath from trolldb.cli import record_messages, record_messages_from_command_line, record_messages_from_config from trolldb.database.mongodb import MongoDB, mongodb_context -from trolldb.test_utils.common import test_app_config -from trolldb.test_utils.mongodb_database import TestDatabase -from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context +from trolldb.test_utils.common import assert_equal, create_config_file, make_test_app_config, test_app_config +from trolldb.test_utils.mongodb_instance import running_prepared_database_context -FILENAME = "20191103_153936-s1b-ew-hh.tiff" @pytest.fixture() -def tmp_filename(tmp_path): - """Create a filename for the messages.""" - return tmp_path / FILENAME - -@pytest.fixture() -def file_message(tmp_filename): +def file_message(tmp_data_filename): """Create a string for a file message.""" return ('pytroll://segment/raster/L2/SAR file a001673@c20969.ad.smhi.se 2019-11-05T13:00:10.366023 v1.01 ' 'application/json {"platform_name": "S1B", "scan_mode": "EW", "type": "GRDM", "data_source": "1SDH", ' '"start_time": "2019-11-03T15:39:36.543000", "end_time": "2019-11-03T15:40:40.821000", "orbit_number": ' '18765, "random_string1": "0235EA", "random_string2": "747D", "uri": ' - f'"{str(tmp_filename)}", "uid": "20191103_153936-s1b-ew-hh.tiff", ' + f'"{str(tmp_data_filename)}", "uid": "20191103_153936-s1b-ew-hh.tiff", ' '"polarization": "hh", "sensor": "sar-c", "format": "GeoTIFF", "pass_direction": "ASCENDING"}') @pytest.fixture() -def del_message(tmp_filename): +def del_message(tmp_data_filename): """Create a string for a delete message.""" - return ('pytroll://segment/raster/L2/SAR delete a001673@c20969.ad.smhi.se 2019-11-05T13:00:10.366023 v1.01 ' + return ('pytroll://deletion del a001673@c20969.ad.smhi.se 2019-11-05T13:00:10.366023 v1.01 ' 'application/json {"platform_name": "S1B", "scan_mode": "EW", "type": "GRDM", "data_source": "1SDH", ' '"start_time": "2019-11-03T15:39:36.543000", "end_time": "2019-11-03T15:40:40.821000", "orbit_number": ' '18765, "random_string1": "0235EA", "random_string2": "747D", "uri": ' - f'"{str(tmp_filename)}", "uid": "20191103_153936-s1b-ew-hh.tiff", ' + f'"{str(tmp_data_filename)}", "uid": "20191103_153936-s1b-ew-hh.tiff", ' '"polarization": "hh", "sensor": "sar-c", "format": "GeoTIFF", "pass_direction": "ASCENDING"}') -@contextmanager -def running_prepared_database(): - """Starts and prepares a database instance for tests.""" - with mongodb_instance_server_process_context(): - TestDatabase.prepare() - yield +async def assert_message(msg, data_filename): + """Documentation to be added.""" + async with mongodb_context(test_app_config.database): + collection = await MongoDB.get_collection("mock_database", "mock_collection") + result = await collection.find_one(dict(scan_mode="EW")) + result.pop("_id") + assert_equal(result, msg.data) + deletion_result = await collection.delete_many({"uri": str(data_filename)}) + assert_equal(deletion_result.deleted_count, 1) -async def test_record_adds_message(tmp_path, file_message, tmp_filename): - """Test that message recording adds a message to the database.""" - msg = Message.decode(file_message) - subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) +async def _record_from_somewhere(config_path: FilePath, message: Any, data_filename, record_from_func, f=False): + """Test that we can record when passed a config file.""" + config_file = create_config_file(config_path) + msg = Message.decode(message) + with running_prepared_database_context(): + with patched_subscriber_recv([message]): + await record_from_func(config_file if not f else [str(config_file)]) + await assert_message(msg, data_filename) - with running_prepared_database(): - with patched_subscriber_recv([file_message]): - await record_messages(subscriber_config) +async def test_record_adds_message(tmp_path, file_message, tmp_data_filename): + """Test that message recording adds a message to the database.""" + await _record_from_somewhere( + tmp_path, file_message, tmp_data_filename, record_messages_from_config + ) - async with mongodb_context(test_app_config.database): - collection = await MongoDB.get_collection("mock_database", "mock_collection") - result = await collection.find_one(dict(scan_mode="EW")) - result.pop("_id") - assert result == msg.data +async def test_record_from_config(tmp_path, file_message, tmp_data_filename): + """Test that we can record when passed a config file.""" + await _record_from_somewhere( + tmp_path, file_message, tmp_data_filename, record_messages_from_config + ) - deletion_result = await collection.delete_many({"uri": str(tmp_filename)}) - assert deletion_result.deleted_count == 1 +async def test_record_cli(tmp_path, file_message, tmp_data_filename): + """Test that we can record when passed a config file.""" + await _record_from_somewhere( + tmp_path, file_message, tmp_data_filename, record_messages_from_command_line, True + ) async def test_record_deletes_message(tmp_path, file_message, del_message): """Test that message recording can delete a record in the database.""" - subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) - - with running_prepared_database(): - + config = make_test_app_config(tmp_path) + with running_prepared_database_context(): with patched_subscriber_recv([file_message, del_message]): + await record_messages(config) - await record_messages(subscriber_config) - - async with mongodb_context(test_app_config.database): + async with mongodb_context(config.database): collection = await MongoDB.get_collection("mock_database", "mock_collection") result = await collection.find_one(dict(scan_mode="EW")) assert result is None - - -async def test_record_from_config(tmp_path, file_message, tmp_filename): - """Test that we can record when passed a config file.""" - config_file = create_config_file(tmp_path) - - msg = Message.decode(file_message) - - with running_prepared_database(): - - with patched_subscriber_recv([file_message]): - - await record_messages_from_config(config_file) - - async with mongodb_context(test_app_config.database): - collection = await MongoDB.get_collection("mock_database", "mock_collection") - - result = await collection.find_one(dict(scan_mode="EW")) - result.pop("_id") - assert result == msg.data - - deletion_result = await collection.delete_many({"uri": str(tmp_filename)}) - - assert deletion_result.deleted_count == 1 - - -def create_config_file(tmp_path): - """Create a config file for tests.""" - config_file = tmp_path / "config.yaml" - subscriber_config = dict(nameserver=False, addresses=[f"ipc://{str(tmp_path)}/in.ipc"], port=3000) - db_config = {"main_database_name": "sat_db", - "main_collection_name": "files", - "url": "mongodb://localhost:27017", - "timeout": 1000} - api_server_config = {"url": "http://localhost:8000"} - - config_dict = dict(subscriber_config=subscriber_config, - database = db_config, - api_server = api_server_config) - with open(config_file, "w") as fd: - fd.write(yaml.dump(config_dict)) - - return config_file - -async def test_record_cli(tmp_path, file_message, tmp_filename): - """Test that we can record when passed a config file.""" - config_file = create_config_file(tmp_path) - - msg = Message.decode(file_message) - - with running_prepared_database(): - - with patched_subscriber_recv([file_message]): - await record_messages_from_command_line([str(config_file)]) - - async with mongodb_context(test_app_config.database): - collection = await MongoDB.get_collection("mock_database", "mock_collection") - - result = await collection.find_one(dict(scan_mode="EW")) - result.pop("_id") - assert result == msg.data - - deletion_result = await collection.delete_many({"uri": str(tmp_filename)}) - - assert deletion_result.deleted_count == 1 From 047f232dcdd26687b717f318a68d8a841f3cf250 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 20 May 2024 23:13:43 +0200 Subject: [PATCH 72/97] Refactor. --- trolldb/api/api.py | 2 +- trolldb/config/config.py | 13 +++++++++--- trolldb/test_utils/common.py | 21 +++++++++++++++---- trolldb/tests/conftest.py | 9 ++++---- trolldb/tests/{test_db.py => test_recoder.py} | 0 5 files changed, 32 insertions(+), 13 deletions(-) rename trolldb/tests/{test_db.py => test_recoder.py} (100%) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index dbd3bca..b0cb39e 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -122,7 +122,7 @@ async def _serve(): @contextmanager @validate_call -def server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2000): +def api_server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2000): """A synchronous context manager to run the API server in a separate process (non-blocking). It uses the `multiprocessing `_ package. The main use case diff --git a/trolldb/config/config.py b/trolldb/config/config.py index eb2518f..22838d6 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -105,15 +105,22 @@ class AppConfig(BaseModel): def as_dict(self) -> dict: """Converts the model to a dictionary, recursively.""" + def _aux(obj: Any) -> Any: + """An auxiliary function to do the conversion based on the type.""" match obj: - case tuple(): + # The type `NamedTuple` has a method named `_asdict` + case tuple() if "_asdict" in dir(obj): return {k: _aux(v) for k, v in obj._asdict().items()} case BaseModel(): - return {k: _aux(v) for k, v in obj.__dict.items()} + return {k: _aux(v) for k, v in obj.__dict__.items()} case dict(): return {k: _aux(v) for k, v in obj.items()} - case int() | str() | list() | bool(): + case list(): + return [_aux(i) for i in obj] + case tuple(): + return (_aux(i) for i in obj) + case int() | float() | str() | bool(): return obj case _: return str(obj) diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 79e8bdb..0b73cd3 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -12,14 +12,26 @@ def make_test_app_config(subscriber_address: FilePath | None = None) -> AppConfig: - """The app configuration when used in testing.""" + """Makes the app configuration when used in testing. + + Args: + subscriber_address: + The address of the subscriber if it is of type ``FilePath``. Otherwise, if it is ``None`` the ``subscriber`` + config will be an empty dictionary. + + Returns: + An object of type :obj:`AppConfig`. + """ return AppConfig( - api_server=APIServerConfig(url=AnyUrl("http://localhost:8080")), + api_server=APIServerConfig( + url=AnyUrl("http://localhost:8080") + ), database=DatabaseConfig( main_database_name="mock_database", main_collection_name="mock_collection", url=AnyUrl("mongodb://localhost:28017"), - timeout=1000), + timeout=1000 + ), subscriber=dict() if subscriber_address is None else dict( nameserver=False, addresses=[f"ipc://{subscriber_address}/in.ipc"], @@ -29,10 +41,11 @@ def make_test_app_config(subscriber_address: FilePath | None = None) -> AppConfi test_app_config = make_test_app_config() +"""The app configs for testing purposes assuming an empty configuration for the subscriber.""" def create_config_file(config_path: FilePath) -> FilePath: - """Create a config file for tests.""" + """Creates a config file for tests.""" config_file = config_path / "config.yaml" with open(config_file, "w") as f: yaml.safe_dump(make_test_app_config(config_path).as_dict(), f) diff --git a/trolldb/tests/conftest.py b/trolldb/tests/conftest.py index 2ea5c70..507f37b 100644 --- a/trolldb/tests/conftest.py +++ b/trolldb/tests/conftest.py @@ -6,25 +6,24 @@ import pytest import pytest_asyncio -from trolldb.api.api import server_process_context +from trolldb.api.api import api_server_process_context from trolldb.database.mongodb import mongodb_context from trolldb.test_utils.common import test_app_config from trolldb.test_utils.mongodb_database import TestDatabase -from trolldb.test_utils.mongodb_instance import mongodb_instance_server_process_context +from trolldb.test_utils.mongodb_instance import running_prepared_database_context @pytest.fixture(scope="session") def _run_mongodb_server_instance(): """Encloses all tests (session scope) in a context manager of a running MongoDB instance (in a separate process).""" - with mongodb_instance_server_process_context(): + with running_prepared_database_context(): yield @pytest.fixture(scope="session") def _test_server_fixture(_run_mongodb_server_instance): """Encloses all tests (session scope) in a context manager of a running API server (in a separate process).""" - TestDatabase.prepare() - with server_process_context(test_app_config, startup_time=2000): + with api_server_process_context(test_app_config, startup_time=2000): yield diff --git a/trolldb/tests/test_db.py b/trolldb/tests/test_recoder.py similarity index 100% rename from trolldb/tests/test_db.py rename to trolldb/tests/test_recoder.py From cd5eac35c2ec475c6c5d30c3545b0d1709babaec Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Mon, 20 May 2024 23:37:49 +0200 Subject: [PATCH 73/97] CI/CD. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 664aec7..b4bb7a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: python -m pip install -e . - name: Test with pytest run: | - pytest --cov=trolldb --cov-report=xml + pytest --asyncio-mode=auto --cov=trolldb --cov-report=xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 with: From c1b726b43836779d59cabb8504732041419d3702 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 00:03:54 +0200 Subject: [PATCH 74/97] Refactor. --- trolldb/tests/test_recoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/trolldb/tests/test_recoder.py b/trolldb/tests/test_recoder.py index 4158469..1ef9e12 100644 --- a/trolldb/tests/test_recoder.py +++ b/trolldb/tests/test_recoder.py @@ -47,13 +47,14 @@ async def assert_message(msg, data_filename): assert_equal(deletion_result.deleted_count, 1) -async def _record_from_somewhere(config_path: FilePath, message: Any, data_filename, record_from_func, f=False): +async def _record_from_somewhere( + config_path: FilePath, message: Any, data_filename, record_from_func, wrap_in_list=False): """Test that we can record when passed a config file.""" config_file = create_config_file(config_path) msg = Message.decode(message) with running_prepared_database_context(): with patched_subscriber_recv([message]): - await record_from_func(config_file if not f else [str(config_file)]) + await record_from_func(config_file if not wrap_in_list else [str(config_file)]) await assert_message(msg, data_filename) From b756c12112c51e2d65f23781e2a27a9960f5e032 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 00:12:29 +0200 Subject: [PATCH 75/97] Clean up. --- trolldb/cli.py | 2 +- trolldb/run_api.py | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 trolldb/run_api.py diff --git a/trolldb/cli.py b/trolldb/cli.py index 7b33e9f..ce9da1c 100644 --- a/trolldb/cli.py +++ b/trolldb/cli.py @@ -47,5 +47,5 @@ async def record_messages_from_command_line(args=None): def run_sync(): - """...""" + """Runs the interface synchronously.""" asyncio.run(record_messages_from_command_line()) diff --git a/trolldb/run_api.py b/trolldb/run_api.py deleted file mode 100644 index b5bc6cf..0000000 --- a/trolldb/run_api.py +++ /dev/null @@ -1,10 +0,0 @@ -"""The main entry point to run the API server according to the configurations given in `config.yaml`. - -Note: - For more information on the API server, see the automatically generated documentation by FastAPI. -""" - -from api.api import run_server - -if __name__ == "__main__": - run_server("config.yaml") From c1d141c47e70f655f6e001e8d445515c30ed6a9d Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 00:27:07 +0200 Subject: [PATCH 76/97] Add model for SubscriberConfig. --- trolldb/config/config.py | 13 ++++++++----- trolldb/template_config.yaml | 9 +++++++++ trolldb/test_utils/common.py | 6 +++--- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index 22838d6..bf108ab 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -10,7 +10,7 @@ import errno import sys -from typing import Any, NamedTuple +from typing import Any, NamedTuple, TypedDict from bson import ObjectId from bson.errors import InvalidId @@ -87,11 +87,14 @@ class DatabaseConfig(NamedTuple): """ -SubscriberConfig = dict[Any, Any] -"""A dictionary to hold all the configurations of the subscriber. +class SubscriberConfig(TypedDict): + """A named tuple to hold all the configurations of the subscriber. -TODO: This has to be moved to the `posttroll` package. -""" + TODO: This has to be moved to the `posttroll` package. + """ + nameserver: bool + addresses: list[str] + port: int class AppConfig(BaseModel): diff --git a/trolldb/template_config.yaml b/trolldb/template_config.yaml index 01c78f7..a958ef7 100644 --- a/trolldb/template_config.yaml +++ b/trolldb/template_config.yaml @@ -22,3 +22,12 @@ database: api_server: # Required url: "http://localhost:8000" + +# Required +subscriber: + # Required + nameserver: False + # Required + addresses: [""] + # Required + port: 3000 diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 0b73cd3..7db7edc 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -8,7 +8,7 @@ from pydantic import AnyUrl, FilePath from urllib3 import BaseHTTPResponse, request -from trolldb.config.config import APIServerConfig, AppConfig, DatabaseConfig +from trolldb.config.config import APIServerConfig, AppConfig, DatabaseConfig, SubscriberConfig def make_test_app_config(subscriber_address: FilePath | None = None) -> AppConfig: @@ -32,9 +32,9 @@ def make_test_app_config(subscriber_address: FilePath | None = None) -> AppConfi url=AnyUrl("mongodb://localhost:28017"), timeout=1000 ), - subscriber=dict() if subscriber_address is None else dict( + subscriber=SubscriberConfig( nameserver=False, - addresses=[f"ipc://{subscriber_address}/in.ipc"], + addresses=[f"ipc://{subscriber_address}/in.ipc"] if subscriber_address is not None else [""], port=3000 ) ) From 4c28ebddac0fefc634946abe7ad7b992e8af16e7 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 10:28:36 +0200 Subject: [PATCH 77/97] Refactor. --- trolldb/api/api.py | 8 ++++---- trolldb/cli.py | 16 +++++++++------- trolldb/config/config.py | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index b0cb39e..9ef0c32 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -30,7 +30,7 @@ from pydantic import FilePath, validate_call from trolldb.api.routes import api_router -from trolldb.config.config import AppConfig, Timeout, from_yaml +from trolldb.config.config import AppConfig, Timeout, parse_config_yaml_file from trolldb.database.mongodb import mongodb_context from trolldb.errors.errors import ResponseError @@ -86,7 +86,7 @@ def run_server(config: AppConfig | FilePath, **kwargs) -> None: """ logger.info("Attempt to run the API server ...") if not isinstance(config, AppConfig): - config = from_yaml(config) + config = parse_config_yaml_file(config) # Concatenate the keyword arguments for the API server in the order of precedence (lower to higher). app = FastAPI(**(config.api_server._asdict() | kwargs | API_INFO)) @@ -143,9 +143,9 @@ def api_server_process_context(config: AppConfig | FilePath, startup_time: Timeo """ logger.info("Attempt to run the API server process in a context manager ...") if not isinstance(config, AppConfig): - config = from_yaml(config) - process = Process(target=run_server, args=(config,)) + config = parse_config_yaml_file(config) + process = Process(target=run_server, args=(config,)) try: process.start() # time.sleep() expects its argument to be in seconds, hence the division by 1000. diff --git a/trolldb/cli.py b/trolldb/cli.py index ce9da1c..a7fcb44 100644 --- a/trolldb/cli.py +++ b/trolldb/cli.py @@ -3,20 +3,22 @@ import argparse import asyncio +from loguru import logger from posttroll.message import Message from posttroll.subscriber import create_subscriber_from_dict_config from pydantic import FilePath -from trolldb.config.config import AppConfig, from_yaml +from trolldb.config.config import AppConfig, parse_config_yaml_file from trolldb.database.mongodb import MongoDB, mongodb_context async def record_messages(config: AppConfig): """Record the metadata of messages into the database.""" async with mongodb_context(config.database): - sub = create_subscriber_from_dict_config(config.subscriber) - collection = await MongoDB.get_collection("mock_database", "mock_collection") - for m in sub.recv(): + collection = await MongoDB.get_collection( + config.database.main_database_name, config.database.main_collection_name + ) + for m in create_subscriber_from_dict_config(config.subscriber).recv(): msg = Message.decode(m) match msg.type: case "file": @@ -24,14 +26,14 @@ async def record_messages(config: AppConfig): case "del": deletion_result = await collection.delete_many({"uri": msg.data["uri"]}) if deletion_result.deleted_count != 1: - raise ValueError("Multiple deletions!") # Replace with logging + logger.error("Recorder found multiple deletions!") # Log some data related to the msg case _: - raise KeyError(f"Don't know what to do with {msg.type} message.") # Replace with logging + logger.error(f"Don't know what to do with {msg.type} message.") async def record_messages_from_config(config_file: FilePath): """Record messages into the database, getting the configuration from a file.""" - config = from_yaml(config_file) + config = parse_config_yaml_file(config_file) await record_messages(config) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index bf108ab..a118426 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -131,7 +131,7 @@ def _aux(obj: Any) -> Any: return _aux(self.__dict__) -def from_yaml(filename: FilePath) -> AppConfig: +def parse_config_yaml_file(filename: FilePath) -> AppConfig: """Parses and validates the configurations from a YAML file. Args: From 4927ebc73990a70ff9532e02ce0eff829b79df9e Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 19:10:01 +0200 Subject: [PATCH 78/97] Address PR comments. --- trolldb/cli.py | 6 +++--- trolldb/test_utils/common.py | 28 +++++++++++++++++++++++++--- trolldb/tests/tests_api/test_api.py | 20 +++++++------------- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/trolldb/cli.py b/trolldb/cli.py index a7fcb44..565307f 100644 --- a/trolldb/cli.py +++ b/trolldb/cli.py @@ -19,16 +19,16 @@ async def record_messages(config: AppConfig): config.database.main_database_name, config.database.main_collection_name ) for m in create_subscriber_from_dict_config(config.subscriber).recv(): - msg = Message.decode(m) + msg = Message.decode(str(m)) match msg.type: case "file": await collection.insert_one(msg.data) case "del": deletion_result = await collection.delete_many({"uri": msg.data["uri"]}) if deletion_result.deleted_count != 1: - logger.error("Recorder found multiple deletions!") # Log some data related to the msg + logger.error("Recorder found multiple deletions!") # TODO: Log some data related to the msg case _: - logger.error(f"Don't know what to do with {msg.type} message.") + logger.debug(f"Don't know what to do with {msg.type} message.") async def record_messages_from_config(config_file: FilePath): diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 7db7edc..d391add 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -67,7 +67,7 @@ def http_get(route: str = "", root: AnyUrl = test_app_config.api_server.url) -> return request("GET", urljoin(root.unicode_string(), route)) -def assert_equal(test: Any, expected: Any, ordered: bool = False) -> None: +def assert_equal(test: Any, expected: Any, ordered: bool = False, silent: bool = False) -> bool: """An auxiliary function to assert the equality of two objects using the ``==`` operator. Examples: @@ -97,6 +97,13 @@ def assert_equal(test: Any, expected: Any, ordered: bool = False) -> None: The object to test against. ordered (Optional, default ``False``): A flag to determine whether the order of items matters in case of a list, a tuple, or a dictionary. + silent (Optional, default ``False``): + A flag to determine whether the assertion should be silent, i.e. simply return the result as a boolean or + it should raise an ``AssertionError``. + + Raises: + AssertionError: + If the ``test`` and ``expected`` are not equal and ``silent=False``. """ def _ordered(obj: Any) -> Any: @@ -109,8 +116,13 @@ def _ordered(obj: Any) -> Any: case _: return obj - if not _ordered(test) == _ordered(expected): - raise AssertionError(f"{test} and {expected} are not equal. The flag `ordered` is set to `{ordered}`.") + if _ordered(test) == _ordered(expected): + return True + + if silent: + return False + + raise AssertionError(f"{test} and {expected} are not equal. The flag `ordered` is set to `{ordered}`.") def compare_by_operator_name(operator: str, left: Any, right: Any) -> Any: @@ -146,3 +158,13 @@ def compare_by_operator_name(operator: str, left: Any, right: Any) -> Any: return left == right case _: raise ValueError(f"Unknown operator: {operator}") + + +def collections_exists(test_collection_names: list[str], expected_collection_name: list[str]) -> bool: + """Checks if the test and expected list of collection names match.""" + return assert_equal(test_collection_names, expected_collection_name, silent=True) + + +def document_ids_are_correct(test_ids: list[str], expected_ids: list[str]) -> bool: + """Checks if the test (retrieved from the API) and expected list of (document) ids match.""" + return assert_equal(test_ids, expected_ids, silent=True) diff --git a/trolldb/tests/tests_api/test_api.py b/trolldb/tests/tests_api/test_api.py index 2d7ecd7..550444f 100644 --- a/trolldb/tests/tests_api/test_api.py +++ b/trolldb/tests/tests_api/test_api.py @@ -10,14 +10,14 @@ import pytest from fastapi import status -from trolldb.test_utils.common import assert_equal, http_get +from trolldb.test_utils.common import assert_equal, collections_exists, document_ids_are_correct, http_get from trolldb.test_utils.mongodb_database import TestDatabase, test_mongodb_context @pytest.mark.usefixtures("_test_server_fixture") def test_root(): """Checks that the server is up and running, i.e. the root routes responds with 200.""" - assert_equal(http_get().status, status.HTTP_200_OK) + assert http_get().status == status.HTTP_200_OK @pytest.mark.usefixtures("_test_server_fixture") @@ -43,7 +43,7 @@ def test_database_names(): @pytest.mark.usefixtures("_test_server_fixture") def test_database_names_negative(): """Checks that the non-existing databases cannot be found.""" - assert_equal(http_get("databases/non_existing_database").status, status.HTTP_404_NOT_FOUND) + assert http_get("databases/non_existing_database").status == status.HTTP_404_NOT_FOUND @pytest.mark.usefixtures("_test_server_fixture") @@ -52,16 +52,13 @@ def test_collections(): with test_mongodb_context() as client: for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names, strict=False): - # Collections exist - assert_equal( + assert collections_exists( http_get(f"databases/{database_name}").json(), [collection_name] ) - - # Document ids are correct - assert_equal( + assert document_ids_are_correct( http_get(f"databases/{database_name}/{collection_name}").json(), - {str(doc["_id"]) for doc in client[database_name][collection_name].find({})} + [str(doc["_id"]) for doc in client[database_name][collection_name].find({})] ) @@ -69,7 +66,4 @@ def test_collections(): def test_collections_negative(): """Checks that the non-existing collections cannot be found.""" for database_name in TestDatabase.database_names: - assert_equal( - http_get(f"databases/{database_name}/non_existing_collection").status, - status.HTTP_404_NOT_FOUND - ) + assert http_get(f"databases/{database_name}/non_existing_collection").status == status.HTTP_404_NOT_FOUND From 3a0098fc5c12df4d87c166b29c9ca859275c7053 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 19:17:14 +0200 Subject: [PATCH 79/97] Address PR comments. --- trolldb/test_utils/mongodb_database.py | 34 ++++---------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index 58c6a3d..f738206 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -2,7 +2,7 @@ from contextlib import contextmanager from datetime import datetime, timedelta -from random import randint, shuffle +from random import choices, randint, shuffle from typing import Iterator from pymongo import MongoClient @@ -36,32 +36,6 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab client.close() -def random_sample(items: list, size: int = 10) -> list: - """Generates a random sample of items from the given list, with repetitions allowed. - - Note: - The length of the output can be larger than the lenght of the given list. See the example. - - Args: - items: - The list of items from which the random sample will be generated. - size (Optional, default ``10``): - The number of elements in the random sample. - - Returns: - A list containing the random sample of elements. - - Example: - >>> items = [1, 2, 3, 4, 5] - >>> random_sample(items, 10) - [2, 4, 1, 5, 3, 4, 2, 1, 3, 5] - """ - last_index = len(items) - 1 - # We suppress ruff (S311) here as we are not generating anything cryptographic here! - indices = [randint(0, last_index) for _ in range(size)] # noqa: S311 - return [items[i] for i in indices] - - class Time: """A static class to enclose functionalities for generating random time stamps.""" @@ -141,10 +115,12 @@ def like_mongodb_document(self) -> dict: class TestDatabase: """A static class which encloses functionalities to prepare and fill the test database with mock data.""" - platform_names = random_sample(["PA", "PB", "PC"]) + # We suppress ruff (S311) here as we are not generating anything cryptographic here! + platform_names = choices(["PA", "PB", "PC"], k=10) # noqa: S311 """Example platform names.""" - sensors = random_sample(["SA", "SB", "SC"]) + # We suppress ruff (S311) here as we are not generating anything cryptographic here! + sensors = choices(["SA", "SB", "SC"], k=10) # noqa: S311 """Example sensor names.""" database_names = [test_app_config.database.main_database_name, "another_mock_database"] From 71ad57d0f885210639f9f08b1dc0ad23e0b2d814 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 19:38:26 +0200 Subject: [PATCH 80/97] Address PR comments. --- trolldb/config/config.py | 37 +++---------------- trolldb/test_utils/common.py | 24 ++++++------ .../{test_recoder.py => test_recorder.py} | 3 +- 3 files changed, 20 insertions(+), 44 deletions(-) rename trolldb/tests/{test_recoder.py => test_recorder.py} (97%) diff --git a/trolldb/config/config.py b/trolldb/config/config.py index a118426..f0ceb51 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -10,7 +10,7 @@ import errno import sys -from typing import Any, NamedTuple, TypedDict +from typing import Any, NamedTuple from bson import ObjectId from bson.errors import InvalidId @@ -87,14 +87,11 @@ class DatabaseConfig(NamedTuple): """ -class SubscriberConfig(TypedDict): - """A named tuple to hold all the configurations of the subscriber. +SubscriberConfig = dict[Any, Any] +"""A dictionary to hold all the configurations of the subscriber. - TODO: This has to be moved to the `posttroll` package. - """ - nameserver: bool - addresses: list[str] - port: int +TODO: This has to be moved to the `posttroll` package. +""" class AppConfig(BaseModel): @@ -106,30 +103,6 @@ class AppConfig(BaseModel): database: DatabaseConfig subscriber: SubscriberConfig - def as_dict(self) -> dict: - """Converts the model to a dictionary, recursively.""" - - def _aux(obj: Any) -> Any: - """An auxiliary function to do the conversion based on the type.""" - match obj: - # The type `NamedTuple` has a method named `_asdict` - case tuple() if "_asdict" in dir(obj): - return {k: _aux(v) for k, v in obj._asdict().items()} - case BaseModel(): - return {k: _aux(v) for k, v in obj.__dict__.items()} - case dict(): - return {k: _aux(v) for k, v in obj.items()} - case list(): - return [_aux(i) for i in obj] - case tuple(): - return (_aux(i) for i in obj) - case int() | float() | str() | bool(): - return obj - case _: - return str(obj) - - return _aux(self.__dict__) - def parse_config_yaml_file(filename: FilePath) -> AppConfig: """Parses and validates the configurations from a YAML file. diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index d391add..4b68e4c 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -8,10 +8,10 @@ from pydantic import AnyUrl, FilePath from urllib3 import BaseHTTPResponse, request -from trolldb.config.config import APIServerConfig, AppConfig, DatabaseConfig, SubscriberConfig +from trolldb.config.config import AppConfig -def make_test_app_config(subscriber_address: FilePath | None = None) -> AppConfig: +def make_test_app_config(subscriber_address: FilePath | None = None) -> dict: """Makes the app configuration when used in testing. Args: @@ -20,27 +20,29 @@ def make_test_app_config(subscriber_address: FilePath | None = None) -> AppConfi config will be an empty dictionary. Returns: - An object of type :obj:`AppConfig`. + A dictionary which resembles an object of type :obj:`AppConfig`. """ - return AppConfig( - api_server=APIServerConfig( - url=AnyUrl("http://localhost:8080") + app_config = dict( + api_server=dict( + url="http://localhost:8080" ), - database=DatabaseConfig( + database=dict( main_database_name="mock_database", main_collection_name="mock_collection", - url=AnyUrl("mongodb://localhost:28017"), + url="mongodb://localhost:28017", timeout=1000 ), - subscriber=SubscriberConfig( + subscriber=dict( nameserver=False, addresses=[f"ipc://{subscriber_address}/in.ipc"] if subscriber_address is not None else [""], port=3000 ) ) + return app_config -test_app_config = make_test_app_config() + +test_app_config = AppConfig(**make_test_app_config()) """The app configs for testing purposes assuming an empty configuration for the subscriber.""" @@ -48,7 +50,7 @@ def create_config_file(config_path: FilePath) -> FilePath: """Creates a config file for tests.""" config_file = config_path / "config.yaml" with open(config_file, "w") as f: - yaml.safe_dump(make_test_app_config(config_path).as_dict(), f) + yaml.safe_dump(make_test_app_config(config_path), f) return config_file diff --git a/trolldb/tests/test_recoder.py b/trolldb/tests/test_recorder.py similarity index 97% rename from trolldb/tests/test_recoder.py rename to trolldb/tests/test_recorder.py index 1ef9e12..aabb67e 100644 --- a/trolldb/tests/test_recoder.py +++ b/trolldb/tests/test_recorder.py @@ -8,6 +8,7 @@ from pydantic import FilePath from trolldb.cli import record_messages, record_messages_from_command_line, record_messages_from_config +from trolldb.config.config import AppConfig from trolldb.database.mongodb import MongoDB, mongodb_context from trolldb.test_utils.common import assert_equal, create_config_file, make_test_app_config, test_app_config from trolldb.test_utils.mongodb_instance import running_prepared_database_context @@ -81,7 +82,7 @@ async def test_record_cli(tmp_path, file_message, tmp_data_filename): async def test_record_deletes_message(tmp_path, file_message, del_message): """Test that message recording can delete a record in the database.""" - config = make_test_app_config(tmp_path) + config = AppConfig(**make_test_app_config(tmp_path)) with running_prepared_database_context(): with patched_subscriber_recv([file_message, del_message]): await record_messages(config) From b5fba56a56841eb3fb6958555353fdad32c8ac0e Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 19:40:31 +0200 Subject: [PATCH 81/97] Address PR comments. --- trolldb/template_config.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/trolldb/template_config.yaml b/trolldb/template_config.yaml index a958ef7..21f661f 100644 --- a/trolldb/template_config.yaml +++ b/trolldb/template_config.yaml @@ -25,9 +25,4 @@ api_server: # Required subscriber: - # Required - nameserver: False - # Required - addresses: [""] - # Required - port: 3000 + # As per the configurations of the posttroll From 22c38b436eb12747dd81ca9b231b641c58c43980 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 19:42:32 +0200 Subject: [PATCH 82/97] Address PR comments. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 182973b..d4b4cde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ classifiers = [ [project.scripts] pytroll-db-recorder = "trolldb.cli:run_sync" - [build-system] requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" From 9bae659699b1eb2412a20598e78bfc7df038940a Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 19:46:11 +0200 Subject: [PATCH 83/97] Address PR comments. --- bin/pytroll-mongo.py | 120 ------------------------------------------- 1 file changed, 120 deletions(-) delete mode 100644 bin/pytroll-mongo.py diff --git a/bin/pytroll-mongo.py b/bin/pytroll-mongo.py deleted file mode 100644 index be1324d..0000000 --- a/bin/pytroll-mongo.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -"""Documentation to be added!""" - -import logging -import os -from threading import Thread - -import yaml -from posttroll.subscriber import Subscribe -from pymongo import MongoClient - -logger = logging.getLogger(__name__) - - -class MongoRecorder: - """A recorder for posttroll file messages.""" - - def __init__(self, - mongo_uri="mongodb://localhost:27017", - db_name="sat_db"): - """Init the recorder.""" - self.db = MongoClient(mongo_uri)[db_name] - self.loop = True - self._recorder = Thread(target=self.record) - - def start(self): - """Start the recording.""" - self._recorder.start() - - def insert_files(self, msg): - """Insert files in the database.""" - self.db.files.insert_one(msg.data) - - def record(self): - """Log stuff.""" - try: - with Subscribe("", addr_listener=True) as sub: - for msg in sub.recv(timeout=1): - if msg: - logger.debug("got msg %s", str(msg)) - if msg.type in ["collection", "file", "dataset"]: - self.insert_files(msg) - if not self.loop: - logger.info("Stop recording") - break - except Exception: - logger.exception("Something went wrong in record") - raise - - def stop(self): - """Stop the machine.""" - self.loop = False - - -log_levels = { - 0: logging.WARN, - 1: logging.INFO, - 2: logging.DEBUG, -} - - -def setup_logging(cmd_args): - """Set up logging.""" - if cmd_args.log_config is not None: - with open(cmd_args.log_config) as fd: - log_dict = yaml.safe_load(fd.read()) - logging.config.dictConfig(log_dict) - return - - root = logging.getLogger("") - root.setLevel(log_levels[cmd_args.verbosity]) - - if cmd_args.log: - fh_ = logging.handlers.TimedRotatingFileHandler( - os.path.join(cmd_args.log), - "midnight", - backupCount=7) - else: - fh_ = logging.StreamHandler() - - formatter = logging.Formatter(LOG_FORMAT) - fh_.setFormatter(formatter) - - root.addHandler(fh_) - - -LOG_FORMAT = "[%(asctime)s %(name)s %(levelname)s] %(message)s" - -if __name__ == "__main__": - import argparse - import time - - parser = argparse.ArgumentParser() - parser.add_argument("-d", "--database", - help="URI to the mongo database (default mongodb://localhost:27017 ).", - default="mongodb://localhost:27017") - parser.add_argument("-l", "--log", - help="The file to log to. stdout otherwise.") - parser.add_argument("-c", "--log-config", - help="Log config file to use instead of the standard logging.") - parser.add_argument("-v", "--verbose", dest="verbosity", action="count", default=0, - help="Verbosity (between 1 and 2 occurrences with more leading to more " - "verbose logging). WARN=0, INFO=1, " - "DEBUG=2. This is overridden by the log config file if specified.") - cmd_args = parser.parse_args() - - logger = logging.getLogger("mongo_recorder") - logger.setLevel(logging.DEBUG) - setup_logging(cmd_args) - logger.info("Starting up.") - - try: - recorder = MongoRecorder(cmd_args.database) - recorder.start() - while True: - time.sleep(1) - except KeyboardInterrupt: - recorder.stop() - # print("Thanks for using pytroll/mongo_recorder. See you soon on www.pytroll.org!") From 425fe9bc87af1cd3a1fa1a4b30e399eac464dc7b Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 19:54:42 +0200 Subject: [PATCH 84/97] Address PR comments. --- trolldb/test_utils/common.py | 9 +++++---- trolldb/tests/tests_api/test_api.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 4b68e4c..8718927 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,6 +1,6 @@ """Common functionalities for testing, shared between tests and other test utility modules.""" -from collections import OrderedDict +from collections import Counter, OrderedDict from typing import Any from urllib.parse import urljoin @@ -73,8 +73,9 @@ def assert_equal(test: Any, expected: Any, ordered: bool = False, silent: bool = """An auxiliary function to assert the equality of two objects using the ``==`` operator. Examples: - - If ``ordered=False`` and the input is a list or a tuple, it will be first converted to a set - so that the order of items therein does not affect the assertion outcome. + - If ``ordered=False`` and the input is a list or a tuple, it will be first converted to an object of type + `counter `_, so that the order of items + therein does not affect the assertion outcome. - If ``ordered=True`` and the input is a dictionary, it will be first converted to an ``OrderedDict``. Note: @@ -112,7 +113,7 @@ def _ordered(obj: Any) -> Any: """An auxiliary function to convert an object to ordered depending on its type and the ``ordered`` flag.""" match obj: case list() | tuple(): - return set(obj) if not ordered else obj + return Counter(obj) if not ordered else obj case dict(): return OrderedDict(obj) if ordered else obj case _: diff --git a/trolldb/tests/tests_api/test_api.py b/trolldb/tests/tests_api/test_api.py index 550444f..eec1c1a 100644 --- a/trolldb/tests/tests_api/test_api.py +++ b/trolldb/tests/tests_api/test_api.py @@ -23,13 +23,13 @@ def test_root(): @pytest.mark.usefixtures("_test_server_fixture") def test_platforms(): """Checks that the retrieved platform names match the expected names.""" - assert_equal(http_get("platforms").json(), TestDatabase.platform_names) + assert set(http_get("platforms").json()) == set(TestDatabase.platform_names) @pytest.mark.usefixtures("_test_server_fixture") def test_sensors(): """Checks that the retrieved sensor names match the expected names.""" - assert_equal(http_get("sensors").json(), TestDatabase.sensors) + assert set(http_get("sensors").json()) == set(TestDatabase.sensors) @pytest.mark.usefixtures("_test_server_fixture") From ba3c1ae5d3a2f6051728e21e0f4c9c716ff81d3b Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 20:08:05 +0200 Subject: [PATCH 85/97] Address PR comments. --- trolldb/test_utils/common.py | 65 +------------------ trolldb/tests/test_recorder.py | 6 +- trolldb/tests/tests_api/test_api.py | 10 +-- .../tests/tests_database/test_pipelines.py | 22 +++---- 4 files changed, 23 insertions(+), 80 deletions(-) diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 8718927..80e08c2 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,6 +1,6 @@ """Common functionalities for testing, shared between tests and other test utility modules.""" -from collections import Counter, OrderedDict +from collections import Counter from typing import Any from urllib.parse import urljoin @@ -69,65 +69,6 @@ def http_get(route: str = "", root: AnyUrl = test_app_config.api_server.url) -> return request("GET", urljoin(root.unicode_string(), route)) -def assert_equal(test: Any, expected: Any, ordered: bool = False, silent: bool = False) -> bool: - """An auxiliary function to assert the equality of two objects using the ``==`` operator. - - Examples: - - If ``ordered=False`` and the input is a list or a tuple, it will be first converted to an object of type - `counter `_, so that the order of items - therein does not affect the assertion outcome. - - If ``ordered=True`` and the input is a dictionary, it will be first converted to an ``OrderedDict``. - - Note: - The rationale behind choosing ``ordered=False`` as the default behaviour is that this function is often used - in combination with API calls and/or querying the database. In such cases, the order of items which are returned - often does not matter. In addition, if the order really matters, one might as well simply use the built-in - ``assert`` statement. - - Note: - Dictionaries by default are unordered objects. - - Warning: - For the purpose of this function, the concept of ordered vs unordered only applies to lists, tuples, and - dictionaries. An object of any other type is assumed as-is, i.e. the default behaviour of Python applies. - For example, conceptually, two strings can be converted to two sets of characters and then be compared with - each other. However, this is not what we do for strings. - - Args: - test: - The object to be tested. - expected: - The object to test against. - ordered (Optional, default ``False``): - A flag to determine whether the order of items matters in case of a list, a tuple, or a dictionary. - silent (Optional, default ``False``): - A flag to determine whether the assertion should be silent, i.e. simply return the result as a boolean or - it should raise an ``AssertionError``. - - Raises: - AssertionError: - If the ``test`` and ``expected`` are not equal and ``silent=False``. - """ - - def _ordered(obj: Any) -> Any: - """An auxiliary function to convert an object to ordered depending on its type and the ``ordered`` flag.""" - match obj: - case list() | tuple(): - return Counter(obj) if not ordered else obj - case dict(): - return OrderedDict(obj) if ordered else obj - case _: - return obj - - if _ordered(test) == _ordered(expected): - return True - - if silent: - return False - - raise AssertionError(f"{test} and {expected} are not equal. The flag `ordered` is set to `{ordered}`.") - - def compare_by_operator_name(operator: str, left: Any, right: Any) -> Any: """Compares two operands given the binary operator name in a string format. @@ -165,9 +106,9 @@ def compare_by_operator_name(operator: str, left: Any, right: Any) -> Any: def collections_exists(test_collection_names: list[str], expected_collection_name: list[str]) -> bool: """Checks if the test and expected list of collection names match.""" - return assert_equal(test_collection_names, expected_collection_name, silent=True) + return Counter(test_collection_names) == Counter(expected_collection_name) def document_ids_are_correct(test_ids: list[str], expected_ids: list[str]) -> bool: """Checks if the test (retrieved from the API) and expected list of (document) ids match.""" - return assert_equal(test_ids, expected_ids, silent=True) + return Counter(test_ids) == Counter(expected_ids) diff --git a/trolldb/tests/test_recorder.py b/trolldb/tests/test_recorder.py index aabb67e..2e6cd34 100644 --- a/trolldb/tests/test_recorder.py +++ b/trolldb/tests/test_recorder.py @@ -10,7 +10,7 @@ from trolldb.cli import record_messages, record_messages_from_command_line, record_messages_from_config from trolldb.config.config import AppConfig from trolldb.database.mongodb import MongoDB, mongodb_context -from trolldb.test_utils.common import assert_equal, create_config_file, make_test_app_config, test_app_config +from trolldb.test_utils.common import create_config_file, make_test_app_config, test_app_config from trolldb.test_utils.mongodb_instance import running_prepared_database_context @@ -42,10 +42,10 @@ async def assert_message(msg, data_filename): collection = await MongoDB.get_collection("mock_database", "mock_collection") result = await collection.find_one(dict(scan_mode="EW")) result.pop("_id") - assert_equal(result, msg.data) + assert result == msg.data deletion_result = await collection.delete_many({"uri": str(data_filename)}) - assert_equal(deletion_result.deleted_count, 1) + assert deletion_result.deleted_count == 1 async def _record_from_somewhere( diff --git a/trolldb/tests/tests_api/test_api.py b/trolldb/tests/tests_api/test_api.py index eec1c1a..1e83372 100644 --- a/trolldb/tests/tests_api/test_api.py +++ b/trolldb/tests/tests_api/test_api.py @@ -7,10 +7,12 @@ against expectations. """ +from collections import Counter + import pytest from fastapi import status -from trolldb.test_utils.common import assert_equal, collections_exists, document_ids_are_correct, http_get +from trolldb.test_utils.common import collections_exists, document_ids_are_correct, http_get from trolldb.test_utils.mongodb_database import TestDatabase, test_mongodb_context @@ -35,9 +37,9 @@ def test_sensors(): @pytest.mark.usefixtures("_test_server_fixture") def test_database_names(): """Checks that the retrieved database names match the expected names.""" - assert_equal(http_get("databases").json(), TestDatabase.database_names) - assert_equal(http_get("databases?exclude_defaults=True").json(), TestDatabase.database_names) - assert_equal(http_get("databases?exclude_defaults=False").json(), TestDatabase.all_database_names) + assert Counter(http_get("databases").json()) == Counter(TestDatabase.database_names) + assert Counter(http_get("databases?exclude_defaults=True").json()) == Counter(TestDatabase.database_names) + assert Counter(http_get("databases?exclude_defaults=False").json()) == Counter(TestDatabase.all_database_names) @pytest.mark.usefixtures("_test_server_fixture") diff --git a/trolldb/tests/tests_database/test_pipelines.py b/trolldb/tests/tests_database/test_pipelines.py index 19e4c97..b993aff 100644 --- a/trolldb/tests/tests_database/test_pipelines.py +++ b/trolldb/tests/tests_database/test_pipelines.py @@ -1,6 +1,6 @@ """Tests for the pipelines and applying comparison operations on them.""" from trolldb.database.piplines import PipelineAttribute, PipelineBooleanDict, Pipelines -from trolldb.test_utils.common import assert_equal, compare_by_operator_name +from trolldb.test_utils.common import compare_by_operator_name def test_pipeline_boolean_dict(): @@ -20,16 +20,16 @@ def test_pipeline_boolean_dict(): def test_pipeline_attribute(): """Tests different comparison operators for a pipeline attribute in a list and as a single item.""" for op in ["$eq", "$gte", "$gt", "$lte", "$lt"]: - assert_equal( - compare_by_operator_name(op, PipelineAttribute("letter"), "A"), - PipelineBooleanDict({"letter": {op: "A"}} if op != "$eq" else {"letter": "A"}) + assert ( + compare_by_operator_name(op, PipelineAttribute("letter"), "A") == + PipelineBooleanDict({"letter": {op: "A"}} if op != "$eq" else {"letter": "A"}) ) - assert_equal( - compare_by_operator_name(op, PipelineAttribute("letter"), ["A", "B"]), - PipelineBooleanDict({"$or": [ - {"letter": {op: "A"} if op != "$eq" else "A"}, - {"letter": {op: "B"} if op != "$eq" else "B"} - ]}) + assert ( + compare_by_operator_name(op, PipelineAttribute("letter"), ["A", "B"]) == + PipelineBooleanDict({"$or": [ + {"letter": {op: "A"} if op != "$eq" else "A"}, + {"letter": {op: "B"} if op != "$eq" else "B"} + ]}) ) @@ -49,4 +49,4 @@ def test_pipelines(): ] for p1, p2 in zip(pipelines, pipelines_literal, strict=False): - assert_equal(p1, p2) + assert p1 == p2 From 6e292a66a48177dbf4333ea117b8e2bcdb8f2675 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 20:13:19 +0200 Subject: [PATCH 86/97] Address PR comments. --- trolldb/test_utils/common.py | 11 ----------- trolldb/tests/tests_api/test_api.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index 80e08c2..cd87534 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,6 +1,5 @@ """Common functionalities for testing, shared between tests and other test utility modules.""" -from collections import Counter from typing import Any from urllib.parse import urljoin @@ -102,13 +101,3 @@ def compare_by_operator_name(operator: str, left: Any, right: Any) -> Any: return left == right case _: raise ValueError(f"Unknown operator: {operator}") - - -def collections_exists(test_collection_names: list[str], expected_collection_name: list[str]) -> bool: - """Checks if the test and expected list of collection names match.""" - return Counter(test_collection_names) == Counter(expected_collection_name) - - -def document_ids_are_correct(test_ids: list[str], expected_ids: list[str]) -> bool: - """Checks if the test (retrieved from the API) and expected list of (document) ids match.""" - return Counter(test_ids) == Counter(expected_ids) diff --git a/trolldb/tests/tests_api/test_api.py b/trolldb/tests/tests_api/test_api.py index 1e83372..2e00376 100644 --- a/trolldb/tests/tests_api/test_api.py +++ b/trolldb/tests/tests_api/test_api.py @@ -12,10 +12,20 @@ import pytest from fastapi import status -from trolldb.test_utils.common import collections_exists, document_ids_are_correct, http_get +from trolldb.test_utils.common import http_get from trolldb.test_utils.mongodb_database import TestDatabase, test_mongodb_context +def collections_exists(test_collection_names: list[str], expected_collection_name: list[str]) -> bool: + """Checks if the test and expected list of collection names match.""" + return Counter(test_collection_names) == Counter(expected_collection_name) + + +def document_ids_are_correct(test_ids: list[str], expected_ids: list[str]) -> bool: + """Checks if the test (retrieved from the API) and expected list of (document) ids match.""" + return Counter(test_ids) == Counter(expected_ids) + + @pytest.mark.usefixtures("_test_server_fixture") def test_root(): """Checks that the server is up and running, i.e. the root routes responds with 200.""" From a1bc1be674535fc12d2d3aa6a15451bc8a2aba0b Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 20:15:30 +0200 Subject: [PATCH 87/97] Address PR comments. --- trolldb/tests/conftest.py | 7 ------- trolldb/tests/test_recorder.py | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/trolldb/tests/conftest.py b/trolldb/tests/conftest.py index 507f37b..69ecd64 100644 --- a/trolldb/tests/conftest.py +++ b/trolldb/tests/conftest.py @@ -33,10 +33,3 @@ async def mongodb_fixture(_run_mongodb_server_instance): TestDatabase.prepare() async with mongodb_context(test_app_config.database): yield - - -@pytest.fixture() -def tmp_data_filename(tmp_path): - """Create a filename for the messages.""" - filename = "20191103_153936-s1b-ew-hh.tiff" - return tmp_path / filename diff --git a/trolldb/tests/test_recorder.py b/trolldb/tests/test_recorder.py index 2e6cd34..5915ece 100644 --- a/trolldb/tests/test_recorder.py +++ b/trolldb/tests/test_recorder.py @@ -36,6 +36,13 @@ def del_message(tmp_data_filename): '"polarization": "hh", "sensor": "sar-c", "format": "GeoTIFF", "pass_direction": "ASCENDING"}') +@pytest.fixture() +def tmp_data_filename(tmp_path): + """Create a filename for the messages.""" + filename = "20191103_153936-s1b-ew-hh.tiff" + return tmp_path / filename + + async def assert_message(msg, data_filename): """Documentation to be added.""" async with mongodb_context(test_app_config.database): From f4517547fa3e383121b205b06cd3ff8ca57a5196 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 20:36:05 +0200 Subject: [PATCH 88/97] Address PR comments. --- trolldb/api/api.py | 5 ++--- trolldb/config/config.py | 8 ++++---- trolldb/database/mongodb.py | 2 +- trolldb/template_config.yaml | 10 +++++----- trolldb/test_utils/common.py | 2 +- trolldb/test_utils/mongodb_database.py | 2 +- trolldb/test_utils/mongodb_instance.py | 4 ++-- trolldb/tests/tests_database/test_mongodb.py | 8 ++++---- 8 files changed, 20 insertions(+), 21 deletions(-) diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 9ef0c32..8992fe5 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -122,7 +122,7 @@ async def _serve(): @contextmanager @validate_call -def api_server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2000): +def api_server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2): """A synchronous context manager to run the API server in a separate process (non-blocking). It uses the `multiprocessing `_ package. The main use case @@ -148,8 +148,7 @@ def api_server_process_context(config: AppConfig | FilePath, startup_time: Timeo process = Process(target=run_server, args=(config,)) try: process.start() - # time.sleep() expects its argument to be in seconds, hence the division by 1000. - time.sleep(startup_time / 1000.) + time.sleep(startup_time) yield process finally: logger.info("Attempt to terminate the API server process in the context manager ...") diff --git a/trolldb/config/config.py b/trolldb/config/config.py index f0ceb51..b43d7fe 100644 --- a/trolldb/config/config.py +++ b/trolldb/config/config.py @@ -20,8 +20,8 @@ from typing_extensions import Annotated from yaml import safe_load -Timeout = Annotated[int, Field(ge=0)] -"""A type hint for the timeout in milliseconds (non-negative int).""" +Timeout = Annotated[float, Field(ge=0)] +"""A type hint for the timeout in seconds (non-negative float).""" def id_must_be_valid(id_like_string: str) -> ObjectId: @@ -82,8 +82,8 @@ class DatabaseConfig(NamedTuple): """The URL of the MongoDB server excluding the port part, e.g. ``"mongodb://localhost:27017"``""" timeout: Timeout - """The timeout in milliseconds (non-negative int), after which an exception is raised if a connection with the - MongoDB instance is not established successfully, e.g. ``1000``. + """The timeout in seconds (non-negative float), after which an exception is raised if a connection with the + MongoDB instance is not established successfully, e.g. ``1.5``. """ diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index a052925..87b2466 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -152,7 +152,7 @@ async def initialize(cls, database_config: DatabaseConfig): # to access the database. cls.__client = AsyncIOMotorClient( database_config.url.unicode_string(), - serverSelectionTimeoutMS=database_config.timeout) + serverSelectionTimeoutMS=database_config.timeout * 1000) __database_names = [] try: diff --git a/trolldb/template_config.yaml b/trolldb/template_config.yaml index 21f661f..71e2370 100644 --- a/trolldb/template_config.yaml +++ b/trolldb/template_config.yaml @@ -6,22 +6,22 @@ # Required database: #Required - main_database_name: "satellite_database" + main_database_name: satellite_database #Required - main_collection_name: "files" + main_collection_name: files #Required - url: "mongodb://localhost:27017" + url: mongodb://localhost:27017 #Required - timeout: 1000 # milliseconds + timeout: 1 # seconds # Required api_server: # Required - url: "http://localhost:8000" + url: http://localhost:8000 # Required subscriber: diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index cd87534..db38568 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -29,7 +29,7 @@ def make_test_app_config(subscriber_address: FilePath | None = None) -> dict: main_database_name="mock_database", main_collection_name="mock_collection", url="mongodb://localhost:28017", - timeout=1000 + timeout=1 ), subscriber=dict( nameserver=False, diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index f738206..74e7fe5 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -29,7 +29,7 @@ def test_mongodb_context(database_config: DatabaseConfig = test_app_config.datab """ client = None try: - client = MongoClient(database_config.url.unicode_string(), connectTimeoutMS=database_config.timeout) + client = MongoClient(database_config.url.unicode_string(), connectTimeoutMS=database_config.timeout * 1000) yield client finally: if client is not None: diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 6f0486b..0a0b207 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -103,7 +103,7 @@ def shutdown_instance(cls): @validate_call def mongodb_instance_server_process_context( database_config: DatabaseConfig = test_app_config.database, - startup_time: Timeout = 2000): + startup_time: Timeout = 2): """A synchronous context manager to run the MongoDB instance in a separate process (non-blocking). It uses the `subprocess `_ package. The main use case is @@ -130,7 +130,7 @@ def mongodb_instance_server_process_context( try: TestMongoInstance.run_instance() - time.sleep(startup_time / 1000) + time.sleep(startup_time) yield finally: TestMongoInstance.shutdown_instance() diff --git a/trolldb/tests/tests_database/test_mongodb.py b/trolldb/tests/tests_database/test_mongodb.py index a03b3bd..78958c1 100644 --- a/trolldb/tests/tests_database/test_mongodb.py +++ b/trolldb/tests/tests_database/test_mongodb.py @@ -19,7 +19,7 @@ async def test_connection_timeout_negative(): """Expect to see the connection attempt times out since the MongoDB URL is invalid.""" - timeout = 3000 + timeout = 3 t1 = time.time() with pytest.raises(SystemExit) as pytest_wrapped_e: async with mongodb_context( @@ -28,7 +28,7 @@ async def test_connection_timeout_negative(): pass t2 = time.time() assert pytest_wrapped_e.value.code == errno.EIO - assert t2 - t1 >= timeout / 1000 + assert t2 - t1 >= timeout @pytest.mark.usefixtures("_run_mongodb_server_instance") @@ -36,7 +36,7 @@ async def test_main_database_negative(): """Expect to fail when giving an invalid name for the main database, given a valid collection name.""" with pytest.raises(SystemExit) as pytest_wrapped_e: async with mongodb_context(DatabaseConfig( - timeout=1000, + timeout=1, url=test_app_config.database.url, main_database_name=" ", main_collection_name=test_app_config.database.main_collection_name)): @@ -49,7 +49,7 @@ async def test_main_collection_negative(): """Expect to fail when giving an invalid name for the main collection, given a valid database name.""" with pytest.raises(SystemExit) as pytest_wrapped_e: async with mongodb_context(DatabaseConfig( - timeout=1000, + timeout=1, url=test_app_config.database.url, main_database_name=test_app_config.database.main_database_name, main_collection_name=" ")): From 9fc425c085fd45adef25952deea4813aefbbefe5 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Tue, 21 May 2024 20:44:10 +0200 Subject: [PATCH 89/97] Address PR comments. --- trolldb/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trolldb/tests/conftest.py b/trolldb/tests/conftest.py index 69ecd64..6a1e80c 100644 --- a/trolldb/tests/conftest.py +++ b/trolldb/tests/conftest.py @@ -23,7 +23,7 @@ def _run_mongodb_server_instance(): @pytest.fixture(scope="session") def _test_server_fixture(_run_mongodb_server_instance): """Encloses all tests (session scope) in a context manager of a running API server (in a separate process).""" - with api_server_process_context(test_app_config, startup_time=2000): + with api_server_process_context(test_app_config, startup_time=2): yield From cbcbe5928d1109767bfbacb571712d395469b8f4 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 10:47:29 +0200 Subject: [PATCH 90/97] Address PR comments. --- trolldb/cli.py | 2 +- trolldb/tests/test_recorder.py | 71 +++++++++++++++------------------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/trolldb/cli.py b/trolldb/cli.py index 565307f..568766a 100644 --- a/trolldb/cli.py +++ b/trolldb/cli.py @@ -43,7 +43,7 @@ async def record_messages_from_command_line(args=None): parser.add_argument( "configuration_file", help="Path to the configuration file") - cmd_args = parser.parse_args(args) + cmd_args = parser.parse_args([str(i) for i in args]) await record_messages_from_config(cmd_args.configuration_file) diff --git a/trolldb/tests/test_recorder.py b/trolldb/tests/test_recorder.py index 5915ece..a10b04b 100644 --- a/trolldb/tests/test_recorder.py +++ b/trolldb/tests/test_recorder.py @@ -1,16 +1,13 @@ """Tests for the message recording into database.""" -from typing import Any - import pytest from posttroll.message import Message from posttroll.testing import patched_subscriber_recv -from pydantic import FilePath +from pytest_lazy_fixtures import lf from trolldb.cli import record_messages, record_messages_from_command_line, record_messages_from_config -from trolldb.config.config import AppConfig from trolldb.database.mongodb import MongoDB, mongodb_context -from trolldb.test_utils.common import create_config_file, make_test_app_config, test_app_config +from trolldb.test_utils.common import AppConfig, create_config_file, make_test_app_config, test_app_config from trolldb.test_utils.mongodb_instance import running_prepared_database_context @@ -43,57 +40,51 @@ def tmp_data_filename(tmp_path): return tmp_path / filename -async def assert_message(msg, data_filename): - """Documentation to be added.""" +@pytest.fixture() +def config_file(tmp_path): + """A fixture to create a config file for the tests.""" + return create_config_file(tmp_path) + + +async def message_in_database_and_delete_count_is_one(msg) -> bool: + """Checks if there is exactly one item in the database which matches the data of the message.""" async with mongodb_context(test_app_config.database): collection = await MongoDB.get_collection("mock_database", "mock_collection") result = await collection.find_one(dict(scan_mode="EW")) result.pop("_id") - assert result == msg.data + deletion_result = await collection.delete_many({"uri": msg.data["uri"]}) + return result == msg.data and deletion_result.deleted_count == 1 - deletion_result = await collection.delete_many({"uri": str(data_filename)}) - assert deletion_result.deleted_count == 1 - -async def _record_from_somewhere( - config_path: FilePath, message: Any, data_filename, record_from_func, wrap_in_list=False): - """Test that we can record when passed a config file.""" - config_file = create_config_file(config_path) - msg = Message.decode(message) +@pytest.mark.parametrize(("function", "args"), [ + (record_messages_from_config, lf("config_file")), + (record_messages_from_command_line, [lf("config_file")]) +]) +async def test_record_from_cli_and_config(tmp_path, file_message, tmp_data_filename, function, args): + """Tests that message recording adds a message to the database either via configs from a file or the CLI.""" + msg = Message.decode(file_message) with running_prepared_database_context(): - with patched_subscriber_recv([message]): - await record_from_func(config_file if not wrap_in_list else [str(config_file)]) - await assert_message(msg, data_filename) - - -async def test_record_adds_message(tmp_path, file_message, tmp_data_filename): - """Test that message recording adds a message to the database.""" - await _record_from_somewhere( - tmp_path, file_message, tmp_data_filename, record_messages_from_config - ) + with patched_subscriber_recv([file_message]): + await function(args) + assert await message_in_database_and_delete_count_is_one(msg) -async def test_record_from_config(tmp_path, file_message, tmp_data_filename): - """Test that we can record when passed a config file.""" - await _record_from_somewhere( - tmp_path, file_message, tmp_data_filename, record_messages_from_config - ) - - -async def test_record_cli(tmp_path, file_message, tmp_data_filename): - """Test that we can record when passed a config file.""" - await _record_from_somewhere( - tmp_path, file_message, tmp_data_filename, record_messages_from_command_line, True - ) +async def test_record_messages(config_file, tmp_path, file_message, tmp_data_filename): + """Tests that message recording adds a message to the database.""" + config = AppConfig(**make_test_app_config(tmp_path)) + msg = Message.decode(file_message) + with running_prepared_database_context(): + with patched_subscriber_recv([file_message]): + await record_messages(config) + assert message_in_database_and_delete_count_is_one(msg) async def test_record_deletes_message(tmp_path, file_message, del_message): - """Test that message recording can delete a record in the database.""" + """Tests that message recording can delete a record in the database.""" config = AppConfig(**make_test_app_config(tmp_path)) with running_prepared_database_context(): with patched_subscriber_recv([file_message, del_message]): await record_messages(config) - async with mongodb_context(config.database): collection = await MongoDB.get_collection("mock_database", "mock_collection") result = await collection.find_one(dict(scan_mode="EW")) From 16610e45718a067dd9fff6cbffc39c15afcd9742 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 10:49:38 +0200 Subject: [PATCH 91/97] CI/CD. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4bb7a3..53e1a23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install ruff pytest pytest-asyncio pytest-cov + python -m pip install ruff pytest pytest-lazy-fixtures pytest-asyncio pytest-cov python -m pip install -e . - name: Test with pytest run: | From 7f6081da7fef0603fd2f1af15af3498c0f6b6829 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 11:28:17 +0200 Subject: [PATCH 92/97] Mock sphinx autodoc imports. --- .gitignore | 1 + docs/source/conf.py | 6 +----- trolldb/api/api.py | 15 +++------------ trolldb/api/routes/common.py | 6 +++--- trolldb/database/mongodb.py | 19 +++++++------------ trolldb/test_utils/common.py | 4 ++-- trolldb/test_utils/mongodb_instance.py | 6 ------ 7 files changed, 17 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 835f613..45291c7 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ coverage.xml # Django stuff: *.log *.pot +log # Sphinx documentation docs/build/ diff --git a/docs/source/conf.py b/docs/source/conf.py index 6f7ab16..84b614c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,14 +11,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. import os -import sys from sphinx.ext import apidoc -sys.path.insert(0, os.path.abspath("../../")) -sys.path.append(os.path.abspath(os.path.dirname(__file__))) -for x in os.walk("../../trolldb"): - sys.path.append(x[0]) +autodoc_mock_imports = ["motor", "pydantic", "fastapi", "uvicorn", "loguru", "pyyaml"] # -- Project information ----------------------------------------------------- diff --git a/trolldb/api/api.py b/trolldb/api/api.py index 8992fe5..461ca9c 100644 --- a/trolldb/api/api.py +++ b/trolldb/api/api.py @@ -2,11 +2,6 @@ This is the main module which is supposed to be imported by the users of the package. -Note: - Functions in this module are decorated with - `pydantic.validate_call `_ - so that their arguments can be validated using the corresponding type hints, when calling the function at runtime. - Note: The following applies to the :obj:`api` package and all its subpackages/modules. @@ -22,6 +17,7 @@ import time from contextlib import contextmanager from multiprocessing import Process +from typing import Union import uvicorn from fastapi import FastAPI, status @@ -52,7 +48,7 @@ @validate_call -def run_server(config: AppConfig | FilePath, **kwargs) -> None: +def run_server(config: Union[AppConfig, FilePath], **kwargs) -> None: """Runs the API server with all the routes and connection to the database. It first creates a FastAPI application and runs it using `uvicorn `_ which is @@ -121,8 +117,7 @@ async def _serve(): @contextmanager -@validate_call -def api_server_process_context(config: AppConfig | FilePath, startup_time: Timeout = 2): +def api_server_process_context(config: Union[AppConfig, FilePath], startup_time: Timeout = 2): """A synchronous context manager to run the API server in a separate process (non-blocking). It uses the `multiprocessing `_ package. The main use case @@ -136,10 +131,6 @@ def api_server_process_context(config: AppConfig | FilePath, startup_time: Timeo The overall time in seconds that is expected for the server and the database connections to be established before actual requests can be sent to the server. For testing purposes ensure that this is sufficiently large so that the tests will not time out. - - Raises: - ValidationError: - If the function is not called with arguments of valid type. """ logger.info("Attempt to run the API server process in a context manager ...") if not isinstance(config, AppConfig): diff --git a/trolldb/api/routes/common.py b/trolldb/api/routes/common.py index f7a995a..b05c86b 100644 --- a/trolldb/api/routes/common.py +++ b/trolldb/api/routes/common.py @@ -1,6 +1,6 @@ """The module with common functions to be used in handling requests related to `databases` and `collections`.""" -from typing import Annotated +from typing import Annotated, Union from fastapi import Depends, Query, Response from motor.motor_asyncio import AsyncIOMotorCollection, AsyncIOMotorDatabase @@ -62,8 +62,8 @@ async def check_collection( async def get_distinct_items_in_collection( - response_or_collection: Response | AsyncIOMotorCollection, - field_name: str) -> Response | list[str]: + response_or_collection: Union[Response, AsyncIOMotorCollection], + field_name: str) -> Union[Response, list[str]]: """An auxiliary function to either return the given response; or return a list of distinct (unique) values. Given the ``field_name`` it conducts a search in all documents of the given collection. The latter behaviour is diff --git a/trolldb/database/mongodb.py b/trolldb/database/mongodb.py index 87b2466..3ec5a79 100644 --- a/trolldb/database/mongodb.py +++ b/trolldb/database/mongodb.py @@ -7,7 +7,7 @@ import errno from contextlib import asynccontextmanager -from typing import Any, AsyncGenerator, Coroutine, TypeVar +from typing import Any, AsyncGenerator, Coroutine, Optional, TypeVar, Union from loguru import logger from motor.motor_asyncio import ( @@ -17,7 +17,7 @@ AsyncIOMotorCursor, AsyncIOMotorDatabase, ) -from pydantic import BaseModel, validate_call +from pydantic import BaseModel from pymongo.collection import _DocumentType from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError @@ -66,7 +66,7 @@ async def get_id(doc: CoroutineDocument) -> str: return str((await doc)["_id"]) -async def get_ids(docs: AsyncIOMotorCommandCursor | AsyncIOMotorCursor) -> list[str]: +async def get_ids(docs: Union[AsyncIOMotorCommandCursor, AsyncIOMotorCursor]) -> list[str]: """Similar to :func:`~MongoDB.get_id` but for a list of documents. Args: @@ -103,8 +103,8 @@ class MongoDB: us, we would like to fail early! """ - __client: AsyncIOMotorClient | None = None - __database_config: DatabaseConfig | None = None + __client: Optional[AsyncIOMotorClient] = None + __database_config: Optional[DatabaseConfig] = None __main_collection: AsyncIOMotorCollection = None __main_database: AsyncIOMotorDatabase = None @@ -223,7 +223,7 @@ def main_database(cls) -> AsyncIOMotorDatabase: async def get_collection( cls, database_name: str, - collection_name: str) -> AsyncIOMotorCollection | ResponseError: + collection_name: str) -> Union[AsyncIOMotorCollection, ResponseError]: """Gets the collection object given its name and the database name in which it resides. Args: @@ -266,7 +266,7 @@ async def get_collection( raise Collections.WrongTypeError @classmethod - async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | ResponseError: + async def get_database(cls, database_name: str) -> Union[AsyncIOMotorDatabase, ResponseError]: """Gets the database object given its name. Args: @@ -292,7 +292,6 @@ async def get_database(cls, database_name: str) -> AsyncIOMotorDatabase | Respon @asynccontextmanager -@validate_call async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: """An asynchronous context manager to connect to the MongoDB client. @@ -305,10 +304,6 @@ async def mongodb_context(database_config: DatabaseConfig) -> AsyncGenerator: Args: database_config: The configuration of the database. - - Raises: - ValidationError: - If the function is not called with arguments of valid type. """ logger.info("Attempt to open the MongoDB context manager ...") try: diff --git a/trolldb/test_utils/common.py b/trolldb/test_utils/common.py index db38568..6e3fb21 100644 --- a/trolldb/test_utils/common.py +++ b/trolldb/test_utils/common.py @@ -1,6 +1,6 @@ """Common functionalities for testing, shared between tests and other test utility modules.""" -from typing import Any +from typing import Any, Optional from urllib.parse import urljoin import yaml @@ -10,7 +10,7 @@ from trolldb.config.config import AppConfig -def make_test_app_config(subscriber_address: FilePath | None = None) -> dict: +def make_test_app_config(subscriber_address: Optional[FilePath] = None) -> dict: """Makes the app configuration when used in testing. Args: diff --git a/trolldb/test_utils/mongodb_instance.py b/trolldb/test_utils/mongodb_instance.py index 0a0b207..1b16f04 100644 --- a/trolldb/test_utils/mongodb_instance.py +++ b/trolldb/test_utils/mongodb_instance.py @@ -9,7 +9,6 @@ from shutil import rmtree from loguru import logger -from pydantic import validate_call from trolldb.config.config import DatabaseConfig, Timeout from trolldb.test_utils.common import test_app_config @@ -100,7 +99,6 @@ def shutdown_instance(cls): @contextmanager -@validate_call def mongodb_instance_server_process_context( database_config: DatabaseConfig = test_app_config.database, startup_time: Timeout = 2): @@ -116,10 +114,6 @@ def mongodb_instance_server_process_context( startup_time: The overall time in seconds that is expected for the MongoDB server instance to run before the database content can be accessed. - - Raises: - ValidationError: - If the function is not called with arguments of valid type. """ TestMongoInstance.port = database_config.url.hosts()[0]["port"] TestMongoInstance.prepare_dirs() From b36dbd294245c95ff1b92e044d14b3fa43c8b2da Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 11:29:10 +0200 Subject: [PATCH 93/97] Mock sphinx autodoc imports. --- docs/requirements.txt | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index a89bb8e..cbf1e36 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,8 +1,2 @@ -sphinx==7.2.6 -sphinx-rtd-theme==2.0.0 -motor==3.4.0 -pydantic==2.6.4 -fastapi==0.110.1 -uvicorn==0.29.0 -loguru==0.7.2 -pyyaml==6.0.1 +sphinx +sphinx-rtd-theme From 5d093602aac0584a356bb5f597bcbfd5e976a0cf Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 12:27:18 +0200 Subject: [PATCH 94/97] Args. --- trolldb/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trolldb/cli.py b/trolldb/cli.py index 568766a..74ae37b 100644 --- a/trolldb/cli.py +++ b/trolldb/cli.py @@ -43,7 +43,7 @@ async def record_messages_from_command_line(args=None): parser.add_argument( "configuration_file", help="Path to the configuration file") - cmd_args = parser.parse_args([str(i) for i in args]) + cmd_args = parser.parse_args(None if args is None else [str(i) for i in args]) await record_messages_from_config(cmd_args.configuration_file) From 12f1a0b5256cc9ad24aef6de3e3fcf725b2ace91 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 12:37:41 +0200 Subject: [PATCH 95/97] Fix a bug with auto-generated docs. --- trolldb/errors/errors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trolldb/errors/errors.py b/trolldb/errors/errors.py index 0d3ba88..95fd6bf 100644 --- a/trolldb/errors/errors.py +++ b/trolldb/errors/errors.py @@ -6,7 +6,7 @@ from collections import OrderedDict from sys import exit -from typing import Literal, Self +from typing import Self from fastapi import Response from fastapi.responses import PlainTextResponse @@ -230,7 +230,7 @@ def sys_exit_log( exit(exit_code) @property - def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], str]]: + def fastapi_descriptor(self) -> dict[StatusCode, dict[str, str]]: """Gets the FastAPI descriptor (dictionary) of the error items stored in :obj:`ResponseError.__dict`. Example: @@ -247,7 +247,7 @@ def fastapi_descriptor(self) -> dict[StatusCode, dict[Literal["description"], st } """ return { - status: {Literal["description"]: _stringify(msg, self.descriptor_delimiter)} + status: {"description": _stringify(msg, self.descriptor_delimiter)} for status, msg in self.__dict.items() } From a936c8e4d52f73d518ced9a0c97e4e0dcc29dc30 Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 12:47:19 +0200 Subject: [PATCH 96/97] pytest warnings. --- trolldb/test_utils/mongodb_database.py | 5 ++--- trolldb/tests/test_recorder.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index 74e7fe5..79eae9e 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -3,7 +3,6 @@ from contextlib import contextmanager from datetime import datetime, timedelta from random import choices, randint, shuffle -from typing import Iterator from pymongo import MongoClient @@ -12,11 +11,11 @@ @contextmanager -def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database) -> Iterator[MongoClient]: +def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database): """A context manager for the MongoDB client given test configurations. Note: - This is based on `Pymongo` and not the `motor` async driver. For testing purposes this is sufficient and we + This is based on `Pymongo` and not the `motor` async driver. For testing purposes this is sufficient, and we do not need async capabilities. Args: diff --git a/trolldb/tests/test_recorder.py b/trolldb/tests/test_recorder.py index a10b04b..cf90da2 100644 --- a/trolldb/tests/test_recorder.py +++ b/trolldb/tests/test_recorder.py @@ -76,7 +76,7 @@ async def test_record_messages(config_file, tmp_path, file_message, tmp_data_fil with running_prepared_database_context(): with patched_subscriber_recv([file_message]): await record_messages(config) - assert message_in_database_and_delete_count_is_one(msg) + assert await message_in_database_and_delete_count_is_one(msg) async def test_record_deletes_message(tmp_path, file_message, del_message): From 167b588b55d3ea3ea59ad46876cca134ae0389da Mon Sep 17 00:00:00 2001 From: Pouria Khalaj Date: Wed, 22 May 2024 12:50:47 +0200 Subject: [PATCH 97/97] pytest warnings. --- trolldb/test_utils/mongodb_database.py | 7 ++++--- trolldb/tests/tests_api/test_api.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/trolldb/test_utils/mongodb_database.py b/trolldb/test_utils/mongodb_database.py index 79eae9e..d8060e9 100644 --- a/trolldb/test_utils/mongodb_database.py +++ b/trolldb/test_utils/mongodb_database.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from datetime import datetime, timedelta from random import choices, randint, shuffle +from typing import Iterator from pymongo import MongoClient @@ -11,7 +12,7 @@ @contextmanager -def test_mongodb_context(database_config: DatabaseConfig = test_app_config.database): +def mongodb_for_test_context(database_config: DatabaseConfig = test_app_config.database) -> Iterator[MongoClient]: """A context manager for the MongoDB client given test configurations. Note: @@ -161,7 +162,7 @@ def reset(cls): This is done by deleting all documents in the collections and then inserting a single empty ``{}`` document in them. """ - with test_mongodb_context() as client: + with mongodb_for_test_context() as client: for db_name, coll_name in zip(cls.database_names, cls.collection_names, strict=False): db = client[db_name] collection = db[coll_name] @@ -171,7 +172,7 @@ def reset(cls): @classmethod def write_mock_date(cls): """Fills databases/collections with mock data.""" - with test_mongodb_context() as client: + with mongodb_for_test_context() as client: # The following function call has side effects! cls.generate_documents() collection = client[ diff --git a/trolldb/tests/tests_api/test_api.py b/trolldb/tests/tests_api/test_api.py index 2e00376..c721345 100644 --- a/trolldb/tests/tests_api/test_api.py +++ b/trolldb/tests/tests_api/test_api.py @@ -13,7 +13,7 @@ from fastapi import status from trolldb.test_utils.common import http_get -from trolldb.test_utils.mongodb_database import TestDatabase, test_mongodb_context +from trolldb.test_utils.mongodb_database import TestDatabase, mongodb_for_test_context def collections_exists(test_collection_names: list[str], expected_collection_name: list[str]) -> bool: @@ -61,7 +61,7 @@ def test_database_names_negative(): @pytest.mark.usefixtures("_test_server_fixture") def test_collections(): """Checks the presence of existing collections and that the ids of documents therein can be correctly retrieved.""" - with test_mongodb_context() as client: + with mongodb_for_test_context() as client: for database_name, collection_name in zip(TestDatabase.database_names, TestDatabase.collection_names, strict=False): assert collections_exists(