From 7b59dbaa810a46e7d2a3ec80464c3e350bd4d339 Mon Sep 17 00:00:00 2001 From: kfilippopolitis Date: Wed, 26 Jun 2024 00:35:33 +0300 Subject: [PATCH] Moved the database specific modules in a folder named databases. Session on the sqlite autocommits. --- mipdb/commands.py | 5 ++-- mipdb/databases/__init__.py | 22 ++++++++++++++++ mipdb/{ => databases}/monetdb.py | 20 --------------- mipdb/{ => databases}/monetdb_tables.py | 4 +-- mipdb/{ => databases}/sqlite.py | 34 +++++++++---------------- mipdb/{ => databases}/sqlite_tables.py | 9 +++---- mipdb/usecases.py | 12 ++++++--- tests/conftest.py | 6 ++--- tests/test_commands.py | 4 +-- tests/test_database.py | 2 +- tests/test_schema.py | 2 +- tests/test_tables.py | 7 ++--- tests/test_usecases.py | 13 +++++----- 13 files changed, 65 insertions(+), 75 deletions(-) create mode 100644 mipdb/databases/__init__.py rename mipdb/{ => databases}/monetdb.py (90%) rename mipdb/{ => databases}/monetdb_tables.py (98%) rename mipdb/{ => databases}/sqlite.py (93%) rename mipdb/{ => databases}/sqlite_tables.py (94%) diff --git a/mipdb/commands.py b/mipdb/commands.py index e3c16d2..5e23088 100644 --- a/mipdb/commands.py +++ b/mipdb/commands.py @@ -4,9 +4,10 @@ import os import glob -from mipdb.monetdb import MonetDB, credentials_from_config +from mipdb.databases import credentials_from_config +from mipdb.databases.monetdb import MonetDB from mipdb.reader import JsonFileReader -from mipdb.sqlite import SQLiteDB +from mipdb.databases.sqlite import SQLiteDB from mipdb.usecases import ( AddDataModel, Cleanup, diff --git a/mipdb/databases/__init__.py b/mipdb/databases/__init__.py new file mode 100644 index 0000000..233e890 --- /dev/null +++ b/mipdb/databases/__init__.py @@ -0,0 +1,22 @@ +import os + +import toml + +CONFIG_PATH = "/home/config.toml" + + +def credentials_from_config(): + try: + return toml.load(os.getenv("CONFIG_PATH", CONFIG_PATH)) + except FileNotFoundError: + return { + "DB_IP": "", + "DB_PORT": "", + "MONETDB_ADMIN_USERNAME": "", + "MONETDB_LOCAL_USERNAME": "", + "MONETDB_LOCAL_PASSWORD": "", + "MONETDB_PUBLIC_USERNAME": "", + "MONETDB_PUBLIC_PASSWORD": "", + "DB_NAME": "", + "SQLITE_DB_PATH": "", + } diff --git a/mipdb/monetdb.py b/mipdb/databases/monetdb.py similarity index 90% rename from mipdb/monetdb.py rename to mipdb/databases/monetdb.py index e174395..4113ca7 100644 --- a/mipdb/monetdb.py +++ b/mipdb/databases/monetdb.py @@ -1,14 +1,11 @@ -import ipaddress from contextlib import contextmanager from typing import Union import sqlalchemy as sql -import toml from mipdb.exceptions import DataBaseError PRIMARYDATA_TABLE = "primary_data" -CONFIG = "/home/config.toml" def handle_errors(func): @@ -133,23 +130,6 @@ def __init__(self, conn: sql.engine.Connection) -> None: super().__init__(conn) -def credentials_from_config(): - try: - return toml.load(CONFIG) - except FileNotFoundError: - return { - "DB_IP": "", - "DB_PORT": "", - "MONETDB_ADMIN_USERNAME": "", - "MONETDB_LOCAL_USERNAME": "", - "MONETDB_LOCAL_PASSWORD": "", - "MONETDB_PUBLIC_USERNAME": "", - "MONETDB_PUBLIC_PASSWORD": "", - "DB_NAME": "", - "SQLITE_DB_PATH": "", - } - - class MonetDB(DBExecutor): """Concrete DataBase object for connecting to a MonetDB instance.""" diff --git a/mipdb/monetdb_tables.py b/mipdb/databases/monetdb_tables.py similarity index 98% rename from mipdb/monetdb_tables.py rename to mipdb/databases/monetdb_tables.py index 78f89e3..dff0b2f 100644 --- a/mipdb/monetdb_tables.py +++ b/mipdb/databases/monetdb_tables.py @@ -1,13 +1,13 @@ from abc import ABC, abstractmethod import json from enum import Enum -from typing import Union, List +from typing import List import sqlalchemy as sql from sqlalchemy import MetaData -from mipdb.monetdb import credentials_from_config from mipdb.data_frame import DATASET_COLUMN_NAME +from mipdb.databases import credentials_from_config from mipdb.dataelements import CommonDataElement from mipdb.exceptions import UserInputError from mipdb.schema import Schema diff --git a/mipdb/sqlite.py b/mipdb/databases/sqlite.py similarity index 93% rename from mipdb/sqlite.py rename to mipdb/databases/sqlite.py index f35f35e..0bed12f 100644 --- a/mipdb/sqlite.py +++ b/mipdb/databases/sqlite.py @@ -1,12 +1,9 @@ -from contextlib import contextmanager from typing import List, Any, Dict from enum import Enum -import json import sqlalchemy as sql from sqlalchemy import MetaData, ForeignKey, inspect from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker from sqlalchemy.orm.exc import MultipleResultsFound @@ -75,7 +72,7 @@ class SQLiteDB: def __init__(self, url: str, echo=False) -> None: self._executor = sql.create_engine(url, echo=echo) - self.Session = sessionmaker(bind=self._executor) + self.Session = sessionmaker(bind=self._executor, autocommit=True) @classmethod def from_config(cls, dbconfig: dict) -> "SQLiteDB": @@ -85,23 +82,20 @@ def from_config(cls, dbconfig: dict) -> "SQLiteDB": @handle_errors def execute(self, query: str, *args, **kwargs) -> None: - conn = self._executor.connect() - conn.execute(sql.text(query), *args, **kwargs) - conn.close() + with self._executor.connect() as conn: + conn = conn.execution_options(autocommit=True) + conn.execute(sql.text(query), *args, **kwargs) @handle_errors def execute_fetchall(self, query: str, *args, **kwargs) -> List[dict]: - conn = self._executor.connect() - result = conn.execute(sql.text(query), *args, **kwargs) - result = result.fetchall() if result else [] - conn.close() - return result + with self._executor.connect() as conn: + result = conn.execute(sql.text(query), *args, **kwargs) + return result.fetchall() if result else [] def insert_values_to_table(self, table: sql.Table, values: List[dict]) -> None: session = self.Session() try: session.execute(table.insert(), values) - session.commit() finally: session.close() @@ -124,7 +118,7 @@ def update_data_model_status(self, status: str, data_model_id: int) -> None: session.query(DataModel).filter( DataModel.data_model_id == data_model_id ).update({"status": status}) - session.commit() + finally: session.close() @@ -163,7 +157,7 @@ def update_dataset_status(self, status: str, dataset_id: int) -> None: session.query(Dataset).filter(Dataset.dataset_id == dataset_id).update( {"status": status} ) - session.commit() + finally: session.close() @@ -244,7 +238,7 @@ def drop_table(self, table: str) -> None: try: table = sql.Table(table, metadata, autoload_with=self._executor) table.drop(bind=self._executor) - session.commit() + finally: session.close() @@ -257,10 +251,6 @@ def delete_from(self, table, where_conditions: Dict[str, Any]) -> None: query = query.filter(getattr(table.c, col) == value) query.delete(synchronize_session=False) - session.commit() - except Exception as e: - session.rollback() - raise finally: session.close() @@ -294,7 +284,7 @@ def set_data_model_properties(self, properties: dict, data_model_id: int) -> Non session.query(DataModel).filter( DataModel.data_model_id == data_model_id ).update({"properties": properties}) - session.commit() + finally: session.close() @@ -304,7 +294,7 @@ def set_dataset_properties(self, properties: dict, dataset_id: int) -> None: session.query(Dataset).filter(Dataset.dataset_id == dataset_id).update( {"properties": properties} ) - session.commit() + finally: session.close() diff --git a/mipdb/sqlite_tables.py b/mipdb/databases/sqlite_tables.py similarity index 94% rename from mipdb/sqlite_tables.py rename to mipdb/databases/sqlite_tables.py index eae49cb..8aa51bb 100644 --- a/mipdb/sqlite_tables.py +++ b/mipdb/databases/sqlite_tables.py @@ -1,12 +1,11 @@ from abc import ABC, abstractmethod -import json import sqlalchemy as sql -from sqlalchemy import Column, Integer, String, JSON, MetaData -from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta +from sqlalchemy import Integer, String, JSON +from sqlalchemy.ext.declarative import declarative_base from mipdb.dataelements import CommonDataElement from mipdb.exceptions import DataBaseError -from mipdb.sqlite import DataModel, Dataset, SQLiteDB +from mipdb.databases.sqlite import DataModel, Dataset METADATA_TABLE = "variables_metadata" PRIMARYDATA_TABLE = "primary_data" @@ -192,6 +191,4 @@ def get_values_from_cdes(cdes): return [{"code": cde.code, "metadata": cde.metadata} for cde in cdes] def insert_values(self, values, db): - # Needs to be overridden because sqlalchemy and monetdb are not cooperating - # well when inserting values to JSON columns db.execute(f'INSERT INTO "{self.name}" VALUES(:code, :metadata)', values) diff --git a/mipdb/usecases.py b/mipdb/usecases.py index 93a6f3d..9a78fc7 100644 --- a/mipdb/usecases.py +++ b/mipdb/usecases.py @@ -4,10 +4,14 @@ import pandas as pd -from mipdb.monetdb import MonetDB +from mipdb.databases.monetdb import MonetDB from mipdb.data_frame_schema import DataFrameSchema from mipdb.exceptions import ForeignKeyError, InvalidDatasetError, UserInputError -from mipdb.monetdb_tables import PrimaryDataTable, TemporaryTable, RECORDS_PER_COPY +from mipdb.databases.monetdb_tables import ( + PrimaryDataTable, + TemporaryTable, + RECORDS_PER_COPY, +) from mipdb.properties import Properties from mipdb.reader import CSVDataFrameReader from mipdb.dataelements import ( @@ -20,8 +24,8 @@ get_dataset_enums, ) from mipdb.schema import Schema -from mipdb.sqlite import SQLiteDB -from mipdb.sqlite_tables import DataModelTable, DatasetsTable, MetadataTable +from mipdb.databases.sqlite import SQLiteDB +from mipdb.databases.sqlite_tables import DataModelTable, DatasetsTable, MetadataTable from mipdb.data_frame import DataFrame, DATASET_COLUMN_NAME LONGITUDINAL = "longitudinal" diff --git a/tests/conftest.py b/tests/conftest.py index 1c6cfdb..4231b15 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,10 @@ import docker from mipdb.commands import get_monetdb_config -from mipdb.monetdb import MonetDB -from mipdb.monetdb_tables import User +from mipdb.databases.monetdb import MonetDB +from mipdb.databases.monetdb_tables import User from mipdb.reader import JsonFileReader -from mipdb.sqlite import SQLiteDB +from mipdb.databases.sqlite import SQLiteDB TEST_DIR = os.path.dirname(os.path.realpath(__file__)) SQLiteDB_PATH = f"{TEST_DIR}/sqlite.db" diff --git a/tests/test_commands.py b/tests/test_commands.py index 5dcdb44..019cd8f 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -20,8 +20,8 @@ from mipdb import validate_dataset from mipdb.commands import validate_folder from mipdb.exceptions import ExitCode -from mipdb.sqlite import Dataset, DataModel -from mipdb.sqlite_tables import DataModelTable +from mipdb.databases.sqlite import Dataset, DataModel +from mipdb.databases.sqlite_tables import DataModelTable from tests.conftest import ( DATASET_FILE, ABSOLUTE_PATH_DATASET_FILE, diff --git a/tests/test_database.py b/tests/test_database.py index 58c5b5a..f17a242 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -6,7 +6,7 @@ import pytest from mipdb.exceptions import DataBaseError -from mipdb.sqlite import DataModel, Dataset +from mipdb.databases.sqlite import DataModel, Dataset from tests.conftest import DATASET_FILE, MONETDB_OPTIONS, SQLiteDB_OPTION from tests.conftest import DATA_MODEL_FILE diff --git a/tests/test_schema.py b/tests/test_schema.py index 309b398..5e2993e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -3,7 +3,7 @@ import pytest from mipdb.exceptions import UserInputError -from mipdb.monetdb import MonetDB +from mipdb.databases.monetdb import MonetDB from mipdb.schema import Schema diff --git a/tests/test_tables.py b/tests/test_tables.py index a9a2394..858f5be 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -1,14 +1,11 @@ -import json - from mipdb.exceptions import DataBaseError import pytest -from mipdb.monetdb_tables import PrimaryDataTable +from mipdb.databases.monetdb_tables import PrimaryDataTable from mipdb.schema import Schema -from mipdb.sqlite_tables import ( +from mipdb.databases.sqlite_tables import ( DataModelTable, - DatasetsTable, MetadataTable, ) from mipdb.dataelements import CommonDataElement, flatten_cdes diff --git a/tests/test_usecases.py b/tests/test_usecases.py index ab8b66f..f02bc9c 100644 --- a/tests/test_usecases.py +++ b/tests/test_usecases.py @@ -1,14 +1,13 @@ -import json from unittest.mock import patch import pandas as pd import pytest -from mipdb.monetdb import MonetDB +from mipdb.databases.monetdb import MonetDB from mipdb.exceptions import ForeignKeyError, DataBaseError, InvalidDatasetError from mipdb.exceptions import UserInputError -from mipdb.sqlite import Dataset -from mipdb.sqlite_tables import DataModelTable, DatasetsTable +from mipdb.databases.sqlite import Dataset +from mipdb.databases.sqlite_tables import DataModelTable, DatasetsTable from mipdb.usecases import ( AddPropertyToDataset, check_unique_longitudinal_dataset_primary_keys, @@ -294,7 +293,7 @@ def test_add_dataset_with_small_record_copy(sqlite_db, monetdb, data_model_metad # Setup InitDB(sqlite_db).execute() AddDataModel(sqlite_db, monetdb).execute(data_model_metadata) - with patch("mipdb.monetdb_tables.RECORDS_PER_COPY", 1): + with patch("mipdb.databases.monetdb_tables.RECORDS_PER_COPY", 1): # Test ImportCSV(sqlite_db, monetdb).execute( csv_path=DATASET_FILE, @@ -316,7 +315,7 @@ def test_add_dataset_with_small_record_copy_with_volume( # Setup InitDB(sqlite_db).execute() AddDataModel(sqlite_db, monetdb).execute(data_model_metadata) - with patch("mipdb.monetdb_tables.RECORDS_PER_COPY", 1): + with patch("mipdb.databases.monetdb_tables.RECORDS_PER_COPY", 1): # Test ImportCSV(sqlite_db, monetdb).execute( csv_path=ABSOLUTE_PATH_DATASET_FILE, @@ -336,7 +335,7 @@ def test_csv_legnth_equals_records_per_copy(sqlite_db, monetdb, data_model_metad # Setup InitDB(sqlite_db).execute() AddDataModel(sqlite_db, monetdb).execute(data_model_metadata) - with patch("mipdb.monetdb_tables.RECORDS_PER_COPY", 5): + with patch("mipdb.databases.monetdb_tables.RECORDS_PER_COPY", 5): # Test ImportCSV(sqlite_db, monetdb).execute( csv_path=ABSOLUTE_PATH_DATASET_FILE,