Moved the database specific modules in a folder named databases.

Session on the sqlite autocommits.

mipdb/commands.py

@@ -4,9 +4,10 @@
 import os
 import glob
 
-from mipdb.monetdb import MonetDB, credentials_from_config
+from mipdb.databases import credentials_from_config
+from mipdb.databases.monetdb import MonetDB
 from mipdb.reader import JsonFileReader
-from mipdb.sqlite import SQLiteDB
+from mipdb.databases.sqlite import SQLiteDB
 from mipdb.usecases import (
     AddDataModel,
     Cleanup,

mipdb/databases/__init__.py

@@ -0,0 +1,22 @@
+import os
+
+import toml
+
+CONFIG_PATH = "/home/config.toml"
+
+
+def credentials_from_config():
+    try:
+        return toml.load(os.getenv("CONFIG_PATH", CONFIG_PATH))
+    except FileNotFoundError:
+        return {
+            "DB_IP": "",
+            "DB_PORT": "",
+            "MONETDB_ADMIN_USERNAME": "",
+            "MONETDB_LOCAL_USERNAME": "",
+            "MONETDB_LOCAL_PASSWORD": "",
+            "MONETDB_PUBLIC_USERNAME": "",
+            "MONETDB_PUBLIC_PASSWORD": "",
+            "DB_NAME": "",
+            "SQLITE_DB_PATH": "",
+        }

mipdb/monetdb.py → mipdb/databases/monetdb.py

@@ -1,14 +1,11 @@
-import ipaddress
 from contextlib import contextmanager
 from typing import Union
 
 import sqlalchemy as sql
-import toml
 
 from mipdb.exceptions import DataBaseError
 
 PRIMARYDATA_TABLE = "primary_data"
-CONFIG = "/home/config.toml"
 
 
 def handle_errors(func):
@@ -133,23 +130,6 @@ def __init__(self, conn: sql.engine.Connection) -> None:
         super().__init__(conn)
 
 
-def credentials_from_config():
-    try:
-        return toml.load(CONFIG)
-    except FileNotFoundError:
-        return {
-            "DB_IP": "",
-            "DB_PORT": "",
-            "MONETDB_ADMIN_USERNAME": "",
-            "MONETDB_LOCAL_USERNAME": "",
-            "MONETDB_LOCAL_PASSWORD": "",
-            "MONETDB_PUBLIC_USERNAME": "",
-            "MONETDB_PUBLIC_PASSWORD": "",
-            "DB_NAME": "",
-            "SQLITE_DB_PATH": "",
-        }
-
-
 class MonetDB(DBExecutor):
     """Concrete DataBase object for connecting to a MonetDB instance."""
 

mipdb/monetdb_tables.py → mipdb/databases/monetdb_tables.py

@@ -1,13 +1,13 @@
 from abc import ABC, abstractmethod
 import json
 from enum import Enum
-from typing import Union, List
+from typing import List
 
 import sqlalchemy as sql
 from sqlalchemy import MetaData
 
-from mipdb.monetdb import credentials_from_config
 from mipdb.data_frame import DATASET_COLUMN_NAME
+from mipdb.databases import credentials_from_config
 from mipdb.dataelements import CommonDataElement
 from mipdb.exceptions import UserInputError
 from mipdb.schema import Schema

mipdb/sqlite.py → mipdb/databases/sqlite.py

@@ -1,12 +1,9 @@
-from contextlib import contextmanager
 from typing import List, Any, Dict
 from enum import Enum
-import json
 
 import sqlalchemy as sql
 from sqlalchemy import MetaData, ForeignKey, inspect
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm.exc import MultipleResultsFound
 
@@ -75,7 +72,7 @@ class SQLiteDB:
 
     def __init__(self, url: str, echo=False) -> None:
         self._executor = sql.create_engine(url, echo=echo)
-        self.Session = sessionmaker(bind=self._executor)
+        self.Session = sessionmaker(bind=self._executor, autocommit=True)
 
     @classmethod
     def from_config(cls, dbconfig: dict) -> "SQLiteDB":
@@ -85,23 +82,20 @@ def from_config(cls, dbconfig: dict) -> "SQLiteDB":
 
     @handle_errors
     def execute(self, query: str, *args, **kwargs) -> None:
-        conn = self._executor.connect()
-        conn.execute(sql.text(query), *args, **kwargs)
-        conn.close()
+        with self._executor.connect() as conn:
+            conn = conn.execution_options(autocommit=True)
+            conn.execute(sql.text(query), *args, **kwargs)
 
     @handle_errors
     def execute_fetchall(self, query: str, *args, **kwargs) -> List[dict]:
-        conn = self._executor.connect()
-        result = conn.execute(sql.text(query), *args, **kwargs)
-        result = result.fetchall() if result else []
-        conn.close()
-        return result
+        with self._executor.connect() as conn:
+            result = conn.execute(sql.text(query), *args, **kwargs)
+            return result.fetchall() if result else []
 
     def insert_values_to_table(self, table: sql.Table, values: List[dict]) -> None:
         session = self.Session()
         try:
             session.execute(table.insert(), values)
-            session.commit()
         finally:
             session.close()
 
@@ -124,7 +118,7 @@ def update_data_model_status(self, status: str, data_model_id: int) -> None:
             session.query(DataModel).filter(
                 DataModel.data_model_id == data_model_id
             ).update({"status": status})
-            session.commit()
+
         finally:
             session.close()
 
@@ -163,7 +157,7 @@ def update_dataset_status(self, status: str, dataset_id: int) -> None:
             session.query(Dataset).filter(Dataset.dataset_id == dataset_id).update(
                 {"status": status}
             )
-            session.commit()
+
         finally:
             session.close()
 
@@ -244,7 +238,7 @@ def drop_table(self, table: str) -> None:
         try:
             table = sql.Table(table, metadata, autoload_with=self._executor)
             table.drop(bind=self._executor)
-            session.commit()
+
         finally:
             session.close()
 
@@ -257,10 +251,6 @@ def delete_from(self, table, where_conditions: Dict[str, Any]) -> None:
                     query = query.filter(getattr(table.c, col) == value)
 
             query.delete(synchronize_session=False)
-            session.commit()
-        except Exception as e:
-            session.rollback()
-            raise
         finally:
             session.close()
 
@@ -294,7 +284,7 @@ def set_data_model_properties(self, properties: dict, data_model_id: int) -> Non
             session.query(DataModel).filter(
                 DataModel.data_model_id == data_model_id
             ).update({"properties": properties})
-            session.commit()
+
         finally:
             session.close()
 
@@ -304,7 +294,7 @@ def set_dataset_properties(self, properties: dict, dataset_id: int) -> None:
             session.query(Dataset).filter(Dataset.dataset_id == dataset_id).update(
                 {"properties": properties}
             )
-            session.commit()
+
         finally:
             session.close()
 

mipdb/sqlite_tables.py → mipdb/databases/sqlite_tables.py

@@ -1,12 +1,11 @@
 from abc import ABC, abstractmethod
-import json
 
 import sqlalchemy as sql
-from sqlalchemy import Column, Integer, String, JSON, MetaData
-from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
+from sqlalchemy import Integer, String, JSON
+from sqlalchemy.ext.declarative import declarative_base
 from mipdb.dataelements import CommonDataElement
 from mipdb.exceptions import DataBaseError
-from mipdb.sqlite import DataModel, Dataset, SQLiteDB
+from mipdb.databases.sqlite import DataModel, Dataset
 
 METADATA_TABLE = "variables_metadata"
 PRIMARYDATA_TABLE = "primary_data"
@@ -192,6 +191,4 @@ def get_values_from_cdes(cdes):
         return [{"code": cde.code, "metadata": cde.metadata} for cde in cdes]
 
     def insert_values(self, values, db):
-        # Needs to be overridden because sqlalchemy and monetdb are not cooperating
-        # well when inserting values to JSON columns
         db.execute(f'INSERT INTO "{self.name}" VALUES(:code, :metadata)', values)

mipdb/usecases.py

@@ -4,10 +4,14 @@
 
 import pandas as pd
 
-from mipdb.monetdb import MonetDB
+from mipdb.databases.monetdb import MonetDB
 from mipdb.data_frame_schema import DataFrameSchema
 from mipdb.exceptions import ForeignKeyError, InvalidDatasetError, UserInputError
-from mipdb.monetdb_tables import PrimaryDataTable, TemporaryTable, RECORDS_PER_COPY
+from mipdb.databases.monetdb_tables import (
+    PrimaryDataTable,
+    TemporaryTable,
+    RECORDS_PER_COPY,
+)
 from mipdb.properties import Properties
 from mipdb.reader import CSVDataFrameReader
 from mipdb.dataelements import (
@@ -20,8 +24,8 @@
     get_dataset_enums,
 )
 from mipdb.schema import Schema
-from mipdb.sqlite import SQLiteDB
-from mipdb.sqlite_tables import DataModelTable, DatasetsTable, MetadataTable
+from mipdb.databases.sqlite import SQLiteDB
+from mipdb.databases.sqlite_tables import DataModelTable, DatasetsTable, MetadataTable
 from mipdb.data_frame import DataFrame, DATASET_COLUMN_NAME
 
 LONGITUDINAL = "longitudinal"

tests/conftest.py

@@ -5,10 +5,10 @@
 import docker
 
 from mipdb.commands import get_monetdb_config
-from mipdb.monetdb import MonetDB
-from mipdb.monetdb_tables import User
+from mipdb.databases.monetdb import MonetDB
+from mipdb.databases.monetdb_tables import User
 from mipdb.reader import JsonFileReader
-from mipdb.sqlite import SQLiteDB
+from mipdb.databases.sqlite import SQLiteDB
 
 TEST_DIR = os.path.dirname(os.path.realpath(__file__))
 SQLiteDB_PATH = f"{TEST_DIR}/sqlite.db"

tests/test_commands.py

@@ -20,8 +20,8 @@
 from mipdb import validate_dataset
 from mipdb.commands import validate_folder
 from mipdb.exceptions import ExitCode
-from mipdb.sqlite import Dataset, DataModel
-from mipdb.sqlite_tables import DataModelTable
+from mipdb.databases.sqlite import Dataset, DataModel
+from mipdb.databases.sqlite_tables import DataModelTable
 from tests.conftest import (
     DATASET_FILE,
     ABSOLUTE_PATH_DATASET_FILE,

tests/test_database.py

@@ -6,7 +6,7 @@
 import pytest
 
 from mipdb.exceptions import DataBaseError
-from mipdb.sqlite import DataModel, Dataset
+from mipdb.databases.sqlite import DataModel, Dataset
 from tests.conftest import DATASET_FILE, MONETDB_OPTIONS, SQLiteDB_OPTION
 from tests.conftest import DATA_MODEL_FILE
 

tests/test_schema.py

@@ -3,7 +3,7 @@
 import pytest
 
 from mipdb.exceptions import UserInputError
-from mipdb.monetdb import MonetDB
+from mipdb.databases.monetdb import MonetDB
 from mipdb.schema import Schema
 
 

tests/test_tables.py

@@ -1,14 +1,11 @@
-import json
-
 from mipdb.exceptions import DataBaseError
 
 import pytest
 
-from mipdb.monetdb_tables import PrimaryDataTable
+from mipdb.databases.monetdb_tables import PrimaryDataTable
 from mipdb.schema import Schema
-from mipdb.sqlite_tables import (
+from mipdb.databases.sqlite_tables import (
     DataModelTable,
-    DatasetsTable,
     MetadataTable,
 )
 from mipdb.dataelements import CommonDataElement, flatten_cdes

tests/test_usecases.py

@@ -1,14 +1,13 @@
-import json
 from unittest.mock import patch
 
 import pandas as pd
 import pytest
 
-from mipdb.monetdb import MonetDB
+from mipdb.databases.monetdb import MonetDB
 from mipdb.exceptions import ForeignKeyError, DataBaseError, InvalidDatasetError
 from mipdb.exceptions import UserInputError
-from mipdb.sqlite import Dataset
-from mipdb.sqlite_tables import DataModelTable, DatasetsTable
+from mipdb.databases.sqlite import Dataset
+from mipdb.databases.sqlite_tables import DataModelTable, DatasetsTable
 from mipdb.usecases import (
     AddPropertyToDataset,
     check_unique_longitudinal_dataset_primary_keys,
@@ -294,7 +293,7 @@ def test_add_dataset_with_small_record_copy(sqlite_db, monetdb, data_model_metad
     # Setup
     InitDB(sqlite_db).execute()
     AddDataModel(sqlite_db, monetdb).execute(data_model_metadata)
-    with patch("mipdb.monetdb_tables.RECORDS_PER_COPY", 1):
+    with patch("mipdb.databases.monetdb_tables.RECORDS_PER_COPY", 1):
         # Test
         ImportCSV(sqlite_db, monetdb).execute(
             csv_path=DATASET_FILE,
@@ -316,7 +315,7 @@ def test_add_dataset_with_small_record_copy_with_volume(
     # Setup
     InitDB(sqlite_db).execute()
     AddDataModel(sqlite_db, monetdb).execute(data_model_metadata)
-    with patch("mipdb.monetdb_tables.RECORDS_PER_COPY", 1):
+    with patch("mipdb.databases.monetdb_tables.RECORDS_PER_COPY", 1):
         # Test
         ImportCSV(sqlite_db, monetdb).execute(
             csv_path=ABSOLUTE_PATH_DATASET_FILE,
@@ -336,7 +335,7 @@ def test_csv_legnth_equals_records_per_copy(sqlite_db, monetdb, data_model_metad
     # Setup
     InitDB(sqlite_db).execute()
     AddDataModel(sqlite_db, monetdb).execute(data_model_metadata)
-    with patch("mipdb.monetdb_tables.RECORDS_PER_COPY", 5):
+    with patch("mipdb.databases.monetdb_tables.RECORDS_PER_COPY", 5):
         # Test
         ImportCSV(sqlite_db, monetdb).execute(
             csv_path=ABSOLUTE_PATH_DATASET_FILE,