#2: Modernize the project

database-replication/app/gen_data.py

@@ -94,7 +94,7 @@ def update_attributes(self, connector: PsqlConnector) -> None:
 
                 fetch_result = cursor.execute(text(sql_script)).fetchone() or []
                 # Current type: sqlalchemy.engine.row.Row
-                fetch_result = fetch_result[0] or {} # Current type: dict
+                fetch_result = fetch_result[0] or {}  # Current type: dict
 
                 new_attributes = fetch_result.get(self.get_name()) or []
 

database-replication/app/psql_connector.py

@@ -3,18 +3,17 @@
 
 import os
 import logging
-from dotenv import load_dotenv
 
 
-load_dotenv()
-
-
-logger = logging.getLogger(__name__)
-f_handler = logging.FileHandler(os.path.dirname(__file__) + f"/logs/{__name__}.log")
-f_handler.setLevel(logging.INFO)
-f_format = logging.Formatter("[%(asctime)s - %(name)s - %(levelname)s ] %(message)s")
-f_handler.setFormatter(f_format)
-logger.addHandler(f_handler)
+# Init logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="[ %(name)s - %(asctime)s %(levelname)s ] %(message)s",
+    handlers=[
+        logging.FileHandler(os.path.dirname(__file__) + f"/logs/{__name__}.log"),
+        logging.StreamHandler(),
+    ],
+)
 
 
 class PsqlConnector:
@@ -31,10 +30,8 @@ def connect(self):
             self.params["port"],
             self.params["database"],
         )
-        logging.debug(f"Creating config string: {conn_info}")
         db_conn = create_engine(conn_info)
         try:
             yield db_conn
         except Exception as e:
             logging.exception(f"Error when connecting to Postgres: {e}")
-            logger.exception(f"Error when connecting to Postgres: {e}")

database-replication/app/ui.py

@@ -1,4 +1,4 @@
-# ruff: noqa: E501
+from sqlalchemy.util import to_list
 import streamlit as st
 import pandas as pd
 from sqlalchemy import text
@@ -15,7 +15,7 @@
 
 # Init logging
 logging.basicConfig(
-    level=logging.INFO,
+    level=logging.NOTSET,
     format="[ %(name)s - %(asctime)s %(levelname)s ] %(message)s",
     handlers=[logging.FileHandler("./logs/streamlit.log"), logging.StreamHandler()],
 )
@@ -86,13 +86,14 @@
 tables = []
 with psql_connector.connect() as engine:
     with engine.connect() as cursor:
-        logging.info("Connected to database")
+        logging.debug("Connected to database")
         sql_script = f"""
             SELECT table_name
             FROM information_schema.tables
             WHERE table_schema = '{st_schema}';
         """
-        tables = cursor.execute(text(sql_script)).fetchall()
+        tables = to_list(cursor.execute(text(sql_script)).fetchall())
+        logging.debug(f"Tables type: {type(tables)}")
         # Remove outlines pattern for each table
         for table in tables:
             tables[tables.index(table)] = table[0]
@@ -105,14 +106,14 @@
 if st.toggle("View first 10 rows"):
     with psql_connector.connect() as engine:
         with engine.connect() as cursor:
-            logging.info("Connected to database")
+            logging.debug("Connected to database")
             sql_script = f"""
                 SELECT *
                 FROM {st_schema}.{st_table}
                 LIMIT 10;
             """
             df = cursor.execute(text(sql_script)).fetchall()
-            # Turn df to pandas dataframe
+            # Turn df to pandas.DataFrame
             df = pd.DataFrame(df)
             st.dataframe(df)
             sql_script = f"""

database-replication/pyproject.toml

@@ -0,0 +1,56 @@
+[project]
+name = "database-replication"
+version = "0.1"
+description = ""
+authors = ["lelouvincx [email protected]"]
+
+[project.dependencies]
+Faker = "^19.6.0"
+streamlit = "^1.26.0"
+SQLAlchemy = "^2.0.20"
+confluent-kafka = "^2.2.0"
+ruff = "^0.0.287"
+black = "^23.9.1"
+pytest = "^7.4.2"
+
+[tool.ruff]
+# Enable rule pycodestyle
+select = ["E"]
+ignore = ["E501", "E101"]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Maximum line length is same as black
+line-length = 88
+src = ["app"]
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+per-file-ignores = {}
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"