refactor: remove stage.py, do DB committing in fewer steps

coursetable · Oct 19, 2024 · 33b4392 · 33b4392
1 parent a68b542
commit 33b4392
Show file tree

Hide file tree

Showing 10 changed files with 176 additions and 300 deletions.
diff --git a/docs/code-structure.md b/docs/code-structure.md
@@ -32,12 +32,4 @@ If the `--snapshot-tables` argument is used, the analyzer will create a CSV file
 
 ## Database connector
 
-The database connector code is located in `ferry/database`.
-
-- `stage.py`: stages the data in the database.
-- `deploy.py`: checks table invariants, and regenerates the database based on the staged data. At this step we also run SQL to generate some derived tables.
-
-TODO:
-
-1. Move the SQL logic to the data analyzer.
-2. Ferry should not regenerate the whole DB each time.
+The database connector code is located in `ferry/database`. It takes the Pandas tables and imports them into the database.
diff --git a/docs/db_diagram.pdf b/docs/db_diagram.pdf
diff --git a/ferry/database/__init__.py b/ferry/database/__init__.py
@@ -18,5 +18,4 @@
     course_flags,
     course_professors,
 )
-from .stage import stage
-from .deploy import deploy
+from .sync_db import sync_db
diff --git a/ferry/database/database.py b/ferry/database/database.py
@@ -21,11 +21,6 @@ def __init__(self, connect_string: str):
                 "keepalives_count": 5,
             },
         )
-        Base.metadata.create_all(self.Engine)
-
-        if any(not table.endswith("_staged") for table in Base.metadata.tables):
-            raise MissingTablesError("Model tables should all end with _staged")
-
         self.Session = sqlalchemy.orm.sessionmaker(bind=self.Engine)
 
 

diff --git a/ferry/database/deploy.py b/ferry/database/deploy.py