update deployment script and makefile

.github/workflows/deployment.sh

@@ -38,3 +38,11 @@ else
     tmux new-session -s $DAGSTER -d
 fi
 
+LOADER="loader"
+
+# Check if the session exists
+if ! tmux has-session -t $LOADER 2>/dev/null; then
+    # Session doesn't exist, create a new one
+    tmux new-session -s $LOADER -d
+fi
+

Makefile

@@ -12,5 +12,9 @@ sleeper:
 
 deploy: init_deploy sleeper dagster streamlit
 
+loader:
+	tmux send-keys -t loader.0 "source .venv/bin/activate && source .env && cd app/ETL && python loader.py" ENTER
+
+
 cancel_deploy:
 	tmux kill-session -t streamlit & tmux kill-session -t dagster

app/ETL/loader.py

@@ -8,9 +8,6 @@
 import os
 from datetime import datetime, timedelta,timezone
 import logging
-from dagster import asset,AssetExecutionContext
-import duckdb
-import pandas as pd
 from helper.source_env import dotenv_path,raw_path
 import time
 
@@ -203,6 +200,5 @@ def dump_to_file(extract_json):
         # extract_json()
         dump_to_file(extract_json())
         logging.info(f'done loading. next iteration in {sleep_time} seconds...')
-        time.sleep(sleep_time) # test
-        # time.sleep(60*30)
+        time.sleep(sleep_time)