From 71c2c42d51397deaa2bede0e2404a133ef41abdc Mon Sep 17 00:00:00 2001
From: luutuankiet <luutuankiet.ftu2@gmail.com>
Date: Wed, 21 Feb 2024 13:03:43 +0700
Subject: [PATCH] update deployment script and makefile

---
 .github/workflows/deployment.sh | 8 ++++++++
 Makefile                        | 4 ++++
 app/ETL/loader.py               | 6 +-----
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/deployment.sh b/.github/workflows/deployment.sh
index 1294fa3..fb084a5 100755
--- a/.github/workflows/deployment.sh
+++ b/.github/workflows/deployment.sh
@@ -38,3 +38,11 @@ else
     tmux new-session -s $DAGSTER -d
 fi
 
+LOADER="loader"
+
+# Check if the session exists
+if ! tmux has-session -t $LOADER 2>/dev/null; then
+    # Session doesn't exist, create a new one
+    tmux new-session -s $LOADER -d
+fi
+
diff --git a/Makefile b/Makefile
index 1c6f856..2dd9829 100644
--- a/Makefile
+++ b/Makefile
@@ -12,5 +12,9 @@ sleeper:
 
 deploy: init_deploy sleeper dagster streamlit
 
+loader:
+	tmux send-keys -t loader.0 "source .venv/bin/activate && source .env && cd app/ETL && python loader.py" ENTER
+
+
 cancel_deploy:
 	tmux kill-session -t streamlit & tmux kill-session -t dagster
\ No newline at end of file
diff --git a/app/ETL/loader.py b/app/ETL/loader.py
index 883dda3..aa2bf15 100644
--- a/app/ETL/loader.py
+++ b/app/ETL/loader.py
@@ -8,9 +8,6 @@
 import os
 from datetime import datetime, timedelta,timezone
 import logging
-from dagster import asset,AssetExecutionContext
-import duckdb
-import pandas as pd
 from helper.source_env import dotenv_path,raw_path
 import time
 
@@ -203,6 +200,5 @@ def dump_to_file(extract_json):
         # extract_json()
         dump_to_file(extract_json())
         logging.info(f'done loading. next iteration in {sleep_time} seconds...')
-        time.sleep(sleep_time) # test
-        # time.sleep(60*30)
+        time.sleep(sleep_time)
         
\ No newline at end of file