diff --git a/.github/workflows/deployment.sh b/.github/workflows/deployment.sh index 1294fa3..fb084a5 100755 --- a/.github/workflows/deployment.sh +++ b/.github/workflows/deployment.sh @@ -38,3 +38,11 @@ else tmux new-session -s $DAGSTER -d fi +LOADER="loader" + +# Check if the session exists +if ! tmux has-session -t $LOADER 2>/dev/null; then + # Session doesn't exist, create a new one + tmux new-session -s $LOADER -d +fi + diff --git a/Makefile b/Makefile index 1c6f856..2dd9829 100644 --- a/Makefile +++ b/Makefile @@ -12,5 +12,9 @@ sleeper: deploy: init_deploy sleeper dagster streamlit +loader: + tmux send-keys -t loader.0 "source .venv/bin/activate && source .env && cd app/ETL && python loader.py" ENTER + + cancel_deploy: tmux kill-session -t streamlit & tmux kill-session -t dagster \ No newline at end of file diff --git a/app/ETL/loader.py b/app/ETL/loader.py index 883dda3..aa2bf15 100644 --- a/app/ETL/loader.py +++ b/app/ETL/loader.py @@ -8,9 +8,6 @@ import os from datetime import datetime, timedelta,timezone import logging -from dagster import asset,AssetExecutionContext -import duckdb -import pandas as pd from helper.source_env import dotenv_path,raw_path import time @@ -203,6 +200,5 @@ def dump_to_file(extract_json): # extract_json() dump_to_file(extract_json()) logging.info(f'done loading. next iteration in {sleep_time} seconds...') - time.sleep(sleep_time) # test - # time.sleep(60*30) + time.sleep(sleep_time) \ No newline at end of file