From 008847f1708164a693c6722976b12a47c84706d6 Mon Sep 17 00:00:00 2001 From: luutuankiet Date: Wed, 28 Aug 2024 01:24:06 +0000 Subject: [PATCH] init setup uses env var prod from secret while bootstrap paths from a shell script init --- .devcontainer/postCreateCommand.sh | 4 +-- .github/workflows/close-preview.yml | 29 +++++++++++++++++++ .github/workflows/deployment.sh | 2 +- .github/workflows/gh_deploy.yml | 41 ++++++++++++++++++++++----- .github/workflows/start-preview.yml | 33 ++++++++++++++++++++-- .vscode/settings.json | 2 +- Makefile | 10 +++---- README.md | 4 +++ app/ETL/loader.py | 4 +-- app/helper/source_env.py | 43 +++++++++++++++++++---------- bootstrap_env.sh | 6 ++++ 11 files changed, 144 insertions(+), 34 deletions(-) diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index d7f5ba5..7c25165 100644 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -48,9 +48,9 @@ apt-get update && apt-get install -y python3-venv # init then source env vars print_section "INIT & SOURCE ENV VARS" -chmod +x ./env.sh +chmod +x ./bootstrap_env.sh -. ./env.sh +. ./bootstrap_env.sh diff --git a/.github/workflows/close-preview.yml b/.github/workflows/close-preview.yml index a793554..df376df 100644 --- a/.github/workflows/close-preview.yml +++ b/.github/workflows/close-preview.yml @@ -12,6 +12,35 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3.4.1 + - name: bootstrap environment variables + id: load-env + run: | + # first step init the file + . ./bootstrap_env.sh + + # Check if .env.bootstrap file exists + if [ -f .env.bootstrap ]; then + # Process each line in the .env.bootstrap file + while IFS= read -r line; do + # Skip empty lines and lines starting with # + if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then + # Export the variable to $GITHUB_ENV + echo "$line" >> $GITHUB_ENV + fi + done < .env.bootstrap + else + echo ".env.bootstrap file not found" + fi + + + - name: Create .env file from secret + run: | + echo "$PROD_ENV" > .env + env: + PROD_ENV: ${{ secrets.PROD_ENV }} + + + - name: Get lightdash version uses: sergeysova/jq-action@v2 diff --git a/.github/workflows/deployment.sh b/.github/workflows/deployment.sh index a50858f..dafb69d 100755 --- a/.github/workflows/deployment.sh +++ b/.github/workflows/deployment.sh @@ -4,7 +4,7 @@ WORKDIR=$(pwd) . $WORKDIR/.venv/bin/activate -. $WORKDIR/env.sh +. $WORKDIR/bootstrap_env.sh # # setup sessions for service diff --git a/.github/workflows/gh_deploy.yml b/.github/workflows/gh_deploy.yml index ebfaaa7..4fcc762 100644 --- a/.github/workflows/gh_deploy.yml +++ b/.github/workflows/gh_deploy.yml @@ -6,8 +6,8 @@ on: env: DBT_VERSION: "1.8.1" - PROJECT_DIR: "./dbt_project" - DW_DBNAME: "gtd_dash" + # PROJECT_DIR: "./dbt_project" + # DW_DBNAME: "gtd_dash" jobs: deploy: @@ -15,6 +15,35 @@ jobs: steps: - uses: actions/checkout@v3 + + - name: bootstrap environment variables + id: load-env + run: | + # first step init the file + . ./bootstrap_env.sh + + # Check if .env.bootstrap file exists + if [ -f .env.bootstrap ]; then + # Process each line in the .env.bootstrap file + while IFS= read -r line; do + # Skip empty lines and lines starting with # + if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then + # Export the variable to $GITHUB_ENV + echo "$line" >> $GITHUB_ENV + fi + done < .env.bootstrap + else + echo ".env.bootstrap file not found" + fi + + + - name: Create .env file from secret + run: | + echo "$PROD_ENV" > .env + env: + PROD_ENV: ${{ secrets.PROD_ENV }} + + # Cache Python packages - name: Cache Python packages @@ -34,8 +63,8 @@ jobs: - name: Install Python dependencies run: | pip install -r requirements.txt - pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-redshift==$DBT_VERSION dbt-snowflake==$DBT_VERSION dbt-bigquery==$DBT_VERSION - dbt deps --project-dir "$PROJECT_DIR" + pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION + dbt deps --project-dir "$DBT_PROJECT_DIR" # Cache npm packages - name: Cache npm packages @@ -63,8 +92,6 @@ jobs: - name: Move credentials to /tmp run: mv googlecredentials.json /tmp || true - - name: Locate dbt_project.yml - run: echo "PROJECT_DIR=$(find . -name "dbt_project.yml" | sed 's/dbt_project.yml//g')" >> $GITHUB_ENV - name: Get lightdash version uses: sergeysova/jq-action@v2 @@ -88,4 +115,4 @@ jobs: LIGHTDASH_PROJECT: ${{ secrets.LIGHTDASH_PROJECT }} LIGHTDASH_URL: ${{ secrets.LIGHTDASH_URL }} GOOGLE_APPLICATION_CREDENTIALS: '/tmp/googlecredentials.json' - run: lightdash deploy --project-dir "$PROJECT_DIR" --profiles-dir . --profile prod || lightdash deploy --project-dir "$PROJECT_DIR" --profiles-dir . \ No newline at end of file + run: lightdash deploy --project-dir "$DBT_PROJECT_DIR" --profiles-dir . --profile prod || lightdash deploy --project-dir "$DBT_PROJECT_DIR" --profiles-dir . \ No newline at end of file diff --git a/.github/workflows/start-preview.yml b/.github/workflows/start-preview.yml index 1b5697f..b06b3e5 100644 --- a/.github/workflows/start-preview.yml +++ b/.github/workflows/start-preview.yml @@ -6,8 +6,6 @@ on: env: DBT_VERSION: "1.8.1" - PROJECT_DIR: "./dbt_project" - DW_DBNAME: "gtd_dash" jobs: preview: @@ -18,6 +16,37 @@ jobs: steps: - uses: actions/checkout@v3 + + - name: bootstrap environment variables + id: load-env + run: | + # first step init the file + . ./bootstrap_env.sh + + # Check if .env.bootstrap file exists + if [ -f .env.bootstrap ]; then + # Process each line in the .env.bootstrap file + while IFS= read -r line; do + # Skip empty lines and lines starting with # + if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then + # Export the variable to $GITHUB_ENV + echo "$line" >> $GITHUB_ENV + fi + done < .env.bootstrap + else + echo ".env.bootstrap file not found" + fi + + + - name: Create .env file from secret + run: | + echo "$PROD_ENV" > .env + env: + PROD_ENV: ${{ secrets.PROD_ENV }} + + + + - name: Cache Python packages uses: actions/cache@v3 with: diff --git a/.vscode/settings.json b/.vscode/settings.json index c06b228..f6ab8a6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -53,5 +53,5 @@ "packages.yml" ] }, - "dbt.perspectiveTheme": "Monokai" + "dbt.perspectiveTheme": "Monokai", } \ No newline at end of file diff --git a/Makefile b/Makefile index f524fb2..5d6501d 100644 --- a/Makefile +++ b/Makefile @@ -3,19 +3,19 @@ init_deploy: dagster: - tmux send-keys -t dagster.0 ". ./.venv/bin/activate && . ./env.sh && dagster dev -m ETL -h 0.0.0.0 -p 60001" ENTER + tmux send-keys -t dagster.0 ". ./.venv/bin/activate && . ./bootstrap_env.sh && dagster dev -m ETL -h 0.0.0.0 -p 60001" ENTER sleeper: sleep 10 init_seed: . ./.venv/bin/activate - . ./env.sh + . ./bootstrap_env.sh python app/helper/scaffold_seeds.py init_dbt: . ./.venv/bin/activate - . ./env.sh + . ./bootstrap_env.sh dbt deps dbt compile @@ -27,7 +27,7 @@ deploy: init_deploy sleeper dagster streamlit deploy-from-scratch: init_seed init_deploy init_dbt sleeper loader dagster streamlit loader: - tmux send-keys -t loader.0 ". ./.venv/bin/activate && . ./env.sh && cd app/ETL && python loader.py" ENTER + tmux send-keys -t loader.0 ". ./.venv/bin/activate && . ./bootstrap_env.sh && cd app/ETL && python loader.py" ENTER cancel_deploy: @@ -40,4 +40,4 @@ loader_rerun: loader_helper loader # command after each reboot the vm -up: init_deploy dagster loader +up: init_deploy sleeper dagster loader \ No newline at end of file diff --git a/README.md b/README.md index 8568293..98e0734 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ MUST already have the seeding csv "list_goal_mapping" in `dbt_project/seeds/list use this command to scaffold the seeds (sourced from google sheet) `make init_seed`, THEN continue with `make deploy-from-scratch` below +for vs code intergrated terminal : add this line to bashrc/zshrc/powershell or whichever your default vs code default terminal uses +. /workspaces/scrape-ticktick/bootstrap_env.sh + + # development clone new repo & install requirements diff --git a/app/ETL/loader.py b/app/ETL/loader.py index 75286a0..1c97a93 100644 --- a/app/ETL/loader.py +++ b/app/ETL/loader.py @@ -39,7 +39,7 @@ from logging.handlers import TimedRotatingFileHandler import time import sys; sys.path.append('..') # to allow import helper which is 1 dir away -from helper.source_env import dotenv_path,raw_path +from helper.source_env import project_dotenv_path,raw_path import asyncio #%% @@ -59,7 +59,7 @@ logger.setLevel(logging.INFO) -cache_path=os.path.join(dotenv_path,'.token-oauth') +cache_path=os.path.join(project_dotenv_path,'.token-oauth') client_id=environ.get('client_id') client_secret=environ.get('client_secret') diff --git a/app/helper/source_env.py b/app/helper/source_env.py index 6b808e4..7e84323 100644 --- a/app/helper/source_env.py +++ b/app/helper/source_env.py @@ -1,28 +1,46 @@ import os from dotenv import load_dotenv import urllib.parse -# from app.ETL.constants import DBT_PROJECT_DIR -# setup paths -current_dir=os.path.dirname(os.path.abspath(__file__)) +def find_project_root(current_dir, marker_file='.env'): + while current_dir != os.path.dirname(current_dir): + if os.path.exists(os.path.join(current_dir, marker_file)): + return current_dir + current_dir = os.path.dirname(current_dir) + raise RuntimeError("Project root with marker file '{}' not found.".format(marker_file)) -# IMPORTANT TO HAVE THIS LOAD FIRST -# source .env from project root to construct dbt paths -project_dotenv_path=os.path.join(current_dir,'..','..','.env') +# Find the project root +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = find_project_root(current_dir) + + +# Load .env.bootstrap + +dotenv_bootstrap_path = os.path.join(project_root, '.env.bootstrap') +load_dotenv(dotenv_bootstrap_path) + + + +# Load .env from project root +project_dotenv_path = os.path.join(project_root, '.env') load_dotenv(project_dotenv_path) -makefile_path = os.path.join(os.environ.get('DAGSTER_HOME'),'Makefile') + + +makefile_path = os.environ.get('MAKEFILE_PATH') makefile_dir = os.path.dirname(makefile_path) -# makefile_path = os.path.join(os.path.abspath(project_dotenv_path),'Makefile') raw_path = os.path.join(current_dir,'..','ETL','raw') -dotenv_path=os.path.join(current_dir,'..','env') -secrets_path = os.path.join(dotenv_path,'.secrets') -service_account_path = os.path.join(dotenv_path,'service_account.json') +ETL_env_path=os.path.join(current_dir,'..','env') +secrets_path = os.path.join(ETL_env_path,'.secrets') +service_account_path = os.path.join(ETL_env_path,'service_account.json') ETL_workdir = os.path.join(current_dir,'..','ETL') +load_dotenv(secrets_path) + + dbt_project_dir = os.environ.get('DBT_PROJECT_DIR') dbt_models_path=os.path.join(dbt_project_dir,'models') @@ -44,6 +62,3 @@ target_schema = os.environ.get('TARGET_SCHEMA','dev') db_url = f'postgresql://{user}:{password_encoded}@{hostname}:{port}/{database}?options=' db_url = db_url + f'-csearch_path=={target_schema}' - - -load_dotenv(secrets_path) \ No newline at end of file diff --git a/bootstrap_env.sh b/bootstrap_env.sh index a014d02..a71fc2a 100755 --- a/bootstrap_env.sh +++ b/bootstrap_env.sh @@ -15,6 +15,8 @@ DAGSTER_HOME="$ENV_WORK_DIR" DAGSTER_DBT_PARSE_PROJECT_ON_LOAD=1 DAGSTER_LOCAL_ARTIFACT_STORAGE_DIR="$ENV_WORK_DIR/dagster_artifacts" +MAKEFILE_PATH="$ENV_WORK_DIR/Makefile" + EOENV } @@ -22,6 +24,10 @@ EOENV source_env() { set -a # Automatically export all variables . ./.env.bootstrap + if [ ! -f .env ]; then + touch .env + fi + . ./.env set +a # Stop automatically exporting variables }