init setup uses env var prod from secret while bootstrap paths from a…

… shell script init

.devcontainer/postCreateCommand.sh

@@ -48,9 +48,9 @@ apt-get update && apt-get install -y python3-venv
 
 # init then source env vars
 print_section "INIT & SOURCE ENV VARS"
-chmod +x ./env.sh
+chmod +x ./bootstrap_env.sh
 
-. ./env.sh
+. ./bootstrap_env.sh
 
 
 

.github/workflows/close-preview.yml

@@ -12,6 +12,35 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
+      - name: bootstrap environment variables
+        id: load-env
+        run: |
+          # first step init the file 
+          . ./bootstrap_env.sh
+
+          # Check if .env.bootstrap file exists
+          if [ -f .env.bootstrap ]; then
+            # Process each line in the .env.bootstrap file
+            while IFS= read -r line; do
+              # Skip empty lines and lines starting with #
+              if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then
+                # Export the variable to $GITHUB_ENV
+                echo "$line" >> $GITHUB_ENV
+              fi
+            done < .env.bootstrap
+          else
+            echo ".env.bootstrap file not found"
+          fi
+
+          
+      - name: Create .env file from secret
+        run: |
+          echo "$PROD_ENV" > .env
+        env:
+          PROD_ENV: ${{ secrets.PROD_ENV }}
+
+
+
 
       - name: Get lightdash version 
         uses: sergeysova/jq-action@v2

.github/workflows/deployment.sh

@@ -4,7 +4,7 @@ WORKDIR=$(pwd)
 
 . $WORKDIR/.venv/bin/activate
 
-. $WORKDIR/env.sh
+. $WORKDIR/bootstrap_env.sh
 
 
 # # setup sessions for service

.github/workflows/gh_deploy.yml

@@ -6,15 +6,44 @@ on:
 
 env:
   DBT_VERSION: "1.8.1"
-  PROJECT_DIR: "./dbt_project"
-  DW_DBNAME: "gtd_dash"
+  # PROJECT_DIR: "./dbt_project"
+  # DW_DBNAME: "gtd_dash"
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
+
+      - name: bootstrap environment variables
+        id: load-env
+        run: |
+          # first step init the file 
+          . ./bootstrap_env.sh
+
+          # Check if .env.bootstrap file exists
+          if [ -f .env.bootstrap ]; then
+            # Process each line in the .env.bootstrap file
+            while IFS= read -r line; do
+              # Skip empty lines and lines starting with #
+              if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then
+                # Export the variable to $GITHUB_ENV
+                echo "$line" >> $GITHUB_ENV
+              fi
+            done < .env.bootstrap
+          else
+            echo ".env.bootstrap file not found"
+          fi
+
+          
+      - name: Create .env file from secret
+        run: |
+          echo "$PROD_ENV" > .env
+        env:
+          PROD_ENV: ${{ secrets.PROD_ENV }}
+
+
 
       # Cache Python packages
       - name: Cache Python packages
@@ -34,8 +63,8 @@ jobs:
       - name: Install Python dependencies
         run: |
           pip install -r requirements.txt
-          pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION dbt-redshift==$DBT_VERSION dbt-snowflake==$DBT_VERSION dbt-bigquery==$DBT_VERSION
-          dbt deps --project-dir "$PROJECT_DIR"
+          pip install dbt-core==$DBT_VERSION dbt-postgres==$DBT_VERSION
+          dbt deps --project-dir "$DBT_PROJECT_DIR"
 
       # Cache npm packages
       - name: Cache npm packages
@@ -63,8 +92,6 @@ jobs:
       - name: Move credentials to /tmp 
         run: mv googlecredentials.json /tmp || true
 
-      - name: Locate dbt_project.yml
-        run: echo "PROJECT_DIR=$(find . -name "dbt_project.yml"  | sed  's/dbt_project.yml//g')" >> $GITHUB_ENV
 
       - name: Get lightdash version 
         uses: sergeysova/jq-action@v2
@@ -88,4 +115,4 @@ jobs:
           LIGHTDASH_PROJECT: ${{ secrets.LIGHTDASH_PROJECT }}          
           LIGHTDASH_URL: ${{ secrets.LIGHTDASH_URL }}      
           GOOGLE_APPLICATION_CREDENTIALS: '/tmp/googlecredentials.json'    
-        run: lightdash deploy --project-dir "$PROJECT_DIR" --profiles-dir . --profile prod || lightdash deploy --project-dir "$PROJECT_DIR" --profiles-dir .
+        run: lightdash deploy --project-dir "$DBT_PROJECT_DIR" --profiles-dir . --profile prod || lightdash deploy --project-dir "$DBT_PROJECT_DIR" --profiles-dir .

.github/workflows/start-preview.yml

@@ -6,8 +6,6 @@ on:
 
 env:
   DBT_VERSION: "1.8.1"
-  PROJECT_DIR: "./dbt_project"
-  DW_DBNAME: "gtd_dash"
 
 jobs:
   preview:
@@ -18,6 +16,37 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+
+      - name: bootstrap environment variables
+        id: load-env
+        run: |
+          # first step init the file 
+          . ./bootstrap_env.sh
+
+          # Check if .env.bootstrap file exists
+          if [ -f .env.bootstrap ]; then
+            # Process each line in the .env.bootstrap file
+            while IFS= read -r line; do
+              # Skip empty lines and lines starting with #
+              if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then
+                # Export the variable to $GITHUB_ENV
+                echo "$line" >> $GITHUB_ENV
+              fi
+            done < .env.bootstrap
+          else
+            echo ".env.bootstrap file not found"
+          fi
+
+          
+      - name: Create .env file from secret
+        run: |
+          echo "$PROD_ENV" > .env
+        env:
+          PROD_ENV: ${{ secrets.PROD_ENV }}
+
+
+
+
       - name: Cache Python packages
         uses: actions/cache@v3
         with:

.vscode/settings.json

@@ -53,5 +53,5 @@
             "packages.yml"
         ]
     },
-    "dbt.perspectiveTheme": "Monokai"
+    "dbt.perspectiveTheme": "Monokai",
 }

Makefile

@@ -3,19 +3,19 @@ init_deploy:
 
 
 dagster:
-	tmux send-keys -t dagster.0 ". ./.venv/bin/activate && . ./env.sh && dagster dev -m ETL -h 0.0.0.0 -p 60001" ENTER
+	tmux send-keys -t dagster.0 ". ./.venv/bin/activate && . ./bootstrap_env.sh && dagster dev -m ETL -h 0.0.0.0 -p 60001" ENTER
 
 sleeper:
 	sleep 10
 
 init_seed:
 	. ./.venv/bin/activate
-	. ./env.sh
+	. ./bootstrap_env.sh
 	python app/helper/scaffold_seeds.py
 
 init_dbt: 
 	. ./.venv/bin/activate 
-	. ./env.sh 
+	. ./bootstrap_env.sh 
 	dbt deps 
 	dbt compile
 
@@ -27,7 +27,7 @@ deploy: init_deploy sleeper dagster streamlit
 deploy-from-scratch: init_seed init_deploy init_dbt sleeper loader dagster streamlit
 
 loader:
-	tmux send-keys -t loader.0 ". ./.venv/bin/activate && . ./env.sh && cd app/ETL && python loader.py" ENTER
+	tmux send-keys -t loader.0 ". ./.venv/bin/activate && . ./bootstrap_env.sh && cd app/ETL && python loader.py" ENTER
 
 
 cancel_deploy:
@@ -40,4 +40,4 @@ loader_rerun: loader_helper loader
 
 
 # command after each reboot the vm
-up: init_deploy dagster loader
+up: init_deploy sleeper dagster loader

README.md

@@ -9,6 +9,10 @@ MUST already have the seeding csv "list_goal_mapping" in `dbt_project/seeds/list
 
 use this command to scaffold the seeds (sourced from google sheet) `make init_seed`, THEN continue with `make deploy-from-scratch` below
 
+for vs code intergrated terminal : add this line to bashrc/zshrc/powershell or whichever your default vs code default terminal uses
+. /workspaces/scrape-ticktick/bootstrap_env.sh
+
+
 
 # development
 clone new repo & install requirements

app/ETL/loader.py

@@ -39,7 +39,7 @@
 from logging.handlers import TimedRotatingFileHandler
 import time
 import sys; sys.path.append('..') # to allow import helper which is 1 dir away
-from helper.source_env import dotenv_path,raw_path
+from helper.source_env import project_dotenv_path,raw_path
 import asyncio
 
 #%%
@@ -59,7 +59,7 @@
 logger.setLevel(logging.INFO)
 
 
-cache_path=os.path.join(dotenv_path,'.token-oauth')
+cache_path=os.path.join(project_dotenv_path,'.token-oauth')
 
 client_id=environ.get('client_id')
 client_secret=environ.get('client_secret')

app/helper/source_env.py

@@ -1,28 +1,46 @@
 import os 
 from dotenv import load_dotenv
 import urllib.parse
-# from app.ETL.constants import DBT_PROJECT_DIR
 
 
 
-# setup paths
-current_dir=os.path.dirname(os.path.abspath(__file__))
+def find_project_root(current_dir, marker_file='.env'):
+    while current_dir != os.path.dirname(current_dir):
+        if os.path.exists(os.path.join(current_dir, marker_file)):
+            return current_dir
+        current_dir = os.path.dirname(current_dir)
+    raise RuntimeError("Project root with marker file '{}' not found.".format(marker_file))
 
-# IMPORTANT TO HAVE THIS LOAD FIRST
-# source .env from project root to construct dbt paths
-project_dotenv_path=os.path.join(current_dir,'..','..','.env')
+# Find the project root
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = find_project_root(current_dir)
+
+
+# Load .env.bootstrap
+
+dotenv_bootstrap_path = os.path.join(project_root, '.env.bootstrap')
+load_dotenv(dotenv_bootstrap_path)
+
+
+
+# Load .env from project root
+project_dotenv_path = os.path.join(project_root, '.env')
 load_dotenv(project_dotenv_path)
 
-makefile_path = os.path.join(os.environ.get('DAGSTER_HOME'),'Makefile')
+
+
+makefile_path = os.environ.get('MAKEFILE_PATH')
 makefile_dir = os.path.dirname(makefile_path)
 
-# makefile_path = os.path.join(os.path.abspath(project_dotenv_path),'Makefile')
 raw_path = os.path.join(current_dir,'..','ETL','raw')
-dotenv_path=os.path.join(current_dir,'..','env')
-secrets_path = os.path.join(dotenv_path,'.secrets')
-service_account_path = os.path.join(dotenv_path,'service_account.json')
+ETL_env_path=os.path.join(current_dir,'..','env')
+secrets_path = os.path.join(ETL_env_path,'.secrets')
+service_account_path = os.path.join(ETL_env_path,'service_account.json')
 ETL_workdir = os.path.join(current_dir,'..','ETL')
 
+load_dotenv(secrets_path)
+
+
 
 dbt_project_dir = os.environ.get('DBT_PROJECT_DIR')
 dbt_models_path=os.path.join(dbt_project_dir,'models')
@@ -44,6 +62,3 @@
 target_schema = os.environ.get('TARGET_SCHEMA','dev')
 db_url = f'postgresql://{user}:{password_encoded}@{hostname}:{port}/{database}?options='
 db_url = db_url + f'-csearch_path=={target_schema}'
-
-
-load_dotenv(secrets_path)

bootstrap_env.sh

@@ -15,13 +15,19 @@ DAGSTER_HOME="$ENV_WORK_DIR"
 DAGSTER_DBT_PARSE_PROJECT_ON_LOAD=1
 DAGSTER_LOCAL_ARTIFACT_STORAGE_DIR="$ENV_WORK_DIR/dagster_artifacts"
 
+MAKEFILE_PATH="$ENV_WORK_DIR/Makefile"
+
 EOENV
 }
 
 # Function to source .env file
 source_env() {
     set -a  # Automatically export all variables
     . ./.env.bootstrap
+    if [ ! -f .env ]; then
+        touch .env
+    fi
+    . ./.env
     set +a  # Stop automatically exporting variables
 }