From 60e92b176993b8e69268bb566cf4f73a6ea3dce8 Mon Sep 17 00:00:00 2001 From: luutuankiet Date: Sat, 17 Feb 2024 12:04:09 +0700 Subject: [PATCH] setup CI --- .env | 18 ++++----- .github/workflows/python_app_ci.yml | 58 ++++++++++++++--------------- .gitignore | 3 +- Dockerfile | 28 ++++++++++++++ 4 files changed, 68 insertions(+), 39 deletions(-) create mode 100644 Dockerfile diff --git a/.env b/.env index 922b227..aaeb7f6 100644 --- a/.env +++ b/.env @@ -1,10 +1,10 @@ -export VIRTUAL_ENV="/workspaces/scrape-ticktick-1/.venv" -export DBT_PROJECT_DIR="/workspaces/scrape-ticktick-1/dbt_project" -export DBT_PROFILES_DIR="/workspaces/scrape-ticktick-1/dbt_project" -export DW_PATH="/workspaces/scrape-ticktick-1/database" -export DAGSTER_HOME="/workspaces/scrape-ticktick-1" -export DBT_PROJECT_DIR="/workspaces/scrape-ticktick-1/dbt_project" -export DBT_PROFILES_DIR="/workspaces/scrape-ticktick-1/dbt_project" -export DW_PATH="/workspaces/scrape-ticktick-1/database" -export PYTHONPATH="/workspaces/scrape-ticktick-1/app" +export VIRTUAL_ENV="/home/ken/dev-main/scrape-ticktick-1/.venv" +export DBT_PROJECT_DIR="/home/ken/dev-main/scrape-ticktick-1/dbt_project" +export DBT_PROFILES_DIR="/home/ken/dev-main/scrape-ticktick-1/dbt_project" +export DW_PATH="/home/ken/dev-main/scrape-ticktick-1/database" +export DAGSTER_HOME="/home/ken/dev-main/scrape-ticktick-1" +export DBT_PROJECT_DIR="/home/ken/dev-main/scrape-ticktick-1/dbt_project" +export DBT_PROFILES_DIR="/home/ken/dev-main/scrape-ticktick-1/dbt_project" +export DW_PATH="/home/ken/dev-main/scrape-ticktick-1/database" +export PYTHONPATH="/home/ken/dev-main/scrape-ticktick-1/app" export motherduck_token="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzZXNzaW9uIjoibHV1dHVhbmtpZXQuZnR1Mi5nbWFpbC5jb20iLCJlbWFpbCI6Imx1dXR1YW5raWV0LmZ0dTJAZ21haWwuY29tIiwidXNlcklkIjoiMjRkOTQxYzktMDg2OC00ZThmLWIwNjMtNDFmNjE2MDYyMjMxIiwiaWF0IjoxNzA3MTU2NzY2LCJleHAiOjE3Mzg3MTQzNjZ9.z-FjZKJ8oAewQKJ2X3_4emnJXszt1q_VaB0ZVOsQEUM" diff --git a/.github/workflows/python_app_ci.yml b/.github/workflows/python_app_ci.yml index b1e9709..1739ee3 100644 --- a/.github/workflows/python_app_ci.yml +++ b/.github/workflows/python_app_ci.yml @@ -1,35 +1,35 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python +# # This workflow will install Python dependencies, run tests and lint with a single version of Python +# # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: Python application +# name: Python application -on: [push] +# on: [push] -permissions: - contents: read +# permissions: +# contents: read -jobs: - build: +# jobs: +# build: - runs-on: ubuntu-latest +# runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.12" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - # - name: Test with pytest - # run: | - # pytest +# steps: +# - uses: actions/checkout@v3 +# - name: Set up Python 3.10 +# uses: actions/setup-python@v3 +# with: +# python-version: "3.12" +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install flake8 pytest +# if [ -f requirements.txt ]; then pip install -r requirements.txt; fi +# - name: Lint with flake8 +# run: | +# # stop the build if there are Python syntax errors or undefined names +# flake8 ./app --count --select=E9,F63,F7,F82 --show-source --statistics +# # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide +# flake8 ./app --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +# # - name: Test with pytest +# # run: | +# # pytest diff --git a/.gitignore b/.gitignore index 20731a3..407bbca 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ history logs storage schedules -dbt_project/target \ No newline at end of file +dbt_project/target +ticktick-gtd/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8715cfc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.10-slim + +COPY requirements.txt . + +RUN apt-get update && apt-get install -y python3-dev gcc && pip install -r requirements.txt + +RUN mkdir -p /opt/dagster/dagster_home /opt/dagster/app + +RUN touch /opt/dagster/dagster_home/dagster.yaml + + +ENV DAGSTER_HOME=/opt/dagster/dagster_home/ + +WORKDIR /opt/dagster/dagster_home/ + +COPY app ./app + +COPY dbt_project ./dbt_project + +COPY env_init.sh . + +RUN chmod +x env_init.sh && . ./env_init.sh + +EXPOSE 3000 + +# RUN /bin/bash -c "source .env" && dbt deps && dbt build + +# ENTRYPOINT ["dagster-webserver","-h","0.0.0.0","-p","3000"]