Merge pull request #948 from dlt-hub/devel

master merge for 0.4.3 release

.github/ISSUE_TEMPLATE/documentation_request.yml

@@ -0,0 +1,28 @@
+---
+name: Documentation Request
+description: Suggest improvements or additions to dlt documentation
+labels: ["kind:documentation"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for contributing to the dlt documentation!
+        For significant updates or new content, open an issue to discuss your ideas. This helps us 
+        coordinate and integrate your suggestions effectively. For simple edits like typos, you can 
+        directly submit a [pull request](https://github.com/dlt-hub/dlt/pulls).
+  - type: textarea
+    attributes:
+      label: Documentation description
+      description: Describe the documentation improvements or additions you'd like to see.
+      placeholder: >
+        What specific documentation changes are you suggesting?
+  - type: dropdown
+    attributes:
+      label: Are you a dlt user?
+      description: Please tell us how you use dlt and how this impacts your need for documentation.
+      multiple: false
+      options:
+        - "Yes, I'm already a dlt user."
+        - "Yes, I run dlt in production."
+        - "Yes, I use it for fun."
+        - "I'm considering using dlt, but the documentation is unclear."

.github/workflows/lint.yml

@@ -25,7 +25,7 @@ jobs:
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}    
+    runs-on: ${{ matrix.os }}
 
     steps:
 
@@ -42,7 +42,7 @@ jobs:
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
-          installer-parallel: true         
+          installer-parallel: true
 
       - name: Load cached venv
         id: cached-poetry-dependencies
@@ -57,7 +57,7 @@ jobs:
 
       - name: Run make lint
         run: |
-          export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows        
+          export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
           make lint
 
       # - name: print envs

.github/workflows/test_dbt_runner.yml

@@ -9,17 +9,8 @@ on:
   workflow_dispatch:
 
 env:
-  # all credentials must be present to be passed to dbt runner
-  DESTINATION__POSTGRES__CREDENTIALS: postgresql://[email protected]:5432/dlt_data
-  DESTINATION__REDSHIFT__CREDENTIALS: postgresql://[email protected]:5439/dlt_ci
-  DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE
-  DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
-
-  DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
-  DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
-  DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
-  DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token
 
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
   RUNTIME__LOG_LEVEL: ERROR
 
 jobs:
@@ -70,6 +61,9 @@ jobs:
         # install dlt with postgres support
         run: poetry install --no-interaction -E postgres -E dbt --with sentry-sdk
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/helpers/dbt_tests -k '(not venv)'
         if: runner.os != 'Windows'

.github/workflows/test_destination_athena.yml

@@ -9,12 +9,7 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
-  DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1
-  DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -72,6 +67,9 @@ jobs:
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/load
         if: runner.os != 'Windows'

.github/workflows/test_destination_athena_iceberg.yml

@@ -9,12 +9,7 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
-  DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1
-  DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -70,7 +65,10 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E  --with sentry-sdk --with pipeline
+        run: poetry install --no-interaction -E --with sentry-sdk --with pipeline
+
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
           poetry run pytest tests/load

.github/workflows/test_destination_bigquery.yml

@@ -9,17 +9,7 @@ on:
   workflow_dispatch:
 
 env:
-  CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
-  CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
-  CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
-  CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token
-  CREDENTIALS__CLIENT_ID: 929384042504-3mtjaj1s7vuvf53j88mgdq4te7akkjm3.apps.googleusercontent.com
-  CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }}
-  CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }}
-
-  # needed for bigquery staging tests
-  # DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
-  # DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -81,8 +71,8 @@ jobs:
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline
 
-      # - name: Install self
-      #   run: poetry install --no-interaction
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
           poetry run pytest tests/helpers/providers tests/load

.github/workflows/test_destination_weaviate.yml → .github/workflows/test_destination_databricks.yml

@@ -1,4 +1,5 @@
-name: test weaviate
+
+name: test databricks
 
 on:
   pull_request:
@@ -8,14 +9,12 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__WEAVIATE__CREDENTIALS__URL:  ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__URL }}
-  DESTINATION__WEAVIATE__CREDENTIALS__API_KEY: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__API_KEY }}
-  DESTINATION__WEAVIATE__CREDENTIALS__ADDITIONAL_HEADERS: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__ADDITIONAL_HEADERS }}
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
 
-  ACTIVE_DESTINATIONS: "[\"weaviate\"]"
+  ACTIVE_DESTINATIONS: "[\"databricks\"]"
   ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
 
 jobs:
@@ -24,20 +23,21 @@ jobs:
     if: ${{ !github.event.pull_request.head.repo.fork }}
 
   run_loader:
-    name: Tests Weaviate loader
+    name: Tests Databricks loader
     needs: get_docs_changes
-    if: !always()
-    # if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os: ["ubuntu-latest"]
+        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
     runs-on: ${{ matrix.os }}
 
     steps:
+
       - name: Check out
         uses: actions/checkout@master
 
@@ -61,19 +61,23 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline
+        run: poetry install --no-interaction -E databricks -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
+
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
-          poetry run pytest tests/load/
+          poetry run pytest tests/load
         if: runner.os != 'Windows'
         name: Run tests Linux/MAC
       - run: |
-          poetry run pytest tests/load/
+          poetry run pytest tests/load
         if: runner.os == 'Windows'
         name: Run tests Windows
         shell: cmd
 
   matrix_job_required_check:
-    name: Weaviate loader tests
+    name: Databricks loader tests
     needs: run_loader
     runs-on: ubuntu-latest
     if: always()

.github/workflows/test_destination_mssql.yml

@@ -9,8 +9,8 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__MSSQL__CREDENTIALS: mssql://[email protected]:1433/dlt_ci
-  DESTINATION__MSSQL__CREDENTIALS__PASSWORD: ${{ secrets.MSSQL_PASSWORD }}
+
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -67,12 +67,15 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
-          poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
+          poetry run pytest tests/load
         if: runner.os != 'Windows'
         name: Run tests Linux/MAC
       - run: |
-          poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
+          poetry run pytest tests/load
         if: runner.os == 'Windows'
         name: Run tests Windows
         shell: cmd

.github/workflows/test_destination_qdrant.yml

@@ -8,8 +8,7 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__QDRANT__CREDENTIALS__LOCATION:  ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__LOCATION }}
-  DESTINATION__QDRANT__CREDENTIALS__API_KEY: ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__API_KEY }}
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -58,6 +57,9 @@ jobs:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - name: Install dependencies
         run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
       - run: |

.github/workflows/test_destination_snowflake.yml

@@ -9,17 +9,7 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE
-  CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
-
-  # needed for snowflake staging tests
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
-  DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
-  DESTINATION__FILESYSTEM__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
-  DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_NAME: dltdata
-  DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -73,6 +63,9 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/load
         if: runner.os != 'Windows'

.github/workflows/test_destination_synapse.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - master
       - devel
-
   workflow_dispatch:
 
 env:
-  DESTINATION__SYNAPSE__CREDENTIALS: ${{ secrets.SYNAPSE_CREDENTIALS }}
-  DESTINATION__SYNAPSE__CREDENTIALS__PASSWORD: ${{ secrets.SYNAPSE_PASSWORD }}
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://cf6086f7d263462088b9fb9f9947caee@o4505514867163136.ingest.sentry.io/4505516212682752
   RUNTIME__LOG_LEVEL: ERROR
@@ -19,19 +17,14 @@ env:
   ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
 
 jobs:
-
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Check source branch name
-      run: |
-        if [[ "${{ github.head_ref }}" != "synapse" ]]; then
-          exit 1
-        fi
+  get_docs_changes:
+    uses: ./.github/workflows/get_docs_changes.yml
+    if: ${{ !github.event.pull_request.head.repo.fork }}
 
   run_loader:
     name: Tests Synapse loader
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'    
     strategy:
       fail-fast: false
       matrix:
@@ -70,14 +63,17 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E synapse -E s3 -E gs -E az --with sentry-sdk --with pipeline
+        run: poetry install --no-interaction -E synapse -E parquet --with sentry-sdk --with pipeline
+
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
+          poetry run pytest tests/load
         if: runner.os != 'Windows'
         name: Run tests Linux/MAC
       - run: |
-          poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
+          poetry run pytest tests/load
         if: runner.os == 'Windows'
         name: Run tests Windows
         shell: cmd