init

.env.sample

@@ -0,0 +1,6 @@
+METHOD=oauth
+PROJECT=test-project
+EXECUTION_PROJECT=test-execution-project
+DATASET=dbt_test
+#GOOGLE_APPLICATION_CREDENTIALS=
+DBT_PROFILES_DIR=

.github/workflows/sqlmesh.yml

@@ -0,0 +1,37 @@
+name: SQLMesh Bot
+run-name: 🚀SQLMesh Bot 🚀
+on:
+  pull_request:
+    types:
+    - synchronize
+    - opened
+# The latest commit is the one that will be used to create the PR environment and deploy to production
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+jobs:
+  sqlmesh:
+    name: SQLMesh Actions Workflow
+    runs-on: ubuntu-latest
+    permissions:
+      # Required to access code in PR
+      contents: write
+      # Required to post comments
+      issues: write
+      # Required to update check runs
+      checks: write
+      # Required to merge
+      pull-requests: write
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v4
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+        with:
+          ref: refs/pull/${{ github.event.issue.pull_request && github.event.issue.number || github.event.pull_request.number  }}/merge
+      - name: Install SQLMesh + Dependencies
+        run: pip install -r requirements.txt
+        shell: bash
+      - name: Run CI/CD Bot
+        run: |
+          sqlmesh_cicd -p ${{ github.workspace }}/dbt_project -p ${{ github.workspace }}/sqlmesh_project github --token ${{ secrets.GITHUB_TOKEN }} run-all

.gitignore

@@ -0,0 +1,8 @@
+.env
+.venv
+*/target/
+.cache
+*.lock
+*.old
+*.db
+logs/*

.python-version

@@ -0,0 +1 @@
+3.11

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 BQ Booster
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,20 @@
+.DEFAULT_GOAL := sqlmesh
+
+plan:
+	sqlmesh -p ./dbt_project -p ./sqlmesh_project plan
+
+test:
+	sqlmesh -p ./dbt_project -p ./sqlmesh_project test
+
+ui:
+	sqlmesh -p ./dbt_project -p ./sqlmesh_project ui
+
+sqlmesh:
+	sqlmesh -p ./dbt_project -p ./sqlmesh_project $(filter-out $@,$(MAKECMDGOALS))
+
+env:
+	export $(cat .env | xargs)
+
+# Prevent make from interpreting the arguments as make targets
+%:
+    @:

README.md

@@ -0,0 +1,12 @@
+# dbt sqlmesh multi project
+
+This project is POC to combine dbt with SQLMesh to create a multi project dbt setup.
+
+## Setup
+
+1. Clone the repo
+2. Install with peotry: `pip install -r requirements.txt`
+3. Copy .env.sample to .env: `cp .env.sample .env`
+4. Update the .env file with your configuration
+5. Export the .env file: `export $(cat .env | xargs)`
+6. Run SQLMesh: `sqlmesh plan`

dbt_project/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

dbt_project/.user.yml

@@ -0,0 +1 @@
+id: 81673b64-3691-402a-a42e-b6085a6702c2

dbt_project/config.py

@@ -0,0 +1,10 @@
+# This file is used to load the sqlmesh configuration
+from pathlib import Path
+
+from sqlmesh.core.config import DuckDBConnectionConfig
+from sqlmesh.dbt.loader import sqlmesh_config
+
+config = sqlmesh_config(
+    Path(__file__).parent,
+    state_connection=DuckDBConnectionConfig(),
+)

dbt_project/dbt_project.yml

@@ -0,0 +1,23 @@
+
+name: 'dbt_sqlmesh_multi_project'
+version: '1.0.0'
+
+profile: 'dbt_sqlmesh_multi_project'
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+
+models:
+  +start: Jan 1 2017
+  dbt_sqlmesh_multi_project:
+    example:
+      +materialized: view

dbt_project/models/example/schema.yml

@@ -0,0 +1,11 @@
+
+version: 2
+
+models:
+  - name: views_per_domain
+    description: "Views per domain"
+    columns:
+      - name: domain
+        description: "The primary key for this table"
+      - name: views_count
+        description: "The number of views"

dbt_project/models/example/views_per_domain.sql

@@ -0,0 +1,4 @@
+{{ config(materialized='table') }}
+SELECT domain, sum(views_count) as views_count
+from {{ ref('views_data') }}
+GROUP BY ALL

dbt_project/models/example/views_per_domain_and_top_browsers.sql

@@ -0,0 +1,22 @@
+{{ config(materialized='table') }}
+{%- call set_sql_header(config) %}
+CREATE TEMP FUNCTION TOP_BROWSERS(browser STRING) AS (
+    IF(
+
+        browser IN UNNEST(
+            [
+                'Chrome',
+                'Safari',
+            ]
+        ),
+        browser,
+        'other'
+    )
+);
+{%- endcall %}
+SELECT 
+domain,
+TOP_BROWSERS(browser) as browser,
+sum(views_count) as views_count
+from {{ ref('views_data') }}
+GROUP BY ALL

dbt_project/profiles.yml

@@ -0,0 +1,15 @@
+dbt_sqlmesh_multi_project:
+  target: dev
+  outputs:
+    dev:
+      type: bigquery
+      method: "{{ env_var('METHOD', 'oauth') }}"
+      project: "{{ env_var('PROJECT', 'project-to-update') }}"
+      execution_project: "{{ env_var('EXECUTION_PROJECT', 'execution-project-to-update') }}"
+      dataset: "{{ env_var('DATASET', 'dbt_my_username') }}"
+      threads: "{{ env_var('THREADS', 8) | as_number }}"
+      keyfile: "{{ env_var('GOOGLE_APPLICATION_CREDENTIALS', 'gcp_credentials.json') }}"
+      timeout_seconds: "{{ env_var('TIMEOUT_SECONDS', 10800) | as_number }}"
+      priority: "{{ env_var('PRIORITY', 'interactive') }}"
+      location: "{{ env_var('LOCATION', 'US') }}"
+      retries: "{{ env_var('RETRIES', 1) | as_number }}"

dbt_project/seeds/schema.yml

@@ -0,0 +1,22 @@
+
+version: 2
+
+seeds:
+  - name: views_data
+    description: "Views"
+    columns:
+      - name: ts
+        description: "The timestamp"
+        type: timestamp
+      - name: domain
+        description: "The domain"
+        type: string
+      - name: os
+        description: "The operating system"
+        type: string
+      - name: browser
+        description: "The browser"
+        type: string
+      - name: views_count
+        description: "The number of views"
+        type: integer

dbt_project/seeds/views_data.csv

@@ -0,0 +1,9 @@
+ts,domain,browser,os,views_count
+2024-11-01 01:07:10,teads.com,Chrome,Android,10
+2024-11-02 01:06:10,teads.com,Safari,macOS,8
+2024-11-01 01:02:10,teads.com,Chrome,iOS,4
+2024-11-01 01:01:10,bqbooster.com,Chrome,Android,3
+2024-11-01 01:03:10,bqbooster.com,Firefox,Windows,5
+2024-11-02 01:07:10,bqbooster.com,Firefox,Android,6
+2024-11-01 01:01:10,kayrnt.fr,Chrome,Android,2
+2024-11-01 02:01:10,kayrnt.fr,Edge,Windows,7

export_env.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# File containing the environment variables
+ENV_FILE=".env"
+
+# Check if the file exists
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "File $ENV_FILE does not exist."
+  exit 1
+fi
+
+# Export each variable in the file
+while IFS= read -r line; do
+  # Skip empty lines and comments
+  [[ -z "$line" || "$line" =~ ^# ]] && continue
+  export "$line"
+done < "$ENV_FILE"
+
+echo "Environment variables exported from $ENV_FILE"

pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "dbt-sqlmesh-multi-project"
+version = "0.1.0"
+description = "dbt SQLMesh multi project"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "agate>=1.9.1",
+    "dbt-bigquery>=1.8.3",
+    "sqlmesh[bigquery,dbt,github,web]>=0.130.1",
+]