Skip to content

Commit

Permalink
setup multi asset for job dump motherduck matching new dagster version
Browse files Browse the repository at this point in the history
  • Loading branch information
luutuankiet committed Feb 26, 2024
1 parent 8ad0e7b commit 409855f
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 18 deletions.
18 changes: 14 additions & 4 deletions app/ETL/EL.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
import os
from dagster import asset,AssetExecutionContext
from dagster import asset,AssetExecutionContext,AssetOut, multi_asset, load_assets_from_modules
from dagster_dbt import get_asset_keys_by_output_name_for_source
import duckdb
import pandas as pd
from helper.source_env import raw_path


@asset(compute_kind='Python')
from . import dbt_assets


@multi_asset(
outs={
name: AssetOut(key=asset_key)
for name, asset_key in get_asset_keys_by_output_name_for_source(
[dbt_assets.ticktick_dbt_assets], "raw_data"
).items()
},
compute_kind='python'
)
def dump_to_motherduck(context: AssetExecutionContext):
motherduck_token=os.environ.get('motherduck_token')

Expand Down
2 changes: 1 addition & 1 deletion app/ETL/definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from dagster_dbt import DbtCliResource

from .constants import DBT_PROJECT_DIR
from. import EL,dbt_assets
from . import EL,dbt_assets

all_assets = load_assets_from_modules([EL,dbt_assets])
ETL_job = define_asset_job("ETL_job",selection=all_assets)
Expand Down
17 changes: 4 additions & 13 deletions dbt_project/models/_source.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,18 @@ version: 2

sources:
- name: raw_data
meta:
dagster:
asset_key : ["dump_to_motherduck"]
description: A replica of the extracted ticktick data.
database: ticktick_gtd
schema: main
tables:
- name: tasks_raw
description: raw tasks data from tickpy
meta:
dagster:
asset_key: ["dump_to_motherduck"]
- name: lists_raw
description: raw lists data from tickpy
meta:
dagster:
asset_key: ["dump_to_motherduck"]
- name: folders_raw
description: raw folders data from tickpy
meta:
dagster:
asset_key: ["dump_to_motherduck"]
description: raw folders data from tickpy
- name: date_seed
description: pre-generated dates
meta:
dagster:
asset_key: ["dump_to_motherduck"]

0 comments on commit 409855f

Please sign in to comment.