diff --git a/.idea/opendbt.iml b/.idea/opendbt.iml index a2d63e9..fb36726 100644 --- a/.idea/opendbt.iml +++ b/.idea/opendbt.iml @@ -4,6 +4,8 @@ + + diff --git a/README.md b/README.md index 142523f..88f476e 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,17 @@ The `opendbt` library extends the capabilities of dbt. It unlocks many customizations, allowing you to tailor dbt to your specific needs and data workflows. -Forexample create custom transformations by customizing existing adapters +Forexample creating custom transformations by customizing existing adapters With `opendbt` you can go beyond the core functionalities of dbt by seamlessly integrating your customized adapter and -providing jinja with -custom Python methods tailored to your advanced needs. +provide jinja with further adapter/python methods. Enabling custom python methods tailored to your advanced needs. # Sample use cases, examples - Use customised adapter, provide jinja with custom python methods - Execute Python Model(Python code) Locally - Enable Model-Level Orchestration Using Airflow +- Create page on Airflow Server to serve DBT docs please see [examples](docs/EXAMPLES.md). diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index 4fa593b..31aa534 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -66,4 +66,20 @@ from opendbt.airflow import OpenDbtAirflowProject p = OpenDbtAirflowProject(resource_type='test', project_dir="/dbt/project_dir", profiles_dir="/dbt/profiles_dir", target='dev', tag="MY_TAG") p.load_dbt_tasks(dag=dag, start_node=start, end_node=end) -``` \ No newline at end of file +``` + +#### Create page on Airflow Server to serve DBT docs + +While its very practical to use airflow for dbt executions, it can be also used to server dbt docs. + +here is how: +**Step-1:** Create python file under airflow `/{airflow}/plugins` directory, with following code. +Adjust the given path to the folder where dbt docs are published + +https://github.com/memiiso/opendbt/blob/154b3e26981d157da70ebb98f1a1576f1fa55832/tests/resources/airflow/plugins/airflow_dbtdocs_page.py#L1-L6 + +**Step-2:** Restart airflow, and check that new link `DBT Docs` is created. +![airflow-dbt-docs-link.png](assets%2Fairflow-dbt-docs-link.png) + +**Step-3:** open the link and browse dbt docs +![airflow-dbt-docs-page.png](assets%2Fairflow-dbt-docs-page.png) \ No newline at end of file diff --git a/docs/assets/airflow-dbt-docs-link.png b/docs/assets/airflow-dbt-docs-link.png new file mode 100644 index 0000000..30893f3 Binary files /dev/null and b/docs/assets/airflow-dbt-docs-link.png differ diff --git a/docs/assets/airflow-dbt-docs-page.png b/docs/assets/airflow-dbt-docs-page.png new file mode 100644 index 0000000..569cf54 Binary files /dev/null and b/docs/assets/airflow-dbt-docs-page.png differ diff --git a/opendbt/airflow/dbtdocs.py b/opendbt/airflow/dbtdocs.py new file mode 100644 index 0000000..ef0c8eb --- /dev/null +++ b/opendbt/airflow/dbtdocs.py @@ -0,0 +1,35 @@ +from pathlib import Path + + +def init_plugins_dbtdocs_page(dbt_docs_dir: Path): + from airflow.plugins_manager import AirflowPlugin + from flask import Blueprint + from flask_appbuilder import BaseView, expose + + class DBTDocsView(BaseView): + # Use it like this if you want to restrict your view to readonly:: + base_permissions = ['can_list', 'can_show'] + default_view = "index" + + @expose("/") + def index(self): + return dbt_docs_dir.joinpath("index.html").read_text() + # return self.render_template("index.html", content="") + + # Creating a flask blueprint to integrate the templates and static folder + bp = Blueprint( + "DBT Docs Plugin", + __name__, + template_folder=dbt_docs_dir.as_posix(), + static_folder=dbt_docs_dir.as_posix(), + static_url_path='/dbtdocsview' + ) + + v_header_menu = {"name": "DBT Docs", "category": "", "view": DBTDocsView()} + + class AirflowDbtDocsPlugin(AirflowPlugin): + name = "DBT Docs Plugin" + flask_blueprints = [bp] + appbuilder_views = [v_header_menu] + + return AirflowDbtDocsPlugin diff --git a/setup.py b/setup.py index e236d60..35b8962 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ 'opendbt = opendbt:main', ], }, - version='0.1.0', + version='0.2.0', packages=find_packages(), author="Memiiso Organization", description='Python opendbt', diff --git a/tests/resources/airflow/docker-compose.yaml b/tests/resources/airflow/docker-compose.yaml index 96792ec..c220b9e 100644 --- a/tests/resources/airflow/docker-compose.yaml +++ b/tests/resources/airflow/docker-compose.yaml @@ -14,6 +14,7 @@ services: - ./airflow/webserver_config.py:/opt/airflow/webserver_config.py - ./airflow/airflow.cfg:/opt/airflow/airflow.cfg - ./dags:/opt/airflow/dags:rw + - ./plugins:/opt/airflow/plugins:rw - ./../dbttest:/opt/dbttest:rw - ./../../../opendbt/macros:/opt/dbttest/macros:rw environment: diff --git a/tests/resources/airflow/plugins/airflow_dbtdocs_page.py b/tests/resources/airflow/plugins/airflow_dbtdocs_page.py new file mode 100644 index 0000000..891d0b2 --- /dev/null +++ b/tests/resources/airflow/plugins/airflow_dbtdocs_page.py @@ -0,0 +1,6 @@ +from pathlib import Path + +from opendbt.airflow import dbtdocs + +# create public page on airflow server to serve DBT docs +airflow_dbtdocs_page = dbtdocs.init_plugins_dbtdocs_page(Path("/opt/dbttest/target")) diff --git a/tests/test_opendbt_project.py b/tests/test_opendbt_project.py index 4db9009..1dd1c02 100644 --- a/tests/test_opendbt_project.py +++ b/tests/test_opendbt_project.py @@ -15,3 +15,7 @@ def test_run_compile(self): def test_run_run(self): dp = OpenDbtProject(project_dir=self.DBTTEST_DIR, profiles_dir=self.DBTTEST_DIR) dp.run(command="run", args=['--select', 'my_first_dbt_model+'], use_subprocess=True) + + def test_run_docs_generate(self): + dp = OpenDbtProject(project_dir=self.DBTTEST_DIR, profiles_dir=self.DBTTEST_DIR) + dp.run(command="docs", args=["generate"])