initializing, adding hydra conf

.github/dependabot.yml

@@ -4,3 +4,13 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    labels:
+      - "part: github_actions"
+
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    labels:
+      - "lang: python"
+      - "part: dependencies"

.github/workflows/build.yml

@@ -1,3 +1,4 @@
+
 name: Build Status
 
 on:
@@ -30,7 +31,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.9]
+        python-version: ["3.11"]
 
     steps:
     - uses: actions/checkout@v4
@@ -59,8 +60,8 @@ jobs:
     - name: Test
       run: make coverage
       if: ${{ matrix.os == 'ubuntu-latest' }}
-
-    - name: Upload test results
+      
+    - name: Upload test results (Python)
       uses: actions/upload-artifact@v4
       with:
         name: pytest-results-${{ matrix.os }}-${{ matrix.python-version }}
@@ -76,7 +77,10 @@ jobs:
 
     - name: Upload coverage
       uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: Make dist
       run: make dist
       if: ${{ matrix.os == 'ubuntu-latest' }}
+

Makefile

@@ -7,7 +7,7 @@ develop:  ## install dependencies and build library
 	python -m pip install -e .[develop]
 
 build:  ## build the python library
-	python setup.py build build_ext --inplace
+	python -m build -w -s
 
 install:  ## install library
 	python -m pip install .
@@ -18,23 +18,23 @@ install:  ## install library
 .PHONY: lint lints fix format
 
 lint:  ## run python linter with ruff
-	python -m isort --check airflow_config setup.py
-	python -m ruff airflow_config setup.py
+	python -m ruff check airflow_config
+	python -m ruff format --check airflow_config
 
 # Alias
 lints: lint
 
 fix:  ## fix python formatting with ruff
-	python -m isort airflow_config setup.py
-	python -m ruff format airflow_config setup.py
+	python -m ruff check --fix airflow_config
+	python -m ruff format airflow_config
 
 # alias
 format: fix
 
 ################
 # Other Checks #
 ################
-.PHONY: check-manifest checks check
+.PHONY: check-manifest checks check annotate
 
 check-manifest:  ## check python sdist manifest with check-manifest
 	check-manifest -v
@@ -44,6 +44,9 @@ checks: check-manifest
 # Alias
 check: checks
 
+annotate:  ## run python type annotation checks with mypy
+	python -m mypy ./airflow_config
+
 #########
 # TESTS #
 #########
@@ -53,7 +56,7 @@ test:  ## run python tests
 	python -m pytest -v airflow_config/tests --junitxml=junit.xml
 
 coverage:  ## run tests and collect test coverage
-	python -m pytest -v airflow_config/tests --junitxml=junit.xml --cov=airflow_config --cov-branch --cov-fail-under=50 --cov-report term-missing --cov-report xml
+	python -m pytest -v airflow_config/tests --junitxml=junit.xml --cov=airflow_config --cov-branch --cov-fail-under=60 --cov-report term-missing --cov-report xml
 
 # Alias
 tests: test
@@ -64,16 +67,16 @@ tests: test
 .PHONY: show-version patch minor major
 
 show-version:  ## show current library version
-	bump2version --dry-run --allow-dirty setup.py --list | grep current | awk -F= '{print $2}'
+	@bump-my-version show current_version
 
 patch:  ## bump a patch version
-	bump2version patch
+	@bump-my-version bump patch
 
 minor:  ## bump a minor version
-	bump2version minor
+	@bump-my-version bump minor
 
 major:  ## bump a major version
-	bump2version major
+	@bump-my-version bump major
 
 ########
 # DIST #

README.md

@@ -2,7 +2,99 @@
 
 [Apache Airflow](https://airflow.apache.org) utilities for for configuration of many DAGs and DAG environments
 
-[![Build Status](https://github.com/airflow-laminar/airflow-config/actions/workflows/build.yml/badge.svg)](https://github.com/airflow-laminar/airflow-config/actions?query=workflow%3A%22Build+Status%22)
+[![Build Status](https://github.com/airflow-laminar/airflow-config/actions/workflows/build.yml/badge.svg?branch=main&event=push)](https://github.com/airflow-laminar/airflow-config/actions/workflows/build.yml)
 [![codecov](https://codecov.io/gh/airflow-laminar/airflow-config/branch/main/graph/badge.svg)](https://codecov.io/gh/airflow-laminar/airflow-config)
 [![License](https://img.shields.io/github/license/airflow-laminar/airflow-config)](https://github.com/airflow-laminar/airflow-config)
 [![PyPI](https://img.shields.io/pypi/v/airflow-config.svg)](https://pypi.python.org/pypi/airflow-config)
+
+## Overview
+
+This library allows for `YAML`-driven configuration of Airflow, including DAGs, Operators, and declaratively defined DAGs (à la [dag-factory](https://github.com/astronomer/dag-factory)). It is built with [Pydantic](https://pydantic.dev), [Hydra](https://hydra.cc), and [OmegaConf](https://omegaconf.readthedocs.io/).
+
+Consider the following basic DAG:
+
+```python
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from datetime import datetime, timedelta
+
+with DAG(
+    dag_id="test-dag",
+    default_args={
+        "depends_on_past": False,
+        "email": ["[email protected]"],
+        "email_on_failure": False,
+        "email_on_retry": False,
+        "retries": 0,
+    },
+    description="test that dag is working properly",
+    schedule=timedelta(minutes=1),
+    start_date=datetime(2024, 1, 1),
+    catchup=False,
+    tags=["utility", "test"],
+):
+    BashOperator(
+        task_id="test-task",
+        bash_command="echo 'test'",
+    )
+```
+
+We can already see many options that we might want to drive centrally via config, perhaps based on some notion of environment (e.g. `dev`, `prod`, etc).
+
+- `"email": ["[email protected]"]`
+- `"email_on_failure": False`
+- `"email_on_retry": False`
+- `"retries": 0`
+- `schedule=timedelta(minutes=1)`
+- `tags=["utility", "test"]`
+
+If we want to change these in our DAG, we need to modify code. Now imagine we have hundreds of DAGs, this can quickly get out of hand, especially since Airflow DAGs are Python code, and we might easily inject a syntax error or a trailing comma or other common problem.
+
+Now consider the alternative, config-driven approach:
+
+`config/config.yaml`
+
+```yaml
+# @package _global_
+_target_: airflow_config.Configuration
+global_:
+  _target_: airflow_config.GlobalDagConfiguration
+  owner: test
+  email: [[email protected]]
+  email_on_failure: false
+  email_on_retry: false
+  retries: 0
+  depends_on_past: false
+  schedule: "01:00"
+  start_date: "2024-01-01"
+  catchup: false
+  tags: ["utility", "test"]
+```
+
+```python
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from airflow_config import DAG, load_config
+
+config = load_config()
+
+with DAG(dag_id="test-dag", description="test that dag is working properly", config=config):
+    BashOperator(
+        task_id="test-task",
+        bash_command="echo 'test'",
+    )
+```
+
+This has a number of benefits:
+
+- Make changes without code changes, with static type validation
+- Make changes across any number of DAGs without having to copy-paste
+- Organize collections of DAGs into groups, e.g. via enviroment like `dev`, `prod`, etc
+
+## Configuration
+
+More documentation coming soon!
+
+## License
+
+This software is licensed under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.

airflow_config/__init__.py

@@ -1 +1,4 @@
 __version__ = "0.1.0"
+
+from .configuration import *
+from .exceptions import *

airflow_config/configuration/__init__.py

@@ -0,0 +1,3 @@
+from .airflow import GlobalDagConfiguration, DagConfiguration
+from .base import Configuration, load_config
+from .python import PythonConfiguration