[FEATURE] Jinja variables file support in execute (#381)

.github/workflows/onpush.yml

@@ -15,25 +15,18 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [ 3.8 ]
+        python-version: [ '3.9' ]
         os: [ ubuntu-latest, windows-latest ]
 
     steps:
       - uses: actions/checkout@v1
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      - uses: actions/cache@v2
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-            ${{ runner.os }}-pip-
+          cache: 'pip' # caching pip dependencies
+          cache-dependency-path: setup.py
 
       - name: Install pip
         run: python -m pip install --upgrade pip

.github/workflows/onrelease.yml

@@ -18,18 +18,11 @@ jobs:
       - uses: actions/checkout@v1
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      - uses: actions/cache@v2
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-            ${{ runner.os }}-pip-
+          cache: 'pip' # caching pip dependencies
+          cache-dependency-path: setup.py
 
       - name: Install pip
         run: python -m pip install --upgrade pip

CHANGELOG.md

@@ -8,10 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 [Please read through the Keep a Changelog (~5min)](https://keepachangelog.com/en/1.0.0/).
 
-## [X.Y.Z] - YYYY-MM-DD
+## [Unreleased] - YYYY-MM-DD
+
+## Added
+
+- `--jinja-variables-file` for `dbx execute`
 
 ## Fixed
 - Support `jobs_api_version` values provided by config in `ApiClient` construction
+- References and wording in the Python template
 
 ----
 > Unreleased changes must be tracked above this line.

dbx/commands/deploy.py

@@ -1,7 +1,7 @@
 import json
-from pathlib import Path
 import shutil
 import tempfile
+from pathlib import Path
 from typing import Dict, Any, Union, Optional
 from typing import List
 
@@ -14,18 +14,17 @@
 from requests.exceptions import HTTPError
 
 from dbx.api.config_reader import ConfigReader
-from dbx.callbacks import verify_jinja_variables_file
+from dbx.utils import dbx_echo
 from dbx.utils.adjuster import adjust_job_definitions
 from dbx.utils.common import (
     prepare_environment,
     parse_multiple,
     get_current_branch_name,
 )
-from dbx.utils import dbx_echo
-from dbx.utils.file_uploader import MlflowFileUploader
-from dbx.utils.options import environment_option, deployment_file_option
 from dbx.utils.dependency_manager import DependencyManager
+from dbx.utils.file_uploader import MlflowFileUploader
 from dbx.utils.job_listing import find_job_by_name
+from dbx.utils.options import environment_option, deployment_file_option, jinja_variables_file_option
 
 
 @click.command(
@@ -110,17 +109,7 @@
     help="""The name of the current branch.
               If not provided or empty, dbx will try to detect the branch name.""",
 )
-@click.option(
-    "--jinja-variables-file",
-    type=click.Path(path_type=Path),
-    default=None,
-    required=False,
-    help="""
-        Path to a file with variables for Jinja template. Only works when Jinja-based deployment file is used.
-        Read more about this functionality in the Jinja2 support doc.
-        """,
-    callback=verify_jinja_variables_file,
-)
+@jinja_variables_file_option
 @debug_option
 @environment_option
 @deployment_file_option

dbx/commands/execute.py

@@ -17,7 +17,7 @@
     handle_package,
     _preprocess_cluster_args,
 )
-from dbx.utils.options import environment_option, deployment_file_option
+from dbx.utils.options import environment_option, deployment_file_option, jinja_variables_file_option
 
 
 @click.command(
@@ -78,6 +78,7 @@
 @environment_option
 @debug_option
 @deployment_file_option
+@jinja_variables_file_option
 def execute(
     environment: str,
     cluster_id: str,
@@ -89,6 +90,7 @@ def execute(
     no_package: bool,
     no_rebuild: bool,
     upload_via_context: bool,
+    jinja_variables_file: Optional[Path],
 ):
     api_client = prepare_environment(environment)
 
@@ -98,7 +100,7 @@ def execute(
 
     handle_package(no_rebuild)
 
-    config_reader = ConfigReader(deployment_file)
+    config_reader = ConfigReader(deployment_file, jinja_variables_file)
 
     deployment = config_reader.get_environment(environment)
 

dbx/templates/projects/python_basic/components/.github/workflows/onpush.yml

@@ -21,10 +21,12 @@ jobs:
     steps:
       - uses: actions/checkout@v1
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.7.5
+          python-version: '3.9'
+          cache: 'pip' # caching pip dependencies
+          cache-dependency-path: setup.py
 
       - name: Install pip
         run: |

dbx/templates/projects/python_basic/components/.github/workflows/onrelease.yml

@@ -12,8 +12,6 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
-      matrix:
-        python-version: [ 3.7 ]
 
     env:
       DATABRICKS_HOST: {{ '${{ secrets.DATABRICKS_HOST }}' }}
@@ -22,10 +20,12 @@ jobs:
     steps:
       - uses: actions/checkout@v1
 
-      - name: Set up Python
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: '3.9'
+          cache: 'pip' # caching pip dependencies
+          cache-dependency-path: setup.py
 
       - name: Install pip
         run: |

dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/common.py

@@ -1,6 +1,5 @@
 from abc import ABC, abstractmethod
 from argparse import ArgumentParser
-from logging import Logger
 from typing import Dict, Any
 import yaml
 import pathlib
@@ -86,7 +85,7 @@ def _read_config(conf_file) -> Dict[str, Any]:
         config = yaml.safe_load(pathlib.Path(conf_file).read_text())
         return config
 
-    def _prepare_logger(self) -> Logger:
+    def _prepare_logger(self):
         log4j_logger = self.spark._jvm.org.apache.log4j  # noqa
         return log4j_logger.LogManager.getLogger(self.__class__.__name__)
 

dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_etl_task.py

@@ -14,9 +14,9 @@ def _write_data(self):
         self.logger.info("Dataset successfully written")
 
     def launch(self):
-        self.logger.info("Launching sample ETL job")
+        self.logger.info("Launching sample ETL task")
         self._write_data()
-        self.logger.info("Sample ETL job finished!")
+        self.logger.info("Sample ETL task finished!")
 
 # if you're using python_wheel_task, you'll need the entrypoint function to be used in setup.py
 def entrypoint():  # pragma: no cover

dbx/utils/options.py

@@ -3,6 +3,8 @@
 import click
 from databricks_cli.configure.provider import DEFAULT_SECTION
 
+from dbx.callbacks import verify_jinja_variables_file
+
 
 def environment_option(f):
     return click.option(
@@ -33,3 +35,17 @@ def deployment_file_option(f):
         help="Path to deployment file.",
         is_eager=True,
     )(f)
+
+
+def jinja_variables_file_option(f):
+    return click.option(
+        "--jinja-variables-file",
+        type=click.Path(path_type=Path),
+        default=None,
+        required=False,
+        help="""
+        Path to a file with variables for Jinja template. Only works when Jinja-based deployment file is used.
+        Read more about this functionality in the Jinja2 support doc.
+        """,
+        callback=verify_jinja_variables_file,
+    )(f)