🔖 Prepare 0.8.1 release (#543)

* improve job search

* provide fix

* provide fixes

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 [Please read through the Keep a Changelog (~5min)](https://keepachangelog.com/en/1.0.0/).
 
+## [0.8.1] - 2022-11-02
+
+## Changed
+- 📖 Reference documentation for deployment file
+- ♻️ Add extensive caching for job listing
+
 ## [0.8.0] - 2022-11-02
 
 ### Changed

dbx/__init__.py

@@ -1 +1 @@
-__version__ = "0.8.0"
+__version__ = "0.8.1"

dbx/api/deployment.py

@@ -5,7 +5,7 @@
 from dbx.api.services.permissions import PermissionsService
 from dbx.api.services.pipelines import NamedPipelinesService
 from dbx.models.deployment import WorkflowList, AnyWorkflow
-from dbx.models.workflow.common.pipeline import Pipeline
+from dbx.models.workflow.common.workflow_types import WorkflowType
 from dbx.utils import dbx_echo
 
 
@@ -14,13 +14,16 @@ def __init__(self, api_client: ApiClient, workflows: WorkflowList):
         super().__init__(api_client)
         self._wfs = workflows
         self._deployment_data = {}
+        self._pipeline_service = NamedPipelinesService(api_client)
+        self._jobs_service = NamedJobsService(api_client)
 
     def _apply_permissions(self, wf: AnyWorkflow):
         PermissionsService(self.api_client).apply(wf)
 
     def _deploy(self, wf: AnyWorkflow):
-        operator_service = NamedPipelinesService if isinstance(wf, Pipeline) else NamedJobsService
-        service_instance = operator_service(self.api_client)
+        service_instance = (
+            self._jobs_service if not wf.workflow_type == WorkflowType.pipeline else self._pipeline_service
+        )
         obj_id = service_instance.find_by_name(wf.name)
 
         if not obj_id:

dbx/api/services/jobs.py

@@ -1,4 +1,5 @@
 import time
+from functools import lru_cache
 from typing import List, Optional, Union
 
 from databricks_cli.sdk import ApiClient, JobsService
@@ -37,24 +38,28 @@ def __init__(self, api_client: ApiClient):
         super().__init__(api_client)
         self._service = JobsService(api_client)
 
-    def find_by_name(self, name: str) -> Optional[int]:
+    @property
+    @lru_cache(maxsize=2000)
+    def all_jobs(self) -> List[JobResponse]:
         offset = 0
         all_jobs = []
-
-        with Console().status(f"Searching for a job with name {name}", spinner="dots"):
+        with Console().status("Listing all available jobs", spinner="dots") as st:
             while True:
-                time.sleep(0.5)  # to avoid bursting out the jobs API
+                time.sleep(0.05)  # to avoid bursting out the jobs API
                 _response = ListJobsResponse(
                     **self._service.list_jobs(
                         limit=self.DEFAULT_LIST_LIMIT, offset=offset, version=self.JOBS_API_VERSION_FOR_SEARCH
                     )
                 )
                 all_jobs.extend(_response.jobs)
+                st.update(f"Listing all available jobs, total jobs checked {len(all_jobs)}")
                 offset += self.DEFAULT_LIST_LIMIT
                 if not _response.has_more:
                     break
+        return all_jobs
 
-        _found_ids = [j.job_id for j in all_jobs if j.settings.name == name]
+    def find_by_name(self, name: str) -> Optional[int]:
+        _found_ids = [j.job_id for j in self.all_jobs if j.settings.name == name]
 
         if len(_found_ids) > 1:
             raise Exception(

docs/reference/deployment.md

@@ -271,7 +271,7 @@ custom:
     runtime_engine: STANDARD
     init_scripts:
       - dbfs:
-        destination: dbfs:/<enter your path>
+          destination: dbfs:/<enter your path>
 
   basic-auto-scale-props: &basic-auto-scale-props
     autoscale:
@@ -285,9 +285,9 @@ custom:
 
   basic-autoscale-cluster: &basic-autoscale-cluster
     new_cluster:
-       <<: # merge these two maps and place them here.
-         - *basic-cluster-props
-         - *basic-auto-scale-props
+      <<: # merge these two maps and place them here.
+        - *basic-cluster-props
+        - *basic-auto-scale-props
 
 environments:
   default:
@@ -298,31 +298,31 @@ environments:
           on_start: [ "[email protected]" ]
           on_success: [ "[email protected]" ]
           on_failure: [ "[email protected]" ]
-          no_alert_for_skipped_runs: false
+
+        no_alert_for_skipped_runs: false
 
         schedule:
-           quartz_cron_expression: "00 25 03 * * ?" #(1)
-           timezone_id: "UTC"
-           pause_status: "PAUSED"
+          quartz_cron_expression: "00 25 03 * * ?" #(1)
+          timezone_id: "UTC"
+          pause_status: "PAUSED"
 
         tags:
           your-key: "your-value"
           your-key1: "your-value1"
 
         format: MULTI_TASK #(2)
 
-        permissions:
-          access_control_list:
-            - user_name: "[email protected]"
-              permission_level: "IS_OWNER"
-            - group_name: "your-group-name"
-              permission_level: "CAN_VIEW"
+        access_control_list:
+          - user_name: "service-principal://some-service-principal"
+            permission_level: "IS_OWNER"
+          - group_name: "your-group-name"
+            permission_level: "CAN_VIEW"
 
         job_clusters:
           - job_cluster_key: "basic-cluster"
-              <<: *basic-static-cluster
+            <<: *basic-static-cluster
           - job_cluster_key: "basic-autoscale-cluster"
-              <<: *basic-autoscale-cluster
+            <<: *basic-autoscale-cluster
 
         tasks:
           - task_key: "your-task-01"
@@ -350,12 +350,28 @@ environments:
             depends_on:
               - task_key: "your-task-01"
 
-          - task_key: "your-task-02"
+          - task_key: "your-task-03"
             job_cluster_key: "basic-cluster"
             notebook_task:
               notebook_path: "/Repos/some/project/notebook"
             depends_on:
               - task_key: "your-task-01"
+
+          - task_key: "example_sql_task"
+            job_cluster_key: "basic-cluster"
+            sql_task:
+              query: "query://some-query-name"
+              warehouse_id: "warehouse://some-warehouse-id"
+
+          - task_key: "example_dbt_task"
+            job_cluster_key: "basic-cluster"
+            dbt_task:
+              project_directory: "/some/project/dir"
+              profiles_directory: "/some/profiles/dir"
+              warehouse_id: "warehouse://some-warehouse-id"
+              commands:
+                - "dbt cmd1"
+                - "dbt cmd2"
 ```
 
 1. Read more about scheduling in [this section](./deployment.md#scheduling-workflows)