CLI to Compare and Apply Configs, Base Datasource Feature Additions

.github/workflows/python-package.yml

@@ -32,6 +32,10 @@ jobs:
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Run Pytest on the tests in tests/
+      run:
+        pytest tests/ -p no:warnings -vv
     - name: Run pytest on tableau-utilities
       run: |
         cd tableau_utilities && pytest -v
+

.gitignore

@@ -4,6 +4,7 @@ sheets.googleapis.com-python.json
 .idea
 .DS_Store
 tmp_tdsx_and_config/
+development_test_files/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -20,6 +20,7 @@ This extra package depends on the tableauhyperapi which is incompatible with App
 #### Locally using pip
 - `cd tableau-utilities`
 - `pip install ./`
+- `pip install --upgrade ./` to overwrite the existing installation without having to uninstall first
 
 #### Confirm installation
 - `which tableau_utilities`

airflow_example/dags/tableau_datasource_update/tableau_datasource_update.py

@@ -314,54 +314,54 @@ def __compare_folders(self, datasource_id, tds_folders, cfg_folders):
     def execute(self, context):
         """ Update Tableau datasource according to config. """
 
-        github_conn = BaseHook.get_connection(self.github_conn_id)
-        config = cfg.Config(
-            githup_token=github_conn.password,
-            repo_name=github_conn.extra_dejson.get('repo_name'),
-            repo_branch=github_conn.extra_dejson.get('repo_branch'),
-            subfolder=github_conn.extra_dejson.get('subfolder')
-        )
-
-        ts = get_tableau_server(self.tableau_conn_id)
-        expected_conn_attrs = self.__set_connection_attributes()
-
-        # Get the ID for each datasource in the config
-        for ds in ts.get.datasources():
-            if ds not in config.datasources:
-                continue
-            config.datasources[ds].id = ds.id
-
-        for datasource in config.datasources:
-            if not datasource.id:
-                logging.error('!! Datasource not found in Tableau Online: %s / %s',
-                              datasource.project_name, datasource.name)
-                continue
-            dsid = datasource.id
+        # github_conn = BaseHook.get_connection(self.github_conn_id)
+        # config = cfg.Config(
+        #     githup_token=github_conn.password,
+        #     repo_name=github_conn.extra_dejson.get('repo_name'),
+        #     repo_branch=github_conn.extra_dejson.get('repo_branch'),
+        #     subfolder=github_conn.extra_dejson.get('subfolder')
+        # )
+        #
+        # ts = get_tableau_server(self.tableau_conn_id)
+        # expected_conn_attrs = self.__set_connection_attributes()
+        #
+        # # Get the ID for each datasource in the config
+        # for ds in ts.get.datasources():
+        #     if ds not in config.datasources:
+        #         continue
+        #     config.datasources[ds].id = ds.id
+        #
+        # # for datasource in config.datasources:
+        #     if not datasource.id:
+        #         logging.error('!! Datasource not found in Tableau Online: %s / %s',
+        #                       datasource.project_name, datasource.name)
+        #         continue
+        #     dsid = datasource.id
             # Set default dict attributes for tasks, for each datasource
             self.tasks[dsid] = {a: [] for a in UPDATE_ACTIONS}
             self.tasks[dsid]['project'] = datasource.project_name
             self.tasks[dsid]['datasource_name'] = datasource.name
-            if not config.in_maintenance_window and AIRFLOW_ENV not in ['STAGING', 'DEV']:
-                self.tasks[dsid]['skip'] = 'Outside maintenance window'
-                logging.info('(SKIP) Outside maintenance window: %s', datasource.name)
-                continue
-            elif datasource.name in EXCLUDED_DATASOURCES:
-                self.tasks[dsid]['skip'] = 'Marked to exclude'
-                logging.info('(SKIP) Marked to exclude: %s', datasource.name)
-                continue
-            logging.info('Checking Datasource: %s', datasource.name)
-            # Download the Datasource for comparison
-            dl_path = f"downloads/{dsid}/"
-            os.makedirs(dl_path, exist_ok=True)
-            ds_path = ts.download.datasource(dsid, file_dir=dl_path, include_extract=False)
-            tds = Datasource(ds_path)
-            # Cleanup downloaded file after assigning the Datasource
-            shutil.rmtree(dl_path, ignore_errors=True)
-            # Add connection task, if there is a difference
-            self.__compare_connection(dsid, datasource.name, tds.connection, expected_conn_attrs)
-            # Add folder tasks, if folders need to be added/deleted
-            self.__compare_folders(dsid, tds.folders_common, datasource.folders)
-            # Add Column tasks, if there are missing columns, or columns need to be updated
+            # if not config.in_maintenance_window and AIRFLOW_ENV not in ['STAGING', 'DEV']:
+            #     self.tasks[dsid]['skip'] = 'Outside maintenance window'
+            #     logging.info('(SKIP) Outside maintenance window: %s', datasource.name)
+            #     continue
+            # elif datasource.name in EXCLUDED_DATASOURCES:
+            #     self.tasks[dsid]['skip'] = 'Marked to exclude'
+            #     logging.info('(SKIP) Marked to exclude: %s', datasource.name)
+            #     continue
+            # logging.info('Checking Datasource: %s', datasource.name)
+            # # Download the Datasource for comparison
+            # dl_path = f"downloads/{dsid}/"
+            # os.makedirs(dl_path, exist_ok=True)
+            # ds_path = ts.download.datasource(dsid, file_dir=dl_path, include_extract=False)
+            # tds = Datasource(ds_path)
+            # # Cleanup downloaded file after assigning the Datasource
+            # shutil.rmtree(dl_path, ignore_errors=True)
+            # # Add connection task, if there is a difference
+            # self.__compare_connection(dsid, datasource.name, tds.connection, expected_conn_attrs)
+            # # Add folder tasks, if folders need to be added/deleted
+            # self.__compare_folders(dsid, tds.folders_common, datasource.folders)
+            # # Add Column tasks, if there are missing columns, or columns need to be updated
             for column in datasource.columns:
                 # Check if the column metadata needs to be updated
                 self.__compare_column_metadata(dsid, tds, column)

setup.py

@@ -12,7 +12,7 @@
     long_description=readme,
     long_description_content_type='text/markdown',
     name="tableau_utilities",
-    version="2.2.11",
+    version="2.2.12",
     requires_python=">=3.8",
     packages=[
         'tableau_utilities',