From 062c0cb10a2414caef7d94b1a1eacd6c2b6fb4bf Mon Sep 17 00:00:00 2001 From: Steven Mi Date: Wed, 9 Aug 2023 15:48:42 +0800 Subject: [PATCH] uploade all python packages --- rocket/rocket.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rocket/rocket.py b/rocket/rocket.py index a877635..a6d93c3 100644 --- a/rocket/rocket.py +++ b/rocket/rocket.py @@ -5,7 +5,7 @@ from rocket.logger import logger from rocket.utils import execute_shell_command, extract_project_name_from_wheel, \ - extract_package_name_from_wheel + extract_python_package_dirs def _add_index_urls_to_cmd(cmd, index_urls): @@ -114,10 +114,12 @@ def _deploy(self): execute_shell_command( f"databricks fs cp --overwrite {self.wheel_path} {self.dbfs_folder}/{self.wheel_file}" ) - package_name = extract_package_name_from_wheel(self.wheel_file) - execute_shell_command( - f"databricks fs cp --recursive --overwrite {self.project_location}/{package_name} {self.dbfs_folder}/{package_name}" - ) + package_dirs = extract_python_package_dirs(self.project_location) + print(package_dirs) + for package_dir in package_dirs: + execute_shell_command( + f"databricks fs cp --recursive --overwrite {package_dir} {self.dbfs_folder}/{os.path.basename(package_dirs)}" + ) except Exception as e: raise Exception( f"Error while copying files to databricks, is your databricks token set and valid? Try to generate a new token and update existing one with `databricks configure --token`. Error details: {e}"