From 788facaee02988132027dbec9b6c7c6664acd326 Mon Sep 17 00:00:00 2001 From: xman1979 Date: Fri, 30 Aug 2024 14:01:29 -0700 Subject: [PATCH 1/4] keep origina tmp slurm submission file --- submitit/core/core.py | 4 ++-- submitit/core/utils.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/submitit/core/core.py b/submitit/core/core.py index d74b005..5b04dff 100644 --- a/submitit/core/core.py +++ b/submitit/core/core.py @@ -925,7 +925,7 @@ def _submit_command(self, command: str) -> Job[tp.Any]: """ tmp_uuid = uuid.uuid4().hex submission_file_path = ( - utils.JobPaths.get_first_id_independent_folder(self.folder) / f"submission_file_{tmp_uuid}.sh" + utils.JobPaths.get_first_id_independent_folder(self.folder) / f".submission_file_{tmp_uuid}.sh" ) with submission_file_path.open("w") as f: f.write(self._make_submission_file_text(command, tmp_uuid)) @@ -935,7 +935,7 @@ def _submit_command(self, command: str) -> Job[tp.Any]: job_id = self._get_job_id_from_submission_command(output) tasks_ids = list(range(self._num_tasks())) job: Job[tp.Any] = self.job_class(folder=self.folder, job_id=job_id, tasks=tasks_ids) - job.paths.move_temporary_file(submission_file_path, "submission_file") + job.paths.copy_temporary_file(submission_file_path, "submission_file") self._write_job_id(job.job_id, tmp_uuid) self._set_job_permissions(job.paths.folder) return job diff --git a/submitit/core/utils.py b/submitit/core/utils.py index ab0bd01..833806d 100644 --- a/submitit/core/utils.py +++ b/submitit/core/utils.py @@ -97,6 +97,10 @@ def move_temporary_file(self, tmp_path: tp.Union[Path, str], name: str) -> None: self.folder.mkdir(parents=True, exist_ok=True) Path(tmp_path).rename(getattr(self, name)) + def copy_temporary_file(self, tmp_path: tp.Union[Path, str], name: str) -> None: + self.folder.mkdir(parents=True, exist_ok=True) + shutil.copyfile(tmp_path, getattr(self, name)) + @staticmethod def get_first_id_independent_folder(folder: tp.Union[Path, str]) -> Path: """Returns the closest folder which is id independent""" From 5a6b02ea1c0e63df009e653466f6b932356ebba7 Mon Sep 17 00:00:00 2001 From: xman1979 Date: Mon, 16 Sep 2024 09:27:47 -0700 Subject: [PATCH 2/4] recreate temporary submission file as symlink --- submitit/core/core.py | 4 ++-- submitit/core/utils.py | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/submitit/core/core.py b/submitit/core/core.py index 5b04dff..3795591 100644 --- a/submitit/core/core.py +++ b/submitit/core/core.py @@ -925,7 +925,7 @@ def _submit_command(self, command: str) -> Job[tp.Any]: """ tmp_uuid = uuid.uuid4().hex submission_file_path = ( - utils.JobPaths.get_first_id_independent_folder(self.folder) / f".submission_file_{tmp_uuid}.sh" + utils.JobPaths.get_first_id_independent_folder(self.folder) / f"submission_file_{tmp_uuid}.sh" ) with submission_file_path.open("w") as f: f.write(self._make_submission_file_text(command, tmp_uuid)) @@ -935,7 +935,7 @@ def _submit_command(self, command: str) -> Job[tp.Any]: job_id = self._get_job_id_from_submission_command(output) tasks_ids = list(range(self._num_tasks())) job: Job[tp.Any] = self.job_class(folder=self.folder, job_id=job_id, tasks=tasks_ids) - job.paths.copy_temporary_file(submission_file_path, "submission_file") + job.paths.move_temporary_file(submission_file_path, "submission_file", keep_as_symlink=True) self._write_job_id(job.job_id, tmp_uuid) self._set_job_permissions(job.paths.folder) return job diff --git a/submitit/core/utils.py b/submitit/core/utils.py index 833806d..d8ea7a0 100644 --- a/submitit/core/utils.py +++ b/submitit/core/utils.py @@ -93,13 +93,11 @@ def _format_id(self, path: tp.Union[Path, str]) -> Path: replaced_path = replaced_path.replace("%a", array_index[0]) return Path(replaced_path.replace("%A", array_id)) - def move_temporary_file(self, tmp_path: tp.Union[Path, str], name: str) -> None: + def move_temporary_file(self, tmp_path: tp.Union[Path, str], name: str, keep_as_symlink: bool = False) -> None: self.folder.mkdir(parents=True, exist_ok=True) Path(tmp_path).rename(getattr(self, name)) - - def copy_temporary_file(self, tmp_path: tp.Union[Path, str], name: str) -> None: - self.folder.mkdir(parents=True, exist_ok=True) - shutil.copyfile(tmp_path, getattr(self, name)) + if keep_as_symlink: + Path(tmp_path).symlink_to(getattr(self, name)) @staticmethod def get_first_id_independent_folder(folder: tp.Union[Path, str]) -> Path: From e946cc76749eeb906c08896f89d3df3343b0d1b0 Mon Sep 17 00:00:00 2001 From: xman1979 Date: Tue, 17 Sep 2024 10:49:08 -0700 Subject: [PATCH 3/4] use submission_file as hidden file --- submitit/core/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submitit/core/core.py b/submitit/core/core.py index 3795591..53a4d9a 100644 --- a/submitit/core/core.py +++ b/submitit/core/core.py @@ -925,7 +925,7 @@ def _submit_command(self, command: str) -> Job[tp.Any]: """ tmp_uuid = uuid.uuid4().hex submission_file_path = ( - utils.JobPaths.get_first_id_independent_folder(self.folder) / f"submission_file_{tmp_uuid}.sh" + utils.JobPaths.get_first_id_independent_folder(self.folder) / f".submission_file_{tmp_uuid}.sh" ) with submission_file_path.open("w") as f: f.write(self._make_submission_file_text(command, tmp_uuid)) From bc6b036382166394feb56f57e10099fad83d459d Mon Sep 17 00:00:00 2001 From: xman1979 Date: Wed, 18 Sep 2024 08:27:42 -0700 Subject: [PATCH 4/4] bump to 1.5.2 --- submitit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submitit/__init__.py b/submitit/__init__.py index 5936c67..5df0a3a 100644 --- a/submitit/__init__.py +++ b/submitit/__init__.py @@ -18,4 +18,4 @@ from .slurm.slurm import SlurmExecutor as SlurmExecutor from .slurm.slurm import SlurmJob as SlurmJob -__version__ = "1.5.1" +__version__ = "1.5.2"