From e974f1b7e564720b4809c802daf52c3293bb8c47 Mon Sep 17 00:00:00 2001 From: Madhur Tandon Date: Wed, 11 Dec 2024 00:44:34 +0530 Subject: [PATCH 1/5] use /usr/bin/python3 by default --- metaflow_extensions/slurm_ext/plugins/slurm/slurm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py b/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py index 0b733ab..b4f06e1 100644 --- a/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py +++ b/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py @@ -115,6 +115,7 @@ def _command(self, environment, code_package_url, step_name, step_cmds, task_spe '${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"} && %s' % cmd_str ) + cmd_str = 'python() { /usr/bin/python3 \\"$@\\"; }; export -f python && %s' % cmd_str return shlex.split('bash -c "%s"' % cmd_str) From 0bf8f2c9baa598c197534aa7f4c08d0ac4aafb39 Mon Sep 17 00:00:00 2001 From: Madhur Tandon Date: Wed, 11 Dec 2024 01:08:19 +0530 Subject: [PATCH 2/5] add path_to_python3 --- metaflow_extensions/slurm_ext/plugins/slurm/slurm.py | 3 ++- metaflow_extensions/slurm_ext/plugins/slurm/slurm_cli.py | 3 +++ metaflow_extensions/slurm_ext/plugins/slurm/slurm_decorator.py | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py b/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py index b4f06e1..2e934ec 100644 --- a/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py +++ b/metaflow_extensions/slurm_ext/plugins/slurm/slurm.py @@ -57,6 +57,7 @@ def __init__( self.datastore = datastore self.metadata = metadata self.environment = environment + self.python3_executable = slurm_access_params.pop("path_to_python3", None) or "/usr/bin/python3" self.slurm_client = SlurmClient(**slurm_access_params) atexit.register(lambda: self.job.kill() if hasattr(self, "job") else None) @@ -115,7 +116,7 @@ def _command(self, environment, code_package_url, step_name, step_cmds, task_spe '${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"} && %s' % cmd_str ) - cmd_str = 'python() { /usr/bin/python3 \\"$@\\"; }; export -f python && %s' % cmd_str + cmd_str = 'python() { %s \\"$@\\"; }; export -f python && %s' % (self.python3_executable, cmd_str) return shlex.split('bash -c "%s"' % cmd_str) diff --git a/metaflow_extensions/slurm_ext/plugins/slurm/slurm_cli.py b/metaflow_extensions/slurm_ext/plugins/slurm/slurm_cli.py index f05c0e5..880ac5d 100644 --- a/metaflow_extensions/slurm_ext/plugins/slurm/slurm_cli.py +++ b/metaflow_extensions/slurm_ext/plugins/slurm/slurm_cli.py @@ -41,6 +41,7 @@ def slurm(): @click.option("--ssh-key-file", help="SSH key file for login node for Slurm.") @click.option("--cert-file", help="Certificate file for login node for Slurm.") @click.option("--remote-workdir", help="Remote working directory for Slurm.") +@click.option("--path-to-python3", help="Path to python3 executable on Slurm.") @click.option( "--cleanup", help="Cleanup created artifacts on Slurm.", is_flag=True, default=False ) @@ -88,6 +89,7 @@ def step( ssh_key_file=None, cert_file=None, remote_workdir=None, + path_to_python3=None, cleanup=False, partition=None, nodes=None, @@ -206,6 +208,7 @@ def _sync_metadata(): "ssh_key_file": ssh_key_file, "cert_file": cert_file, "remote_workdir": remote_workdir, + "path_to_python3": path_to_python3, "cleanup": cleanup, }, ) diff --git a/metaflow_extensions/slurm_ext/plugins/slurm/slurm_decorator.py b/metaflow_extensions/slurm_ext/plugins/slurm/slurm_decorator.py index 3754086..0168801 100644 --- a/metaflow_extensions/slurm_ext/plugins/slurm/slurm_decorator.py +++ b/metaflow_extensions/slurm_ext/plugins/slurm/slurm_decorator.py @@ -41,6 +41,8 @@ class SlurmDecorator(StepDecorator): remote_workdir : str, optional, default None Working directory on the remote Slurm cluster where job files will be stored. If not specified, defaults to METAFLOW_SLURM_REMOTE_WORKDIR environment variable. + path_to_python3: str, optional, default None + The path to the python3 executable on the Slurm cluster. cleanup : bool, default False If True, cleanup created artifacts on Slurm after job completion. This doesn't delete the log files. @@ -63,6 +65,7 @@ class SlurmDecorator(StepDecorator): "ssh_key_file": None, "cert_file": None, "remote_workdir": None, + "path_to_python3": None, "cleanup": False, "partition": None, "nodes": None, From 5ac2f982b064da1b5e56cc2df993b0614274abb0 Mon Sep 17 00:00:00 2001 From: Madhur Tandon Date: Wed, 11 Dec 2024 01:18:18 +0530 Subject: [PATCH 3/5] update docs --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index afcab92..0789e27 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,13 @@ PS -- If you are on the [Outerbounds](https://outerbounds.com/) platform, the au - The extension runs workloads via shell scripts and `sbatch` in a linux native environment - i.e. the workloads are NOT run inside docker containers - - As such, the compute instances should not have `python2` installed and both `python` and `python3` should refer to a python version above 3.8 preferrably. + - As such, the compute instances should have `python3` installed (above 3.8 preferrably) + - If the default `python` points to `python2`, one can use the `path_to_python3` argument of the decorator i.e. + +```py +@slurm( + path_to_python3="/usr/bin/python3", +) +``` ### Fin. From 6c7619763ee7fa3ad6a6b64fc390806fda88fc67 Mon Sep 17 00:00:00 2001 From: Madhur Tandon Date: Wed, 11 Dec 2024 01:20:50 +0530 Subject: [PATCH 4/5] release 0.0.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a477f0f..8703f2b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_namespace_packages -version = "0.0.1" +version = "0.0.2" setup( name="metaflow-slurm", From 445753404e3a81c1700cc6dac99ac9a4d2f865b0 Mon Sep 17 00:00:00 2001 From: Madhur Tandon Date: Wed, 11 Dec 2024 01:21:51 +0530 Subject: [PATCH 5/5] change version in extension too --- metaflow_extensions/slurm_ext/toplevel/toplevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow_extensions/slurm_ext/toplevel/toplevel.py b/metaflow_extensions/slurm_ext/toplevel/toplevel.py index 7233eb0..62310bb 100644 --- a/metaflow_extensions/slurm_ext/toplevel/toplevel.py +++ b/metaflow_extensions/slurm_ext/toplevel/toplevel.py @@ -1,2 +1,2 @@ __mf_extensions__ = "slurm" -__version__ = "0.0.1" +__version__ = "0.0.2"