Skip to content

Commit bee77b6

Browse files
authored
Merge pull request #3 from outerbounds/custom-python-path
use `/usr/bin/python3` by default
2 parents 2e14f1d + 4457534 commit bee77b6

File tree

6 files changed

+18
-3
lines changed

6 files changed

+18
-3
lines changed

README.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,13 @@ PS -- If you are on the [Outerbounds](https://outerbounds.com/) platform, the au
8585

8686
- The extension runs workloads via shell scripts and `sbatch` in a linux native environment
8787
- i.e. the workloads are NOT run inside docker containers
88-
- As such, the compute instances should not have `python2` installed and both `python` and `python3` should refer to a python version above 3.8 preferrably.
88+
- As such, the compute instances should have `python3` installed (above 3.8 preferrably)
89+
- If the default `python` points to `python2`, one can use the `path_to_python3` argument of the decorator i.e.
90+
91+
```py
92+
@slurm(
93+
path_to_python3="/usr/bin/python3",
94+
)
95+
```
8996

9097
### Fin.

metaflow_extensions/slurm_ext/plugins/slurm/slurm.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ def __init__(
5757
self.datastore = datastore
5858
self.metadata = metadata
5959
self.environment = environment
60+
self.python3_executable = slurm_access_params.pop("path_to_python3", None) or "/usr/bin/python3"
6061
self.slurm_client = SlurmClient(**slurm_access_params)
6162
atexit.register(lambda: self.job.kill() if hasattr(self, "job") else None)
6263

@@ -115,6 +116,7 @@ def _command(self, environment, code_package_url, step_name, step_cmds, task_spe
115116
'${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"} && %s'
116117
% cmd_str
117118
)
119+
cmd_str = 'python() { %s \\"$@\\"; }; export -f python && %s' % (self.python3_executable, cmd_str)
118120

119121
return shlex.split('bash -c "%s"' % cmd_str)
120122

metaflow_extensions/slurm_ext/plugins/slurm/slurm_cli.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ def slurm():
4141
@click.option("--ssh-key-file", help="SSH key file for login node for Slurm.")
4242
@click.option("--cert-file", help="Certificate file for login node for Slurm.")
4343
@click.option("--remote-workdir", help="Remote working directory for Slurm.")
44+
@click.option("--path-to-python3", help="Path to python3 executable on Slurm.")
4445
@click.option(
4546
"--cleanup", help="Cleanup created artifacts on Slurm.", is_flag=True, default=False
4647
)
@@ -88,6 +89,7 @@ def step(
8889
ssh_key_file=None,
8990
cert_file=None,
9091
remote_workdir=None,
92+
path_to_python3=None,
9193
cleanup=False,
9294
partition=None,
9395
nodes=None,
@@ -206,6 +208,7 @@ def _sync_metadata():
206208
"ssh_key_file": ssh_key_file,
207209
"cert_file": cert_file,
208210
"remote_workdir": remote_workdir,
211+
"path_to_python3": path_to_python3,
209212
"cleanup": cleanup,
210213
},
211214
)

metaflow_extensions/slurm_ext/plugins/slurm/slurm_decorator.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ class SlurmDecorator(StepDecorator):
4141
remote_workdir : str, optional, default None
4242
Working directory on the remote Slurm cluster where job files will be stored.
4343
If not specified, defaults to METAFLOW_SLURM_REMOTE_WORKDIR environment variable.
44+
path_to_python3: str, optional, default None
45+
The path to the python3 executable on the Slurm cluster.
4446
cleanup : bool, default False
4547
If True, cleanup created artifacts on Slurm after job completion. This doesn't
4648
delete the log files.
@@ -63,6 +65,7 @@ class SlurmDecorator(StepDecorator):
6365
"ssh_key_file": None,
6466
"cert_file": None,
6567
"remote_workdir": None,
68+
"path_to_python3": None,
6669
"cleanup": False,
6770
"partition": None,
6871
"nodes": None,
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
__mf_extensions__ = "slurm"
2-
__version__ = "0.0.1"
2+
__version__ = "0.0.2"

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from setuptools import setup, find_namespace_packages
22

3-
version = "0.0.1"
3+
version = "0.0.2"
44

55
setup(
66
name="metaflow-slurm",

0 commit comments

Comments
 (0)