Skip to content

Commit

Permalink
Merge pull request #3 from outerbounds/custom-python-path
Browse files Browse the repository at this point in the history
use `/usr/bin/python3` by default
  • Loading branch information
savingoyal authored Dec 10, 2024
2 parents 2e14f1d + 4457534 commit bee77b6
Show file tree
Hide file tree
Showing 6 changed files with 18 additions and 3 deletions.
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,13 @@ PS -- If you are on the [Outerbounds](https://outerbounds.com/) platform, the au

- The extension runs workloads via shell scripts and `sbatch` in a linux native environment
- i.e. the workloads are NOT run inside docker containers
- As such, the compute instances should not have `python2` installed and both `python` and `python3` should refer to a python version above 3.8 preferrably.
- As such, the compute instances should have `python3` installed (above 3.8 preferrably)
- If the default `python` points to `python2`, one can use the `path_to_python3` argument of the decorator i.e.

```py
@slurm(
path_to_python3="/usr/bin/python3",
)
```

### Fin.
2 changes: 2 additions & 0 deletions metaflow_extensions/slurm_ext/plugins/slurm/slurm.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ def __init__(
self.datastore = datastore
self.metadata = metadata
self.environment = environment
self.python3_executable = slurm_access_params.pop("path_to_python3", None) or "/usr/bin/python3"
self.slurm_client = SlurmClient(**slurm_access_params)
atexit.register(lambda: self.job.kill() if hasattr(self, "job") else None)

Expand Down Expand Up @@ -115,6 +116,7 @@ def _command(self, environment, code_package_url, step_name, step_cmds, task_spe
'${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"} && %s'
% cmd_str
)
cmd_str = 'python() { %s \\"$@\\"; }; export -f python && %s' % (self.python3_executable, cmd_str)

return shlex.split('bash -c "%s"' % cmd_str)

Expand Down
3 changes: 3 additions & 0 deletions metaflow_extensions/slurm_ext/plugins/slurm/slurm_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def slurm():
@click.option("--ssh-key-file", help="SSH key file for login node for Slurm.")
@click.option("--cert-file", help="Certificate file for login node for Slurm.")
@click.option("--remote-workdir", help="Remote working directory for Slurm.")
@click.option("--path-to-python3", help="Path to python3 executable on Slurm.")
@click.option(
"--cleanup", help="Cleanup created artifacts on Slurm.", is_flag=True, default=False
)
Expand Down Expand Up @@ -88,6 +89,7 @@ def step(
ssh_key_file=None,
cert_file=None,
remote_workdir=None,
path_to_python3=None,
cleanup=False,
partition=None,
nodes=None,
Expand Down Expand Up @@ -206,6 +208,7 @@ def _sync_metadata():
"ssh_key_file": ssh_key_file,
"cert_file": cert_file,
"remote_workdir": remote_workdir,
"path_to_python3": path_to_python3,
"cleanup": cleanup,
},
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ class SlurmDecorator(StepDecorator):
remote_workdir : str, optional, default None
Working directory on the remote Slurm cluster where job files will be stored.
If not specified, defaults to METAFLOW_SLURM_REMOTE_WORKDIR environment variable.
path_to_python3: str, optional, default None
The path to the python3 executable on the Slurm cluster.
cleanup : bool, default False
If True, cleanup created artifacts on Slurm after job completion. This doesn't
delete the log files.
Expand All @@ -63,6 +65,7 @@ class SlurmDecorator(StepDecorator):
"ssh_key_file": None,
"cert_file": None,
"remote_workdir": None,
"path_to_python3": None,
"cleanup": False,
"partition": None,
"nodes": None,
Expand Down
2 changes: 1 addition & 1 deletion metaflow_extensions/slurm_ext/toplevel/toplevel.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
__mf_extensions__ = "slurm"
__version__ = "0.0.1"
__version__ = "0.0.2"
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from setuptools import setup, find_namespace_packages

version = "0.0.1"
version = "0.0.2"

setup(
name="metaflow-slurm",
Expand Down

0 comments on commit bee77b6

Please sign in to comment.