From 87b77e191d94626b905ade94b6767e08b6c30ef8 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 11 Apr 2024 16:36:29 +0200 Subject: [PATCH 01/13] Add access_on_submission_command sched config option --- docs/config_reference.rst | 14 ++++++++++++++ docs/manpage.rst | 15 +++++++++++++++ reframe/core/schedulers/slurm.py | 7 ++++++- reframe/frontend/cli.py | 7 +++++++ reframe/schemas/config.json | 2 ++ 5 files changed, 44 insertions(+), 1 deletion(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 2632e76833..0269d80981 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -365,6 +365,20 @@ System Partition Configuration .. warning:: This option is broken in 4.0. + +.. py:attribute:: systems.partitions.sched_options.access_on_submission_command + + :required: No + :default: ``false`` + + Normally, ReFrame will pass the :attr:`~config.systems.partitions.access` options to the job script only. + When this attribute is ``true`` the options are passed verbatim also in the submission command. + + This option is currently relevant for the Slurm backends only. + + .. versionadded:: 4.6.0 + + .. py:attribute:: systems.partitions.sched_options.ssh_hosts :required: No diff --git a/docs/manpage.rst b/docs/manpage.rst index 86365b6a77..56013c1444 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1209,6 +1209,21 @@ Here is an alphabetical list of the environment variables recognized by ReFrame. Whenever an environment variable is associated with a configuration option, its default value is omitted as it is the same. +.. envvar:: RFM_ACCESS_ON_SUBMISSION_COMMAND + + Pass access options in the submission command (only for Slurm). + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.access_on_submission_command` + ================================== ================== + +.. versionadded:: 4.6.0 + + .. envvar:: RFM_AUTODETECT_FQDN Use the fully qualified domain name as the hostname. diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 779e270bf0..3481a43781 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -140,6 +140,7 @@ def __init__(self): self._submit_timeout = self.get_option('job_submit_timeout') self._use_nodes_opt = self.get_option('use_nodes_option') self._resubmit_on_errors = self.get_option('resubmit_on_errors') + self._access_on_submission_command = self.get_option('access_on_submission_command') def make_job(self, *args, **kwargs): return _SlurmJob(*args, **kwargs) @@ -250,7 +251,11 @@ def emit_preamble(self, job): return list(filter(None, preamble)) def submit(self, job): - cmd = f'sbatch {job.script_filename}' + cmd_opts = ( + ' '.join(job.sched_access) if self._access_on_submission_command + else '' + ) + cmd = f'sbatch {cmd_opts} {job.script_filename}' intervals = itertools.cycle([1, 2, 3]) while True: try: diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f1d77cc231..d6d8f0e576 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -576,6 +576,13 @@ def main(): ) # Options not associated with command-line arguments + argparser.add_argument( + dest='access_on_submission_command', + envvar='RFM_ACCESS_ON_SUBMISSION_COMMAND', + configvar='systems*/sched_options/access_on_submission_command', + action='store_true', + help='Pass access options in the submission command (only for Slurm)' + ) argparser.add_argument( dest='autodetect_fqdn', envvar='RFM_AUTODETECT_FQDN', diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 4eadc997da..4cbeb90ccc 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -109,6 +109,7 @@ "sched_options": { "type": "object", "properties": { + "access_on_submission_command": {"type": "boolean"}, "hosts": { "type": "array", "items": {"type": "string"} @@ -625,6 +626,7 @@ "systems/partitions/time_limit": null, "systems/partitions/devices": [], "systems/partitions/extras": {}, + "systems*/sched_options/access_on_submission_command": false, "systems*/sched_options/ssh_hosts": [], "systems*/sched_options/ignore_reqnodenotavail": false, "systems*/sched_options/job_submit_timeout": 60, From d38d712c135a0448c7378ca78fba315155ebe4f8 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 11 Apr 2024 17:08:47 +0200 Subject: [PATCH 02/13] Split long line --- reframe/core/schedulers/slurm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 3481a43781..247376f60c 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -140,7 +140,9 @@ def __init__(self): self._submit_timeout = self.get_option('job_submit_timeout') self._use_nodes_opt = self.get_option('use_nodes_option') self._resubmit_on_errors = self.get_option('resubmit_on_errors') - self._access_on_submission_command = self.get_option('access_on_submission_command') + self._access_on_submission_command = self.get_option( + 'access_on_submission_command' + ) def make_job(self, *args, **kwargs): return _SlurmJob(*args, **kwargs) From caede17caa134ac73c0487c6cb6385056059ae70 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 18 Apr 2024 10:13:07 +0200 Subject: [PATCH 03/13] Address PR comments --- docs/config_reference.rst | 2 +- docs/manpage.rst | 4 ++-- reframe/core/schedulers/lsf.py | 11 ++++++++--- reframe/core/schedulers/oar.py | 12 +++++++++--- reframe/core/schedulers/pbs.py | 17 +++++++++++++++-- reframe/core/schedulers/slurm.py | 31 ++++++++++++++++--------------- reframe/frontend/cli.py | 6 +++--- reframe/schemas/config.json | 4 ++-- 8 files changed, 56 insertions(+), 31 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 805d41f0b1..cacf62ca3d 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -366,7 +366,7 @@ System Partition Configuration This option is broken in 4.0. -.. py:attribute:: systems.partitions.sched_options.access_on_submission_command +.. py:attribute:: systems.partitions.sched_options.sched_access_in_submit :required: No :default: ``false`` diff --git a/docs/manpage.rst b/docs/manpage.rst index d7311edd2a..b6b3586d69 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1214,7 +1214,7 @@ Here is an alphabetical list of the environment variables recognized by ReFrame. Whenever an environment variable is associated with a configuration option, its default value is omitted as it is the same. -.. envvar:: RFM_ACCESS_ON_SUBMISSION_COMMAND +.. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT Pass access options in the submission command (only for Slurm). @@ -1223,7 +1223,7 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== Associated command line option N/A - Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.access_on_submission_command` + Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit` ================================== ================== .. versionadded:: 4.6.0 diff --git a/reframe/core/schedulers/lsf.py b/reframe/core/schedulers/lsf.py index 2813e43250..5523aab74f 100644 --- a/reframe/core/schedulers/lsf.py +++ b/reframe/core/schedulers/lsf.py @@ -57,8 +57,9 @@ def emit_preamble(self, job): f'{self._prefix} -W {int(job.time_limit // 60)}' ) - for opt in job.sched_access: - preamble.append(f'{self._prefix} {opt}') + if not self._sched_access_in_submit: + for opt in job.sched_access: + preamble.append(f'{self._prefix} {opt}') # emit the rest of the options options = job.options + job.cli_options @@ -76,7 +77,11 @@ def emit_preamble(self, job): def submit(self, job): with open(job.script_filename, 'r') as fp: - completed = _run_strict('bsub', stdin=fp) + cmd_opts = ( + ' '.join(job.sched_access) if self._sched_access_in_submit + else '' + ) + completed = _run_strict('bsub {cmd_opts}', stdin=fp) jobid_match = re.search(r'^Job <(?P\S+)> is submitted', completed.stdout) if not jobid_match: diff --git a/reframe/core/schedulers/oar.py b/reframe/core/schedulers/oar.py index 34b1524021..f72fe9baec 100644 --- a/reframe/core/schedulers/oar.py +++ b/reframe/core/schedulers/oar.py @@ -88,8 +88,11 @@ def emit_preamble(self, job): num_nodes=num_nodes, num_tasks_per_node=num_tasks_per_node, )] + if not self._sched_access_in_submit: + options += job.sched_access + # Emit the rest of the options - options += job.sched_access + job.options + job.cli_options + options += job.options + job.cli_options for opt in options: if opt.startswith('#'): preamble.append(opt) @@ -101,9 +104,12 @@ def emit_preamble(self, job): def submit(self, job): # OAR batch submission mode needs full path to the job script job_script_fullpath = os.path.join(job.workdir, job.script_filename) - + cmd_opts = ( + ' '.join(job.sched_access) if self._sched_access_in_submit + else '' + ) # OAR needs -S to submit job in batch mode - cmd = f'oarsub -S {job_script_fullpath}' + cmd = f'oarsub {cmd_opts} -S {job_script_fullpath}' completed = _run_strict(cmd, timeout=self._submit_timeout) jobid_match = re.search(r'.*OAR_JOB_ID=(?P\S+)', completed.stdout) diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index 1cd25cabd6..afd0aaa357 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -76,6 +76,9 @@ class PbsJobScheduler(sched.JobScheduler): def __init__(self): self._prefix = '#PBS' self._submit_timeout = self.get_option('job_submit_timeout') + self._sched_access_in_submit = self.get_option( + 'sched_access_in_submit' + ) def _emit_lselect_option(self, job): num_tasks = job.num_tasks or 1 @@ -92,7 +95,12 @@ def _emit_lselect_option(self, job): # Options starting with `-` are emitted in separate lines rem_opts = [] verb_opts = [] - for opt in (*job.sched_access, *job.options, *job.cli_options): + if self._sched_access_in_submit: + all_opts = (*job.options, *job.cli_options) + else: + all_opts = (*job.sched_access, *job.options, *job.cli_options) + + for opt in all_opts: if opt.startswith('-'): rem_opts.append(opt) elif opt.startswith('#'): @@ -139,9 +147,14 @@ def filternodes(self, job, nodes): 'node filtering') def submit(self, job): + cmd_opts = ( + ' '.join(job.sched_access) if self._sched_access_in_submit + else '' + ) # `-o` and `-e` options are only recognized in command line by the PBS # Slurm wrappers. - cmd = f'qsub -o {job.stdout} -e {job.stderr} {job.script_filename}' + cmd = (f'qsub {cmd_opts} -o {job.stdout} -e {job.stderr} ' + f'{job.script_filename}') completed = _run_strict(cmd, timeout=self._submit_timeout) jobid_match = re.search(r'^(?P\S+)', completed.stdout) if not jobid_match: diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 5456782c8f..8d35f58c13 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -140,8 +140,8 @@ def __init__(self): self._submit_timeout = self.get_option('job_submit_timeout') self._use_nodes_opt = self.get_option('use_nodes_option') self._resubmit_on_errors = self.get_option('resubmit_on_errors') - self._access_on_submission_command = self.get_option( - 'access_on_submission_command' + self._sched_access_in_submit = self.get_option( + 'sched_access_in_submit' ) def make_job(self, *args, **kwargs): @@ -212,21 +212,22 @@ def emit_preamble(self, job): ) ) - for opt in job.sched_access: - if not opt.strip().startswith(('-C', '--constraint')): - preamble.append('%s %s' % (self._prefix, opt)) - - # To avoid overriding a constraint that's passed into `sched_access`, - # we AND it with the `--constraint` option passed either in `options` - # or in `cli_options` constraints = [] constraint_parser = ArgumentParser() constraint_parser.add_argument('-C', '--constraint') - parsed_options, _ = constraint_parser.parse_known_args( - job.sched_access - ) - if parsed_options.constraint: - constraints.append(parsed_options.constraint.strip()) + if not job._sched_access_in_submit: + for opt in job.sched_access: + if not opt.strip().startswith(('-C', '--constraint')): + preamble.append('%s %s' % (self._prefix, opt)) + + # To avoid overriding a constraint that's passed into + # `sched_access`, we AND it with the `--constraint` option + # passed either in `options` or in `cli_options` + parsed_options, _ = constraint_parser.parse_known_args( + job.sched_access + ) + if parsed_options.constraint: + constraints.append(parsed_options.constraint.strip()) # NOTE: Here last of the passed --constraint job options is taken # into account in order to respect the behavior of slurm. @@ -263,7 +264,7 @@ def emit_preamble(self, job): def submit(self, job): cmd_opts = ( - ' '.join(job.sched_access) if self._access_on_submission_command + ' '.join(job.sched_access) if self._sched_access_in_submit else '' ) cmd = f'sbatch {cmd_opts} {job.script_filename}' diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index d6d8f0e576..f51305f4af 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -577,9 +577,9 @@ def main(): # Options not associated with command-line arguments argparser.add_argument( - dest='access_on_submission_command', - envvar='RFM_ACCESS_ON_SUBMISSION_COMMAND', - configvar='systems*/sched_options/access_on_submission_command', + dest='sched_access_in_submit', + envvar='RFM_SCHED_ACCESS_IN_SUBMIT', + configvar='systems*/sched_options/sched_access_in_submit', action='store_true', help='Pass access options in the submission command (only for Slurm)' ) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 4cbeb90ccc..638cfaf7be 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -109,7 +109,7 @@ "sched_options": { "type": "object", "properties": { - "access_on_submission_command": {"type": "boolean"}, + "sched_access_in_submit": {"type": "boolean"}, "hosts": { "type": "array", "items": {"type": "string"} @@ -626,7 +626,7 @@ "systems/partitions/time_limit": null, "systems/partitions/devices": [], "systems/partitions/extras": {}, - "systems*/sched_options/access_on_submission_command": false, + "systems*/sched_options/sched_access_in_submit": false, "systems*/sched_options/ssh_hosts": [], "systems*/sched_options/ignore_reqnodenotavail": false, "systems*/sched_options/job_submit_timeout": 60, From 9dd1c04630802a9fb4410887b0d83feba88b352b Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 18 Apr 2024 10:57:58 +0200 Subject: [PATCH 04/13] Update docs --- docs/config_reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index cacf62ca3d..57adc4402a 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -374,7 +374,7 @@ System Partition Configuration Normally, ReFrame will pass the :attr:`~config.systems.partitions.access` options to the job script only. When this attribute is ``true`` the options are passed verbatim also in the submission command. - This option is currently relevant for the Slurm backends only. + This option is relevant for the LSF, OAR, PBS and Slurm backends. .. versionadded:: 4.6.0 From 8ec61db6ba4852addd6b91705a282157dd5236ee Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 18 Apr 2024 10:58:59 +0200 Subject: [PATCH 05/13] Small fix --- docs/manpage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index b6b3586d69..2b56ef1080 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1216,7 +1216,7 @@ Whenever an environment variable is associated with a configuration option, its .. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT - Pass access options in the submission command (only for Slurm). + Pass access options in the submission command (relevant for LSF, OAR, PBS and Slurm). .. table:: :align: left From 064a28902519ed35c2156a6643e86ac4a710484e Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 18 Apr 2024 13:28:10 +0200 Subject: [PATCH 06/13] Add _sched_access_in_submit in oar and lsf squeduler --- reframe/core/schedulers/lsf.py | 3 +++ reframe/core/schedulers/oar.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/reframe/core/schedulers/lsf.py b/reframe/core/schedulers/lsf.py index 5523aab74f..f2885334f6 100644 --- a/reframe/core/schedulers/lsf.py +++ b/reframe/core/schedulers/lsf.py @@ -27,6 +27,9 @@ class LsfJobScheduler(PbsJobScheduler): def __init__(self): self._prefix = '#BSUB' self._submit_timeout = self.get_option('job_submit_timeout') + self._sched_access_in_submit = self.get_option( + 'sched_access_in_submit' + ) def _format_option(self, var, option): if var is not None: diff --git a/reframe/core/schedulers/oar.py b/reframe/core/schedulers/oar.py index f72fe9baec..f6f43dcf0b 100644 --- a/reframe/core/schedulers/oar.py +++ b/reframe/core/schedulers/oar.py @@ -60,6 +60,9 @@ class OarJobScheduler(PbsJobScheduler): def __init__(self): self._prefix = '#OAR' self._submit_timeout = self.get_option('job_submit_timeout') + self._sched_access_in_submit = self.get_option( + 'sched_access_in_submit' + ) def emit_preamble(self, job): # host is de-facto nodes and core is number of cores requested per node From d33a49de92b683ace21e43ae7817818e108b13c5 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 18 Apr 2024 13:34:11 +0200 Subject: [PATCH 07/13] Fix typo --- reframe/core/schedulers/slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 8d35f58c13..3dd7d06b9d 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -215,7 +215,7 @@ def emit_preamble(self, job): constraints = [] constraint_parser = ArgumentParser() constraint_parser.add_argument('-C', '--constraint') - if not job._sched_access_in_submit: + if not self._sched_access_in_submit: for opt in job.sched_access: if not opt.strip().startswith(('-C', '--constraint')): preamble.append('%s %s' % (self._prefix, opt)) From 49df2f7f5e2f22d193c12c07389cd889b95bfa9d Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Fri, 19 Apr 2024 17:35:12 +0200 Subject: [PATCH 08/13] Update docs/config_reference.rst Co-authored-by: Vasileios Karakasis --- docs/config_reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 57adc4402a..cbe7c8ab26 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -372,7 +372,7 @@ System Partition Configuration :default: ``false`` Normally, ReFrame will pass the :attr:`~config.systems.partitions.access` options to the job script only. - When this attribute is ``true`` the options are passed verbatim also in the submission command. + When this attribute is ``true`` the options are passed in the submission command instead. This option is relevant for the LSF, OAR, PBS and Slurm backends. From 4cecb14bdd3438c10e646c0ef58b204bf6c13546 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Fri, 19 Apr 2024 17:35:26 +0200 Subject: [PATCH 09/13] Update reframe/core/schedulers/lsf.py Co-authored-by: Vasileios Karakasis --- reframe/core/schedulers/lsf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reframe/core/schedulers/lsf.py b/reframe/core/schedulers/lsf.py index f2885334f6..8fb36990ef 100644 --- a/reframe/core/schedulers/lsf.py +++ b/reframe/core/schedulers/lsf.py @@ -80,11 +80,11 @@ def emit_preamble(self, job): def submit(self, job): with open(job.script_filename, 'r') as fp: - cmd_opts = ( - ' '.join(job.sched_access) if self._sched_access_in_submit - else '' - ) - completed = _run_strict('bsub {cmd_opts}', stdin=fp) + cmd_parts = ['bsub'] + if self._sched_access_in_submit: + cmd_parts += job.sched_access + + completed = _run_strict(' '.join(cmd_parts), stdin=fp) jobid_match = re.search(r'^Job <(?P\S+)> is submitted', completed.stdout) if not jobid_match: From f084223e123cf0279ed35e5cdc83b0af832bc5e8 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Fri, 19 Apr 2024 18:20:08 +0200 Subject: [PATCH 10/13] Refactor job submission --- reframe/core/schedulers/lsf.py | 6 ++++-- reframe/core/schedulers/oar.py | 11 ++++++----- reframe/core/schedulers/pbs.py | 12 ++++++------ reframe/core/schedulers/slurm.py | 11 ++++++----- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/reframe/core/schedulers/lsf.py b/reframe/core/schedulers/lsf.py index 8fb36990ef..73b6593f3b 100644 --- a/reframe/core/schedulers/lsf.py +++ b/reframe/core/schedulers/lsf.py @@ -83,8 +83,10 @@ def submit(self, job): cmd_parts = ['bsub'] if self._sched_access_in_submit: cmd_parts += job.sched_access - - completed = _run_strict(' '.join(cmd_parts), stdin=fp) + + cmd = ' '.join(cmd_parts) + completed = _run_strict(cmd, stdin=fp) + jobid_match = re.search(r'^Job <(?P\S+)> is submitted', completed.stdout) if not jobid_match: diff --git a/reframe/core/schedulers/oar.py b/reframe/core/schedulers/oar.py index f6f43dcf0b..06733bf600 100644 --- a/reframe/core/schedulers/oar.py +++ b/reframe/core/schedulers/oar.py @@ -107,12 +107,13 @@ def emit_preamble(self, job): def submit(self, job): # OAR batch submission mode needs full path to the job script job_script_fullpath = os.path.join(job.workdir, job.script_filename) - cmd_opts = ( - ' '.join(job.sched_access) if self._sched_access_in_submit - else '' - ) + cmd_parts = ['oarsub'] + if self._sched_access_in_submit: + cmd_parts += job.sched_access + # OAR needs -S to submit job in batch mode - cmd = f'oarsub {cmd_opts} -S {job_script_fullpath}' + cmd_parts += ['-S', job_script_fullpath] + cmd = ' '.join(cmd_parts) completed = _run_strict(cmd, timeout=self._submit_timeout) jobid_match = re.search(r'.*OAR_JOB_ID=(?P\S+)', completed.stdout) diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index afd0aaa357..a15bea5df9 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -147,14 +147,14 @@ def filternodes(self, job, nodes): 'node filtering') def submit(self, job): - cmd_opts = ( - ' '.join(job.sched_access) if self._sched_access_in_submit - else '' - ) + cmd_parts = ['qsub'] + if self._sched_access_in_submit: + cmd_parts += job.sched_access + # `-o` and `-e` options are only recognized in command line by the PBS # Slurm wrappers. - cmd = (f'qsub {cmd_opts} -o {job.stdout} -e {job.stderr} ' - f'{job.script_filename}') + cmd_parts += ['-o', job.stdout, '-e', job.stderr, job.script_filename] + cmd = ' '.join(cmd_parts) completed = _run_strict(cmd, timeout=self._submit_timeout) jobid_match = re.search(r'^(?P\S+)', completed.stdout) if not jobid_match: diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 3dd7d06b9d..c8d370742e 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -263,11 +263,12 @@ def emit_preamble(self, job): return list(filter(None, preamble)) def submit(self, job): - cmd_opts = ( - ' '.join(job.sched_access) if self._sched_access_in_submit - else '' - ) - cmd = f'sbatch {cmd_opts} {job.script_filename}' + cmd_parts = ['sbatch'] + if self._sched_access_in_submit: + cmd_parts += job.sched_access + + cmd_parts += [job.script_filename] + cmd = ' '.join(cmd_parts) intervals = itertools.cycle([1, 2, 3]) while True: try: From 809cba47f96537027e7419946f9d04252e11e029 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 13 Jun 2024 23:12:57 +0200 Subject: [PATCH 11/13] Combine Slurm constraints also when `sched_access_in_submit` is set --- reframe/core/schedulers/slurm.py | 57 ++++++++++++++------------------ unittests/test_schedulers.py | 23 +++++++++++++ 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index c8d370742e..62d2cd2e38 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -212,40 +212,33 @@ def emit_preamble(self, job): ) ) - constraints = [] - constraint_parser = ArgumentParser() - constraint_parser.add_argument('-C', '--constraint') - if not self._sched_access_in_submit: - for opt in job.sched_access: - if not opt.strip().startswith(('-C', '--constraint')): - preamble.append('%s %s' % (self._prefix, opt)) - - # To avoid overriding a constraint that's passed into - # `sched_access`, we AND it with the `--constraint` option - # passed either in `options` or in `cli_options` - parsed_options, _ = constraint_parser.parse_known_args( - job.sched_access - ) - if parsed_options.constraint: - constraints.append(parsed_options.constraint.strip()) - - # NOTE: Here last of the passed --constraint job options is taken - # into account in order to respect the behavior of slurm. - parsed_options, _ = constraint_parser.parse_known_args( - job.options + job.cli_options - ) - if parsed_options.constraint: - constraints.append(parsed_options.constraint.strip()) - - if constraints: - if len(constraints) == 1: - constr = constraints[0] + # Combine constraints in `sched_access` + # + # We AND the constraints defined in `sched_access` with those in + # either the `job.options` or `job.cli_options`. We essentially "move" + # the option from the source option list to `sched_access` as if the + # user has specified all the constraint in `sched_access`. We can then + # move with the preamble generation or the submission normally. + c_parser = ArgumentParser() + c_parser.add_argument('-C', '--constraint') + access, access_other = c_parser.parse_known_args(job.sched_access) + job_opts, other_job_opts = c_parser.parse_known_args(job.options) + cli_opts, other_cli_opts = c_parser.parse_known_args(job.cli_options) + if access.constraint and (job_opts.constraint or cli_opts.constraint): + constraints = [access.constraint] + if job_opts.constraint: + constraints.append(job_opts.constraint) + job.options = other_job_opts else: - # Parenthesize the constraints prior to joining them with `&` - # to make sure that precedence is respected. - constr = '&'.join(f'({c})' for c in constraints) + constraints.append(cli_opts.constraint) + job._cli_options = other_cli_opts + + arg = '&'.join(f'({c.strip()})' for c in constraints) + job._sched_access = [f'--constraint={arg}'] - preamble.append(self._format_option(constr, '--constraint={0}')) + if not self._sched_access_in_submit: + for opt in job.sched_access: + preamble.append(f'{self._prefix} {opt}') preamble.append(self._format_option(hint, '--hint={0}')) prefix_patt = re.compile(r'(#\w+)') diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 8c4ebea5c9..851657acd0 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -617,6 +617,7 @@ def test_combined_access_constraint(make_job, slurm_only): with open(job.script_filename) as fp: script_content = fp.read() + print(script_content) assert re.search(r'(?m)--constraint=\(c1\)&\(c2&c3\)$', script_content) assert re.search(r'(?m)--constraint=(c1|c2&c3)$', script_content) is None @@ -644,6 +645,28 @@ def test_combined_access_verbatim_constraint(make_job, slurm_only): assert re.search(r'(?m)^#SBATCH -C c3$', script_content) +def test_sched_access_in_submit(make_job): + job = make_job(sched_access=['--constraint=c1', '--foo=bar']) + job.options = ['--constraint=c2', '--xyz'] + job.scheduler._sched_access_in_submit = True + + if job.scheduler.registered_name in ('local', 'ssh'): + pytest.skip(f'not relevant for this scheduler backend') + + prepare_job(job) + with open(job.script_filename) as fp: + script_content = fp.read() + + print(script_content) + assert '--xyz' in script_content + assert '--foo=bar' not in script_content + if job.scheduler.registered_name in ('slurm', 'squeue'): + # Constraints are combined in `sched_access` for Slurm backends + assert '--constraint' not in script_content + else: + assert '--constraint=c1' not in script_content + + def test_guess_num_tasks(minimal_job, scheduler): minimal_job.num_tasks = 0 if scheduler.registered_name == 'local': From dd528fcea623b742bbb07367d083c4d752419707 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 13 Jun 2024 23:29:45 +0200 Subject: [PATCH 12/13] Update version annotation in docs --- docs/config_reference.rst | 2 +- docs/manpage.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 47f66a013a..c8715dbd36 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -378,7 +378,7 @@ System Partition Configuration This option is relevant for the LSF, OAR, PBS and Slurm backends. - .. versionadded:: 4.6.0 + .. versionadded:: 4.7 .. py:attribute:: systems.partitions.sched_options.ssh_hosts diff --git a/docs/manpage.rst b/docs/manpage.rst index 387c27f9f0..926aa67fba 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1238,7 +1238,7 @@ Whenever an environment variable is associated with a configuration option, its Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit` ================================== ================== -.. versionadded:: 4.6.0 +.. versionadded:: 4.7 .. envvar:: RFM_AUTODETECT_FQDN From e79279a8093ce01b057a05a6fc18d4fa4bb59bff Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 13 Jun 2024 23:31:53 +0200 Subject: [PATCH 13/13] Skip `_sched_access_in_submit` unit test for Flux scheduler --- unittests/test_schedulers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 93a40f1640..609efc8f68 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -651,7 +651,7 @@ def test_sched_access_in_submit(make_job): job.options = ['--constraint=c2', '--xyz'] job.scheduler._sched_access_in_submit = True - if job.scheduler.registered_name in ('local', 'ssh'): + if job.scheduler.registered_name in ('flux', 'local', 'ssh'): pytest.skip(f'not relevant for this scheduler backend') prepare_job(job)