Merge branch 'master' into develop

ci-scripts/dockerfiles/reframe-lmod77.dockerfile

@@ -7,7 +7,7 @@ FROM ghcr.io/reframe-hpc/lmod:7.7
 
 # Install ReFrame unit test requirements
 RUN apt-get -y update && \
-    apt-get -y install gcc make python3 python3-pip
+    apt-get -y install gcc git make python3 python3-pip
 
 # ReFrame user
 RUN useradd -ms /bin/bash rfmuser

docs/config_reference.rst

@@ -1707,7 +1707,7 @@ Result storage configuration
 .. py:attribute:: storage.enable
 
    :required: No
-   :default: ``true``
+   :default: ``false``
 
    Enable results storage.
 

docs/manpage.rst

@@ -477,6 +477,10 @@ Options controlling ReFrame output
 
    .. versionadded:: 3.1
 
+   .. warning::
+
+      Running a test with :option:`--dont-restage` on a stage directory that was created with a different ReFrame version is undefined behaviour.
+
 .. option:: --keep-stage-files
 
    Keep test stage directories even for tests that finish successfully.

docs/tutorial.rst

@@ -180,16 +180,17 @@ These files are located by default under ``perflogs/<system>/<partition>/<testna
 In our example, this translates to ``perflogs/generic/default/stream_test.log``.
 The information that is being logged is fully configurable and we will cover this in the :ref:`logging` section.
 
-Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session and compare them (by default) to their last obtained performance.
+Finally, you can also use the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session and (optionally) compare them to past results (see :ref:`querying-past-results` for more on comparing performance results).
 
 .. code-block:: console
 
-   ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑
-   │ name        │ sysenv                  │ pvar     │    pval │ punit   │ pdiff   │ job_nodelist   │ result   │
-   ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥
-   │ stream_test │ generic:default+builtin │ copy_bw  │ 40292.1 │ MB/s    │ -0.04%  │ myhost         │ pass     │
-   │ stream_test │ generic:default+builtin │ triad_bw │ 30564.7 │ MB/s    │ +0.12%  │ myhost         │ pass     │
-   ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙
+   ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━┑
+   │ name        │ sysenv                  │ job_nodelist   │ pvar     │ punit   │    pval │ result   │
+   ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━┥
+   │ stream_test │ generic:default+builtin │ myhost         │ copy_bw  │ MB/s    │ 17154.1 │ pass     │
+   ├─────────────┼─────────────────────────┼────────────────┼──────────┼─────────┼─────────┼──────────┤
+   │ stream_test │ generic:default+builtin │ myhost         │ triad_bw │ MB/s    │ 13664.8 │ pass     │
+   ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━┙
 
 
 Inspecting the test artifacts

reframe/core/config.py

@@ -30,15 +30,22 @@
 
 
 def _match_option(opt, opt_map):
+    def _copy_if_mutable(item):
+        for c in (dict, list):
+            if isinstance(item, c):
+                return c(item)
+
+        return item
+
     if isinstance(opt, list):
         opt = '/'.join(opt)
 
     if opt in opt_map:
-        return opt_map[opt]
+        return _copy_if_mutable(opt_map[opt])
 
     for k, v in opt_map.items():
         if fnmatch.fnmatchcase(opt, k):
-            return v
+            return _copy_if_mutable(v)
 
     raise KeyError(opt)
 

reframe/core/pipeline.py

@@ -20,6 +20,7 @@
 import numbers
 import os
 import shutil
+from pathlib import Path
 
 import reframe.core.fields as fields
 import reframe.core.hooks as hooks
@@ -1779,6 +1780,19 @@ def setup(self, partition, environ, **job_opts):
         self._setup_container_platform()
         self._resolve_fixtures()
 
+    def _mark_stagedir(self):
+        (Path(self.stagedir) / '.rfm_mark').touch()
+
+    def _requires_stagedir_contents(self):
+        '''Return true if the contents of the stagedir need to be generated'''
+
+        # Every time the stage directory is created a fresh mark is created.
+        # Normally, this is wiped out before running the test, unless
+        # `--dont-restage` is passed. In this case, we want to leave the
+        # existing stagedir untouched.
+
+        return not (Path(self.stagedir) / '.rfm_mark').exists()
+
     def _copy_to_stagedir(self, path):
         self.logger.debug(f'Copying {path} to stage directory')
         self.logger.debug(f'Symlinking files: {self.readonly_files}')
@@ -1788,13 +1802,16 @@ def _copy_to_stagedir(self, path):
             )
         except (OSError, ValueError, TypeError) as e:
             raise PipelineError('copying of files failed') from e
+        else:
+            self._mark_stagedir()
 
     def _clone_to_stagedir(self, url):
         self.logger.debug(f'Cloning URL {url} into stage directory')
         osext.git_clone(
             self.sourcesdir, self._stagedir,
             timeout=rt.runtime().get_option('general/0/git_timeout')
         )
+        self._mark_stagedir()
 
     @final
     def compile(self):
@@ -1832,11 +1849,12 @@ def compile(self):
                     f'interpreted as relative to it'
                 )
 
-            if osext.is_url(self.sourcesdir):
-                self._clone_to_stagedir(self.sourcesdir)
-            else:
-                self._copy_to_stagedir(os.path.join(self._prefix,
-                                                    self.sourcesdir))
+            if self._requires_stagedir_contents():
+                if osext.is_url(self.sourcesdir):
+                    self._clone_to_stagedir(self.sourcesdir)
+                else:
+                    self._copy_to_stagedir(os.path.join(self._prefix,
+                                                        self.sourcesdir))
 
         # Set executable (only if hasn't been provided)
         if not hasattr(self, 'executable'):
@@ -2628,7 +2646,7 @@ def run(self):
         The resources of the test are copied to the stage directory and the
         rest of execution is delegated to the :func:`RegressionTest.run()`.
         '''
-        if self.sourcesdir:
+        if self.sourcesdir and self._requires_stagedir_contents():
             if osext.is_url(self.sourcesdir):
                 self._clone_to_stagedir(self.sourcesdir)
             else:

reframe/frontend/cli.py

@@ -941,9 +941,12 @@ def restrict_logging():
                 # We lexically split the mode options, because otherwise spaces
                 # will be treated as part of the option argument;
                 # see GH bug #1554
-                mode_args = list(
-                    itertools.chain.from_iterable(shlex.split(m)
-                                                  for m in mode_args))
+                mode_args = list(itertools.chain.from_iterable(
+                    shlex.split(osext.expandvars(arg)) for arg in mode_args)
+                )
+                printer.debug(f'Expanding execution mode {options.mode!r}: '
+                              f'{" ".join(mode_args)}')
+
                 # Parse the mode's options and reparse the command-line
                 options = argparser.parse_args(mode_args,
                                                suppress_required=True)

reframe/frontend/executors/policies.py

@@ -183,7 +183,7 @@ def on_task_compile_exit(self, task):
         pass
 
     def on_task_skip(self, task):
-        msg = str(task.exc_info[1])
+        msg = f'{task.info()} [{task.exc_info[1]}]'
         self.printer.status('SKIP', msg, just='right')
 
     def on_task_abort(self, task):
@@ -438,13 +438,13 @@ def _advance_all(self, tasks, timeout=None):
             num_progressed += bump_state(t)
             new_state = t.state
 
+            if self._pipeline_statistics:
+                self._update_pipeline_progress(old_state, new_state, 1)
+
             t_elapsed = time.time() - t_init
             if timeout and t_elapsed > timeout and num_progressed:
                 break
 
-            if self._pipeline_statistics:
-                self._update_pipeline_progress(old_state, new_state, 1)
-
         getlogger().debug2(f'Bumped {num_progressed} test(s)')
 
     def _advance_startup(self, task):
@@ -605,7 +605,7 @@ def on_task_compile_exit(self, task):
         self._pollctl.reset_snooze_time()
 
     def on_task_skip(self, task):
-        msg = str(task.exc_info[1])
+        msg = f'{task.info()} [{task.exc_info[1]}]'
         self.printer.status('SKIP', msg, just='right')
 
     def on_task_abort(self, task):

unittests/resources/config/settings.py

@@ -171,7 +171,6 @@ def hostname():
                     'launcher': 'local',
                     'environs': ['e0', 'e1', 'e2']
                 }
-
             ]
         },
         {
@@ -194,6 +193,27 @@ def hostname():
                     'environs': ['builtin']
                 }
             ]
+        },
+        {
+            'name': 'sys3',
+            'descr': 'System required for testing default extras',
+            'hostnames': [r'sys3\d+'],
+            'partitions': [
+                {
+                    'name': 'part1',
+                    'descr': 'Login nodes',
+                    'scheduler': 'local',
+                    'launcher': 'local',
+                    'environs': ['builtin']
+                },
+                {
+                    'name': 'part2',
+                    'descr': 'Compute nodes',
+                    'scheduler': 'slurm',
+                    'launcher': 'srun',
+                    'environs': ['builtin']
+                }
+            ]
         }
     ],
     'environments': [
@@ -255,6 +275,11 @@ def hostname():
                 '-p builtin',
                 '-S local=1'
             ]
+        },
+        {
+            'name': 'env_vars',
+            'options': ['-n', '${TEST_NAME_PATTERN}',
+                        '-S', '${VAR}=${VAL}']
         }
     ],
     'logging': [

unittests/test_cli.py

@@ -339,23 +339,42 @@ def test_check_sanity_failure(run_reframe, tmp_path, run_action):
 
 
 def test_dont_restage(run_reframe, tmp_path):
-    run_reframe(
-        checkpath=['unittests/resources/checks/frontend_checks.py'],
-        more_options=['-n', 'SanityFailureCheck']
-    )
+    # Here we test four properties of `--dont-restage`
+    # 1. If the stage directory has not been populated before, it will
+    # 2. If the stage directory has been populated, it will stay untouched
+    # 3. The sourcesdir will not be copied again
+    # 4. When combined with `--max-retries`, the stage directory is reused
+
+    returncode = run_reframe(
+        checkpath=['unittests/resources/checks/'],
+        more_options=['-n', 'SanityFailureCheck', '-n', '^HelloTest$',
+                      '--dont-restage', '--keep-stage-files']
+    )[0]
+    # Assert property (1)
+    assert returncode != 0
 
-    # Place a random file in the test's stage directory and rerun with
-    # `--dont-restage` and `--max-retries`
+    stagedir_runonly = (tmp_path / 'stage' / 'generic' / 'default' /
+                        'builtin' / 'SanityFailureCheck')
     stagedir = (tmp_path / 'stage' / 'generic' / 'default' /
-                'builtin' / 'SanityFailureCheck')
+                'builtin' / 'HelloTest')
+
+    # Place a new file in the stagedir to test (2)
     (stagedir / 'foobar').touch()
+    (stagedir_runonly / 'foobar').touch()
+
+    # Remove a not-needed file to test (2) and (3)
+    (stagedir / 'Makefile').unlink()
+    (stagedir_runonly / 'Makefile').unlink()
+
+    # Use `--max-retries` to test (4)
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
         more_options=['-n', 'SanityFailureCheck',
                       '--dont-restage', '--max-retries=1']
     )
-    assert os.path.exists(stagedir / 'foobar')
-    assert not os.path.exists(f'{stagedir}_retry1')
+    assert os.path.exists(stagedir_runonly / 'foobar')
+    assert not os.path.exists(stagedir_runonly / 'Makefile')
+    assert not os.path.exists(f'{stagedir_runonly}_retry1')
 
     # And some standard assertions
     assert 'Traceback' not in stdout
@@ -542,6 +561,20 @@ def test_execution_modes(run_reframe, run_action):
     assert 'Ran 1/1 test case' in stdout
 
 
+def test_execution_modes_envvar_expansion(run_reframe, monkeypatch):
+    monkeypatch.setenv('TEST_NAME_PATTERN', '^HelloTest$')
+    monkeypatch.setenv('VAR', 'x')
+    monkeypatch.setenv('VAL', '1')
+    returncode, stdout, stderr = run_reframe(
+        mode='env_vars', action='list'
+    )
+    assert returncode == 0
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert "the following variables were not set: 'x'" in stdout
+    assert 'Found 1 check(s)' in stdout
+
+
 def test_invalid_mode_error(run_reframe):
     mode = 'invalid'
     returncode, stdout, stderr = run_reframe(

unittests/test_config.py

@@ -239,7 +239,7 @@ def test_select_subconfig(site_config):
     site_config.select_subconfig('testsys')
     assert len(site_config['systems']) == 1
     assert len(site_config['systems'][0]['partitions']) == 2
-    assert len(site_config['modes']) == 1
+    assert len(site_config['modes']) == 2
     assert site_config.get('systems/0/name') == 'testsys'
     assert site_config.get('systems/0/descr') == 'Fake system for unit tests'
     assert site_config.get('systems/0/hostnames') == ['testsys']
@@ -510,6 +510,18 @@ def test_system_create(site_config):
     system = System.create(site_config)
     assert system.partitions[0].container_runtime == 'Docker'
 
+    # Test correct extras when partitions have no extras set
+    # See: https://github.com/reframe-hpc/reframe/issues/3371
+    site_config.select_subconfig('sys3:part1')
+    system = System.create(site_config)
+    assert system.partitions[0].extras == {'scheduler': 'local',
+                                           'launcher': 'local'}
+
+    site_config.select_subconfig('sys3:part2')
+    system = System.create(site_config)
+    assert system.partitions[0].extras == {'scheduler': 'slurm',
+                                           'launcher': 'srun'}
+
 
 def test_yaml_bindings(monkeypatch):
     import os

unittests/test_pipeline.py

@@ -763,6 +763,23 @@ def validate(self):
     _run(MyTest(), *local_exec_ctx)
 
 
+def test_sourcesdir_git(local_exec_ctx):
+    @test_util.custom_prefix('unittests/resources/checks')
+    class MyTest(rfm.RunOnlyRegressionTest):
+        sourcesdir = 'https://github.com/reframe-hpc/ci-hello-world.git'
+        executable = 'true'
+        valid_systems = ['*']
+        valid_prog_environs = ['*']
+        keep_files = ['README.md']
+
+        @sanity_function
+        def validate(self):
+            print(self.stagedir)
+            return sn.assert_true(os.path.exists('README.md'))
+
+    _run(MyTest(), *local_exec_ctx)
+
+
 def test_sourcesdir_none_generated_sources(local_exec_ctx):
     @test_util.custom_prefix('unittests/resources/checks')
     class MyTest(rfm.RegressionTest):

unittests/test_utility.py

@@ -428,8 +428,8 @@ def test_is_interactive(monkeypatch):
 
 
 def test_is_url():
-    repo_https = 'https://github.com/reframe-hpc/reframe.git'
-    repo_ssh = '[email protected]:reframe-hpc/reframe.git'
+    repo_https = 'https://github.com/reframe-hpc/ci-hello-world.git'
+    repo_ssh = '[email protected]:reframe-hpc/ci-hello-world.git'
     assert osext.is_url(repo_https)
     assert not osext.is_url(repo_ssh)