Merge remote-tracking branch 'upstream/develop' into feat/asyncio_exec

.github/workflows/main.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -46,6 +46,24 @@ jobs:
       run: |
         ./test_reframe.py
 
+  unittest-py37:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.7']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies and docs
+      run: |
+        ./bootstrap.sh +docs
+    - name: Generic Unittests
+      run: |
+        ./test_reframe.py
+
   unittest-macos:
     runs-on: macos-latest
     strategy:
@@ -120,7 +138,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+<<<<<<< HEAD
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+=======
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+>>>>>>> upstream/develop
     steps:
       - uses: actions/checkout@v4
       - name: Setup up Python ${{ matrix.python-version }}
@@ -143,7 +165,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+<<<<<<< HEAD
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+=======
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+>>>>>>> upstream/develop
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

reframe/__init__.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.8.0-dev.1'
+VERSION = '4.8.0-dev.2'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

reframe/core/schedulers/slurm.py

@@ -238,7 +238,7 @@ def emit_preamble(self, job):
                 job._cli_options = other_cli_opts
 
             arg = '&'.join(f'({c.strip()})' for c in constraints)
-            job._sched_access = [f'--constraint={arg}']
+            job._sched_access = [f'--constraint={arg}'] + access_other
 
         if not self._sched_access_in_submit:
             for opt in job.sched_access:
@@ -248,8 +248,10 @@ def emit_preamble(self, job):
         prefix_patt = re.compile(r'(#\w+)')
         for opt in job.options + job.cli_options:
             if opt.strip().startswith(('-C', '--constraint')):
-                # Constraints are already processed
-                continue
+                if access.constraint:
+                    # Constraints are already combined with the `sched_access`
+                    # and processed
+                    continue
 
             if not prefix_patt.match(opt):
                 preamble.append('%s %s' % (self._prefix, opt))

reframe/frontend/reporting/storage.py

@@ -233,26 +233,41 @@ def _decode_sessions(self, results, sess_filter):
 
         Return a map of session uuids to decoded session data
         '''
+        sess_info_patt = re.compile(
+            r'\"session_info\":\s+(?P<sess_info>\{.*?\})'
+        )
+
+        def _extract_sess_info(s):
+            return sess_info_patt.search(s).group('sess_info')
+
+        @time_function
+        def _mass_json_decode(json_objs):
+            data = '[' + ','.join(json_objs) + ']'
+            getlogger().debug(
+                f'decoding JSON raw data of length {len(data)}'
+            )
+            return json.loads(data)
+
+        session_infos = {}
         sessions = {}
         for uuid, json_blob in results:
             sessions.setdefault(uuid, json_blob)
+            session_infos.setdefault(uuid, _extract_sess_info(json_blob))
 
-        # Join all sessions and decode them at once
-        reports_blob = '[' + ','.join(sessions.values()) + ']'
-        getprofiler().enter_region('json decode')
-        reports = json.loads(reports_blob)
-        getprofiler().exit_region()
-
-        # Reindex and filter sessions based on their decoded data
-        sessions.clear()
-        for rpt in reports:
+        # Find the UUIDs to decode fully by inspecting only the session info
+        uuids = []
+        for sess_info in _mass_json_decode(session_infos.values()):
             try:
-                if self._db_filter_json(sess_filter, rpt['session_info']):
-                    sessions[rpt['session_info']['uuid']] = rpt
+                if self._db_filter_json(sess_filter, sess_info):
+                    uuids.append(sess_info['uuid'])
             except Exception:
                 continue
 
-        return sessions
+        # Decode selected sessions
+        reports = _mass_json_decode(sessions[uuid] for uuid in uuids)
+
+        # Return only the selected sessions
+        return {rpt['session_info']['uuid']: rpt for rpt in reports}
 
     @time_function
     def _fetch_testcases_raw(self, condition):

requirements.txt

@@ -1,7 +1,8 @@
 archspec==0.2.5
 argcomplete==3.1.2; python_version < '3.8'
-argcomplete==3.5.1; python_version >= '3.8'
+argcomplete==3.5.3; python_version >= '3.8'
 autopep8==2.0.4
+argcomplete==3.5.3; python_version >= '3.8'
 filelock==3.4.1; python_version == '3.6'
 filelock==3.12.2; python_version == '3.7'
 filelock==3.16.1; python_version >= '3.8'

unittests/test_schedulers.py

@@ -452,6 +452,13 @@ def test_prepare_without_smt(fake_job, slurm_only):
         assert re.search(r'--hint=nomultithread', fp.read()) is not None
 
 
+def test_prepare_with_constraints(fake_job, slurm_only):
+    fake_job.options = ['--constraint=foo']
+    prepare_job(fake_job)
+    with open(fake_job.script_filename) as fp:
+        assert re.search(r'#SBATCH --constraint=foo', fp.read()) is not None
+
+
 def test_prepare_nodes_option(make_exec_ctx, make_job, slurm_only):
     make_exec_ctx(test_util.TEST_CONFIG_FILE, 'testsys')
     job = make_job(sched_opts={'part_name': 'gpu'})
@@ -625,42 +632,47 @@ def test_no_empty_lines_in_preamble(minimal_job):
 
 
 def test_combined_access_constraint(make_job, slurm_only):
-    job = make_job(sched_access=['--constraint=c1'])
+    job = make_job(sched_access=['--constraint=c1', '-A acct', '-p part'])
     job.options = ['-C c2&c3']
     prepare_job(job)
     with open(job.script_filename) as fp:
         script_content = fp.read()
 
-    print(script_content)
+    assert re.search('-A acct', script_content)
+    assert re.search('-p part', script_content)
     assert re.search(r'(?m)--constraint=\(c1\)&\(c2&c3\)$', script_content)
     assert re.search(r'(?m)--constraint=(c1|c2&c3)$', script_content) is None
 
 
 def test_combined_access_multiple_constraints(make_job, slurm_only):
-    job = make_job(sched_access=['--constraint=c1'])
+    job = make_job(sched_access=['--constraint=c1', '-A acct', '-p part'])
     job.options = ['--constraint=c2', '-C c3']
     prepare_job(job)
     with open(job.script_filename) as fp:
         script_content = fp.read()
 
+    assert re.search('-A acct', script_content)
+    assert re.search('-p part', script_content)
     assert re.search(r'(?m)--constraint=\(c1\)&\(c3\)$', script_content)
     assert re.search(r'(?m)--constraint=(c1|c2|c3)$', script_content) is None
 
 
 def test_combined_access_verbatim_constraint(make_job, slurm_only):
-    job = make_job(sched_access=['--constraint=c1'])
+    job = make_job(sched_access=['--constraint=c1', '-A acct', '-p part'])
     job.options = ['#SBATCH --constraint=c2', '#SBATCH -C c3']
     prepare_job(job)
     with open(job.script_filename) as fp:
         script_content = fp.read()
 
+    assert re.search('-A acct', script_content)
+    assert re.search('-p part', script_content)
     assert re.search(r'(?m)--constraint=c1$', script_content)
     assert re.search(r'(?m)^#SBATCH --constraint=c2$', script_content)
     assert re.search(r'(?m)^#SBATCH -C c3$', script_content)
 
 
 def test_sched_access_in_submit(make_job):
-    job = make_job(sched_access=['--constraint=c1', '--foo=bar'])
+    job = make_job(sched_access=['--constraint=c1', '-A acct'])
     job.options = ['--constraint=c2', '--xyz']
     job.scheduler._sched_access_in_submit = True
 
@@ -673,7 +685,7 @@ def test_sched_access_in_submit(make_job):
 
     print(script_content)
     assert '--xyz' in script_content
-    assert '--foo=bar' not in script_content
+    assert '-A acct' not in script_content
     if job.scheduler.registered_name in ('slurm', 'squeue'):
         # Constraints are combined in `sched_access` for Slurm backends
         assert '--constraint' not in script_content