Tooling: Track and verify CTS state

ci/confirm_cts_state.py

@@ -0,0 +1,66 @@
+from argparse import ArgumentParser
+from operator import itemgetter
+import os
+import subprocess
+import sys
+
+import pandas as pd
+
+parser = ArgumentParser()
+parser.add_argument('cts_root', type=str, help='SYCL-CTS repository path')
+parser.add_argument('cts_build_dir', type=str, help='SYCL-CTS + SimSYCL build directory')
+args = parser.parse_args()
+cts_root = os.path.realpath(args.cts_root)
+cts_build_dir = os.path.realpath(args.cts_build_dir)
+
+state_file = pd.read_csv('ci/cts_state.csv', delimiter=';')
+tests_in_state_file = set(state_file['suite'])
+
+tests_dir = os.path.join(cts_root, 'tests')
+tests_in_cts = set(t for t in os.listdir(tests_dir)
+                   if os.path.isdir(os.path.join(tests_dir, t))
+                   and t not in ['common', 'extension'])
+
+n_build_failed = 0
+n_run_failed = 0
+n_passed = 0
+changed = []
+for test in sorted(tests_in_cts):
+    status_in_state_file = state_file['status'][state_file['suite'] == test].values
+    status_in_state_file = status_in_state_file[0] if status_in_state_file.size > 0 else 'not in list'
+    if status_in_state_file == 'not applicable': continue
+
+    print('testing', test, end='... ', flush=True)
+    r = subprocess.run(['cmake', '--build', cts_build_dir, '--target', f'test_{test}'],
+                       cwd=cts_root, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    if r.returncode == 0:
+        r = subprocess.run(os.path.join(cts_build_dir, 'bin', f'test_{test}'), cwd=cts_root,
+                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        if r.returncode == 0:
+            status_now = 'passed'
+            n_passed += 1
+        else:
+            status_now = 'run failed'
+            n_run_failed += 1
+    else:
+        status_now = 'build failed'
+        n_build_failed += 1
+
+    if status_now == status_in_state_file:
+        print(status_now)
+    else:
+        print(f'{status_now}, but was {status_in_state_file}')
+        changed.append((test, status_in_state_file, status_now))
+
+print(f'\n{n_passed} passed, {n_run_failed} failed to run, {n_build_failed} failed to build')
+
+for test in tests_in_state_file - tests_in_cts:
+    status_in_state_file = state_file['status'][state_file['suite'] == test].values[0]
+    changed.append((test, status_in_state_file, 'not in SYCL-CTS'))
+
+if changed:
+    print(f'\n{len(changed)} change(s) compared to cts_state.csv:')
+    changed.sort(key=itemgetter(0))
+    for test, status_in_state_file, status_now in changed:
+        print(f'  - {test}: {status_in_state_file} -> {status_now}')
+    sys.exit(1)

ci/cts_state.csv

@@ -0,0 +1,76 @@
+suite;status;reason
+accessor_basic;run failed;iterators: SimSYCL check failed: m_accessor != nullptr
+accessor_generic;run failed;everything
+accessor_legacy;build failed;image targets, sycl::atomic NYI
+accessor_placeholder;run failed;SimSYCL check failed: *m_required
+address_space;passed;
+atomic;build failed;deprecated atomic types (return values of accessor<atomic>::operator[])
+atomic_fence;passed;
+atomic_ref;passed;
+atomic_ref_stress;build failed;CTS bug - atomicity_with_host_code receives wrong number of template arguments
+bit_cast;passed;
+buffer;run failed;sub-buffers NYI
+context;passed;
+cuda_interop;not applicable;
+device;run failed;sub-devices NYI
+device_event;passed;
+device_selector;passed;
+error;passed;
+event;run failed;async_handler NYI, Limitation: no asynchronicity between application thread and host tasks
+exception_handling;build failed;sub-devices NYI
+exceptions;run failed;"async" error handling NYI
+full_feature_set;passed;
+function_objects;passed;
+group;passed;
+group_functions;run failed;group scan defective?
+handler;run failed;nd_item with offset defective
+header;passed;
+hierarchical;passed;
+h_item;passed;
+host_accessor;run failed;(investigate)
+host_task;passed;
+id;passed;
+image;build failed;images NYI
+image_accessor;build failed;images NYI
+invoke;run failed;parallel_for 2D / 3D short-hands NYI
+is_device_copyable;passed;
+item;passed;
+kernel;build failed;kernel bundle global functions NYI
+kernel_args;build failed;samplers NYI
+kernel_bundle;build failed;device_image NYI, ...
+language;passed;
+local_accessor;run failed;SimSYCL check (investigate)
+marray_arithmetic_assignment;passed;
+marray_arithmetic_binary;passed;
+marray_basic;passed;
+marray_bitwise;passed;
+marray_pre_post;passed;
+marray_relational;passed;
+math_builtin_api;build failed;math functions incomplete (only using std)
+multi_ptr;build failed;multi_ptr<legacy> == element* is ambiguous (incorrect in CTS / DPC++?)
+namespace;passed;
+nd_item;passed;
+nd_range;passed;
+opencl_interop;not applicable;
+optional_kernel_features;run failed;incorrect exception codes thrown (investigate)
+platform;passed;
+pointers;passed;
+property;passed;
+queue;run failed;property iterator (investigate)?
+range;passed;
+reduction;build failed;reductions on `span` NYI
+sampler;build failed;samplers / images NYI
+scalars;passed;
+spec_constants;build failed;use_kernel_bundle() and associated checking NYI
+stream;build failed;sycl:stream NYI
+sub_group;passed;
+sycl_external;run failed;UBSan: reached an unreachable program point in kernel_between_aspects, Clang compiler bug?
+usm;passed;Limitation: SimSYCL cannot communicate from main thread to kernel through SHMEM
+vector_alias;passed;
+vector_api;run failed;convert() tests fail because rounding modes are NYI
+vector_constructors;passed;
+vector_deduction_guides;passed;
+vector_load_store;passed;
+vector_operators;passed;
+vector_swizzle_assignment;passed;
+vector_swizzles;passed;