Skip to content

Commit

Permalink
Tooling: Track and verify CTS state
Browse files Browse the repository at this point in the history
  • Loading branch information
fknorr committed Dec 28, 2024
1 parent 6692407 commit d70a80c
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 0 deletions.
66 changes: 66 additions & 0 deletions ci/confirm_cts_state.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
from argparse import ArgumentParser
from operator import itemgetter
import os
import subprocess
import sys

import pandas as pd

parser = ArgumentParser()
parser.add_argument('cts_root', type=str, help='SYCL-CTS repository path')
parser.add_argument('cts_build_dir', type=str, help='SYCL-CTS + SimSYCL build directory')
args = parser.parse_args()
cts_root = os.path.realpath(args.cts_root)
cts_build_dir = os.path.realpath(args.cts_build_dir)

state_file = pd.read_csv('ci/cts_state.csv', delimiter=';')
tests_in_state_file = set(state_file['suite'])

tests_dir = os.path.join(cts_root, 'tests')
tests_in_cts = set(t for t in os.listdir(tests_dir)
if os.path.isdir(os.path.join(tests_dir, t))
and t not in ['common', 'extension'])

n_build_failed = 0
n_run_failed = 0
n_passed = 0
changed = []
for test in sorted(tests_in_cts):
status_in_state_file = state_file['status'][state_file['suite'] == test].values
status_in_state_file = status_in_state_file[0] if status_in_state_file.size > 0 else 'not in list'
if status_in_state_file == 'not applicable': continue

print('testing', test, end='... ', flush=True)
r = subprocess.run(['cmake', '--build', cts_build_dir, '--target', f'test_{test}'],
cwd=cts_root, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
if r.returncode == 0:
r = subprocess.run(os.path.join(cts_build_dir, 'bin', f'test_{test}'), cwd=cts_root,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
if r.returncode == 0:
status_now = 'passed'
n_passed += 1
else:
status_now = 'run failed'
n_run_failed += 1
else:
status_now = 'build failed'
n_build_failed += 1

if status_now == status_in_state_file:
print(status_now)
else:
print(f'{status_now}, but was {status_in_state_file}')
changed.append((test, status_in_state_file, status_now))

print(f'\n{n_passed} passed, {n_run_failed} failed to run, {n_build_failed} failed to build')

for test in tests_in_state_file - tests_in_cts:
status_in_state_file = state_file['status'][state_file['suite'] == test].values[0]
changed.append((test, status_in_state_file, 'not in SYCL-CTS'))

if changed:
print(f'\n{len(changed)} change(s) compared to cts_state.csv:')
changed.sort(key=itemgetter(0))
for test, status_in_state_file, status_now in changed:
print(f' - {test}: {status_in_state_file} -> {status_now}')
sys.exit(1)
76 changes: 76 additions & 0 deletions ci/cts_state.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
suite;status;reason
accessor_basic;run failed;accessor implicit conversions
accessor_generic;run failed;everything
accessor_legacy;build failed;image targets, sycl::atomic NYI
accessor_placeholder;run failed;conditions check (investigate)
address_space;passed;
atomic;build failed;deprecated atomic types (return values of accessor<atomic>::operator[])
atomic_fence;passed;
atomic_ref;passed;
atomic_ref_stress;passed;
bit_cast;passed;
buffer;run failed;sub-buffers NYI
context;passed;
cuda_interop;not applicable;
device;run failed;sub-devices NYI
device_event;passed;
device_selector;passed;
error;passed;
event;run failed;async_handler NYI, Limitation: no asynchronicity between application thread and host tasks
exception_handling;build failed;sub-devices NYI
exceptions;run failed;"async" error handling NYI
full_feature_set;passed;
function_objects;passed;
group;passed;
group_functions;run failed;group scan defective?
handler;run failed;hierarchical parallel for requires known local range
header;passed;
hierarchical;passed;
h_item;passed;
host_accessor;run failed;reference semantics (copy equality), ...
host_task;passed;
id;passed;
image;build failed;images NYI
image_accessor;build failed;images NYI
invoke;run failed;parallel_for 2D / 3D short-hands NYI
is_device_copyable;passed;
item;passed;
kernel;build failed;kernel bundle global functions NYI
kernel_args;build failed;samplers NYI
kernel_bundle;build failed;device_image NYI, ...
language;passed;
local_accessor;run failed;reference semantics (copy equality), ...
marray_arithmetic_assignment;passed;
marray_arithmetic_binary;passed;
marray_basic;passed;
marray_bitwise;passed;
marray_pre_post;passed;
marray_relational;passed;
math_builtin_api;build failed;math functions incomplete (only using std)
multi_ptr;build failed;multi_ptr<legacy> == element* is ambiguous (incorrect in CTS / DPC++?)
namespace;passed;
nd_item;passed;
nd_range;passed;
opencl_interop;not applicable;
optional_kernel_features;run failed;incorrect exception codes thrown (investigate)
platform;passed;
pointers;passed;
property;passed;
queue;passed;
range;passed;
reduction;build failed;reductions on `span` NYI
sampler;build failed;samplers / images NYI
scalars;passed;
spec_constants;build failed;use_kernel_bundle() and associated checking NYI
stream;build failed;sycl:stream NYI
sub_group;passed;
sycl_external;run failed;UBSan: reached an unreachable program point in kernel_between_aspects, Clang compiler bug?
usm;passed;Limitation: SimSYCL cannot communicate from main thread to kernel through SHMEM
vector_alias;passed;
vector_api;run failed;convert() tests fail because rounding modes are NYI
vector_constructors;passed;
vector_deduction_guides;passed;
vector_load_store;passed;
vector_operators;passed;
vector_swizzle_assignment;passed;
vector_swizzles;passed;

0 comments on commit d70a80c

Please sign in to comment.