-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
2 changed files
with
142 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
from argparse import ArgumentParser | ||
from operator import itemgetter | ||
import os | ||
import subprocess | ||
import sys | ||
|
||
import pandas as pd | ||
|
||
parser = ArgumentParser() | ||
parser.add_argument('cts_root', type=str, help='SYCL-CTS repository path') | ||
parser.add_argument('cts_build_dir', type=str, help='SYCL-CTS + SimSYCL build directory') | ||
args = parser.parse_args() | ||
cts_root = os.path.realpath(args.cts_root) | ||
cts_build_dir = os.path.realpath(args.cts_build_dir) | ||
|
||
state_file = pd.read_csv('ci/cts_state.csv', delimiter=';') | ||
tests_in_state_file = set(state_file['suite']) | ||
|
||
tests_dir = os.path.join(cts_root, 'tests') | ||
tests_in_cts = set(t for t in os.listdir(tests_dir) | ||
if os.path.isdir(os.path.join(tests_dir, t)) | ||
and t not in ['common', 'extension']) | ||
|
||
n_build_failed = 0 | ||
n_run_failed = 0 | ||
n_passed = 0 | ||
changed = [] | ||
for test in sorted(tests_in_cts): | ||
status_in_state_file = state_file['status'][state_file['suite'] == test].values | ||
status_in_state_file = status_in_state_file[0] if status_in_state_file.size > 0 else 'not in list' | ||
if status_in_state_file == 'not applicable': continue | ||
|
||
print('testing', test, end='... ', flush=True) | ||
r = subprocess.run(['cmake', '--build', cts_build_dir, '--target', f'test_{test}'], | ||
cwd=cts_root, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) | ||
if r.returncode == 0: | ||
r = subprocess.run(os.path.join(cts_build_dir, 'bin', f'test_{test}'), cwd=cts_root, | ||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) | ||
if r.returncode == 0: | ||
status_now = 'passed' | ||
n_passed += 1 | ||
else: | ||
status_now = 'run failed' | ||
n_run_failed += 1 | ||
else: | ||
status_now = 'build failed' | ||
n_build_failed += 1 | ||
|
||
if status_now == status_in_state_file: | ||
print(status_now) | ||
else: | ||
print(f'{status_now}, but was {status_in_state_file}') | ||
changed.append((test, status_in_state_file, status_now)) | ||
|
||
print(f'\n{n_passed} passed, {n_run_failed} failed to run, {n_build_failed} failed to build') | ||
|
||
for test in tests_in_state_file - tests_in_cts: | ||
status_in_state_file = state_file['status'][state_file['suite'] == test].values[0] | ||
changed.append((test, status_in_state_file, 'not in SYCL-CTS')) | ||
|
||
if changed: | ||
print(f'\n{len(changed)} change(s) compared to cts_state.csv:') | ||
changed.sort(key=itemgetter(0)) | ||
for test, status_in_state_file, status_now in changed: | ||
print(f' - {test}: {status_in_state_file} -> {status_now}') | ||
sys.exit(1) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
suite;status;reason | ||
accessor_basic;run failed;accessor implicit conversions | ||
accessor_generic;run failed;everything | ||
accessor_legacy;build failed;image targets, sycl::atomic NYI | ||
accessor_placeholder;run failed;conditions check (investigate) | ||
address_space;passed; | ||
atomic;build failed;deprecated atomic types (return values of accessor<atomic>::operator[]) | ||
atomic_fence;passed; | ||
atomic_ref;passed; | ||
atomic_ref_stress;passed; | ||
bit_cast;passed; | ||
buffer;run failed;sub-buffers NYI | ||
context;passed; | ||
cuda_interop;not applicable; | ||
device;run failed;sub-devices NYI | ||
device_event;passed; | ||
device_selector;passed; | ||
error;passed; | ||
event;run failed;async_handler NYI, Limitation: no asynchronicity between application thread and host tasks | ||
exception_handling;build failed;sub-devices NYI | ||
exceptions;run failed;"async" error handling NYI | ||
full_feature_set;passed; | ||
function_objects;passed; | ||
group;passed; | ||
group_functions;run failed;group scan defective? | ||
handler;run failed;hierarchical parallel for requires known local range | ||
header;passed; | ||
hierarchical;passed; | ||
h_item;passed; | ||
host_accessor;run failed;reference semantics (copy equality), ... | ||
host_task;passed; | ||
id;passed; | ||
image;build failed;images NYI | ||
image_accessor;build failed;images NYI | ||
invoke;run failed;parallel_for 2D / 3D short-hands NYI | ||
is_device_copyable;passed; | ||
item;passed; | ||
kernel;build failed;kernel bundle global functions NYI | ||
kernel_args;build failed;samplers NYI | ||
kernel_bundle;build failed;device_image NYI, ... | ||
language;passed; | ||
local_accessor;run failed;reference semantics (copy equality), ... | ||
marray_arithmetic_assignment;passed; | ||
marray_arithmetic_binary;passed; | ||
marray_basic;passed; | ||
marray_bitwise;passed; | ||
marray_pre_post;passed; | ||
marray_relational;passed; | ||
math_builtin_api;build failed;math functions incomplete (only using std) | ||
multi_ptr;build failed;multi_ptr<legacy> == element* is ambiguous (incorrect in CTS / DPC++?) | ||
namespace;passed; | ||
nd_item;passed; | ||
nd_range;passed; | ||
opencl_interop;not applicable; | ||
optional_kernel_features;run failed;incorrect exception codes thrown (investigate) | ||
platform;passed; | ||
pointers;passed; | ||
property;passed; | ||
queue;passed; | ||
range;passed; | ||
reduction;build failed;reductions on `span` NYI | ||
sampler;build failed;samplers / images NYI | ||
scalars;passed; | ||
spec_constants;build failed;use_kernel_bundle() and associated checking NYI | ||
stream;build failed;sycl:stream NYI | ||
sub_group;passed; | ||
sycl_external;run failed;UBSan: reached an unreachable program point in kernel_between_aspects, Clang compiler bug? | ||
usm;passed;Limitation: SimSYCL cannot communicate from main thread to kernel through SHMEM | ||
vector_alias;passed; | ||
vector_api;run failed;convert() tests fail because rounding modes are NYI | ||
vector_constructors;passed; | ||
vector_deduction_guides;passed; | ||
vector_load_store;passed; | ||
vector_operators;passed; | ||
vector_swizzle_assignment;passed; | ||
vector_swizzles;passed; |