Skip to content

Add CTS macro for marking tests as known failures in code #2392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Add CTS macro for marking tests as known failures in code #2392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions scripts/core/INTRO.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,13 @@ no valid platforms, then the tests will fail. Command line arguments take priori

A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.

.. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES

A boolean option to enable running tests which have been marked as known
failures using the :c:macro:`UUR_KNOWN_FAILURE_ON` macro. Enabled when the
environment variable is set to any of the following values: ``1``, ``on``,
``ON``, ``yes``, ``YES``, ``true``, ``TRUE``.

Service identifiers
---------------------

Expand Down
2 changes: 2 additions & 0 deletions test/conformance/kernel/urKernelSetArgValue.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// See LICENSE.TXT
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include "uur/known_failure.h"
#include <uur/fixtures.h>

struct urKernelSetArgValueTest : uur::urKernelTest {
Expand Down Expand Up @@ -45,6 +46,7 @@ TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentIndex) {
}

TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentSize) {
UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE,
urKernelSetArgValue(kernel, 2, 0, nullptr, &arg_value));
}
2 changes: 2 additions & 0 deletions test/conformance/memory/urMemImageCreate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
// See LICENSE.TXT
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "uur/known_failure.h"
#include <uur/fixtures.h>
#include <uur/raii.h>

Expand Down Expand Up @@ -188,6 +189,7 @@ TEST_P(urMemImageCreateTest, InvalidNullPointerImageFormat) {
}

TEST_P(urMemImageCreateTest, InvalidSize) {
UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});

uur::raii::Mem image_handle = nullptr;

Expand Down
2 changes: 2 additions & 0 deletions test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
// See LICENSE.TXT
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "uur/known_failure.h"
#include <uur/fixtures.h>
#include <vector>

Expand Down Expand Up @@ -90,6 +91,7 @@ UUR_TEST_SUITE_P(
uur::deviceTestWithParamPrinter<ur_image_format_t>);

TEST_P(urMemImageCreateTestWithImageFormatParam, Success) {
UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
ur_image_channel_order_t channel_order =
std::get<1>(GetParam()).channelOrder;
ur_image_channel_type_t channel_type = std::get<1>(GetParam()).channelType;
Expand Down
2 changes: 2 additions & 0 deletions test/conformance/program/urProgramGetFunctionPointer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// See LICENSE.TXT
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include "uur/known_failure.h"
#include <uur/fixtures.h>

struct urProgramGetFunctionPointerTest : uur::urProgramTest {
Expand All @@ -20,6 +21,7 @@ struct urProgramGetFunctionPointerTest : uur::urProgramTest {
UUR_INSTANTIATE_KERNEL_TEST_SUITE_P(urProgramGetFunctionPointerTest);

TEST_P(urProgramGetFunctionPointerTest, Success) {
UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
void *function_pointer = nullptr;
ASSERT_SUCCESS(urProgramGetFunctionPointer(
device, program, function_name.data(), &function_pointer));
Expand Down
117 changes: 117 additions & 0 deletions test/conformance/testing/include/uur/known_failure.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright (C) 2024 Intel Corporation
// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
// See LICENSE.TXT
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#ifndef UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
#define UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED

#include "uur/utils.h"
#include <string>
#include <string_view>
#include <vector>

namespace uur {
struct DeviceMatcher {
DeviceMatcher(uint32_t adapterVersion, ur_platform_backend_t backend,
std::vector<std::string> deviceNames)
: adapterVersion(adapterVersion), backend(backend),
deviceNames(deviceNames) {}

uint32_t adapterVersion;
ur_platform_backend_t backend;
std::vector<std::string> deviceNames;
};

struct OpenCL : DeviceMatcher {
OpenCL(std::initializer_list<std::string> il)
: DeviceMatcher(1, UR_PLATFORM_BACKEND_OPENCL, {il.begin(), il.end()}) {
}
};

struct LevelZero : DeviceMatcher {
LevelZero(std::initializer_list<std::string> il)
: DeviceMatcher(1, UR_PLATFORM_BACKEND_LEVEL_ZERO,
{il.begin(), il.end()}) {}
};

struct LevelZeroV2 : DeviceMatcher {
LevelZeroV2(std::initializer_list<std::string> il)
: DeviceMatcher(2, UR_PLATFORM_BACKEND_LEVEL_ZERO,
{il.begin(), il.end()}) {}
};

struct CUDA : DeviceMatcher {
CUDA(std::initializer_list<std::string> il)
: DeviceMatcher(1, UR_PLATFORM_BACKEND_CUDA, {il.begin(), il.end()}) {}
};

struct HIP : DeviceMatcher {
HIP(std::initializer_list<std::string> il)
: DeviceMatcher(1, UR_PLATFORM_BACKEND_HIP, {il.begin(), il.end()}) {}
};

struct NativeCPU : DeviceMatcher {
NativeCPU(std::initializer_list<std::string> il)
: DeviceMatcher(1, UR_PLATFORM_BACKEND_NATIVE_CPU,
{il.begin(), il.end()}) {}
};

inline bool isKnownFailureOn(ur_adapter_handle_t adapter,
ur_platform_handle_t platform,
ur_device_handle_t device,
std::vector<DeviceMatcher> deviceMatchers) {
for (const auto &deviceMatcher : deviceMatchers) {
uint32_t adapterVersion = 0;
urAdapterGetInfo(adapter, UR_ADAPTER_INFO_VERSION,
sizeof(adapterVersion), &adapterVersion, nullptr);
ur_platform_backend_t backend = UR_PLATFORM_BACKEND_UNKNOWN;
uur::GetPlatformInfo<ur_platform_backend_t>(
platform, UR_PLATFORM_INFO_BACKEND, backend);
if (deviceMatcher.adapterVersion != adapterVersion &&
deviceMatcher.backend != backend) {
continue;
}
if (deviceMatcher.deviceNames.empty()) {
return true;
}
std::string deviceName;
uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME,
deviceName);
for (const auto &unsupportedDeviceName : deviceMatcher.deviceNames) {
if (deviceName.find(unsupportedDeviceName) != std::string::npos) {
return true;
}
}
}
return false;
}
} // namespace uur

#define UUR_KNOWN_FAILURE_ON(...) \
if (uur::isKnownFailureOn(adapter, platform, device, {__VA_ARGS__})) { \
using namespace std::literals; \
std::string platformName; \
uur::GetPlatformInfo<std::string>(platform, UR_PLATFORM_INFO_NAME, \
platformName); \
std::string deviceName; \
uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME, \
deviceName); \
const char *alsoRunKnownFailures = \
std::getenv("UR_CTS_ALSO_RUN_KNOWN_FAILURES"); \
if (alsoRunKnownFailures && (alsoRunKnownFailures == "1"sv || \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could getting and checking the environment variable be done in a separate function? Just to avoid having a huge macro in each failing test.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've done this in another branch which is based on top of the work to support multiple adapters/paltforms/devices at once. This work will likely get rolled into those changes.

alsoRunKnownFailures == "yes"sv || \
alsoRunKnownFailures == "YES"sv || \
alsoRunKnownFailures == "on"sv || \
alsoRunKnownFailures == "ON"sv || \
alsoRunKnownFailures == "true"sv || \
alsoRunKnownFailures == "TRUE"sv)) { \
std::cerr << "Known failure on: " << platformName << ", " \
<< deviceName << "\n"; \
} else { \
GTEST_SKIP() << "Known failure on: " << platformName << ", " \
<< deviceName; \
} \
}

#endif // UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
6 changes: 3 additions & 3 deletions test/conformance/testing/include/uur/utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,12 @@ ur_result_t GetInfo(ObjectTy object, InfoTy info, Callable cb, T &out_value) {

// special case for strings
if constexpr (std::is_same_v<std::string, T>) {
std::vector<char> data(size);
result = cb(object, info, size, data.data(), nullptr);
std::string value(size, '\0');
result = cb(object, info, size, value.data(), nullptr);
if (result != UR_RESULT_SUCCESS) {
return result;
}
out_value = std::string(data.data(), data.size());
out_value = value.substr(0, value.find_last_of('\0'));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A malfunctioning adapter could return a string that isn't null terminated, resulting in a crash here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll chuck a std::min(value.find_last_of('\0'), value.size()) on this.

return UR_RESULT_SUCCESS;
} else {
if (size != sizeof(T)) {
Expand Down
Loading