Add XCTExpectFailure

Makefile

@@ -14,8 +14,7 @@ test-debug:
 test: test-debug
 	@swift test -c release
 
-test-linux: test-debug
-	@swift test -c release
+test-linux: test
 
 test-linux-docker:
 	@docker run \

Sources/XCTestDynamicOverlay/XCTExpectFailure.swift

@@ -0,0 +1,184 @@
+import Foundation
+
+#if DEBUG && canImport(ObjectiveC)
+  /// Instructs the test to expect a failure in an upcoming assertion, with options to customize
+  /// expected failure checking and handling.
+  ///
+  /// - Parameters:
+  ///   - failureReason: An optional string that describes why the test expects a failure.
+  ///   - enabled: A Boolean value that indicates whether the test checks for the expected failure.
+  ///   - strict: A Boolean value that indicates whether the test reports an error if the expected
+  ///     failure doesn’t occur.
+  ///   - failingBlock: A block of test code and assertions where the test expects a failure.
+  @_disfavoredOverload
+  public func XCTExpectFailure<R>(
+    _ failureReason: String? = nil,
+    enabled: Bool? = nil,
+    strict: Bool? = nil,
+    failingBlock: () throws -> R,
+    issueMatcher: ((_XCTIssue) -> Bool)? = nil
+  ) rethrows -> R {
+    guard enabled ?? true
+    else { return try failingBlock() }
+    guard
+      let XCTExpectedFailureOptions = NSClassFromString("XCTExpectedFailureOptions")
+        as Any as? NSObjectProtocol,
+      let options = strict ?? true
+        ? XCTExpectedFailureOptions
+          .perform(NSSelectorFromString("alloc"))?.takeUnretainedValue()
+          .perform(NSSelectorFromString("init"))?.takeUnretainedValue()
+        : XCTExpectedFailureOptions
+          .perform(NSSelectorFromString("nonStrictOptions"))?.takeUnretainedValue(),
+      let functionBlockPointer = dlsym(dlopen(nil, RTLD_LAZY), "XCTExpectFailureWithOptionsInBlock")
+    else {
+      let errorString = dlerror().map { charPointer in String(cString: charPointer) }
+        ?? "Unknown error"
+      assertionFailure(
+        "Failed to get symbol for XCTExpectFailureWithOptionsInBlock with error: \(errorString)."
+      )
+      return try failingBlock()
+    }
+
+    if let issueMatcher = issueMatcher {
+      let issueMatcher: @convention(block) (AnyObject) -> Bool = { issue in
+        issueMatcher(_XCTIssue(issue))
+      }
+      options.setValue(issueMatcher, forKey: "issueMatcher")
+    }
+
+    let XCTExpectFailureWithOptionsInBlock = unsafeBitCast(
+      functionBlockPointer,
+      to: (@convention(c) (String?, AnyObject, () -> Void) -> Void).self
+    )
+
+    var result: Result<R, Error>!
+    XCTExpectFailureWithOptionsInBlock(failureReason, options) {
+      result = Result { try failingBlock() }
+    }
+    return try result._rethrowGet()
+  }
+
+  /// Instructs the test to expect a failure in an upcoming assertion, with options to customize
+  /// expected failure checking and handling.
+  ///
+  /// - Parameters:
+  ///   - failureReason: An optional string that describes why the test expects a failure.
+  ///   - enabled: A Boolean value that indicates whether the test checks for the expected failure.
+  ///   - strict: A Boolean value that indicates whether the test reports an error if the expected
+  ///     failure doesn’t occur.
+  @_disfavoredOverload
+  public func XCTExpectFailure(
+    _ failureReason: String? = nil,
+    enabled: Bool? = nil,
+    strict: Bool? = nil,
+    issueMatcher: ((_XCTIssue) -> Bool)? = nil
+  ) {
+    guard enabled ?? true
+    else { return }
+    guard
+      let XCTExpectedFailureOptions = NSClassFromString("XCTExpectedFailureOptions")
+        as Any as? NSObjectProtocol,
+      let options = strict ?? true
+        ? XCTExpectedFailureOptions
+          .perform(NSSelectorFromString("alloc"))?.takeUnretainedValue()
+          .perform(NSSelectorFromString("init"))?.takeUnretainedValue()
+        : XCTExpectedFailureOptions
+          .perform(NSSelectorFromString("nonStrictOptions"))?.takeUnretainedValue(),
+      let functionBlockPointer = dlsym(dlopen(nil, RTLD_LAZY), "XCTExpectFailureWithOptions")
+    else {
+      let errorString = dlerror().map { charPointer in String(cString: charPointer) }
+        ?? "Unknown error"
+      assertionFailure(
+        "Failed to get symbol for XCTExpectFailureWithOptionsInBlock with error: \(errorString)."
+      )
+      return
+    }
+
+    if let issueMatcher = issueMatcher {
+      let issueMatcher: @convention(block) (AnyObject) -> Bool = { issue in
+        issueMatcher(_XCTIssue(issue))
+      }
+      options.setValue(issueMatcher, forKey: "issueMatcher")
+    }
+
+    let XCTExpectFailureWithOptions = unsafeBitCast(
+      functionBlockPointer,
+      to: (@convention(c) (String?, AnyObject) -> Void).self
+    )
+
+    XCTExpectFailureWithOptions(failureReason, options)
+  }
+
+  public struct _XCTIssue: /*CustomStringConvertible, */Equatable, Hashable {
+    public var type: IssueType
+    public var compactDescription: String
+    public var detailedDescription: String?
+
+    // NB: This surface are has been left unimplemented for now. We can consider adopting more of it
+    //     in the future:
+    //
+    // var sourceCodeContext: XCTSourceCodeContext
+    // var associatedError: Error?
+    // var attachments: [XCTAttachment]
+    // mutating func add(XCTAttachment)
+    //
+    // public var description: String {
+    //   """
+    //   \(self.type.description) \
+    //   at \
+    //   \(self.sourceCodeContext.location.fileURL.lastPathComponent):\
+    //   \(self.sourceCodeContext.location.lineNumber): \
+    //   \(self.compactDescription)
+    //   """
+    // }
+
+    init(_ issue: AnyObject) {
+      self.type = IssueType(rawValue: issue.value(forKey: "type") as! Int)!
+      self.compactDescription = issue.value(forKey: "compactDescription") as! String
+      self.detailedDescription = issue.value(forKey: "detailedDescription") as? String
+    }
+
+    public enum IssueType: Int, Sendable {
+      case assertionFailure = 0
+      case performanceRegression = 3
+      case system = 4
+      case thrownError = 1
+      case uncaughtException = 2
+      case unmatchedExpectedFailure = 5
+
+      var description: String {
+        switch self {
+        case .assertionFailure:
+          return "Assertion Failure"
+        case .performanceRegression:
+          return "Performance Regression"
+        case .system:
+          return "System Error"
+        case .thrownError:
+          return "Thrown Error"
+        case .uncaughtException:
+          return "Uncaught Exception"
+        case .unmatchedExpectedFailure:
+          return "Unmatched ExpectedFailure"
+        }
+      }
+    }
+  }
+
+  @rethrows
+  private protocol _ErrorMechanism {
+    associatedtype Output
+    func get() throws -> Output
+  }
+  extension _ErrorMechanism {
+    func _rethrowError() rethrows -> Never {
+      _ = try _rethrowGet()
+      fatalError()
+    }
+    @usableFromInline
+    func _rethrowGet() rethrows -> Output {
+      return try get()
+    }
+  }
+  extension Result: _ErrorMechanism {}
+#endif

Tests/XCTestDynamicOverlayTests/GeneratePlaceholderTests.swift

@@ -1,5 +1,4 @@
-// TODO: https://github.com/apple/swift-corelibs-xctest/issues/438
-#if !os(Linux) && !os(Windows)
+#if DEBUG && !os(Linux) && !os(Windows)
   import Foundation
   import XCTest
   import XCTestDynamicOverlay

Tests/XCTestDynamicOverlayTests/TestHelpers.swift

@@ -5,6 +5,38 @@ func MyXCTFail(_ message: String) {
   XCTFail(message)
 }
 
+#if DEBUG && canImport(ObjectiveC)
+  func MyXCTExpectFailure(
+    _ failureReason: String,
+    enabled: Bool = true,
+    strict: Bool = true,
+    failingBlock: () -> Void,
+    issueMatcher: ((_XCTIssue) -> Bool)? = nil
+  ) {
+    XCTExpectFailure(
+      failureReason,
+      enabled: enabled,
+      strict: strict,
+      failingBlock: failingBlock,
+      issueMatcher: issueMatcher
+    )
+  }
+
+  func MyXCTExpectFailure(
+    _ failureReason: String,
+    enabled: Bool = true,
+    strict: Bool = true,
+    issueMatcher: ((_XCTIssue) -> Bool)? = nil
+  ) {
+    XCTExpectFailure(
+      failureReason,
+      enabled: enabled,
+      strict: strict,
+      issueMatcher: issueMatcher
+    )
+}
+#endif
+
 struct Client {
   var p00: () -> Int
   var p01: () throws -> Int

Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift

@@ -1,5 +1,4 @@
-// TODO: https://github.com/apple/swift-corelibs-xctest/issues/438
-#if !os(Linux) && !os(Windows)
+#if DEBUG && !os(Linux) && !os(Windows)
   import XCTest
 
   final class UnimplementedTests: XCTestCase {
@@ -11,7 +10,7 @@
           Unimplemented: f00 …
 
             Defined at:
-              XCTestDynamicOverlayTests/TestHelpers.swift:66
+              XCTestDynamicOverlayTests/TestHelpers.swift:70
           """
       }
 
@@ -22,7 +21,7 @@
           Unimplemented: f01 …
 
             Defined at:
-              XCTestDynamicOverlayTests/TestHelpers.swift:67
+              XCTestDynamicOverlayTests/TestHelpers.swift:71
 
             Invoked with:
               ""
@@ -36,7 +35,7 @@
           Unimplemented: f02 …
 
             Defined at:
-              XCTestDynamicOverlayTests/TestHelpers.swift:68
+              XCTestDynamicOverlayTests/TestHelpers.swift:72
 
             Invoked with:
               ("", 42)
@@ -50,7 +49,7 @@
           Unimplemented: f03 …
 
             Defined at:
-              XCTestDynamicOverlayTests/TestHelpers.swift:69
+              XCTestDynamicOverlayTests/TestHelpers.swift:73
 
             Invoked with:
               ("", 42, 1.2)
@@ -64,7 +63,7 @@
           Unimplemented: f04 …
 
             Defined at:
-              XCTestDynamicOverlayTests/TestHelpers.swift:70
+              XCTestDynamicOverlayTests/TestHelpers.swift:74
 
             Invoked with:
               ("", 42, 1.2, [1, 2])
@@ -80,7 +79,7 @@
           Unimplemented: f05 …
 
             Defined at:
-              XCTestDynamicOverlayTests/TestHelpers.swift:71
+              XCTestDynamicOverlayTests/TestHelpers.swift:75
 
             Invoked with:
               ("", 42, 1.2, [1, 2], XCTestDynamicOverlayTests.User(id: DEADBEEF-DEAD-BEEF-DEAD-BEEFDEADBEEF))

Tests/XCTestDynamicOverlayTests/XCTContextTests.swift

@@ -1,5 +1,4 @@
-// TODO: https://github.com/apple/swift-corelibs-xctest/issues/438
-#if !os(Linux) && !os(Windows)
+#if DEBUG && !os(Linux) && !os(Windows)
   import XCTest
   import XCTestDynamicOverlay
 

Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift

@@ -0,0 +1,30 @@
+import XCTest
+
+#if DEBUG && canImport(ObjectiveC)
+  final class XCTExpectFailureTests: XCTestCase {
+    func testXCTDynamicOverlayWithBlockShouldFail() async throws {
+      MyXCTExpectFailure("This is expected to pass.", strict: false) {
+        XCTAssertEqual(42, 42)
+      }
+
+      MyXCTExpectFailure("This is expected to pass.", strict: true) {
+        XCTAssertEqual(42, 1729)
+      } issueMatcher: {
+        $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"#
+      }
+
+      if ProcessInfo.processInfo.environment["TEST_FAILURE"] != nil {
+        MyXCTExpectFailure("This is expected to fail!", strict: true) {
+          XCTAssertEqual(42, 42)
+        }
+      }
+    }
+
+    func testXCTDynamicOverlayShouldFail() async throws {
+      MyXCTExpectFailure("This is expected to pass.", strict: true) {
+        $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"#
+      }
+      XCTAssertEqual(42, 1729)
+    }
+  }
+#endif

Tests/XCTestDynamicOverlayTests/XCTFailTests.swift

@@ -1,6 +1,6 @@
 import XCTest
 
-final class XCTestDynamicOverlayTests: XCTestCase {
+final class XCTFailTests: XCTestCase {
   func testXCTFailShouldFail() async throws {
     if ProcessInfo.processInfo.environment["TEST_FAILURE"] != nil {
       MyXCTFail("This is expected to fail!")