From d950a03d4edb63387262fa862378d10e0c3a9760 Mon Sep 17 00:00:00 2001 From: Zev Eisenberg Date: Mon, 10 Apr 2023 09:59:50 -0400 Subject: [PATCH 01/10] Add XCTExpectFailure. --- .../XCTExpectFailure.swift | 103 ++++++++++++++++++ Sources/XCTestDynamicOverlay/XCTFail.swift | 1 + 2 files changed, 104 insertions(+) create mode 100644 Sources/XCTestDynamicOverlay/XCTExpectFailure.swift diff --git a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift new file mode 100644 index 00000000..4034a658 --- /dev/null +++ b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift @@ -0,0 +1,103 @@ +import Foundation + +#if DEBUG + #if canImport(ObjectiveC) + /// Instructs the test to expect a failure in an upcoming assertion, with options to customize expected failure checking and handling. + /// - Parameters: + /// - failureReason: An optional string that describes why the test expects a failure. + /// - strict: A Boolean value that indicates whether the test reports an error if the expected failure doesn’t occur. + /// - failingBlock: A block of test code and assertions where the test expects a failure. + @_transparent @_disfavoredOverload + public func XCTExpectFailure( + _ failureReason: String? = nil, + strict: Bool = true, + failingBlock: () -> Void + ) { + guard + let XCTExpectedFailureOptions = NSClassFromString("XCTExpectedFailureOptions") + as Any as? NSObjectProtocol, + let options = strict + ? XCTExpectedFailureOptions + .perform(NSSelectorFromString("alloc"))?.takeUnretainedValue() + .perform(NSSelectorFromString("init"))?.takeUnretainedValue() + : XCTExpectedFailureOptions + .perform(NSSelectorFromString("nonStrictOptions"))?.takeUnretainedValue() + else { return } + + guard + let functionBlockPointer = dlsym( + dlopen(nil, RTLD_LAZY), "XCTExpectFailureWithOptionsInBlock") + else { + let errorString = + dlerror().map { charPointer in + String(cString: charPointer) + } ?? "Unknown error" + fatalError( + "Failed to get symbol for XCTExpectFailureWithOptionsInBlock with error: \(errorString).") + } + + let XCTExpectFailureWithOptionsInBlock = unsafeBitCast( + functionBlockPointer, + to: (@convention(c) (String?, AnyObject, () -> Void) -> Void).self + ) + + XCTExpectFailureWithOptionsInBlock(failureReason, options, failingBlock) + } + #elseif canImport(XCTest) + // NB: It seems to be safe to import XCTest on Linux + @_exported import func XCTest.XCTExpectFailure + #else + @_disfavoredOverload + public func XCTExpectFailure( + _ failureReason: String? = nil, + strict: Bool = true, + failingBlock: () -> Void + ) { + print(noop(message: failureReason)) + } + #endif +#else + + /// Instructs the test to expect a failure in an upcoming assertion, with options to customize expected failure checking and handling. + /// - Parameters: + /// - failureReason: An optional string that describes why the test expects a failure. + /// - strict: A Boolean value that indicates whether the test reports an error if the expected failure doesn’t occur. + /// - failingBlock: A block of test code and assertions where the test expects a failure. + @_disfavoredOverload + public func XCTExpectFailure( + _ failureReason: String? = nil, + strict: Bool = true, + failingBlock: () -> Void + ) { + print(noop(message: failureReason)) + } +#endif + +// Rule-of-threes: this is also used in XCTFail.swift. If you need it in a third place, consider refactoring. +private func noop(message: String, file: StaticString? = nil, line: UInt? = nil) -> String { + let fileAndLine: String + if let file = file, let line = line { + fileAndLine = """ + : + ┃ + ┃ \(file):\(line) + ┃ + ┃ … + """ + } else { + fileAndLine = "\n┃ " + } + + return """ + XCTExpectFailure: \(message) + + ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┉┅ + ┃ ⚠︎ Warning: This XCTExpectFailure was ignored + ┃ + ┃ XCTExpectFailure was invoked in a non-DEBUG environment\(fileAndLine)and so was ignored. Be sure to run tests with + ┃ the DEBUG=1 flag set in order to dynamically + ┃ load XCTExpectFailure. + ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┉┅ + ▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄ + """ +} diff --git a/Sources/XCTestDynamicOverlay/XCTFail.swift b/Sources/XCTestDynamicOverlay/XCTFail.swift index 4b6f7eba..13838417 100644 --- a/Sources/XCTestDynamicOverlay/XCTFail.swift +++ b/Sources/XCTestDynamicOverlay/XCTFail.swift @@ -160,6 +160,7 @@ import Foundation } #endif +// Rule-of-threes: this is also used in XCTExpectFailure.swift. If you need it in a third place, consider refactoring. private func noop(message: String, file: StaticString? = nil, line: UInt? = nil) -> String { let fileAndLine: String if let file = file, let line = line { From d3d203151332190a78c53de1abbb7b28266799f8 Mon Sep 17 00:00:00 2001 From: Zev Eisenberg Date: Mon, 10 Apr 2023 10:16:32 -0400 Subject: [PATCH 02/10] Add tests for XCTExpectFailure. --- Makefile | 2 +- .../XCTExpectFailure.swift | 4 ++-- .../TestHelpers.swift | 4 ++++ .../UnimplementedTests.swift | 12 ++++++------ .../XCTExpectFailureTests.swift | 19 +++++++++++++++++++ .../XCTFailTests.swift | 2 +- 6 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift diff --git a/Makefile b/Makefile index 63169bb4..212a0132 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ test-debug: || (echo "$(FAIL) expected $(XCT_FAIL) to be called with $(EXPECTED)" >&2 && exit 1) test: test-debug - @swift test -c release | grep '⚠︎ Warning: This XCTFail was ignored' || exit 1 + @swift test -c release | grep '⚠︎ Warning: This (XCTFail|XCTExpectFailure) was ignored' || exit 1 test-linux: test-debug @swift test -c release diff --git a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift index 4034a658..abde29fb 100644 --- a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift +++ b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift @@ -74,7 +74,7 @@ import Foundation #endif // Rule-of-threes: this is also used in XCTFail.swift. If you need it in a third place, consider refactoring. -private func noop(message: String, file: StaticString? = nil, line: UInt? = nil) -> String { +private func noop(message: String?, file: StaticString? = nil, line: UInt? = nil) -> String { let fileAndLine: String if let file = file, let line = line { fileAndLine = """ @@ -89,7 +89,7 @@ private func noop(message: String, file: StaticString? = nil, line: UInt? = nil) } return """ - XCTExpectFailure: \(message) + XCTExpectFailure: \(message ?? "") ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┉┅ ┃ ⚠︎ Warning: This XCTExpectFailure was ignored diff --git a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift index 779e34cc..f6247673 100644 --- a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift +++ b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift @@ -5,6 +5,10 @@ func MyXCTFail(_ message: String) { XCTFail(message) } +func MyXCTExpectFailure(strict: Bool, message: String, failingBlock: () -> Void) { + XCTExpectFailure(message, strict: strict, failingBlock: failingBlock) +} + struct Client { var p00: () -> Int var p01: () throws -> Int diff --git a/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift b/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift index 53a7c345..185724ad 100644 --- a/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift +++ b/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift @@ -10,7 +10,7 @@ Unimplemented: f00 … Defined at: - XCTestDynamicOverlayTests/TestHelpers.swift:66 + XCTestDynamicOverlayTests/TestHelpers.swift:70 """ } @@ -21,7 +21,7 @@ Unimplemented: f01 … Defined at: - XCTestDynamicOverlayTests/TestHelpers.swift:67 + XCTestDynamicOverlayTests/TestHelpers.swift:71 Invoked with: "" @@ -35,7 +35,7 @@ Unimplemented: f02 … Defined at: - XCTestDynamicOverlayTests/TestHelpers.swift:68 + XCTestDynamicOverlayTests/TestHelpers.swift:72 Invoked with: ("", 42) @@ -49,7 +49,7 @@ Unimplemented: f03 … Defined at: - XCTestDynamicOverlayTests/TestHelpers.swift:69 + XCTestDynamicOverlayTests/TestHelpers.swift:73 Invoked with: ("", 42, 1.2) @@ -63,7 +63,7 @@ Unimplemented: f04 … Defined at: - XCTestDynamicOverlayTests/TestHelpers.swift:70 + XCTestDynamicOverlayTests/TestHelpers.swift:74 Invoked with: ("", 42, 1.2, [1, 2]) @@ -79,7 +79,7 @@ Unimplemented: f05 … Defined at: - XCTestDynamicOverlayTests/TestHelpers.swift:71 + XCTestDynamicOverlayTests/TestHelpers.swift:75 Invoked with: ("", 42, 1.2, [1, 2], XCTestDynamicOverlayTests.User(id: DEADBEEF-DEAD-BEEF-DEAD-BEEFDEADBEEF)) diff --git a/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift b/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift new file mode 100644 index 00000000..c600cfa8 --- /dev/null +++ b/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift @@ -0,0 +1,19 @@ +import XCTest + +final class XCTExpectFailureTests: XCTestCase { + func testXCTDynamicOverlayShouldFail() async throws { + MyXCTExpectFailure(strict: false, message: "This is expected to pass.") { + XCTAssertEqual(42, 42) + } + + MyXCTExpectFailure(strict: true, message: "This is expected to pass.") { + XCTAssertEqual(42, 1729) + } + + if ProcessInfo.processInfo.environment["TEST_FAILURE"] != nil { + MyXCTExpectFailure(strict: true, message: "This is expected to fail!") { + XCTAssertEqual(42, 42) + } + } + } +} diff --git a/Tests/XCTestDynamicOverlayTests/XCTFailTests.swift b/Tests/XCTestDynamicOverlayTests/XCTFailTests.swift index a9a2fd2f..aba583b2 100644 --- a/Tests/XCTestDynamicOverlayTests/XCTFailTests.swift +++ b/Tests/XCTestDynamicOverlayTests/XCTFailTests.swift @@ -1,6 +1,6 @@ import XCTest -final class XCTestDynamicOverlayTests: XCTestCase { +final class XCTFailTests: XCTestCase { func testXCTFailShouldFail() async throws { if ProcessInfo.processInfo.environment["TEST_FAILURE"] != nil { MyXCTFail("This is expected to fail!") From 1d88778050b006744c1e975cea0cab833df0a43a Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Mon, 22 Jan 2024 21:52:47 -0800 Subject: [PATCH 03/10] Fix --- .../XCTExpectFailure.swift | 247 ++++++++++++------ .../TestHelpers.swift | 30 ++- .../XCTExpectFailureTests.swift | 17 +- 3 files changed, 203 insertions(+), 91 deletions(-) diff --git a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift index abde29fb..b74bf40a 100644 --- a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift +++ b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift @@ -1,103 +1,180 @@ import Foundation -#if DEBUG - #if canImport(ObjectiveC) - /// Instructs the test to expect a failure in an upcoming assertion, with options to customize expected failure checking and handling. - /// - Parameters: - /// - failureReason: An optional string that describes why the test expects a failure. - /// - strict: A Boolean value that indicates whether the test reports an error if the expected failure doesn’t occur. - /// - failingBlock: A block of test code and assertions where the test expects a failure. - @_transparent @_disfavoredOverload - public func XCTExpectFailure( - _ failureReason: String? = nil, - strict: Bool = true, - failingBlock: () -> Void - ) { - guard - let XCTExpectedFailureOptions = NSClassFromString("XCTExpectedFailureOptions") - as Any as? NSObjectProtocol, - let options = strict - ? XCTExpectedFailureOptions - .perform(NSSelectorFromString("alloc"))?.takeUnretainedValue() - .perform(NSSelectorFromString("init"))?.takeUnretainedValue() - : XCTExpectedFailureOptions - .perform(NSSelectorFromString("nonStrictOptions"))?.takeUnretainedValue() - else { return } +#if DEBUG && canImport(ObjectiveC) + /// Instructs the test to expect a failure in an upcoming assertion, with options to customize + /// expected failure checking and handling. + /// - Parameters: + /// - failureReason: An optional string that describes why the test expects a failure. + /// - enabled: A Boolean value that indicates whether the test checks for the expected failure. + /// - strict: A Boolean value that indicates whether the test reports an error if the expected + /// failure doesn’t occur. + /// - failingBlock: A block of test code and assertions where the test expects a failure. + @_disfavoredOverload + public func XCTExpectFailure( + _ failureReason: String? = nil, + enabled: Bool? = nil, + strict: Bool? = nil, + failingBlock: () throws -> R, + issueMatcher: ((_XCTIssue) -> Bool)? = nil + ) rethrows -> R { + guard enabled ?? true + else { return try failingBlock() } + guard + let XCTExpectedFailureOptions = NSClassFromString("XCTExpectedFailureOptions") + as Any as? NSObjectProtocol, + let options = strict ?? true + ? XCTExpectedFailureOptions + .perform(NSSelectorFromString("alloc"))?.takeUnretainedValue() + .perform(NSSelectorFromString("init"))?.takeUnretainedValue() + : XCTExpectedFailureOptions + .perform(NSSelectorFromString("nonStrictOptions"))?.takeUnretainedValue(), + let functionBlockPointer = dlsym(dlopen(nil, RTLD_LAZY), "XCTExpectFailureWithOptionsInBlock") + else { + let errorString = dlerror().map { charPointer in String(cString: charPointer) } + ?? "Unknown error" + assertionFailure( + "Failed to get symbol for XCTExpectFailureWithOptionsInBlock with error: \(errorString)." + ) + return try failingBlock() + } - guard - let functionBlockPointer = dlsym( - dlopen(nil, RTLD_LAZY), "XCTExpectFailureWithOptionsInBlock") - else { - let errorString = - dlerror().map { charPointer in - String(cString: charPointer) - } ?? "Unknown error" - fatalError( - "Failed to get symbol for XCTExpectFailureWithOptionsInBlock with error: \(errorString).") + if let issueMatcher = issueMatcher { + let issueMatcher: @convention(block) (AnyObject) -> Bool = { issue in + issueMatcher(_XCTIssue(issue)) } + options.setValue(issueMatcher, forKey: "issueMatcher") + } - let XCTExpectFailureWithOptionsInBlock = unsafeBitCast( - functionBlockPointer, - to: (@convention(c) (String?, AnyObject, () -> Void) -> Void).self - ) + let XCTExpectFailureWithOptionsInBlock = unsafeBitCast( + functionBlockPointer, + to: (@convention(c) (String?, AnyObject, () -> Void) -> Void).self + ) - XCTExpectFailureWithOptionsInBlock(failureReason, options, failingBlock) + var result: Result! + XCTExpectFailureWithOptionsInBlock(failureReason, options) { + result = Result { try failingBlock() } } - #elseif canImport(XCTest) - // NB: It seems to be safe to import XCTest on Linux - @_exported import func XCTest.XCTExpectFailure - #else - @_disfavoredOverload - public func XCTExpectFailure( - _ failureReason: String? = nil, - strict: Bool = true, - failingBlock: () -> Void - ) { - print(noop(message: failureReason)) - } - #endif -#else + return try result._rethrowGet() + } - /// Instructs the test to expect a failure in an upcoming assertion, with options to customize expected failure checking and handling. + /// Instructs the test to expect a failure in an upcoming assertion, with options to customize + /// expected failure checking and handling. /// - Parameters: /// - failureReason: An optional string that describes why the test expects a failure. - /// - strict: A Boolean value that indicates whether the test reports an error if the expected failure doesn’t occur. - /// - failingBlock: A block of test code and assertions where the test expects a failure. + /// - enabled: A Boolean value that indicates whether the test checks for the expected failure. + /// - strict: A Boolean value that indicates whether the test reports an error if the expected + /// failure doesn’t occur. @_disfavoredOverload public func XCTExpectFailure( _ failureReason: String? = nil, - strict: Bool = true, - failingBlock: () -> Void + enabled: Bool? = nil, + strict: Bool? = nil, + issueMatcher: ((_XCTIssue) -> Bool)? = nil ) { - print(noop(message: failureReason)) - } -#endif + guard enabled ?? true + else { return } + guard + let XCTExpectedFailureOptions = NSClassFromString("XCTExpectedFailureOptions") + as Any as? NSObjectProtocol, + let options = strict ?? true + ? XCTExpectedFailureOptions + .perform(NSSelectorFromString("alloc"))?.takeUnretainedValue() + .perform(NSSelectorFromString("init"))?.takeUnretainedValue() + : XCTExpectedFailureOptions + .perform(NSSelectorFromString("nonStrictOptions"))?.takeUnretainedValue(), + let functionBlockPointer = dlsym(dlopen(nil, RTLD_LAZY), "XCTExpectFailureWithOptions") + else { + let errorString = dlerror().map { charPointer in String(cString: charPointer) } + ?? "Unknown error" + assertionFailure( + "Failed to get symbol for XCTExpectFailureWithOptionsInBlock with error: \(errorString)." + ) + return + } + + if let issueMatcher = issueMatcher { + let issueMatcher: @convention(block) (AnyObject) -> Bool = { issue in + issueMatcher(_XCTIssue(issue)) + } + options.setValue(issueMatcher, forKey: "issueMatcher") + } + + let XCTExpectFailureWithOptions = unsafeBitCast( + functionBlockPointer, + to: (@convention(c) (String?, AnyObject) -> Void).self + ) -// Rule-of-threes: this is also used in XCTFail.swift. If you need it in a third place, consider refactoring. -private func noop(message: String?, file: StaticString? = nil, line: UInt? = nil) -> String { - let fileAndLine: String - if let file = file, let line = line { - fileAndLine = """ - : - ┃ - ┃ \(file):\(line) - ┃ - ┃ … - """ - } else { - fileAndLine = "\n┃ " + XCTExpectFailureWithOptions(failureReason, options) } - return """ - XCTExpectFailure: \(message ?? "") - ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┉┅ - ┃ ⚠︎ Warning: This XCTExpectFailure was ignored - ┃ - ┃ XCTExpectFailure was invoked in a non-DEBUG environment\(fileAndLine)and so was ignored. Be sure to run tests with - ┃ the DEBUG=1 flag set in order to dynamically - ┃ load XCTExpectFailure. - ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┉┅ - ▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄ - """ -} + public struct _XCTIssue: /*CustomStringConvertible, */Equatable, Hashable { + public var type: IssueType + public var compactDescription: String + public var detailedDescription: String? + + // var sourceCodeContext: XCTSourceCodeContext + // var associatedError: Error? + // var attachments: [XCTAttachment] + // mutating func add(XCTAttachment) + // + // public var description: String { + // """ + // \(self.type.description) \ + // at \ + // \(self.sourceCodeContext.location.fileURL.lastPathComponent):\ + // \(self.sourceCodeContext.location.lineNumber): \ + // \(self.compactDescription) + // """ + // } + + init(_ issue: AnyObject) { + self.type = IssueType(rawValue: issue.value(forKey: "type") as! Int)! + self.compactDescription = issue.value(forKey: "compactDescription") as! String + self.detailedDescription = issue.value(forKey: "detailedDescription") as? String + } + + public enum IssueType: Int, Sendable { + case assertionFailure = 0 + case performanceRegression = 3 + case system = 4 + case thrownError = 1 + case uncaughtException = 2 + case unmatchedExpectedFailure = 5 + + var description: String { + switch self { + case .assertionFailure: + return "Assertion Failure" + case .performanceRegression: + return "Performance Regression" + case .system: + return "System Error" + case .thrownError: + return "Thrown Error" + case .uncaughtException: + return "Uncaught Exception" + case .unmatchedExpectedFailure: + return "Unmatched ExpectedFailure" + } + } + } + } + + @rethrows + private protocol _ErrorMechanism { + associatedtype Output + func get() throws -> Output + } + extension _ErrorMechanism { + func _rethrowError() rethrows -> Never { + _ = try _rethrowGet() + fatalError() + } + @usableFromInline + func _rethrowGet() rethrows -> Output { + return try get() + } + } + extension Result: _ErrorMechanism {} +#endif diff --git a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift index f6247673..49e25c87 100644 --- a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift +++ b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift @@ -5,8 +5,34 @@ func MyXCTFail(_ message: String) { XCTFail(message) } -func MyXCTExpectFailure(strict: Bool, message: String, failingBlock: () -> Void) { - XCTExpectFailure(message, strict: strict, failingBlock: failingBlock) +func MyXCTExpectFailure( + _ failureReason: String, + enabled: Bool = true, + strict: Bool = true, + failingBlock: () -> Void, + issueMatcher: ((_XCTIssue) -> Bool)? = nil +) { + XCTExpectFailure( + failureReason, + enabled: enabled, + strict: strict, + failingBlock: failingBlock, + issueMatcher: issueMatcher + ) +} + +func MyXCTExpectFailure( + _ failureReason: String, + enabled: Bool = true, + strict: Bool = true, + issueMatcher: ((_XCTIssue) -> Bool)? = nil +) { + XCTExpectFailure( + failureReason, + enabled: enabled, + strict: strict, + issueMatcher: issueMatcher + ) } struct Client { diff --git a/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift b/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift index c600cfa8..8ba96881 100644 --- a/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift +++ b/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift @@ -1,19 +1,28 @@ import XCTest final class XCTExpectFailureTests: XCTestCase { - func testXCTDynamicOverlayShouldFail() async throws { - MyXCTExpectFailure(strict: false, message: "This is expected to pass.") { + func testXCTDynamicOverlayWithBlockShouldFail() async throws { + MyXCTExpectFailure("This is expected to pass.", strict: false) { XCTAssertEqual(42, 42) } - MyXCTExpectFailure(strict: true, message: "This is expected to pass.") { + MyXCTExpectFailure("This is expected to pass.", strict: true) { XCTAssertEqual(42, 1729) + } issueMatcher: { + $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"# } if ProcessInfo.processInfo.environment["TEST_FAILURE"] != nil { - MyXCTExpectFailure(strict: true, message: "This is expected to fail!") { + MyXCTExpectFailure("This is expected to fail!", strict: true) { XCTAssertEqual(42, 42) } } } + + func testXCTDynamicOverlayShouldFail() async throws { + MyXCTExpectFailure("This is expected to pass.", strict: true) { + $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"# + } + XCTAssertEqual(42, 1729) + } } From 015115da907200fedd66d977ab56c032b09d03d6 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Tue, 23 Jan 2024 09:36:32 -0800 Subject: [PATCH 04/10] wip --- .../XCTExpectFailureTests.swift | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift b/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift index 8ba96881..914441e8 100644 --- a/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift +++ b/Tests/XCTestDynamicOverlayTests/XCTExpectFailureTests.swift @@ -1,28 +1,30 @@ import XCTest -final class XCTExpectFailureTests: XCTestCase { - func testXCTDynamicOverlayWithBlockShouldFail() async throws { - MyXCTExpectFailure("This is expected to pass.", strict: false) { - XCTAssertEqual(42, 42) - } +#if DEBUG && canImport(ObjectiveC) + final class XCTExpectFailureTests: XCTestCase { + func testXCTDynamicOverlayWithBlockShouldFail() async throws { + MyXCTExpectFailure("This is expected to pass.", strict: false) { + XCTAssertEqual(42, 42) + } - MyXCTExpectFailure("This is expected to pass.", strict: true) { - XCTAssertEqual(42, 1729) - } issueMatcher: { - $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"# - } + MyXCTExpectFailure("This is expected to pass.", strict: true) { + XCTAssertEqual(42, 1729) + } issueMatcher: { + $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"# + } - if ProcessInfo.processInfo.environment["TEST_FAILURE"] != nil { - MyXCTExpectFailure("This is expected to fail!", strict: true) { - XCTAssertEqual(42, 42) + if ProcessInfo.processInfo.environment["TEST_FAILURE"] != nil { + MyXCTExpectFailure("This is expected to fail!", strict: true) { + XCTAssertEqual(42, 42) + } } } - } - func testXCTDynamicOverlayShouldFail() async throws { - MyXCTExpectFailure("This is expected to pass.", strict: true) { - $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"# + func testXCTDynamicOverlayShouldFail() async throws { + MyXCTExpectFailure("This is expected to pass.", strict: true) { + $0.compactDescription == #"XCTAssertEqual failed: ("42") is not equal to ("1729")"# + } + XCTAssertEqual(42, 1729) } - XCTAssertEqual(42, 1729) } -} +#endif From b28af93781f6a599f9aa797059537e64be2418f3 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Tue, 23 Jan 2024 09:49:48 -0800 Subject: [PATCH 05/10] wip --- Sources/XCTestDynamicOverlay/XCTExpectFailure.swift | 3 +++ Sources/XCTestDynamicOverlay/XCTFail.swift | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift index b74bf40a..916c36e7 100644 --- a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift +++ b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift @@ -113,6 +113,9 @@ import Foundation public var compactDescription: String public var detailedDescription: String? + // NB: This surface are has been left unimplemented for now. We can consider adopting more of it + // in the future: + // // var sourceCodeContext: XCTSourceCodeContext // var associatedError: Error? // var attachments: [XCTAttachment] diff --git a/Sources/XCTestDynamicOverlay/XCTFail.swift b/Sources/XCTestDynamicOverlay/XCTFail.swift index 13838417..4b6f7eba 100644 --- a/Sources/XCTestDynamicOverlay/XCTFail.swift +++ b/Sources/XCTestDynamicOverlay/XCTFail.swift @@ -160,7 +160,6 @@ import Foundation } #endif -// Rule-of-threes: this is also used in XCTExpectFailure.swift. If you need it in a third place, consider refactoring. private func noop(message: String, file: StaticString? = nil, line: UInt? = nil) -> String { let fileAndLine: String if let file = file, let line = line { From 6cf2de8d6d1c417c4a8a2aed52ef234a790f1fd3 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Tue, 23 Jan 2024 09:50:59 -0800 Subject: [PATCH 06/10] wip --- .../TestHelpers.swift | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift index 49e25c87..2d6df0d1 100644 --- a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift +++ b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift @@ -5,35 +5,37 @@ func MyXCTFail(_ message: String) { XCTFail(message) } -func MyXCTExpectFailure( - _ failureReason: String, - enabled: Bool = true, - strict: Bool = true, - failingBlock: () -> Void, - issueMatcher: ((_XCTIssue) -> Bool)? = nil -) { - XCTExpectFailure( - failureReason, - enabled: enabled, - strict: strict, - failingBlock: failingBlock, - issueMatcher: issueMatcher - ) -} +#if canImport(ObjectiveC) + func MyXCTExpectFailure( + _ failureReason: String, + enabled: Bool = true, + strict: Bool = true, + failingBlock: () -> Void, + issueMatcher: ((_XCTIssue) -> Bool)? = nil + ) { + XCTExpectFailure( + failureReason, + enabled: enabled, + strict: strict, + failingBlock: failingBlock, + issueMatcher: issueMatcher + ) + } -func MyXCTExpectFailure( - _ failureReason: String, - enabled: Bool = true, - strict: Bool = true, - issueMatcher: ((_XCTIssue) -> Bool)? = nil -) { - XCTExpectFailure( - failureReason, - enabled: enabled, - strict: strict, - issueMatcher: issueMatcher - ) + func MyXCTExpectFailure( + _ failureReason: String, + enabled: Bool = true, + strict: Bool = true, + issueMatcher: ((_XCTIssue) -> Bool)? = nil + ) { + XCTExpectFailure( + failureReason, + enabled: enabled, + strict: strict, + issueMatcher: issueMatcher + ) } +#endif struct Client { var p00: () -> Int From 88d33f4c655aca39655abf0e40c8d8d3dfa00455 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Tue, 23 Jan 2024 09:56:41 -0800 Subject: [PATCH 07/10] wip --- Sources/XCTestDynamicOverlay/XCTExpectFailure.swift | 3 ++- Tests/XCTestDynamicOverlayTests/TestHelpers.swift | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift index 916c36e7..ad32816e 100644 --- a/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift +++ b/Sources/XCTestDynamicOverlay/XCTExpectFailure.swift @@ -3,6 +3,7 @@ import Foundation #if DEBUG && canImport(ObjectiveC) /// Instructs the test to expect a failure in an upcoming assertion, with options to customize /// expected failure checking and handling. + /// /// - Parameters: /// - failureReason: An optional string that describes why the test expects a failure. /// - enabled: A Boolean value that indicates whether the test checks for the expected failure. @@ -59,6 +60,7 @@ import Foundation /// Instructs the test to expect a failure in an upcoming assertion, with options to customize /// expected failure checking and handling. + /// /// - Parameters: /// - failureReason: An optional string that describes why the test expects a failure. /// - enabled: A Boolean value that indicates whether the test checks for the expected failure. @@ -107,7 +109,6 @@ import Foundation XCTExpectFailureWithOptions(failureReason, options) } - public struct _XCTIssue: /*CustomStringConvertible, */Equatable, Hashable { public var type: IssueType public var compactDescription: String diff --git a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift index 2d6df0d1..7ca2c0b9 100644 --- a/Tests/XCTestDynamicOverlayTests/TestHelpers.swift +++ b/Tests/XCTestDynamicOverlayTests/TestHelpers.swift @@ -5,7 +5,7 @@ func MyXCTFail(_ message: String) { XCTFail(message) } -#if canImport(ObjectiveC) +#if DEBUG && canImport(ObjectiveC) func MyXCTExpectFailure( _ failureReason: String, enabled: Bool = true, From eda6e8f7028bafad57a1d444f6d94f27cd7db470 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Tue, 23 Jan 2024 10:12:18 -0800 Subject: [PATCH 08/10] wip --- Tests/XCTestDynamicOverlayTests/XCTContextTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift b/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift index ecb673bd..907bc936 100644 --- a/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift +++ b/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift @@ -1,5 +1,5 @@ // TODO: https://github.com/apple/swift-corelibs-xctest/issues/438 -#if !os(Linux) && !os(Windows) +#if DEBUG && !os(Linux) && !os(Windows) import XCTest import XCTestDynamicOverlay From 7eeebf84aafead183c3d622d704b95a58f5630aa Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Tue, 23 Jan 2024 10:15:07 -0800 Subject: [PATCH 09/10] wip --- Tests/XCTestDynamicOverlayTests/GeneratePlaceholderTests.swift | 3 +-- Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift | 3 +-- Tests/XCTestDynamicOverlayTests/XCTContextTests.swift | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Tests/XCTestDynamicOverlayTests/GeneratePlaceholderTests.swift b/Tests/XCTestDynamicOverlayTests/GeneratePlaceholderTests.swift index b22d7157..819bf8d4 100644 --- a/Tests/XCTestDynamicOverlayTests/GeneratePlaceholderTests.swift +++ b/Tests/XCTestDynamicOverlayTests/GeneratePlaceholderTests.swift @@ -1,5 +1,4 @@ -// TODO: https://github.com/apple/swift-corelibs-xctest/issues/438 -#if !os(Linux) && !os(Windows) +#if DEBUG && !os(Linux) && !os(Windows) import Foundation import XCTest import XCTestDynamicOverlay diff --git a/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift b/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift index 633ee1be..4a16cbfa 100644 --- a/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift +++ b/Tests/XCTestDynamicOverlayTests/UnimplementedTests.swift @@ -1,5 +1,4 @@ -// TODO: https://github.com/apple/swift-corelibs-xctest/issues/438 -#if !os(Linux) && !os(Windows) +#if DEBUG && !os(Linux) && !os(Windows) import XCTest final class UnimplementedTests: XCTestCase { diff --git a/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift b/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift index 907bc936..9d982407 100644 --- a/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift +++ b/Tests/XCTestDynamicOverlayTests/XCTContextTests.swift @@ -1,4 +1,3 @@ -// TODO: https://github.com/apple/swift-corelibs-xctest/issues/438 #if DEBUG && !os(Linux) && !os(Windows) import XCTest import XCTestDynamicOverlay From ec39d2dbaa0a205a74059f79b12cca6401ba264d Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Tue, 23 Jan 2024 10:24:19 -0800 Subject: [PATCH 10/10] wip --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 391b94cf..c8166e6e 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,7 @@ test-debug: test: test-debug @swift test -c release -test-linux: test-debug - @swift test -c release +test-linux: test test-linux-docker: @docker run \