Skip to content

Commit ace2130

Browse files
stephencelisstephencelis
authored
Finesse host application warning (#49)
* Finesse host application warning Just a lil cleanup work from #48. * wip * wip * wip * wip
1 parent cdcc63c commit ace2130

File tree

2 files changed

+96
-118
lines changed

2 files changed

+96
-118
lines changed

‎Sources/XCTestDynamicOverlay/XCTFail.swift‎

Lines changed: 79 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
#if DEBUG
2-
import Foundation
1+
import Foundation
32

3+
#if DEBUG
44
#if canImport(ObjectiveC)
55
/// This function generates a failure immediately and unconditionally.
66
///
@@ -12,7 +12,8 @@
1212
/// results.
1313
@_disfavoredOverload
1414
public func XCTFail(_ message: String = "") {
15-
let message = appendHostAppWarningIfNeeded(message)
15+
var message = message
16+
attachHostApplicationWarningIfNeeded(&message)
1617
guard
1718
let currentTestCase = XCTCurrentTestCase,
1819
let XCTIssue = NSClassFromString("XCTIssue")
@@ -46,7 +47,8 @@
4647
/// results.
4748
@_disfavoredOverload
4849
public func XCTFail(_ message: String = "", file: StaticString, line: UInt) {
49-
let message = appendHostAppWarningIfNeeded(message)
50+
var message = message
51+
attachHostApplicationWarningIfNeeded(&message)
5052
_XCTFailureHandler(nil, true, "\(file)", line, "\(message.isEmpty ? "failed" : message)", nil)
5153
}
5254

@@ -57,6 +59,79 @@
5759
dlsym(dlopen(nil, RTLD_LAZY), "_XCTFailureHandler"),
5860
to: XCTFailureHandler.self
5961
)
62+
63+
private func attachHostApplicationWarningIfNeeded(_ message: inout String) {
64+
guard
65+
_XCTIsTesting,
66+
Bundle.main.bundleIdentifier != "com.apple.dt.xctest.tool"
67+
else { return }
68+
69+
let callStack = Thread.callStackSymbols
70+
71+
// Detect when synchronous test exists in stack.
72+
guard callStack.allSatisfy({ frame in !frame.contains(" XCTestCore ") })
73+
else { return }
74+
75+
// Detect when asynchronous test exists in stack.
76+
guard callStack.allSatisfy({ frame in !isTestFrame(frame) })
77+
else { return }
78+
79+
let displayName =
80+
Bundle.main.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String
81+
?? Bundle.main.object(forInfoDictionaryKey: kCFBundleNameKey as String) as? String
82+
?? "Unknown host application"
83+
84+
let bundleIdentifier = Bundle.main.bundleIdentifier ?? "Unknown bundle identifier"
85+
86+
if !message.contains(where: \.isNewline) {
87+
message.append(" …")
88+
}
89+
90+
message.append("""
91+
92+
93+
┏━━━━━━━━━━━━━━━━━┉┅
94+
┃ ⚠︎ Warning:
95+
┃
96+
┃ This failure was emitted from a host application outside the test stack.
97+
┃
98+
┃ Host application:
99+
┃ \(displayName) (\(bundleIdentifier))
100+
┃
101+
┃ The host application may have emitted this failure when it first launched,
102+
┃ outside this current test that happens to be running.
103+
┃
104+
┃ Consider setting the test target's host application to "None," or prevent
105+
┃ the host application from performing the code path that emits failure.
106+
┗━━━━━━━━━━━━━━━━━┉┅
107+
▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄
108+
109+
For more information (and workarounds), see "Testing gotchas":
110+
111+
https://pointfreeco.github.io/swift-dependencies/main/documentation/dependencies/testing#testing-gotchas
112+
"""
113+
)
114+
}
115+
116+
func isTestFrame(_ frame: String) -> Bool {
117+
// Regular expression to detect and demangle an XCTest case frame:
118+
//
119+
// 1. `(?<=\$s)`: Starts with "$s" (stable mangling)
120+
// 2. `\d{1,3}`: Some numbers (the class name length or the module name length)
121+
// 3. `.*`: The class name, or module name + class name length + class name
122+
// 4. `C`: The class type identifier
123+
// 5. `(?=\d{1,3}test.*yy(Ya)?K?F)`: The function name length, a function that starts with
124+
// `test`, has no arguments (`y`), returns Void (`y`), and is a function (`F`), potentially
125+
// async (`Ya`), throwing (`K`), or both.
126+
let mangledTestFrame = #"(?<=\$s)\d{1,3}.*C(?=\d{1,3}test.*yy(Ya)?K?F)"#
127+
128+
guard let XCTestCase = NSClassFromString("XCTestCase")
129+
else { return false }
130+
131+
return frame.range(of: mangledTestFrame, options: .regularExpression)
132+
.map { (_typeByName(String(frame[$0])) as? NSObject.Type)?.isSubclass(of: XCTestCase) ?? false }
133+
?? false
134+
}
60135
#elseif canImport(XCTest)
61136
// NB: It seems to be safe to import XCTest on Linux
62137
@_exported import func XCTest.XCTFail
@@ -66,83 +141,6 @@
66141
@_disfavoredOverload
67142
public func XCTFail(_ message: String = "", file: StaticString, line: UInt) {}
68143
#endif
69-
70-
func appendHostAppWarningIfNeeded(_ originalMessage: String) -> String {
71-
guard _XCTIsTesting else { return originalMessage }
72-
if Bundle.main.bundleIdentifier == "com.apple.dt.xctest.tool" // Apple platforms
73-
|| Bundle.main.bundleIdentifier == nil // Linux
74-
{
75-
// XCTesting is providing a default host app.
76-
return originalMessage
77-
}
78-
79-
if Thread.callStackSymbols.contains(where: { $0.range(of: "XCTestCore") != nil }) {
80-
// We are apparently performing a sync test
81-
return originalMessage
82-
}
83-
84-
if testCaseSubclass(callStackSymbols: Thread.callStackSymbols) != nil {
85-
// We are apparently performing an async test.
86-
// We're matching a `() -> ()` function that starts with `test`, from a `XCTestCase` subclass
87-
return originalMessage
88-
}
89-
90-
let message = """
91-
Warning! This failure occurred while running tests hosted by the main app.
92-
93-
Testing using the main app as a host can lead to false positive test failures created by the \
94-
app accessing unimplemented values itself when it is spun up.
95-
96-
- Test host: \(Bundle.main.bundleIdentifier ?? "Unknown")
97-
98-
You can find more information and workarounds in the "Testing/Testing Gotchas" section of \
99-
Dependencies' documentation at \
100-
https://pointfreeco.github.io/swift-dependencies/main/documentation/dependencies/testing/.
101-
"""
102-
103-
return [originalMessage, "", message].joined(separator: "\n")
104-
}
105-
106-
// (?<=\$s): Starts with "$s" (stable mangling);
107-
// \d{1,3}: Some numbers (the class name length or the module name length);
108-
// .*: The class name, or module name + class name length + class name;
109-
// C: The class type identifier;
110-
// (?=\d{1,3}test.*yy(Ya)?K?F): Followed by the function name length, function that starts with
111-
// `test`, has no arguments (y), returns Void (y), and is a function (F), potentially async (Ya),
112-
// throwing (K), or both.
113-
private let testCaseRegex = #"(?<=\$s)\d{1,3}.*C(?=\d{1,3}test.*yy(Ya)?K?F)"#
114-
115-
func testCaseSubclass(callStackSymbols: [String]) -> Any.Type? {
116-
for frame in callStackSymbols {
117-
var startIndex = frame.startIndex
118-
while startIndex != frame.endIndex {
119-
if let range = frame.range(
120-
of: testCaseRegex,
121-
options: .regularExpression,
122-
range: startIndex..<frame.endIndex,
123-
locale: nil
124-
) {
125-
if let testCase = testCase(mangledName: String(frame[range])) {
126-
return testCase
127-
}
128-
startIndex = range.upperBound
129-
} else {
130-
break
131-
}
132-
}
133-
}
134-
return nil
135-
}
136-
137-
private func testCase(mangledName: String) -> Any.Type? {
138-
if let object = _typeByName(mangledName) as? NSObject.Type,
139-
NSClassFromString("XCTestCase").map(object.isSubclass(of:)) == true
140-
{
141-
return object
142-
}
143-
return nil
144-
}
145-
146144
#else
147145
/// This function generates a failure immediately and unconditionally.
148146
///

‎Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift‎

Lines changed: 17 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,23 @@
1-
import XCTest
1+
#if DEBUG && canImport(ObjectiveC)
2+
import XCTest
23

3-
@testable import XCTestDynamicOverlay
4+
@testable import XCTestDynamicOverlay
45

5-
final class HostAppCallStackTests: XCTestCase {
6-
func testIsAbleToDetectTest() {
7-
XCTAssertEqual(
8-
testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
9-
ObjectIdentifier(HostAppCallStackTests.self)
10-
)
11-
}
12-
13-
func testIsAbleToDetectAsyncTest() async {
14-
XCTAssertEqual(
15-
testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
16-
ObjectIdentifier(HostAppCallStackTests.self)
17-
)
18-
}
6+
final class HostAppCallStackTests: XCTestCase {
7+
func testIsAbleToDetectTest() {
8+
XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame))
9+
}
1910

20-
func testIsAbleToDetectThrowingTest() throws {
21-
XCTAssertEqual(
22-
testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
23-
ObjectIdentifier(HostAppCallStackTests.self)
24-
)
25-
}
11+
func testIsAbleToDetectAsyncTest() async {
12+
XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame))
13+
}
2614

27-
func testIsAbleToDetectAsyncThrowingTest() async throws {
28-
XCTAssertEqual(
29-
testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
30-
ObjectIdentifier(HostAppCallStackTests.self)
31-
)
32-
}
15+
func testIsAbleToDetectThrowingTest() throws {
16+
XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame))
17+
}
3318

34-
#if !os(Linux)
35-
func testFailDoesNotAppendHostAppWarningFromATest() {
36-
XCTExpectFailure {
37-
XCTestDynamicOverlay.XCTFail("foo")
38-
} issueMatcher: {
39-
$0.compactDescription == "foo"
40-
}
19+
func testIsAbleToDetectAsyncThrowingTest() async throws {
20+
XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame))
4121
}
42-
#endif
43-
}
22+
}
23+
#endif

0 commit comments

Comments
 (0)