Show a warning when some failure is generated outside of a test by the host app. (#48)

* wip

* wip

* Improve fuzzy matching heuristics

* wip

* wip

* wip

* wip

* wip

* Better test output

* fix linux

---------

Co-authored-by: Stephen Celis <[email protected]>

Author: tgrapperon

Makefile

@@ -6,7 +6,8 @@ EXPECTED_STRING = This is expected to fail!
 EXPECTED = \033[31m\"$(EXPECTED_STRING)\"\033[0m
 
 test:
-	@TEST_FAILURE=true swift test 2>&1 | grep '$(EXPECTED_STRING)' > /dev/null \
+	@swift build --build-tests \
+		&& TEST_FAILURE=true swift test 2>&1 | grep '$(EXPECTED_STRING)' > /dev/null \
 		&& (echo "$(PASS) $(XCT_FAIL) was called with $(EXPECTED)" && exit) \
 		|| (echo "$(FAIL) expected $(XCT_FAIL) to be called with $(EXPECTED)" >&2 && exit 1)
 

Sources/XCTestDynamicOverlay/XCTFail.swift

@@ -1,7 +1,7 @@
 #if DEBUG
-  #if canImport(ObjectiveC)
-    import Foundation
+  import Foundation
 
+  #if canImport(ObjectiveC)
     /// This function generates a failure immediately and unconditionally.
     ///
     /// Dynamically creates and records an `XCTIssue` under the hood that captures the source code
@@ -12,6 +12,7 @@
     ///   results.
     @_disfavoredOverload
     public func XCTFail(_ message: String = "") {
+      let message = appendHostAppWarningIfNeeded(message)
       guard
         let currentTestCase = XCTCurrentTestCase,
         let XCTIssue = NSClassFromString("XCTIssue")
@@ -45,6 +46,7 @@
     ///   results.
     @_disfavoredOverload
     public func XCTFail(_ message: String = "", file: StaticString, line: UInt) {
+      let message = appendHostAppWarningIfNeeded(message)
       _XCTFailureHandler(nil, true, "\(file)", line, "\(message.isEmpty ? "failed" : message)", nil)
     }
 
@@ -64,6 +66,83 @@
     @_disfavoredOverload
     public func XCTFail(_ message: String = "", file: StaticString, line: UInt) {}
   #endif
+
+  func appendHostAppWarningIfNeeded(_ originalMessage: String) -> String {
+    guard _XCTIsTesting else { return originalMessage }
+    if Bundle.main.bundleIdentifier == "com.apple.dt.xctest.tool"  // Apple platforms
+      || Bundle.main.bundleIdentifier == nil  // Linux
+    {
+      // XCTesting is providing a default host app.
+      return originalMessage
+    }
+
+    if Thread.callStackSymbols.contains(where: { $0.range(of: "XCTestCore") != nil }) {
+      // We are apparently performing a sync test
+      return originalMessage
+    }
+
+    if testCaseSubclass(callStackSymbols: Thread.callStackSymbols) != nil {
+      // We are apparently performing an async test.
+      // We're matching a `() -> ()` function that starts with `test`, from a `XCTestCase` subclass
+      return originalMessage
+    }
+
+    let message = """
+      Warning! This failure occurred while running tests hosted by the main app.
+
+      Testing using the main app as a host can lead to false positive test failures created by the \
+      app accessing unimplemented values itself when it is spun up.
+
+        - Test host: \(Bundle.main.bundleIdentifier ?? "Unknown")
+
+      You can find more information and workarounds in the "Testing/Testing Gotchas" section of \
+      Dependencies' documentation at \
+      https://pointfreeco.github.io/swift-dependencies/main/documentation/dependencies/testing/.
+      """
+
+    return [originalMessage, "", message].joined(separator: "\n")
+  }
+
+  // (?<=\$s): Starts with "$s" (stable mangling);
+  // \d{1,3}: Some numbers (the class name length or the module name length);
+  // .*: The class name, or module name + class name length + class name;
+  // C: The class type identifier;
+  // (?=\d{1,3}test.*yy(Ya)?K?F): Followed by the function name length, function that starts with
+  // `test`, has no arguments (y), returns Void (y), and is a function (F), potentially async (Ya),
+  // throwing (K), or both.
+  private let testCaseRegex = #"(?<=\$s)\d{1,3}.*C(?=\d{1,3}test.*yy(Ya)?K?F)"#
+
+  func testCaseSubclass(callStackSymbols: [String]) -> Any.Type? {
+    for frame in callStackSymbols {
+      var startIndex = frame.startIndex
+      while startIndex != frame.endIndex {
+        if let range = frame.range(
+          of: testCaseRegex,
+          options: .regularExpression,
+          range: startIndex..<frame.endIndex,
+          locale: nil
+        ) {
+          if let testCase = testCase(mangledName: String(frame[range])) {
+            return testCase
+          }
+          startIndex = range.upperBound
+        } else {
+          break
+        }
+      }
+    }
+    return nil
+  }
+
+  private func testCase(mangledName: String) -> Any.Type? {
+    if let object = _typeByName(mangledName) as? NSObject.Type,
+      NSClassFromString("XCTestCase").map(object.isSubclass(of:)) == true
+    {
+      return object
+    }
+    return nil
+  }
+
 #else
   /// This function generates a failure immediately and unconditionally.
   ///

Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift

@@ -0,0 +1,43 @@
+import XCTest
+
+@testable import XCTestDynamicOverlay
+
+final class HostAppCallStackTests: XCTestCase {
+  func testIsAbleToDetectTest() {
+    XCTAssertEqual(
+      testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
+      ObjectIdentifier(HostAppCallStackTests.self)
+    )
+  }
+
+  func testIsAbleToDetectAsyncTest() async {
+    XCTAssertEqual(
+      testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
+      ObjectIdentifier(HostAppCallStackTests.self)
+    )
+  }
+
+  func testIsAbleToDetectThrowingTest() throws {
+    XCTAssertEqual(
+      testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
+      ObjectIdentifier(HostAppCallStackTests.self)
+    )
+  }
+
+  func testIsAbleToDetectAsyncThrowingTest() async throws {
+    XCTAssertEqual(
+      testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init),
+      ObjectIdentifier(HostAppCallStackTests.self)
+    )
+  }
+
+  #if !os(Linux)
+    func testFailDoesNotAppendHostAppWarningFromATest() {
+      XCTExpectFailure {
+        XCTestDynamicOverlay.XCTFail("foo")
+      } issueMatcher: {
+        $0.compactDescription == "foo"
+      }
+    }
+  #endif
+}