diff --git a/Sentry.xcodeproj/project.pbxproj b/Sentry.xcodeproj/project.pbxproj index a429a966987..e33eadf78c5 100644 --- a/Sentry.xcodeproj/project.pbxproj +++ b/Sentry.xcodeproj/project.pbxproj @@ -749,6 +749,7 @@ A8F17B2E2901765900990B25 /* SentryRequest.m in Sources */ = {isa = PBXBuildFile; fileRef = A8F17B2D2901765900990B25 /* SentryRequest.m */; }; A8F17B342902870300990B25 /* SentryHttpStatusCodeRange.m in Sources */ = {isa = PBXBuildFile; fileRef = A8F17B332902870300990B25 /* SentryHttpStatusCodeRange.m */; }; D4009EB22D771BC20007AF30 /* SentryFileIOTrackerSwiftHelpersTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4009EB12D771BB90007AF30 /* SentryFileIOTrackerSwiftHelpersTests.swift */; }; + D40A826B2EAA62E900843F37 /* SentryAccessibilityEnabler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D40A82652EAA62E600843F37 /* SentryAccessibilityEnabler.swift */; }; D41415A72DEEE532003B14D5 /* SentryRedactViewHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = D41415A62DEEE532003B14D5 /* SentryRedactViewHelper.swift */; }; D4291A6D2DD62ACE00772088 /* SentryDispatchFactoryTests.m in Sources */ = {isa = PBXBuildFile; fileRef = D4291A6C2DD62AC800772088 /* SentryDispatchFactoryTests.m */; }; D42ADEEF2E9CF43200753166 /* SentrySessionReplayEnvironmentChecker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D42ADEE92E9CF42800753166 /* SentrySessionReplayEnvironmentChecker.swift */; }; @@ -771,6 +772,7 @@ D4411DD52E02B74900EA4987 /* ArrayAccessesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4411DD42E02B74100EA4987 /* ArrayAccessesTests.swift */; }; D44311312EB22812006CABE4 /* SentryUIRedactBuilderTests+ReactNative.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4AF7D212E93FFCA004F0F59 /* SentryUIRedactBuilderTests+ReactNative.swift */; }; D44B16722DE464AD006DBDB3 /* TestDispatchFactoryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D44B16712DE464A9006DBDB3 /* TestDispatchFactoryTests.swift */; }; + D450A8672EB4CDC9002DB0B4 /* SentryAccessibilityRedactBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = D450A8662EB4CDC9002DB0B4 /* SentryAccessibilityRedactBuilder.swift */; }; D451ED5D2D92ECD200C9BEA8 /* SentryOnDemandReplayError.swift in Sources */ = {isa = PBXBuildFile; fileRef = D451ED5C2D92ECD200C9BEA8 /* SentryOnDemandReplayError.swift */; }; D451ED5F2D92ECDE00C9BEA8 /* SentryReplayFrame.swift in Sources */ = {isa = PBXBuildFile; fileRef = D451ED5E2D92ECDE00C9BEA8 /* SentryReplayFrame.swift */; }; D452FC592DDB4B1700AFF56F /* SentryWatchdogTerminationBreadcrumbProcessor.m in Sources */ = {isa = PBXBuildFile; fileRef = D452FC572DDB4B1700AFF56F /* SentryWatchdogTerminationBreadcrumbProcessor.m */; }; @@ -816,6 +818,9 @@ D4AF00232D2E931000F5F3D7 /* SentryNSFileManagerSwizzling.h in Headers */ = {isa = PBXBuildFile; fileRef = D4AF00222D2E931000F5F3D7 /* SentryNSFileManagerSwizzling.h */; }; D4AF00252D2E93C400F5F3D7 /* SentryNSFileManagerSwizzlingTests.m in Sources */ = {isa = PBXBuildFile; fileRef = D4AF00242D2E93C400F5F3D7 /* SentryNSFileManagerSwizzlingTests.m */; }; D4AF7D2A2E940493004F0F59 /* SentryUIRedactBuilderTests+Common.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4AF7D292E940492004F0F59 /* SentryUIRedactBuilderTests+Common.swift */; }; + D4AF7E7D2E9537FD004F0F59 /* SentryUIRedactBuilderProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4AF7E772E9537F5004F0F59 /* SentryUIRedactBuilderProtocol.swift */; }; + D4AF7E7F2E953B43004F0F59 /* SentryMLRedactBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4AF7E7E2E953B3E004F0F59 /* SentryMLRedactBuilder.swift */; }; + D4AF7E812E953D10004F0F59 /* SentryMaskingModel.mlpackage in Sources */ = {isa = PBXBuildFile; fileRef = D4AF7E802E953D10004F0F59 /* SentryMaskingModel.mlpackage */; }; D4B0DC7F2DA9257A00DE61B6 /* SentryRenderVideoResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4B0DC7E2DA9257200DE61B6 /* SentryRenderVideoResult.swift */; }; D4B339F92EA7823000359F3A /* SentryTestUtilsDynamic.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D84DAD4D2B17428D003CF120 /* SentryTestUtilsDynamic.framework */; }; D4B339FA2EA7823000359F3A /* SentryTestUtilsDynamic.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = D84DAD4D2B17428D003CF120 /* SentryTestUtilsDynamic.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; @@ -2108,6 +2113,7 @@ A8F17B2D2901765900990B25 /* SentryRequest.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryRequest.m; sourceTree = ""; }; A8F17B332902870300990B25 /* SentryHttpStatusCodeRange.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryHttpStatusCodeRange.m; sourceTree = ""; }; D4009EB12D771BB90007AF30 /* SentryFileIOTrackerSwiftHelpersTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryFileIOTrackerSwiftHelpersTests.swift; sourceTree = ""; }; + D40A82652EAA62E600843F37 /* SentryAccessibilityEnabler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryAccessibilityEnabler.swift; sourceTree = ""; }; D41415A62DEEE532003B14D5 /* SentryRedactViewHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryRedactViewHelper.swift; sourceTree = ""; }; D41909922D48FFF6002B83D0 /* SentryNSDictionarySanitize+Tests.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "SentryNSDictionarySanitize+Tests.h"; sourceTree = ""; }; D41909942D490006002B83D0 /* SentryNSDictionarySanitize+Tests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = "SentryNSDictionarySanitize+Tests.m"; sourceTree = ""; }; @@ -2130,6 +2136,7 @@ D43B26D92D70A60E007747FD /* SentrySpanDataKey.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentrySpanDataKey.m; sourceTree = ""; }; D4411DD42E02B74100EA4987 /* ArrayAccessesTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ArrayAccessesTests.swift; sourceTree = ""; }; D44B16712DE464A9006DBDB3 /* TestDispatchFactoryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestDispatchFactoryTests.swift; sourceTree = ""; }; + D450A8662EB4CDC9002DB0B4 /* SentryAccessibilityRedactBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryAccessibilityRedactBuilder.swift; sourceTree = ""; }; D451ED5C2D92ECD200C9BEA8 /* SentryOnDemandReplayError.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryOnDemandReplayError.swift; sourceTree = ""; }; D451ED5E2D92ECDE00C9BEA8 /* SentryReplayFrame.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryReplayFrame.swift; sourceTree = ""; }; D452FC562DDB4B1700AFF56F /* SentryWatchdogTerminationBreadcrumbProcessor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = SentryWatchdogTerminationBreadcrumbProcessor.h; path = ../include/SentryWatchdogTerminationBreadcrumbProcessor.h; sourceTree = ""; }; @@ -2181,6 +2188,9 @@ D4AF00242D2E93C400F5F3D7 /* SentryNSFileManagerSwizzlingTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryNSFileManagerSwizzlingTests.m; sourceTree = ""; }; D4AF7D212E93FFCA004F0F59 /* SentryUIRedactBuilderTests+ReactNative.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SentryUIRedactBuilderTests+ReactNative.swift"; sourceTree = ""; }; D4AF7D292E940492004F0F59 /* SentryUIRedactBuilderTests+Common.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SentryUIRedactBuilderTests+Common.swift"; sourceTree = ""; }; + D4AF7E772E9537F5004F0F59 /* SentryUIRedactBuilderProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryUIRedactBuilderProtocol.swift; sourceTree = ""; }; + D4AF7E7E2E953B3E004F0F59 /* SentryMLRedactBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryMLRedactBuilder.swift; sourceTree = ""; }; + D4AF7E802E953D10004F0F59 /* SentryMaskingModel.mlpackage */ = {isa = PBXFileReference; lastKnownFileType = folder.mlpackage; path = SentryMaskingModel.mlpackage; sourceTree = ""; }; D4B0DC7E2DA9257200DE61B6 /* SentryRenderVideoResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryRenderVideoResult.swift; sourceTree = ""; }; D4BCA0C22DA93C25009E49AB /* SentrySessionReplayIntegration+Test.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "SentrySessionReplayIntegration+Test.h"; sourceTree = ""; }; D4C5F5992D4249E0002A9BF6 /* DataSentryTracingIntegrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataSentryTracingIntegrationTests.swift; sourceTree = ""; }; @@ -4956,11 +4966,16 @@ children = ( D4CD2A7C2DE9F91900DA9F59 /* SentryRedactRegion.swift */, D4CD2A7D2DE9F91900DA9F59 /* SentryRedactRegionType.swift */, + D40A82652EAA62E600843F37 /* SentryAccessibilityEnabler.swift */, + D450A8662EB4CDC9002DB0B4 /* SentryAccessibilityRedactBuilder.swift */, FA67DCDF2DDBD4EA00896B02 /* SentryDefaultMaskRenderer.swift */, FA67DCE02DDBD4EA00896B02 /* SentryDefaultViewRenderer.swift */, FA67DCE12DDBD4EA00896B02 /* SentryGraphicsImageRenderer.swift */, FA67DCE22DDBD4EA00896B02 /* SentryMaskRenderer.swift */, FA67DCE32DDBD4EA00896B02 /* SentryMaskRendererV2.swift */, + D4AF7E7E2E953B3E004F0F59 /* SentryMLRedactBuilder.swift */, + D4AF7E802E953D10004F0F59 /* SentryMaskingModel.mlpackage */, + D4AF7E772E9537F5004F0F59 /* SentryUIRedactBuilderProtocol.swift */, FA67DCE42DDBD4EA00896B02 /* SentryViewPhotographer.swift */, FA67DCE52DDBD4EA00896B02 /* SentryViewRenderer.swift */, FA67DCE62DDBD4EA00896B02 /* SentryViewRendererV2.swift */, @@ -5757,6 +5772,7 @@ 623FD9042D3FA92700803EDA /* NSNumberDecodableWrapper.swift in Sources */, 63FE718D20DA4C1100CDBAE8 /* SentryCrashReportStore.c in Sources */, 7BA0C0482805600A003E0326 /* SentryTransportAdapter.m in Sources */, + D4AF7E812E953D10004F0F59 /* SentryMaskingModel.mlpackage in Sources */, 63FE712920DA4C1000CDBAE8 /* SentryCrashCPU_arm.c in Sources */, 03F84D3427DD4191008FE43F /* SentryThreadMetadataCache.cpp in Sources */, 62862B1E2B1DDC35009B16E3 /* SentryDelayedFrame.m in Sources */, @@ -5782,6 +5798,7 @@ 8E133FA225E72DEF00ABD0BF /* SentrySamplingContext.m in Sources */, F451FAA62E0B304E0050ACF2 /* LoadValidator.swift in Sources */, D81988C02BEBFFF70020E36C /* SentryReplayRecording.swift in Sources */, + D4AF7E7D2E9537FD004F0F59 /* SentryUIRedactBuilderProtocol.swift in Sources */, 7B6C5F8726034395007F7DFF /* SentryWatchdogTerminationLogic.m in Sources */, 63FE708D20DA4C1000CDBAE8 /* SentryCrashReportFilterBasic.m in Sources */, F40E42352EA1887000E53876 /* SentryFramesTracker.swift in Sources */, @@ -5958,12 +5975,14 @@ 849B8F9C2C6E906900148E1F /* SentryUserFeedbackThemeConfiguration.swift in Sources */, F452437E2DE60B71003E8F50 /* SentryUseNSExceptionCallstackWrapper.m in Sources */, 63FE70D320DA4C1000CDBAE8 /* SentryCrashMonitor_AppState.c in Sources */, + D4AF7E7F2E953B43004F0F59 /* SentryMLRedactBuilder.swift in Sources */, FA1841832E4B457F005DEDC7 /* SentryApplication.swift in Sources */, 849B8F9D2C6E906900148E1F /* SentryUserFeedbackWidgetConfiguration.swift in Sources */, 620467AC2D3FFD230025F06C /* SentryNSErrorCodable.swift in Sources */, 639FCFA51EBC809A00778193 /* SentryStacktrace.m in Sources */, D84D2CC32C29AD120011AF8A /* SentrySessionReplay.swift in Sources */, F4FE9DFD2E622CD70014FED5 /* SentryDefaultObjCRuntimeWrapper.swift in Sources */, + D40A826B2EAA62E900843F37 /* SentryAccessibilityEnabler.swift in Sources */, F4FE9DFE2E622CD70014FED5 /* SentryObjCRuntimeWrapper.swift in Sources */, 84A898542E163072009A551E /* SentryProfileConfiguration.m in Sources */, 849B8F9B2C6E906900148E1F /* SentryUserFeedbackIntegrationDriver.swift in Sources */, @@ -6013,6 +6032,7 @@ 63FE713120DA4C1100CDBAE8 /* SentryCrashDynamicLinker.c in Sources */, 03F84D3527DD4191008FE43F /* SentryThreadHandle.cpp in Sources */, 0A2D8DA9289BC905008720F6 /* SentryViewHierarchyProviderHelper.m in Sources */, + D450A8672EB4CDC9002DB0B4 /* SentryAccessibilityRedactBuilder.swift in Sources */, D84D2CDD2C2BF7370011AF8A /* SentryReplayEvent.swift in Sources */, D8BC28CC2BFF78220054DA4D /* SentryRRWebTouchEvent.swift in Sources */, D452FC592DDB4B1700AFF56F /* SentryWatchdogTerminationBreadcrumbProcessor.m in Sources */, diff --git a/Sources/Sentry/SentrySessionReplayIntegration.m b/Sources/Sentry/SentrySessionReplayIntegration.m index cc2be17baab..01fbcbb4312 100644 --- a/Sources/Sentry/SentrySessionReplayIntegration.m +++ b/Sources/Sentry/SentrySessionReplayIntegration.m @@ -127,10 +127,19 @@ - (void)setupWith:(SentryReplayOptions *)replayOptions viewRenderer = [[SentryDefaultViewRenderer alloc] init]; } + id redactBuilder; + if (experimentalOptions.sessionReplayMaskingStrategy == kSessionReplayMaskingStrategyAccessibilty) { + redactBuilder = [[SentryAccessibilityRedactBuilder alloc] initWithOptions:replayOptions]; + } else if(experimentalOptions.sessionReplayMaskingStrategy == kSessionReplayMaskingStrategyMachineLearning) { + redactBuilder = [[SentryMLRedactBuilder alloc] initWithOptions:replayOptions]; + } else { + redactBuilder = [[SentryUIRedactBuilder alloc] initWithOptions:replayOptions]; + } + // We are using the flag for the view renderer V2 also for the mask renderer V2, as it would // just introduce another option without affecting the SDK user experience. _viewPhotographer = [[SentryViewPhotographer alloc] initWithRenderer:viewRenderer - redactOptions:replayOptions + redactBuilder:redactBuilder enableMaskRendererV2:enableViewRendererV2]; if (touchTracker) { diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryAccessibilityEnabler.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryAccessibilityEnabler.swift new file mode 100644 index 00000000000..e9d8f1368d1 --- /dev/null +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryAccessibilityEnabler.swift @@ -0,0 +1,97 @@ +// +// SentryAccessibilityEnabler.swift +// Sentry +// +// Enables accessibility temporarily for redaction purposes. +// Based on ASAccessibilityEnabler from AccessibilitySnapshot. +// + +import Foundation +import Darwin + +@objcMembers +public final class SentryAccessibilityEnabler: NSObject { + private typealias AXSAutomationEnabled = @convention(c) () -> Int32 + private typealias AXSSetAutomationEnabled = @convention(c) (Int32) -> Void + + private var getAutomationEnabled: AXSAutomationEnabled? + private var setAutomationEnabled: AXSSetAutomationEnabled? + private var previousValue: Int32 = 0 + private var handle: UnsafeMutableRawPointer? + + public override init() { + self.getAutomationEnabled = nil + self.setAutomationEnabled = nil + self.previousValue = 0 + self.handle = nil + super.init() + } + + /// Enables accessibility automation by calling the private _AXSSetAutomationEnabled API. + /// - Returns: true if accessibility was successfully enabled, false otherwise. + public func enable() -> Bool { + if handle != nil { + // Already loaded + return true + } + + // Load the private accessibility dylib + guard let h = loadDylib(path: "/usr/lib/libAccessibility.dylib") else { + NSLog("[Sentry] Failed to load libAccessibility.dylib") + return false + } + handle = h + + // Resolve function pointers to private APIs + guard let symEnabled = dlsym(h, "_AXSAutomationEnabled"), + let symSetEnabled = dlsym(h, "_AXSSetAutomationEnabled") else { + NSLog("[Sentry] Failed to find accessibility automation functions") + dlclose(h) + handle = nil + return false + } + + getAutomationEnabled = unsafeBitCast(symEnabled, to: AXSAutomationEnabled.self) + setAutomationEnabled = unsafeBitCast(symSetEnabled, to: AXSSetAutomationEnabled.self) + + // Save current state and enable accessibility + previousValue = getAutomationEnabled?() ?? 0 + setAutomationEnabled?(1) + + return true + } + + /// Disables accessibility automation and restores the previous state. + public func disable() { + guard let handle else { + // Not enabled + return + } + + // Restore previous state + if let setAutomationEnabled { + setAutomationEnabled(previousValue) + } + + // Clean up + dlclose(handle) + self.handle = nil + self.getAutomationEnabled = nil + self.setAutomationEnabled = nil + } + + private func loadDylib(path: String) -> UnsafeMutableRawPointer? { + // Handle simulator vs device paths + let environment = ProcessInfo.processInfo.environment + if let simulatorRoot = environment["IPHONE_SIMULATOR_ROOT"] { + let fullPath = simulatorRoot + path + return dlopen((fullPath as NSString).fileSystemRepresentation, RTLD_LOCAL) + } else { + return dlopen((path as NSString).fileSystemRepresentation, RTLD_LOCAL) + } + } + + deinit { + disable() + } +} diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryAccessibilityRedactBuilder.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryAccessibilityRedactBuilder.swift new file mode 100644 index 00000000000..acfe3205143 --- /dev/null +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryAccessibilityRedactBuilder.swift @@ -0,0 +1,239 @@ +#if canImport(UIKit) && !SENTRY_NO_UIKIT +#if os(iOS) || os(tvOS) +import UIKit +@_implementationOnly import _SentryPrivate + +@objcMembers +@_spi(Private) public class SentryAccessibilityRedactBuilder: NSObject, SentryUIRedactBuilderProtocol { + + private let options: SentryRedactOptions + + required public init(options: SentryRedactOptions) { + self.options = options + super.init() + } + + // Accessibility element with its frame in image coordinates + private struct AccessibilityElement { + let frame: CGRect + let label: String? + let value: String? + let traits: UIAccessibilityTraits + let isSecureTextEntry: Bool + } + + private static let processingQueue = DispatchQueue(label: "io.sentry.redaction.accessibility", qos: .userInitiated) + + // Parse accessibility elements from the view hierarchy + private func parseAccessibilityElements(in root: UIView, image: UIImage) -> [AccessibilityElement] { + var result: [AccessibilityElement] = [] + + // Compute transform from view (root) coordinates into image coordinates + let rootBounds = root.bounds + let sx = image.size.width / max(rootBounds.width, 1) + let sy = image.size.height / max(rootBounds.height, 1) + // let scaleTransform = CGAffineTransform(scaleX: sx, y: sy) + + // Recursively parse accessibility hierarchy + func parseHierarchy(from object: NSObject) { + // Skip if accessibility is hidden + guard !object.accessibilityElementsHidden else { + return + } + + // Skip hidden or zero-sized views + if let view = object as? UIView { + if view.isHidden || view.frame.size == .zero || view.alpha <= 0 { + return + } + } + + // If this object is an accessibility element, add it + if object.isAccessibilityElement { + // Determine the element's frame in root coordinates. Prefer presentation layers + // for UIView-backed elements to better match in-flight animations/transitions. + let frameInRoot: CGRect = { + if let view = object as? UIView { + let rootLayer = root.layer.presentation() ?? root.layer + let viewLayer = view.layer.presentation() ?? view.layer + let rectInViewLayer = viewLayer.bounds + return viewLayer.convert(rectInViewLayer, to: rootLayer) + } else { + // Fallback to accessibilityFrame (screen coordinates) for non-UIView elements + let accessibilityFrame = object.accessibilityFrame + return root.convert(accessibilityFrame, from: nil) + } + }() + + // Scale the size to match image coordinates + let scaledSize = CGSize( + width: frameInRoot.width * sx, + height: frameInRoot.height * sy + ) + + // Scale the position (center point) to match image coordinates + let scaledCenter = CGPoint( + x: (frameInRoot.minX + frameInRoot.width / 2) * sx, + y: (frameInRoot.minY + frameInRoot.height / 2) * sy + ) + + // Create frame with scaled dimensions + let frameInImage = CGRect( + x: scaledCenter.x - scaledSize.width / 2, + y: scaledCenter.y - scaledSize.height / 2, + width: scaledSize.width, + height: scaledSize.height + ) + + // Check for secure text entry on the main thread (before moving to background queue) + var isSecure = false + if let textField = object as? UITextField { + isSecure = textField.isSecureTextEntry + } else if let textView = object as? UITextView { + isSecure = textView.isSecureTextEntry + } + + result.append(AccessibilityElement( + frame: frameInImage, + label: object.accessibilityLabel, + value: object.accessibilityValue, + traits: object.accessibilityTraits, + isSecureTextEntry: isSecure + )) + } else if let accessibilityElements = object.accessibilityElements as? [NSObject] { // If it has an accessibilityElements array, parse those + for element in accessibilityElements { + parseHierarchy(from: element) + } + } else if let view = object as? UIView { // Otherwise, recurse into subviews + // Check for modal views + let subviewsToParse: [UIView] + if let lastModalView = view.subviews.last(where: { $0.accessibilityViewIsModal }) { + subviewsToParse = [lastModalView] + } else { + subviewsToParse = view.subviews + } + + for subview in subviewsToParse { + parseHierarchy(from: subview) + } + } + } + + parseHierarchy(from: root) + return result + } + + public func addIgnoreClass(_ ignoreClass: AnyClass) { + // no-op + } + + public func addRedactClass(_ redactClass: AnyClass) { + // no-op + } + + public func addIgnoreClasses(_ ignoreClasses: [AnyClass]) { + // no-op + } + + public func addRedactClasses(_ redactClasses: [AnyClass]) { + // no-op + } + + public func setIgnoreContainerClass(_ ignoreContainerClass: AnyClass) { + // no-op + } + + public func setRedactContainerClass(_ redactContainerClass: AnyClass) { + // no-op + } + + public func redactRegionsFor(view: UIView, image: UIImage, callback: @escaping ([SentryRedactRegion]?, Error?) -> Void) { + // Enable accessibility automation temporarily to ensure SwiftUI populates accessibility properties + let accessibilityEnabler = SentryAccessibilityEnabler() + let accessibilityEnabled = accessibilityEnabler.enable() + + if !accessibilityEnabled { + // Log warning but continue - some elements may still be accessible + print("[Sentry] Warning: Failed to enable accessibility automation. SwiftUI text may not be detected.") + } + + // Give the system a moment to update accessibility properties + // SwiftUI needs time to populate accessibility info after automation is enabled + DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { [weak self] in + guard let self = self else { + accessibilityEnabler.disable() + return + } + + // Capture a lightweight snapshot of accessibility elements on the main thread + let elements = self.parseAccessibilityElements(in: view, image: image) + + // Disable accessibility automation as soon as we're done capturing + accessibilityEnabler.disable() + + // Process on a background queue to avoid blocking UI + Self.processingQueue.async { + var regions: [SentryRedactRegion] = [] + + for element in elements { + // Determine if we should redact this element + let shouldRedact = self.shouldRedact(element: element) + + if shouldRedact { + let rect = element.frame + let size = rect.size + + // The transform should position the center of the rect + // Then offset by -anchorPoint (which is -size/2 for centered anchor) + let center = CGPoint(x: rect.midX, y: rect.midY) + let anchorOffset = CGPoint(x: size.width / 2, y: size.height / 2) + + var transform = CGAffineTransform(translationX: center.x, y: center.y) + transform = transform.translatedBy(x: -anchorOffset.x, y: -anchorOffset.y) + + let description = [element.label, element.value] + .compactMap { $0 } + .joined(separator: " ") + + regions.append(SentryRedactRegion( + size: size, + transform: transform, + type: .redact, + color: nil, // Let the renderer choose the color + name: description.isEmpty ? "Accessibility Element" : description + )) + } + } + + DispatchQueue.main.async { + callback(regions, nil) + } + } + } + } + + // Determine if an accessibility element should be redacted + private func shouldRedact(element: AccessibilityElement) -> Bool { + let traits = element.traits + + // Always redact secure text fields + if element.isSecureTextEntry { + return true + } + + // If maskAllText is enabled, redact all text-containing elements + if options.maskAllText { + // Check if element has static text, text entry, or keyboard key traits + let isTextElement = traits.contains(.staticText) || + traits.contains(.keyboardKey) || + traits.contains(.searchField) || + (element.label != nil || element.value != nil) + return isTextElement + } + + return false + } +} + +#endif // os(iOS) || os(tvOS) +#endif // canImport(UIKit) && !SENTRY_NO_UIKIT diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryMLRedactBuilder.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryMLRedactBuilder.swift new file mode 100644 index 00000000000..0f17a2bc7ae --- /dev/null +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryMLRedactBuilder.swift @@ -0,0 +1,461 @@ +#if canImport(UIKit) && !SENTRY_NO_UIKIT +#if os(iOS) || os(tvOS) +import UIKit +import CoreML +@_implementationOnly import _SentryPrivate +import Vision + +@objcMembers +@_spi(Private) public class SentryMLRedactBuilder: NSObject, SentryUIRedactBuilderProtocol { + private static let modelInputSize: CGFloat = 640.0 + private let mlModel: MLModel + private let queue: DispatchQueue + private let classNames: [Int: String] + + required public init(options: SentryRedactOptions) { + // Load the model at initialization time + guard #available(iOS 17.0, *) else { + fatalError( + "[Sentry] Warning: The iOS 17.0 ML model is not available. " + + "Please update your SDK to a version that supports it." + ) + } + + var loadedClassNames: [Int: String] = [:] + + do { + // Try to load the compiled model from the bundle + guard let modelURL = Bundle(for: Self.self).url(forResource: "SentryMaskingModel", withExtension: "mlmodelc") else { + fatalError("[Sentry] Warning: Could not find SentryMaskingModel.mlmodelc in bundle") + } + + let config = MLModelConfiguration() + self.mlModel = try MLModel(contentsOf: modelURL, configuration: config) + + // Parse class names from model metadata + if let metadata = self.mlModel.modelDescription.metadata[.creatorDefinedKey] as? [String: String], + let namesString = metadata["names"] { + // Parse Python dict string like: "{0: 'BackgroundImage', 1: 'Bottom_Navigation', ...}" + let cleaned = namesString + .replacingOccurrences(of: "{", with: "") + .replacingOccurrences(of: "}", with: "") + .replacingOccurrences(of: "'", with: "") + + let pairs = cleaned.components(separatedBy: ", ") + for pair in pairs { + let parts = pair.components(separatedBy: ": ") + if parts.count == 2, + let classId = Int(parts[0].trimmingCharacters(in: .whitespaces)), + !parts[1].isEmpty { + loadedClassNames[classId] = parts[1].trimmingCharacters(in: .whitespaces) + } + } + print("[Sentry] Loaded \(loadedClassNames.count) class names from model metadata") + } + } catch { + fatalError("[Sentry] Warning: Failed to load ML model: \(error)") + } + + self.classNames = loadedClassNames + // Run ML inference asynchronously on a background queue to avoid blocking + queue = DispatchQueue.global(qos: .userInitiated) + super.init() + } + + public func addIgnoreClass(_ ignoreClass: AnyClass) {} + + public func addRedactClass(_ redactClass: AnyClass) {} + + public func addIgnoreClasses(_ ignoreClasses: [AnyClass]) {} + + public func addRedactClasses(_ redactClasses: [AnyClass]) {} + + public func setIgnoreContainerClass(_ containerClass: AnyClass) {} + + public func setRedactContainerClass(_ containerClass: AnyClass) {} + + public func redactRegionsFor(view: UIView, image: UIImage, callback: @escaping ([SentryRedactRegion]?, Error?) -> Void) { + guard #available(iOS 17.0, *) else { + callback([], nil) + return + } + + // Debug: Print view hierarchy (comment out for production) + if let viewDescription = view.value(forKey: "recursiveDescription") as? String { + print("[Sentry] View Hierarchy:\n\(viewDescription)") + } + + queue.async { [weak self] in + guard let self = self else { + callback([], nil) + return + } + + do { + // Store original size for scaling back + let originalSize = image.size + + // Calculate letterbox/pillarbox scaling to maintain aspect ratio + let targetSize = Self.modelInputSize + let widthRatio = targetSize / originalSize.width + let heightRatio = targetSize / originalSize.height + let scaleFactor = min(widthRatio, heightRatio) + + let scaledWidth = originalSize.width * scaleFactor + let scaledHeight = originalSize.height * scaleFactor + + // Calculate offsets for centering in 640x640 + let xOffset = (targetSize - scaledWidth) / 2.0 + let yOffset = (targetSize - scaledHeight) / 2.0 + + print("[Sentry] Letterbox info:") + print(" Original: \(originalSize.width)x\(originalSize.height)") + print(" Scaled to: \(scaledWidth)x\(scaledHeight) (scale: \(scaleFactor))") + print(" Centered at: x=\(xOffset), y=\(yOffset) in 640x640") + + // Resize image to 640x640 with letterboxing (maintains aspect ratio) +// let resizedImage = self.resizeImage( +// image, +// targetSize: targetSize +// ) + + // Convert to CVPixelBuffer +// guard let pixelBuffer = self.createPixelBuffer(from: resizedImage) else { +// print("[Sentry] Warning: Failed to create pixel buffer") +// callback([], nil) +// return +// } + + // Create input feature dictionary +// let inputFeatures: [String: Any] = [ +// "image": pixelBuffer, +// "iouThreshold": 0.7, +// "confidenceThreshold": 0.25 +// ] + + // let input = try MLDictionaryFeatureProvider(dictionary: inputFeatures) + + // Run prediction - this is the expensive operation + // let output = try self.mlModel.prediction(from: input) + + let model = try VNCoreMLModel(for: self.mlModel) + model.inputImageFeatureName = "image" + + let request = VNCoreMLRequest(model: model) { request, error in + guard let observations = request.results as? [VNRecognizedObjectObservation] else { + fatalError() + } + var regions = [SentryRedactRegion]() + let renderer = UIGraphicsImageRenderer(size: image.size) + let masked = renderer.image { context in + image.draw(at: .zero) + + context.cgContext.setStrokeColor(UIColor.red.cgColor) + context.cgContext.setLineWidth(2) + + for observation in observations { + let rect = Self.denormalizeRect(observation.boundingBox, imageSize: image.size) + context.cgContext.stroke(rect) + regions.append(SentryRedactRegion( + size: rect.size, + transform: CGAffineTransformMakeTranslation(rect.origin.x, rect.origin.y), + type: .redact, + color: .red.withAlphaComponent(0.3), + name: observation.description + )) + } + } + print(masked) + + callback(regions, nil) + } + request.imageCropAndScaleOption = .scaleFill + + let tempUrl = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(UUID().uuidString) + FileManager.default.createFile(atPath: tempUrl.path, contents: nil, attributes: nil) + try image.pngData()!.write(to: tempUrl) + let ciImage = CIImage(contentsOf: tempUrl)! + // let handler = VNImageRequestHandler(url: tempUrl) + let handler = VNImageRequestHandler(ciImage: ciImage, orientation: .up) + try handler.perform([request]) + + +// // Parse the output +// guard let coordinatesFeature = output.featureValue(for: "coordinates"), +// let confidenceFeature = output.featureValue(for: "confidence") else { +// SentrySDKLog.error("[Sentry] Warning: Could not extract coordinates or confidence from model output") +// callback([], nil) +// return +// } +// +// // Convert MLMultiArray to regions +// let regions = self.parseModelOutput( +// coordinates: coordinatesFeature.multiArrayValue, +// confidence: confidenceFeature.multiArrayValue, +// scaleFactor: scaleFactor, +// xOffset: xOffset, +// yOffset: yOffset, +// originalSize: originalSize, +// classNames: self.classNames +// ) +// +// // Debug: Print region summary +// print("[Sentry] Generated \(regions.count) redact regions from \(coordinatesFeature.multiArrayValue?.shape[0].intValue ?? 0) detections") +// for (idx, region) in regions.enumerated() { +// print(" Region \(idx): \(region.name) at (\(region.transform.tx), \(region.transform.ty)) size=\(region.size)") +// } + +// callback(regions, nil) + } catch { + SentrySDKLog.error("[Sentry] Warning: ML inference failed: \(error)") + callback(nil, error) + } + } + } + + private func resizeImage( + _ image: UIImage, + targetSize: CGFloat, + ) -> UIImage { + // let scaledWidth = image.size.width / image.size.height * targetSize + let size = CGSize(width: targetSize, height: targetSize) + UIGraphicsBeginImageContextWithOptions(size, true, 1.0) + defer { UIGraphicsEndImageContext() } + + UIRectFill(CGRect(origin: .zero, size: size)) + + image.draw(in: CGRect( + x: 0, + y: 0, + width: size.width, + height: size.height + )) + + return UIGraphicsGetImageFromCurrentImageContext() ?? image + } + + private func resizeImageWithLetterbox( + _ image: UIImage, + targetSize: CGFloat, + scaledWidth: CGFloat, + scaledHeight: CGFloat, + xOffset: CGFloat, + yOffset: CGFloat + ) -> UIImage { + let size = CGSize(width: targetSize, height: targetSize) + UIGraphicsBeginImageContextWithOptions(size, true, 1.0) + defer { UIGraphicsEndImageContext() } + + // Fill with black (letterbox padding) + UIColor.black.setFill() + UIRectFill(CGRect(origin: .zero, size: size)) + + // Draw image centered with aspect ratio maintained + image.draw(in: CGRect( + x: xOffset, + y: yOffset, + width: scaledWidth, + height: scaledHeight + )) + + return UIGraphicsGetImageFromCurrentImageContext() ?? image + } + + func createPixelBuffer(from uiImage: UIImage) -> CVPixelBuffer? { + // Normalize orientation first (renders to .up) + let fmt = UIGraphicsImageRendererFormat() + fmt.scale = uiImage.scale + let normalized = UIGraphicsImageRenderer(size: uiImage.size, format: fmt).image { _ in + uiImage.draw(in: CGRect(origin: .zero, size: uiImage.size)) + } + guard let cgImage = normalized.cgImage else { return nil } + + let width = cgImage.width + let height = cgImage.height + + let attrs: [CFString: Any] = [ + kCVPixelBufferCGImageCompatibilityKey: true, + kCVPixelBufferCGBitmapContextCompatibilityKey: true, + kCVPixelBufferIOSurfacePropertiesKey: [:], // helps with pools/Metal + kCVPixelBufferMetalCompatibilityKey: true + ] + + var pixelBuffer: CVPixelBuffer? + guard CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + attrs as CFDictionary, + &pixelBuffer + ) == kCVReturnSuccess, let buffer = pixelBuffer else { return nil } + + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + + guard let base = CVPixelBufferGetBaseAddress(buffer) else { return nil } + + let bytesPerRow = CVPixelBufferGetBytesPerRow(buffer) + let colorSpace = CGColorSpaceCreateDeviceRGB() + let bitmapInfo = CGBitmapInfo.byteOrder32Little + .union(CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue)) + + guard let ctx = CGContext( + data: base, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: bytesPerRow, + space: colorSpace, + bitmapInfo: bitmapInfo.rawValue + ) else { return nil } + + // Flip for CoreGraphics coordinate system + ctx.translateBy(x: 0, y: CGFloat(height)) + ctx.scaleBy(x: 1, y: -1) + + ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height)) + + return buffer + } + + private func parseModelOutput( + coordinates: MLMultiArray?, + confidence: MLMultiArray?, + scaleFactor: CGFloat, + xOffset: CGFloat, + yOffset: CGFloat, + originalSize: CGSize, + classNames: [Int: String] + ) -> [SentryRedactRegion] { + guard let coords = coordinates, let confs = confidence else { + return [] + } + + var regions: [SentryRedactRegion] = [] + + // The model outputs: coordinates [N, 4] where each row is [x_center, y_center, width, height] + // YOLO models output the CENTER of the bounding box, not the top-left corner + // Coordinates are normalized (0-1) relative to the 640x640 input + + let numBoxes = coords.shape[0].intValue + let modelSize = Self.modelInputSize + + for i in 0.. 0 && heightNorm > 0 else { + continue + } + + // Convert normalized coordinates to pixels in 640x640 space + let xCenter640 = xCenterNorm * modelSize + let yCenter640 = yCenterNorm * modelSize + let width640 = widthNorm * modelSize + let height640 = heightNorm * modelSize + + // Subtract letterbox offsets to get coordinates in the letterboxed image space + let xCenterLetterbox = xCenter640 - xOffset + let yCenterLetterbox = yCenter640 - yOffset + + // Scale back to original image size + let xCenterOriginal = xCenterLetterbox / scaleFactor + let yCenterOriginal = yCenterLetterbox / scaleFactor + let widthOriginal = width640 / scaleFactor + let heightOriginal = height640 / scaleFactor + + var detectedClasses: [(name: String, confidence: Float)] = [] + if confs.shape.count >= 2 { + let numClasses = confs.shape[1].intValue + for classId in 0.. 0 { + detectedClasses += [ + (name: classNames[classId] ?? "Unknown", confidence: floor(conf * 100)) + ] + } + } + } + if detectedClasses.isEmpty { + continue; + } + let className = detectedClasses.sorted { lhs, rhs in + lhs.confidence > rhs.confidence + }.map { (name, confidence) in + "\(name) (\(confidence)%)" + }.joined(separator: ", ") + + // Debug: Print conversion for ALL boxes + print("[Sentry] Box \(i) \(className):") + print(" Normalized: center=(\(xCenterNorm), \(yCenterNorm)) size=(\(widthNorm), \(heightNorm))") + print(" 640x640: center=(\(xCenter640), \(yCenter640))") + print(" After removing letterbox: center=(\(xCenterLetterbox), \(yCenterLetterbox))") + print(" Original space: center=(\(xCenterOriginal), \(yCenterOriginal))") + + // Create transform for this region using clamped coordinates + let transform = CGAffineTransform(translationX: xCenterOriginal, y: yCenterOriginal) + + // Create redact region with clamped size and descriptive name + let region = SentryRedactRegion( + size: CGSize(width: widthOriginal, height: heightOriginal), + transform: transform, + type: .redact, + color: UIColor.red, + name: className + ) + + regions.append(region) + } + + return regions + } + +// /// Convert normalized bounding box (0-1) to actual pixel coordinates +// /// - Parameters: +// /// - normalizedRect: CGRect with normalized coordinates (0-1) from Vision (bottom-left origin) +// /// - imageSize: Size of the image +// /// - Returns: CGRect in pixel coordinates (top-left origin for UIKit) +// static func denormalizeRect(_ normalizedRect: CGRect, imageSize: CGSize) -> CGRect { +// // When using .centerCrop, Vision crops to the center square of the image +// // The crop size is min(width, height) +// let cropSize = min(imageSize.width, imageSize.height) +// +// // Calculate crop offsets (how much was cropped from each side) +// let xOffset = (imageSize.width - cropSize) / 2.0 +// let yOffset = (imageSize.height - cropSize) / 2.0 +// +// // Scale normalized coordinates to the crop region +// let x = normalizedRect.origin.x * cropSize + xOffset +// let width = normalizedRect.width * cropSize +// let height = normalizedRect.height * cropSize +// +// // Vision uses bottom-left origin, convert to top-left (UIKit) +// // Within the crop region, flip the y coordinate +// let yInCrop = (1 - normalizedRect.origin.y - normalizedRect.height) * cropSize +// let y = yInCrop + yOffset +// +// return CGRect(x: x, y: y, width: width, height: height) +// } + + static func denormalizeRect(_ normalizedRect: CGRect, imageSize: CGSize) -> CGRect { + // ScaleFill: Image is stretched to model input size, no cropping + // Simply scale normalized coordinates to image dimensions + let x = normalizedRect.origin.x * imageSize.width + let width = normalizedRect.width * imageSize.width + let height = normalizedRect.height * imageSize.height + + // Vision uses bottom-left origin, convert to top-left (UIKit) + let y = (1 - normalizedRect.origin.y - normalizedRect.height) * imageSize.height + + return CGRect(x: x, y: y, width: width, height: height) + } +} + +#endif // os(iOS) || os(tvOS) +#endif // canImport(UIKit) && !SENTRY_NO_UIKIT diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/FeatureDescriptions.json b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/FeatureDescriptions.json new file mode 100644 index 00000000000..24ac6f69d03 --- /dev/null +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/FeatureDescriptions.json @@ -0,0 +1,27 @@ +{ + "Inputs" : { + "image" : { + "MLFeatureShortDescription" : "Input image" + }, + "confidenceThreshold" : { + "MLFeatureShortDescription" : "(optional) Confidence threshold override (default: 0.25)" + }, + "iouThreshold" : { + "MLFeatureShortDescription" : "(optional) IoU threshold override (default: 0.7)" + } + }, + "Outputs" : { + "coordinates" : { + "MLFeatureShortDescription" : "Boxes × [x, y, width, height] (relative to image size)" + }, + "confidence" : { + "MLFeatureShortDescription" : "Boxes × Class confidence (see user-defined metadata \"classes\")" + } + }, + "TrainingInputs" : { + + }, + "States" : { + + } +} \ No newline at end of file diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/Metadata.json b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/Metadata.json new file mode 100644 index 00000000000..d3d6d35c176 --- /dev/null +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/Metadata.json @@ -0,0 +1,19 @@ +{ + "MLModelVersionStringKey" : "8.3.205", + "MLModelDescriptionKey" : "Ultralytics best model trained on .\/data\/dataset.yaml", + "MLModelCreatorDefinedKey" : { + "stride" : "32", + "IoU threshold" : "0.7", + "Confidence threshold" : "0.25", + "docs" : "https:\/\/docs.ultralytics.com", + "task" : "detect", + "channels" : "3", + "imgsz" : "[640, 640]", + "date" : "2025-10-07T11:40:15.382578", + "args" : "{'batch': 1, 'half': False, 'int8': True, 'nms': True}", + "batch" : "1", + "names" : "{0: 'BackgroundImage', 1: 'Bottom_Navigation', 2: 'Card', 3: 'CheckBox', 4: 'Checkbox', 5: 'CheckedTextView', 6: 'Drawer', 7: 'EditText', 8: 'Icon', 9: 'Image', 10: 'Map', 11: 'Modal', 12: 'Multi_Tab', 13: 'PageIndicator', 14: 'Remember', 15: 'Spinner', 16: 'Switch', 17: 'Text', 18: 'TextButton', 19: 'Toolbar', 20: 'UpperTaskBar'}" + }, + "MLModelAuthorKey" : "Ultralytics", + "MLModelLicenseKey" : "AGPL-3.0 License (https:\/\/ultralytics.com\/license)" +} \ No newline at end of file diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/model.mlmodel b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/model.mlmodel new file mode 100644 index 00000000000..27c0b763ffc Binary files /dev/null and b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/model.mlmodel differ diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/weights/weight.bin b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/weights/weight.bin new file mode 100644 index 00000000000..eb000a48318 Binary files /dev/null and b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Data/com.apple.CoreML/weights/weight.bin differ diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Manifest.json b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Manifest.json new file mode 100644 index 00000000000..7bb360ffd3e --- /dev/null +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryMaskingModel.mlpackage/Manifest.json @@ -0,0 +1,30 @@ +{ + "fileFormatVersion": "1.0.0", + "itemInfoEntries": { + "14BC4834-7C94-40B3-9ED3-01064D56F7DE": { + "author": "com.apple.CoreML", + "description": "External FeatureDescription Overlay", + "name": "FeatureDescriptions.json", + "path": "com.apple.CoreML/FeatureDescriptions.json" + }, + "39119436-A185-454C-9355-FAE9D78AD25E": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" + }, + "99D8118F-826E-4FA8-9BC2-3A3A6F0398CB": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" + }, + "F16C0EF2-758E-46D2-A287-F207B193E2A6": { + "author": "com.apple.CoreML", + "description": "External Metadata Overlay", + "name": "Metadata.json", + "path": "com.apple.CoreML/Metadata.json" + } + }, + "rootModelIdentifier": "99D8118F-826E-4FA8-9BC2-3A3A6F0398CB" +} diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryRedactRegion.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryRedactRegion.swift index 9f39cdda5d3..38e106807de 100644 --- a/Sources/Swift/Core/Tools/ViewCapture/SentryRedactRegion.swift +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryRedactRegion.swift @@ -4,7 +4,7 @@ import Foundation import ObjectiveC.NSObjCRuntime import UIKit -struct SentryRedactRegion: Equatable { +@objc @_spi(Private) public class SentryRedactRegion: NSObject { let size: CGSize let transform: CGAffineTransform let type: SentryRedactRegionType @@ -22,6 +22,37 @@ struct SentryRedactRegion: Equatable { func canReplace(as other: SentryRedactRegion) -> Bool { size == other.size && transform == other.transform && type == other.type } + + public override func isEqual(_ object: Any?) -> Bool { + guard let other = object as? SentryRedactRegion else { + return false + } + guard other.size == self.size else { + return false + } + guard other.color == self.color else { + return false + } + guard other.type == self.type else { + return false + } + guard other.color == self.color else { + return false + } + guard other.name == self.name else { + return false + } + return true + } +} + +extension SentryRedactRegion: Encodable {} + +extension UIColor: @retroactive Encodable { + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(self.cgColor.components) + } } #endif #endif diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryScreenshotSource.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryScreenshotSource.swift index 590928ad503..9f95dd65149 100644 --- a/Sources/Swift/Core/Tools/ViewCapture/SentryScreenshotSource.swift +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryScreenshotSource.swift @@ -16,7 +16,7 @@ import UIKit assertionFailure("Use init(photographer:) instead") self.init(photographer: SentryViewPhotographer( renderer: SentryDefaultViewRenderer(), - redactOptions: SentryRedactDefaultOptions(), + redactBuilder: SentryUIRedactBuilder(options: SentryRedactDefaultOptions()), enableMaskRendererV2: false )) } diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryUIRedactBuilder.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryUIRedactBuilder.swift index e79c902ea6d..130cf3053e1 100644 --- a/Sources/Swift/Core/Tools/ViewCapture/SentryUIRedactBuilder.swift +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryUIRedactBuilder.swift @@ -4,12 +4,14 @@ import Foundation import ObjectiveC.NSObjCRuntime import UIKit +@_implementationOnly import _SentryPrivate #if os(iOS) import PDFKit import WebKit #endif -final class SentryUIRedactBuilder { +@objcMembers +@_spi(Private) public class SentryUIRedactBuilder: NSObject, SentryUIRedactBuilderProtocol { // MARK: - Types /// Type used to represented a view that needs to be redacted @@ -119,7 +121,7 @@ final class SentryUIRedactBuilder { /// /// - note: On iOS, views such as `WKWebView` and `UIWebView` are always redacted, and controls like /// `UISlider` and `UISwitch` are ignored by default. - init(options: SentryRedactOptions) { + required public init(options: SentryRedactOptions) { var redactClasses = Set() if options.maskAllText { @@ -208,7 +210,9 @@ final class SentryUIRedactBuilder { } redactClassesIdentifiers = redactClasses - + + super.init() + // didSet doesn't run during initialization, so we need to manually build the optimization structures rebuildOptimizedLookups() } @@ -303,34 +307,34 @@ final class SentryUIRedactBuilder { } /// Adds a class to the ignore list. - func addIgnoreClass(_ ignoreClass: AnyClass) { + public func addIgnoreClass(_ ignoreClass: AnyClass) { ignoreClassesIdentifiers.insert(ClassIdentifier(class: ignoreClass)) } /// Adds a class to the redact list. - func addRedactClass(_ redactClass: AnyClass) { + public func addRedactClass(_ redactClass: AnyClass) { redactClassesIdentifiers.insert(ClassIdentifier(class: redactClass)) } /// Adds multiple classes to the ignore list. - func addIgnoreClasses(_ ignoreClasses: [AnyClass]) { + public func addIgnoreClasses(_ ignoreClasses: [AnyClass]) { ignoreClasses.forEach(addIgnoreClass(_:)) } /// Adds multiple classes to the redact list. - func addRedactClasses(_ redactClasses: [AnyClass]) { + public func addRedactClasses(_ redactClasses: [AnyClass]) { redactClasses.forEach(addRedactClass(_:)) } /// Marks a container class whose direct children should be ignored (unmasked). - func setIgnoreContainerClass(_ containerClass: AnyClass) { + public func setIgnoreContainerClass(_ containerClass: AnyClass) { ignoreContainerClassIdentifier = ObjectIdentifier(containerClass) } /// Marks a container class whose subtree should be force‑redacted. /// /// Note: We also add the container class to the redact list so the container itself becomes a region. - func setRedactContainerClass(_ containerClass: AnyClass) { + public func setRedactContainerClass(_ containerClass: AnyClass) { let id = ObjectIdentifier(containerClass) redactContainerClassIdentifier = id redactClassesIdentifiers.insert(ClassIdentifier(class: containerClass)) @@ -364,7 +368,11 @@ final class SentryUIRedactBuilder { /// /// The function returns the redaction regions in reverse order from what was found in the hierarchy, /// so clip regions are applied first before drawing a redact mask on lower views. - func redactRegionsFor(view: UIView) -> [SentryRedactRegion] { + public func redactRegionsFor( + view: UIView, + image: UIImage, + callback: @escaping ([SentryRedactRegion]?, (any Error)?) -> Void + ) { var redactingRegions = [SentryRedactRegion]() self.mapRedactRegion( @@ -387,7 +395,7 @@ final class SentryUIRedactBuilder { } //The swiftUI type needs to appear first in the list so it always get masked - return (otherRegions + swiftUIRedact).reversed() + return callback((otherRegions + swiftUIRedact).reversed(), nil) } private func shouldIgnore(view: UIView) -> Bool { diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryUIRedactBuilderProtocol.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryUIRedactBuilderProtocol.swift new file mode 100644 index 00000000000..019205d382f --- /dev/null +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryUIRedactBuilderProtocol.swift @@ -0,0 +1,17 @@ +#if canImport(UIKit) && !SENTRY_NO_UIKIT +#if os(iOS) || os(tvOS) +import UIKit + +@objc @_spi(Private) public protocol SentryUIRedactBuilderProtocol { + init(options: SentryRedactOptions) + func redactRegionsFor(view: UIView, image: UIImage, callback: @escaping ([SentryRedactRegion]?, Error?) -> Void) + func addIgnoreClass(_ ignoreClass: AnyClass) + func addRedactClass(_ redactClass: AnyClass) + func addIgnoreClasses(_ ignoreClasses: [AnyClass]) + func addRedactClasses(_ redactClasses: [AnyClass]) + func setIgnoreContainerClass(_ containerClass: AnyClass) + func setRedactContainerClass(_ containerClass: AnyClass) + +} +#endif // canImport(UIKit) && !SENTRY_NO_UIKIT +#endif // os(iOS) || os(tvOS) diff --git a/Sources/Swift/Core/Tools/ViewCapture/SentryViewPhotographer.swift b/Sources/Swift/Core/Tools/ViewCapture/SentryViewPhotographer.swift index 8356e10f10e..e8ce1595246 100644 --- a/Sources/Swift/Core/Tools/ViewCapture/SentryViewPhotographer.swift +++ b/Sources/Swift/Core/Tools/ViewCapture/SentryViewPhotographer.swift @@ -8,7 +8,7 @@ import UIKit @objcMembers @_spi(Private) public class SentryViewPhotographer: NSObject, SentryViewScreenshotProvider { - private let redactBuilder: SentryUIRedactBuilder + private let redactBuilder: SentryUIRedactBuilderProtocol private let maskRenderer: SentryMaskRenderer private let dispatchQueue = SentryDispatchQueueWrapper() @@ -18,18 +18,17 @@ import UIKit /// /// - Parameters: /// - renderer: Implementation of the view renderer. - /// - redactOptions: Options provided to redact sensitive information. - /// - enableMaskRendererV2: Flag to enable experimental view renderer. + /// - redactBuilder: Implementation of the redact builder /// - Note: The option `enableMaskRendererV2` is an internal flag, which is not part of the public API. /// Therefore, it is not part of the the `redactOptions` parameter, to not further expose it. public init( renderer: SentryViewRenderer, - redactOptions: SentryRedactOptions, + redactBuilder: SentryUIRedactBuilderProtocol, enableMaskRendererV2: Bool ) { self.renderer = renderer self.maskRenderer = enableMaskRendererV2 ? SentryMaskRendererV2() : SentryDefaultMaskRenderer() - redactBuilder = SentryUIRedactBuilder(options: redactOptions) + self.redactBuilder = redactBuilder super.init() } @@ -37,28 +36,39 @@ import UIKit // Define a helper variable for the size, so the view is not accessed in the async block let viewSize = view.bounds.size - // The redact regions are expected to be thread-safe data structures - let redactRegions = redactBuilder.redactRegionsFor(view: view) - // The render method is synchronous and must be called on the main thread. // This is because the render method accesses the view hierarchy which is managed from the main thread. let renderedScreenshot = renderer.render(view: view) - dispatchQueue.dispatchAsync { [maskRenderer] in - // The mask renderer does not need to be on the main thread. - // Moving it to a background thread to avoid blocking the main thread, therefore reducing the performance - // impact/lag of the user interface. - let maskedScreenshot = maskRenderer.maskScreenshot(screenshot: renderedScreenshot, size: viewSize, masking: redactRegions) - - onComplete(maskedScreenshot) - } + // The redact regions are expected to be thread-safe data structures + redactBuilder + .redactRegionsFor(view: view, image: renderedScreenshot) { [dispatchQueue, maskRenderer] (redactRegions: [SentryRedactRegion]?, error: Error?) in + if let error = error { + print(error) + } + dispatchQueue.dispatchAsync { [maskRenderer] in + // The mask renderer does not need to be on the main thread. + // Moving it to a background thread to avoid blocking the main thread, therefore reducing the performance + // impact/lag of the user interface. + let maskedScreenshot = maskRenderer.maskScreenshot(screenshot: renderedScreenshot, size: viewSize, masking: redactRegions ?? []) + + onComplete(maskedScreenshot) + } + } } public func image(view: UIView) -> UIImage { let viewSize = view.bounds.size - let redactRegions = redactBuilder.redactRegionsFor(view: view) let renderedScreenshot = renderer.render(view: view) - let maskedScreenshot = maskRenderer.maskScreenshot(screenshot: renderedScreenshot, size: viewSize, masking: redactRegions) + let dispatchGroup = DispatchGroup() + var redactRegions: [SentryRedactRegion]? + redactBuilder.redactRegionsFor(view: view, image: renderedScreenshot, callback: { regions, error in + redactRegions = regions + dispatchGroup.leave() + }) + dispatchGroup.enter() + dispatchGroup.wait() + let maskedScreenshot = maskRenderer.maskScreenshot(screenshot: renderedScreenshot, size: viewSize, masking: redactRegions ?? []) return maskedScreenshot } diff --git a/Sources/Swift/Integrations/SessionReplay/Preview/SentryMaskingPreviewView.swift b/Sources/Swift/Integrations/SessionReplay/Preview/SentryMaskingPreviewView.swift index 4140ba606b2..3a6c9620029 100644 --- a/Sources/Swift/Integrations/SessionReplay/Preview/SentryMaskingPreviewView.swift +++ b/Sources/Swift/Integrations/SessionReplay/Preview/SentryMaskingPreviewView.swift @@ -27,7 +27,7 @@ import UIKit public init(redactOptions: SentryRedactOptions) { self.photographer = SentryViewPhotographer( renderer: PreviewRenderer(), - redactOptions: redactOptions, + redactBuilder: SentryUIRedactBuilder(options: redactOptions), enableMaskRendererV2: false ) super.init(frame: .zero) diff --git a/Sources/Swift/SentryDependencyContainer.swift b/Sources/Swift/SentryDependencyContainer.swift index 395bb32a62e..67d334ea259 100644 --- a/Sources/Swift/SentryDependencyContainer.swift +++ b/Sources/Swift/SentryDependencyContainer.swift @@ -207,7 +207,8 @@ extension SentryFileManager: SentryFileManagerProtocol { } // The options could be null here, but this is a general issue in the dependency // container and will be fixed in a future refactoring. guard let options = SentrySDKInternal.options else { - return nil + // return nil + preconditionFailure() } let viewRenderer: SentryViewRenderer @@ -217,9 +218,14 @@ extension SentryFileManager: SentryFileManagerProtocol { } viewRenderer = SentryDefaultViewRenderer() } + let redactBuilder: SentryUIRedactBuilderProtocol + redactBuilder = SentryUIRedactBuilder(options: RedactWrapper( + SentryDependencyContainerSwiftHelper.redactOptions(options)) + ) + let photographer = SentryViewPhotographer( renderer: viewRenderer, - redactOptions: RedactWrapper(SentryDependencyContainerSwiftHelper.redactOptions(options)), + redactBuilder: redactBuilder, enableMaskRendererV2: SentryDependencyContainerSwiftHelper.viewRendererV2Enabled(options)) return SentryScreenshotSource(photographer: photographer) } diff --git a/Sources/Swift/SentryExperimentalOptions.swift b/Sources/Swift/SentryExperimentalOptions.swift index 47eaf011be5..e4a98555d7f 100644 --- a/Sources/Swift/SentryExperimentalOptions.swift +++ b/Sources/Swift/SentryExperimentalOptions.swift @@ -29,6 +29,8 @@ public final class SentryExperimentalOptions: NSObject { */ public var enableSessionReplayInUnreliableEnvironment = false + public var sessionReplayMaskingStrategy: SessionReplayMaskingStrategy = .viewHierarchy + @_spi(Private) public func validateOptions(_ options: [String: Any]?) { } } @@ -50,3 +52,15 @@ extension Options { // swiftlint:enable force_cast } } + +@objc +public enum SessionReplayMaskingStrategy: Int { + @objc(kSessionReplayMaskingStrategyViewHierarchy) + case viewHierarchy = 0 + + @objc(kSessionReplayMaskingStrategyAccessibilty) + case accessibility + + @objc(kSessionReplayMaskingStrategyMachineLearning) + case machineLearning +} diff --git a/develop-docs/VIEW_MASKING_STRATEGIES.md b/develop-docs/VIEW_MASKING_STRATEGIES.md new file mode 100644 index 00000000000..84808a7d399 --- /dev/null +++ b/develop-docs/VIEW_MASKING_STRATEGIES.md @@ -0,0 +1,228 @@ +# Sensitive View Masking for Screenshot & Session Replay + +## Overview + +This document explores approaches of a robust, extensible approach to detecting and masking sensitive content in screenshots and session replay frames captured on iOS, with a particular focus on SwiftUI. +Traditional UIKit-based masking via view traversal often fails for SwiftUI because text and values are frequently rendered by opaque layers and not represented by discrete, discoverable `UIView` subclasses (anymore). + +## Background & Motivation + +SDK features like session replay and screenshots may capture PII if left unredacted. Shipping a defensible, default-on redaction solution is required for privacy (GDPR/CCPA), regulatory compliance, and user trust. +While UIKit exposes rich view metadata, SwiftUI does not, therefore we need to move away from class-name heuristics toward alternatives such as semantic information from accessibility or pixel-based analysis using machine-learning. + +### Goals + +- Detect and mask sensitive regions without requiring app developer annotations. +- Support UIKit, SwiftUI, and mixed apps incl. hybrid apps built with React Native. +- Minimize maintenance cost across OS updates and new UI frameworks. +- Keep performance overhead low and predictable. + +### Requirements + +- No private APIs in production App Store builds. +- No dependence on globally toggled system services (e.g., VoiceOver) in production. +- Work reliably in background-capable scenarios and during animations (e.g., during view transitions). +- Capture edge-cases such as frames taken during view transitions. +- Be configurable via user-provided include/exclude class lists. + +## Processing Pipeline Overview + +The process when taking a screenshot or session replay frame is as follows: + +```mermaid +flowchart TD + A[**Root View/Window**] + B[**Full-Screen Screenshot**
UIImage from root view/window] + C[**Region Detection Engine**
Accessibility, View Hierarchy, ML] + D[**Mask Renderer**
Solid overlay of redact regions] + E[**Masked Output Image**] + + A --> B --> C --> D --> E +``` + +This document focuses on the **Region Detection Engine** and how we calculate the `SentryRedactRegion` geometry. + +## View Hierarchy-Based Redaction (Default Implementation) + +The `SentryUIRedactBuilder` is the current default implementation for identifying which areas of a view hierarchy should be masked during screenshot or session replay capture. +It is highly configurable and built to handle both UIKit and modern hybrid/SwiftUI scenarios, minimizing risk of privacy leaks while reducing the chance of costly masking mistakes (such as over-masking backgrounds or missing PII embedded in complex render trees). + +Some of the key features of the `SentryUIRedactBuilder` are: + +- Maintains configured sets of classes to redact and ignore (optionally gated by backing `CALayer` type to handle edge cases) and supports container overrides (force-redact or force-ignore). +- Traverses the Core Animation layer tree from the root view’s presentation layer to match runtime geometry during animations. +- Creates `SentryRedactRegion` entries for matched views, inserts clip regions for opaque/clip-to-bounds cases, and handles ordering so masks apply correctly. +- Applies special handling for `UIImageView` (skip tiny/bundle images), SwiftUI render surfaces, and known web/pdf/video views. + +### Class-Based Redaction and Ignoring + +At the heart of the builder is the concept of **class identifiers**: + +Views to mask ("redact") and views to explicitly "ignore" are stored as _sets of string identifiers_ (not `AnyClass` directly). +This deliberate design avoids referencing Objective-C class objects at runtime, which can trigger class initializers (`+initialize`) and crash if done off the main thread. +Instead, each class name is stored and compared as a string via `type(of: view).description()`. + +The builder supports: + +- **Unmasked (ignored) views:** e.g. standard system controls such as `UISlider`, `UISwitch`, or explicit user configuration, will have their subtrees excluded from redaction. +- **Masked (redacted) views:** e.g. `UILabel`, `UITextView`, `UIImageView` (with heuristics), and known hybrid/SwiftUI/React Native renderers. +- **Granular overrides:** The system allows the user to register container classes to force redaction or allow-list (ignore) their direct children, or marking instances of views to be ignored or redacted. + +### Layer Tree Traversal & Presentation-State Geometry + +The redaction builder walks the Core Animation layer tree, not just the UIKit `subviews`, to match real-time, animated geometry. This allows the builder to correctly handle views that are hidden, alpha-transparent, or have a non-zero frame size, and also cases where views use multiple layers. + +Each view’s masking eligibility is checked not only based on its type but, when necessary, with a secondary filter on the underlying `CALayer` type. +This allows disambiguation for cases like SwiftUI rendering, where the same view class serves both as a text/image renderer and as a generic structural element. + +During traversal, if the system encounters opaque views that completely cover previously marked redaction regions, it can remove or bypass those earlier masks to prevent the creation of unnecessary or hidden mask layers. The rules that determine which parts of the interface to mask are flexible and can be combined: for instance, a `UIImageView` may be masked or left visible depending on specific heuristics, such as whether its image comes from a bundle asset or if the image is very small. + +### Special Case Handling + +**SwiftUI/Hybrid Views:** + +Many hybrid frameworks and SwiftUI components use generic or non-public view types to render content. The builder uses a combination of: + +- (a) string-matched class IDs (e.g., `"SwiftUI._UIGraphicsView"`) +- (b) layer class filters (e.g., only masking when `.layerId == "SwiftUI.ImageLayer"`) + +React Native text/image renderers are also included explicitly to cover cross-platform apps. + +**Known risk classes:** + +A special case exists for `"CameraUI.ChromeSwiftUIView"` (Xcode 16/iOS 26+) because accessing its layers directly can cause crashes. It is therefore ignored in subtree traversal. + +### Clip Handling and Z-Ordering + +Redact regions are assembled in **reverse z-order** (to match real on-screen compositing). +When clips (because of `clipsToBounds` or opaque/covering views) are encountered, logical "clip begin"/"clip end" markers are inserted so the renderer can avoid over-masking subregions or incorrectly masking nested content. + +### Container Overrides + +The builder supports registering _ignore containers_ (marking all direct children as safe/unmasked) or _redact containers_ (force-masking entire subtrees). + +### Thread Safety and Side Effects Avoidance + +All class-matching and hierarchy operations are intentionally free of logic that would cause UIKit classes to initialize on background threads. As access to class objects is limited to the main thread, the redaction calculations are performed on the main thread while further processing is performed on a background queue using the thread-safe `SentryRedactRegion` type. + +### Redact Region Output + +At the end, the builder outputs a collection of `SentryRedactRegion` items, each corresponding to a geometric region in the screenshot to be masked. For debugging, entries also include the view’s class name and other relevant metadata. + +### Known Limitations and Safeguards + +Some UIKit/private class names or render layers can change across iOS versions. The design allows quick update of class/layer rules but requires regular review on new OS releases. +Some "decoration" views (e.g., `"_UICollectionViewListLayoutSectionBackgroundColorDecorationView"`) get special handling to prevent over-eager region suppression. + +## Accessibility-Based Redaction: + +The `SentryAccessibilityRedactBuilder` is an alternative implementation for identifying which areas of a view hierarchy should be masked during screenshot or session replay capture. + +### The Accessibility Framework + +The Accessibility framework on iOS is a system of APIs and runtime services that allow apps to expose their interface semantics and content to users with disabilities, as well as to automated tools. Developers annotate UI elements (such as buttons, images, and custom views) with accessibility properties like labels, traits, hints, and values, either directly in Interface Builder or programmatically. The framework constructs an accessibility "tree" that mirrors key elements of the UI, describing their position, state, and purpose. This metadata lets assistive technologies like VoiceOver describe app contents, navigate hierarchies, and let users interact with the interface through non-visual means. + +VoiceOver, iOS’s built-in screen reader, relies on the Accessibility framework to give spoken feedback about what appears on screen and to support alternative input mechanisms like swipe and tap gestures. When VoiceOver is enabled, it queries the accessibility tree of visible UI elements, reads out each element’s label, value, and role, and listens for user gestures to perform navigation or activation. Apple encourages developers to annotate their UI with accessibility properties tailored to their app’s content and user experience to improve the VoiceOver experience for users with disabilities. + +**Example:** + +```swift +// UIKit +let button = UIButton(type: .system) +button.accessibilityLabel = "Login" +button.accessibilityValue = "Click to login" +button.accessibilityTraits = .button +button.accessibilityHint = "Login to your account" +button.accessibilityIdentifier = "loginButton" +button.isAccessibilityElement = true + +// SwiftUI +struct LoginView: View { + var body: some View { + Button(action: { + print("Login button tapped") + }) { + Text("Login") + } + .accessibilityLabel("Login") + .accessibilityValue("Click to login") + .accessibilityTraits(.button) + .accessibilityHint("Login to your account") + .accessibilityIdentifier("loginButton") + .isAccessibilityElement(true) + } +``` + +In addition to assistive use cases, the Accessibility framework also powers automated UI testing tools such as XCTest’s UI test APIs. UI tests interact with an app not by directly manipulating view instances, but by simulating user actions (like taps or swipes) on accessibility elements. The framework exposes a stable, semantic interface layer that remains consistent even if the underlying view implementation changes, making tests more robust and maintainable. This unified approach means the same accessibility information that enables inclusive apps for users with disabilities also enables reliable, semantic-driven testing and automation. + +**Example:** + +```swift +let app = XCUIApplication() + +let loginButton = app.buttons["loginButton"] +guard loginButton.waitForExistence(timeout: 10) else { + XCTFail("Login button not found") + return +} +loginButton.tap() +``` + +To explore the Accessibility information available in a view hierarchy, we can use the the _Accessibility Inspector_ included in Xcode. + +### Accessing Accessibility Information from SDK + +To access the information from the SDK, we have two options available: + +1. Access the tree directly using the `UIAccessibility` framework. +2. Traversing the view hierarchy and accessing the accessibility information from the views. + +#### Accessing the Accessibility Tree Directly + +The `UIAccessibility` framework provides a way to access the Accessibility tree directly. + +```swift +let accessibilityElement = UIAccessibilityElement(accessibilityContainer: view) +let accessibilityFrame = accessibilityElement.accessibilityFrame +``` + +#### Traversing the View Hierarchy + +The SDK traverses the view hierarchy and accesses the accessibility information from the views. + +