diff --git a/Cotabby.xcodeproj/project.pbxproj b/Cotabby.xcodeproj/project.pbxproj index 4e2cf28d..078d040e 100644 --- a/Cotabby.xcodeproj/project.pbxproj +++ b/Cotabby.xcodeproj/project.pbxproj @@ -31,7 +31,6 @@ 258EAFB0292290C88520E915 /* SystemSettingsWindowLocator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5976600F428C1265121D4C0C /* SystemSettingsWindowLocator.swift */; }; 25D4FC8D191A50F63E6391F9 /* ModelAndPresentationValueTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03766F6253FF17639230C0F6 /* ModelAndPresentationValueTests.swift */; }; 26E0331E9E2F92FAE531BDEE /* ActivationIndicatorController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D84D4528EEC9EFEB8AE8E318 /* ActivationIndicatorController.swift */; }; - 29A11716A212C8E66202D527 /* CaretGeometrySourceCacheTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C6ECC3A26862471FAAF1F906 /* CaretGeometrySourceCacheTests.swift */; }; 2C6159231472A849F15BD0AE /* ScreenFrameReader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5484C8A04B9C00CF79D589EB /* ScreenFrameReader.swift */; }; 2DF5A3826AAB99C279EBB8DE /* InputMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B81DD30EB657368AACE9625A /* InputMonitor.swift */; }; 2EE05B312C990104BE934772 /* GhostFontSizeStabilizerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 335BF59EE80F3A0143B79740 /* GhostFontSizeStabilizerTests.swift */; }; @@ -121,7 +120,6 @@ B00FDD3DEE0B73FF5136C91C /* FocusTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C9FDF029F7828CAF3FE8850 /* FocusTracker.swift */; }; B0828FF0D7EE110C0B23DB94 /* TagsInputView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 58C0F017699EE44C81C095CA /* TagsInputView.swift */; }; B0B115C6EBAC37FF6115B4BE /* SuggestionCoordinator+Lifecycle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78E280F4F39A9D86840800D2 /* SuggestionCoordinator+Lifecycle.swift */; }; - B2CABC9F92368B4FCB7BF2C4 /* CaretGeometrySourceCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED63EBDD04B6E9EBF7E7D411 /* CaretGeometrySourceCache.swift */; }; B2F7589B8D32ACF97BB642AB /* HuggingFaceModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = A520809E71697E3BB9A8139C /* HuggingFaceModels.swift */; }; B93AB7E845086F6FBB068369 /* SuggestionRequestFactoryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = EE94342B888A5A2CCF66BC93 /* SuggestionRequestFactoryTests.swift */; }; BB6325CA50F97B18B9725918 /* SuggestionTextNormalizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = B424E2AC97C99D335B0D5751 /* SuggestionTextNormalizer.swift */; }; @@ -288,7 +286,6 @@ C046CB4F3CB4BFE9391DB5DE /* AXTextGeometryResolverTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AXTextGeometryResolverTests.swift; sourceTree = ""; }; C05B0439348261163B37C508 /* SuggestionAvailabilityEvaluatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionAvailabilityEvaluatorTests.swift; sourceTree = ""; }; C1C5DE0F3FF63545000E2453 /* DisplayCoordinateConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DisplayCoordinateConverterTests.swift; sourceTree = ""; }; - C6ECC3A26862471FAAF1F906 /* CaretGeometrySourceCacheTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaretGeometrySourceCacheTests.swift; sourceTree = ""; }; C7B2D34A6F3AC9DFD61350F7 /* CotabbyDebugOptions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CotabbyDebugOptions.swift; sourceTree = ""; }; CA942A53B7C09D1F4EC57239 /* SuggestionInteractionState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionInteractionState.swift; sourceTree = ""; }; CB58035EFFD65B767949BAE6 /* AXTextGeometryResolver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AXTextGeometryResolver.swift; sourceTree = ""; }; @@ -311,7 +308,6 @@ E7F42112F14026E6253BB865 /* PermissionAndContextModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PermissionAndContextModelTests.swift; sourceTree = ""; }; EAAE6B395FAB604DF059280A /* KeyCodeLabels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyCodeLabels.swift; sourceTree = ""; }; EBEDE359CFD99EF906FA50BC /* IndicatorIconImageProcessor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IndicatorIconImageProcessor.swift; sourceTree = ""; }; - ED63EBDD04B6E9EBF7E7D411 /* CaretGeometrySourceCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaretGeometrySourceCache.swift; sourceTree = ""; }; ED8672B87CEC72BE3978C6BB /* CotabbyTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CotabbyTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; EE94342B888A5A2CCF66BC93 /* SuggestionRequestFactoryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionRequestFactoryTests.swift; sourceTree = ""; }; EFD89799BB82AF7A92559AEB /* ClipboardContentDistillerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClipboardContentDistillerTests.swift; sourceTree = ""; }; @@ -435,7 +431,6 @@ isa = PBXGroup; children = ( CB58035EFFD65B767949BAE6 /* AXTextGeometryResolver.swift */, - ED63EBDD04B6E9EBF7E7D411 /* CaretGeometrySourceCache.swift */, 04E25414C307A20B6F9F20EC /* FocusSnapshotResolver.swift */, 5C9FDF029F7828CAF3FE8850 /* FocusTracker.swift */, ); @@ -493,7 +488,6 @@ A168A7B6A7AD11559B60C56B /* ApplicationBundleMetadataTests.swift */, C046CB4F3CB4BFE9391DB5DE /* AXTextGeometryResolverTests.swift */, 18D990E515E1AE4F312F4E95 /* BundledRuntimeLocatorTests.swift */, - C6ECC3A26862471FAAF1F906 /* CaretGeometrySourceCacheTests.swift */, EFD89799BB82AF7A92559AEB /* ClipboardContentDistillerTests.swift */, 90B0D133AB77A2503FB08827 /* ClipboardRelevanceFilterTests.swift */, AF1E065C7FFB697FCEB2FA5C /* CotabbyTestFixtures.swift */, @@ -747,7 +741,6 @@ C4C6734678797669055988E0 /* AppUpdateManager.swift in Sources */, 66C23A7C2FCDE0266FF425F8 /* ApplicationBundleMetadata.swift in Sources */, 3CBBC3BFAC0DC8952EE24EF7 /* BundledRuntimeLocator.swift in Sources */, - B2CABC9F92368B4FCB7BF2C4 /* CaretGeometrySourceCache.swift in Sources */, 6E01052209B73D7361C12CEF /* ClipboardContentDistiller.swift in Sources */, D2CAA59F69BCBC07DDA2B0D8 /* ClipboardContextProvider.swift in Sources */, 157A55EB796BEB7819B90D5D /* ClipboardRelevanceFilter.swift in Sources */, @@ -862,7 +855,6 @@ 6D0E79CF3C1A8CE53046FCE5 /* AXTextGeometryResolverTests.swift in Sources */, A36481222BB5B2A67349D389 /* ApplicationBundleMetadataTests.swift in Sources */, 58AC3193D846FDE88513377D /* BundledRuntimeLocatorTests.swift in Sources */, - 29A11716A212C8E66202D527 /* CaretGeometrySourceCacheTests.swift in Sources */, 8865B95FE84198D70390DF80 /* ClipboardContentDistillerTests.swift in Sources */, BFCA7FAFDAEBF586AB615567 /* ClipboardRelevanceFilterTests.swift in Sources */, 5E10EFC426217CB7218A5847 /* CotabbyTestFixtures.swift in Sources */, diff --git a/Cotabby/App/Coordinators/SuggestionCoordinator+Input.swift b/Cotabby/App/Coordinators/SuggestionCoordinator+Input.swift index 838dfeab..18c39bff 100644 --- a/Cotabby/App/Coordinators/SuggestionCoordinator+Input.swift +++ b/Cotabby/App/Coordinators/SuggestionCoordinator+Input.swift @@ -1,4 +1,3 @@ -import CoreGraphics import Foundation import Logging @@ -25,7 +24,7 @@ extension SuggestionCoordinator { func handleFocusSnapshotChange(_ snapshot: FocusSnapshot) { CotabbyLogger.suggestion.trace( - "Focus snapshot changed: app=\(snapshot.applicationName) capability=\(snapshot.capability.shortLabel) \(focusDiagnostics(for: snapshot))" + "Focus snapshot changed: app=\(snapshot.applicationName) capability=\(snapshot.capability.shortLabel)" ) // Start capturing visual context for a newly focused input even when predictions are // temporarily disabled by transient field states (e.g., "text is selected" or "secure @@ -194,41 +193,3 @@ extension SuggestionCoordinator { } } } - -private extension SuggestionCoordinator { - /// Website editors often expose unstable AX trees, so this log keeps the next repro actionable - /// without writing the user's message text into the debug file. - func focusDiagnostics(for snapshot: FocusSnapshot) -> String { - let inspection = snapshot.inspection - guard let context = snapshot.context else { - return "reason=\"\(snapshot.capability.summary)\" " - + "focusedRole=\(inspection?.focusedRoleSummary ?? "n/a") " - + "resolvedRole=\(inspection?.resolvedRoleSummary ?? "n/a") " - + "missing=\(inspection?.missingCapabilitySummary ?? "n/a")" - } - - let textLength = context.precedingText.count + context.trailingText.count - return "role=\(context.role)/\(context.subrole ?? "n/a") " - + "selection=\(context.selection.location)+\(context.selection.length) " - + "textLength=\(textLength) before=\(context.precedingText.count) after=\(context.trailingText.count) " - + "secure=\(context.isSecure) sequence=\(context.focusChangeSequence) " - + "caret=\(context.caretQuality.label):\(context.caretSource) " - + "caretRect=\(formatRect(context.caretRect)) inputRect=\(formatOptionalRect(context.inputFrameRect)) " - + "resolvedRole=\(inspection?.resolvedRoleSummary ?? "n/a") " - + "missing=\(inspection?.missingCapabilitySummary ?? "n/a")" - } - - func formatOptionalRect(_ rect: CGRect?) -> String { - rect.map(formatRect) ?? "nil" - } - - func formatRect(_ rect: CGRect) -> String { - String( - format: "(x=%.0f,y=%.0f,w=%.0f,h=%.0f)", - rect.origin.x, - rect.origin.y, - rect.width, - rect.height - ) - } -} diff --git a/Cotabby/Services/Focus/AXTextGeometryResolver.swift b/Cotabby/Services/Focus/AXTextGeometryResolver.swift index d8b0ddad..0e63a9dd 100644 --- a/Cotabby/Services/Focus/AXTextGeometryResolver.swift +++ b/Cotabby/Services/Focus/AXTextGeometryResolver.swift @@ -31,18 +31,6 @@ struct CaretGeometryResult { @MainActor struct AXTextGeometryResolver { - /// Remembers the text-run leaves of the focused field so per-keystroke caret resolution can - /// re-read them instead of re-walking the tree. Nil disables caching (tests, non-focus callers). - /// - /// Exposed (not private) so `FocusSnapshotResolver` can adopt this exact instance for its own - /// deep-walk fast path. If the two types held separate caches, an injected resolver would - /// silently lose run-walk caching while deep-walk caching kept working. - let cache: CaretGeometrySourceCache? - - init(cache: CaretGeometrySourceCache? = nil) { - self.cache = cache - } - /// Resolves the full input frame for workflows that need the whole field bounds, such as /// screenshot cropping and field-level diagnostics. This stays separate from caret resolution /// because not every consumer wants the same geometry contract. @@ -65,8 +53,7 @@ struct AXTextGeometryResolver { supportsBoundsForRange: Bool, supportsFrame: Bool, cocoaAnchorFrame: CGRect?, - textValue: String? = nil, - focusChangeSequence: UInt64? = nil + textValue: String? = nil ) -> CaretGeometryResult? { // Branch 1: Zero-length BoundsForRange at the caret position — ideal case. // Gated on `supportsBoundsForRange` because the API is a synchronous cross-process @@ -107,20 +94,6 @@ struct AXTextGeometryResolver { ) } - // Branch 1.6: Ancestor-owned AXTextMarker selection. - // Chromium contenteditable fields can focus a nested text entry node while the page-wide - // text marker space is owned by an ancestor such as AXWebArea. Walking upward finds that - // owner without depending on it appearing in the shallow candidate list. - if let ancestorMarkerRect = resolveCaretFromAncestorTextMarkerRange( - of: element, - cocoaAnchorFrame: cocoaAnchorFrame - ) { - return CaretGeometryResult( - rect: normalizedCaretRect(fromZeroLengthRangeRect: ancestorMarkerRect), - quality: .exact - ) - } - // Branch 2: BoundsForRange on the character before the caret, then shift to its trailing edge. // Same gate and anchor validation as Branch 1. if supportsBoundsForRange, @@ -134,15 +107,7 @@ struct AXTextGeometryResolver { fromAccessibilityRect: rect, anchorFrame: cocoaAnchorFrame ) - if rectIsNearAnchor(cocoaRect, anchor: cocoaAnchorFrame), - !looksLikeOversizedSingleCharacterRange(cocoaRect, anchor: cocoaAnchorFrame), - !looksLikeStaleLineStartRange( - cocoaRect, - anchor: cocoaAnchorFrame, - text: textValue, - selection: selection - ), - !looksLikeMultilineRangeUnion(cocoaRect, in: element) { + if rectIsNearAnchor(cocoaRect, anchor: cocoaAnchorFrame) { return CaretGeometryResult( rect: CGRect( x: cocoaRect.maxX, y: cocoaRect.minY, width: 2, height: cocoaRect.height), @@ -159,8 +124,7 @@ struct AXTextGeometryResolver { if let result = resolveCaretFromChildTextRuns( element: element, parentSelection: selection, - parentText: parentText, - focusChangeSequence: focusChangeSequence + parentText: parentText ) { return result } @@ -223,55 +187,6 @@ struct AXTextGeometryResolver { return cocoaRect.minX + estimatedWidth } - private func resolveCaretFromAncestorTextMarkerRange( - of element: AXUIElement, - cocoaAnchorFrame: CGRect? - ) -> CGRect? { - let maxAncestorDepth = 16 - var currentElement = element - var seen = Set() - - for _ in 0.. Bool { - guard !rect.isEmpty else { - return false - } - - guard let anchorFrame, !anchorFrame.isEmpty else { - return true - } - - let tolerance: CGFloat = 80 - return anchorFrame - .insetBy(dx: -tolerance, dy: -tolerance) - .contains(CGPoint(x: rect.midX, y: rect.midY)) - } - /// Walks AXStaticText children of a text container to find the one containing the caret, /// then estimates caret position proportionally within that child's AXFrame. This is the /// primary caret resolution path for Gmail, Outlook, and other Chromium editors where @@ -279,99 +194,20 @@ struct AXTextGeometryResolver { private func resolveCaretFromChildTextRuns( element: AXUIElement, parentSelection: NSRange, - parentText: String, - focusChangeSequence: UInt64? + parentText: String ) -> CaretGeometryResult? { let parentTextLength = (parentText as NSString).length guard parentSelection.location <= parentTextLength else { return nil } - // Per-line runs omit the line breaks the field value keeps, so a caret near the end of a - // multi-line message sits past the summed run length by the number of breaks above it. Size - // the past-all-runs tolerance to that glue so the fallback still anchors to the last run - // instead of collapsing to the whole-field estimate, while a larger overshoot still rejects - // an incomplete or label-polluted run list. - let caretLocation = min(parentSelection.location, parentTextLength) - let newlinesBeforeCaret = (parentText as NSString).substring(to: caretLocation) - .unicodeScalars - .reduce(into: 0) { count, scalar in - if CharacterSet.newlines.contains(scalar) { count += 1 } - } - let missingTextTolerance = max(2, newlinesBeforeCaret) - - let fieldKey: CaretGeometrySourceCache.FieldKey? - if let focusChangeSequence, cache != nil { - fieldKey = CaretGeometrySourceCache.FieldKey( - containerIdentifier: AXHelper.elementIdentity(for: element), - focusChangeSequence: focusChangeSequence - ) - } else { - fieldKey = nil - } - - // Fast path: re-read the cached line leaves instead of re-walking the tree. A successful - // caret map means the field's line structure is intact — including after the caret jumped - // lines, since the offset simply maps to a different cached run. A nil map (e.g. a line the - // cache predates) falls through to a fresh walk that refreshes the cache. - if let fieldKey, let cache, - let cachedElements = cache.textRunElements(for: fieldKey), - let runs = textRuns(fromElements: cachedElements), !runs.isEmpty, - let result = caretResult(fromRuns: runs, caretOffset: parentSelection.location) { - return result - } - - // Slow path: discover the leaves with a bounded walk, then map. The resolved field's leaves - // are cached once after candidate selection by `cacheTextRunSources` — deliberately not here - // — so a non-winning candidate probed on the same poll cannot evict the focused field's entry - // and force a re-walk every keystroke. - let elements = collectStaticTextElements(from: element) - guard !elements.isEmpty, - let runs = textRuns(fromElements: elements), !runs.isEmpty else { - return nil - } - return caretResult(fromRuns: runs, caretOffset: parentSelection.location) - } - - /// Populates the text-run cache for the resolved field once candidate selection is done. - /// - /// `FocusSnapshotResolver` probes every candidate with the run cache read-only: each candidate - /// has a distinct cache identity and the cache holds a single field, so letting every probe write - /// would let a non-winning candidate evict the focused field's leaves on the same poll and force - /// a re-walk every keystroke. The resolver instead calls this once for the winner. Already-warm - /// fields are a no-op; the walk runs only when the entry is cold (first poll on a field) or a - /// line change moved the caret off the cached leaves. - func cacheTextRunSources(for element: AXUIElement, focusChangeSequence: UInt64) { - guard let cache else { - return - } - - let fieldKey = CaretGeometrySourceCache.FieldKey( - containerIdentifier: AXHelper.elementIdentity(for: element), - focusChangeSequence: focusChangeSequence - ) - guard cache.textRunElements(for: fieldKey) == nil else { - return - } + let textRuns = collectStaticTextRuns(from: element) - let elements = collectStaticTextElements(from: element) - guard !elements.isEmpty, - let runs = textRuns(fromElements: elements), !runs.isEmpty else { - return - } - cache.store(textRunElements: elements, for: fieldKey) - } + guard !textRuns.isEmpty else { return nil } - /// Maps a caret offset onto ordered text runs: walk cumulative text length to find the - /// containing run, then interpolate proportionally inside its frame. Returns nil when the offset - /// overshoots the runs by more than a small tolerance — the signal that the run list is stale or - /// incomplete, so the caller should re-walk rather than anchor the overlay to an unrelated tail. - private func caretResult( - fromRuns textRuns: [(text: String, frame: CGRect)], - caretOffset: Int - ) -> CaretGeometryResult? { - // Average character width measured directly from the child frames — the actual rendered - // font, not a guess. Aggregated across runs so one short run doesn't skew it. + // Derive the average character width from the child frames — this is a direct measurement + // of the actual rendered font, not a guess. We aggregate across all children so a single + // short run doesn't skew the estimate. var totalChars = 0 var totalWidth: CGFloat = 0 for run in textRuns { @@ -380,11 +216,13 @@ struct AXTextGeometryResolver { } let charWidth: CGFloat? = totalChars > 0 ? totalWidth / CGFloat(totalChars) : nil - // AX selections use UTF-16 offsets, so match on NSString length. + // Find which child contains the caret by matching parent selection against cumulative + // text lengths. AX selections use UTF-16 offsets, so we match on NSString length. + let caretOffset = parentSelection.location var cumulative = 0 for run in textRuns { let runLen = (run.text as NSString).length - if caretOffset < cumulative + runLen { + if caretOffset <= cumulative + runLen { let localOffset = caretOffset - cumulative let fraction = runLen > 0 ? CGFloat(localOffset) / CGFloat(runLen) : 1.0 let cocoaFrame = AXHelper.cocoaRect(fromAccessibilityRect: run.frame) @@ -399,16 +237,9 @@ struct AXTextGeometryResolver { cumulative += runLen } - // A tiny overrun can happen when browsers omit newline glue from child text runs. A large - // overrun means the run list is incomplete or polluted by non-editor labels (Gmail tracking - // banners are exposed this way), so anchoring to the last run would jump the overlay to - // unrelated UI. - let missingTextTolerance = 2 - guard caretOffset <= cumulative + missingTextTolerance, let lastRun = textRuns.last else { - return nil - } - - let lastFrame = AXHelper.cocoaRect(fromAccessibilityRect: lastRun.frame) + // Caret is past all children (e.g. newline not included in child text). + // Use the last child's trailing edge. + let lastFrame = AXHelper.cocoaRect(fromAccessibilityRect: textRuns.last!.frame) return CaretGeometryResult( rect: CGRect(x: lastFrame.maxX, y: lastFrame.minY, width: 2, height: lastFrame.height), quality: .derived, @@ -420,12 +251,12 @@ struct AXTextGeometryResolver { /// anonymous containers, etc.). Walking only one child level misses those runs and forces /// Branch 3 (`AXFrame`) fallback. We scan descendants in pre-order so cumulative text length /// still tracks visual reading order in most editor trees. - private func collectStaticTextElements(from root: AXUIElement) -> [AXUIElement] { + private func collectStaticTextRuns(from root: AXUIElement) -> [(text: String, frame: CGRect)] { let maxDepth = 8 let maxNodes = 300 var visitedNodes = 0 var seen = Set() - var elements: [AXUIElement] = [] + var runs: [(text: String, frame: CGRect)] = [] func walk(_ element: AXUIElement, depth: Int) { guard depth <= maxDepth, visitedNodes < maxNodes else { @@ -445,7 +276,7 @@ struct AXTextGeometryResolver { !text.isEmpty, let frame = AXHelper.rectValue(for: "AXFrame" as CFString, on: element), !frame.isEmpty { - elements.append(element) + runs.append((text, frame)) } guard depth < maxDepth else { @@ -461,49 +292,7 @@ struct AXTextGeometryResolver { walk(child, depth: 1) } - return elements - } - - /// Reads the current text and frame of each leaf and returns them in visual reading order. - /// Returns nil when a leaf no longer reports the static-text role — the signal that a cached - /// element list is stale and the caller must re-walk. Leaves that read empty are skipped rather - /// than treated as stale, since a line can be momentarily empty mid-edit. - private func textRuns(fromElements elements: [AXUIElement]) -> [(text: String, frame: CGRect)]? { - var runs: [(text: String, frame: CGRect)] = [] - for element in elements { - guard AXHelper.stringValue(for: kAXRoleAttribute as CFString, on: element) - == kAXStaticTextRole as String else { - return nil - } - guard let text = AXHelper.stringValue(for: kAXValueAttribute as CFString, on: element), - !text.isEmpty, - let frame = AXHelper.rectValue(for: "AXFrame" as CFString, on: element), - !frame.isEmpty else { - continue - } - runs.append((text, frame)) - } - - return runs.sorted { lhs, rhs in - let lhsFrame = AXHelper.cocoaRect(fromAccessibilityRect: lhs.frame) - let rhsFrame = AXHelper.cocoaRect(fromAccessibilityRect: rhs.frame) - // Bucket midY into fixed slots so every pair of runs on the same visual line maps to - // the same bucket. A direct `abs(Δ) > tolerance` comparison is non-transitive (A~B, B~C, - // but A≁C), which yields an undefined sort order and can map cumulative offsets to the - // wrong run — placing ghost text on the wrong line. - let lineTolerance: CGFloat = 4 - let lhsBucket = (lhsFrame.midY / lineTolerance).rounded(.toNearestOrAwayFromZero) - let rhsBucket = (rhsFrame.midY / lineTolerance).rounded(.toNearestOrAwayFromZero) - if lhsBucket != rhsBucket { - return lhsBucket > rhsBucket - } - return lhsFrame.minX < rhsFrame.minX - } - } - - /// One-shot run collection for callers that don't cache (e.g. line-height estimation). - private func collectStaticTextRuns(from root: AXUIElement) -> [(text: String, frame: CGRect)] { - textRuns(fromElements: collectStaticTextElements(from: root)) ?? [] + return runs } /// Confirms a BoundsForRange result actually belongs to the focused field's neighborhood. @@ -530,77 +319,6 @@ struct AXTextGeometryResolver { return expanded.contains(CGPoint(x: cocoaRect.midX, y: cocoaRect.midY)) } - /// `AXBoundsForRange(location - 1, 1)` should describe one rendered character. Chromium can - /// instead return the full wrapped text-run containing that character, which looks plausible - /// by position but is hundreds of points wide. Deriving a caret from that rectangle pins the - /// overlay to the run's far edge, so reject it and let text-run fallbacks try a tighter source. - func looksLikeOversizedSingleCharacterRange(_ rect: CGRect, anchor: CGRect?) -> Bool { - guard !rect.isEmpty else { - return false - } - - let absoluteMaxCharacterWidth: CGFloat = 80 - guard let anchor, !anchor.isEmpty else { - return rect.width > absoluteMaxCharacterWidth - } - - return rect.width > min(anchor.width * 0.35, absoluteMaxCharacterWidth) - } - - /// Outlook-on-the-web has a different Chromium failure from Gmail: the one-character range - /// can be normal height and very narrow, but pinned to the editable field's left edge even as - /// the selection offset advances. That shape passes width/height guards, yet deriving a caret - /// from it parks ghost text on the wrong line. When the plain text says the caret is already - /// inside the current logical line, reject this left-edge range and let child text-run geometry - /// or frame fallback try next. - func looksLikeStaleLineStartRange( - _ rect: CGRect, - anchor: CGRect?, - text: String?, - selection: NSRange - ) -> Bool { - guard let anchor, !anchor.isEmpty, let text, !text.isEmpty, !rect.isEmpty else { - return false - } - - let nsText = text as NSString - guard selection.location > 1, selection.location <= nsText.length else { - return false - } - - let prefix = nsText.substring(to: selection.location) - let currentLinePrefix = prefix.components(separatedBy: .newlines).last ?? prefix - guard (currentLinePrefix as NSString).length > 1 else { - return false - } - - let leftEdgeTolerance: CGFloat = 4 - let rangeTrailingEdge = rect.maxX - return abs(rangeTrailingEdge - anchor.minX) <= leftEdgeTolerance - } - - /// Chromium can answer `BoundsForRange(location - 1, 1)` with the union of multiple rendered - /// lines. That rectangle is still near the field, so halo validation cannot catch it, but its - /// height is much larger than the real text-run line height exposed by nearby `AXStaticText` - /// descendants. When we can measure those descendants, reject the union and let later fallbacks - /// search for marker geometry or use a conservative frame estimate. - private func looksLikeMultilineRangeUnion(_ rect: CGRect, in element: AXUIElement) -> Bool { - let lineHeights = collectStaticTextRuns(from: element) - .map { AXHelper.cocoaRect(fromAccessibilityRect: $0.frame).height } - .filter { $0 >= 8 } - .sorted() - - guard !lineHeights.isEmpty else { - return false - } - - guard let largestObservedLineHeight = lineHeights.last else { - return false - } - - return rect.height > max(largestObservedLineHeight * 1.8, largestObservedLineHeight + 24) - } - /// Some browser-based editors return a full line fragment for a zero-length range instead of /// a narrow caret box. Collapse those wide rects back down to a caret-like anchor. private func normalizedCaretRect(fromZeroLengthRangeRect rect: CGRect) -> CGRect { diff --git a/Cotabby/Services/Focus/CaretGeometrySourceCache.swift b/Cotabby/Services/Focus/CaretGeometrySourceCache.swift deleted file mode 100644 index 68f544b6..00000000 --- a/Cotabby/Services/Focus/CaretGeometrySourceCache.swift +++ /dev/null @@ -1,95 +0,0 @@ -import ApplicationServices -import Foundation - -/// File overview: -/// Remembers the AX leaf elements that produced caret geometry for the currently focused field so -/// the resolver can re-read them directly instead of re-discovering them with a fresh tree walk on -/// every keystroke. -/// -/// Why this exists: in Chromium editors the caret rect comes from per-line `AXStaticText` leaves, -/// and finding them means a bounded-but-real BFS (`collectStaticTextRuns`) on each focus poll. The -/// leaf *elements* are stable across keystrokes within a field even though their text and frames -/// change, so caching the element references turns the per-keystroke crawl into a handful of -/// attribute reads. The cache is intentionally a reference type: the resolvers are value-type -/// structs, so their own state cannot survive across calls. -/// -/// Correctness contract: the cache is keyed by `(container identity, focusChangeSequence)`. The -/// sequence is the authoritative field-switch signal (`elementIdentifier` alone is `CFHash`-based -/// and can collide across recycled nodes), so a key match guarantees "same field, same focus." Even -/// on a match, callers must re-validate the cached elements before trusting them — a stale element -/// read simply misses and forces a re-walk. -@MainActor -final class CaretGeometrySourceCache { - struct FieldKey: Equatable { - let containerIdentifier: String - let focusChangeSequence: UInt64 - } - - private var runKey: FieldKey? - private var textRunElements: [AXUIElement]? - - private var deepKey: FieldKey? - private var deepSourceElement: AXUIElement? - - /// Releasing the cache's storage (CF element references, strings, ints) is thread-safe, so the - /// deinit does not need main-actor isolation. Marking it `nonisolated` keeps the rest of the type - /// main-actor isolated while preventing the compiler from emitting an isolated-deinit hop. With - /// our macOS 14 deployment target that hop runs through the back-deployment shim - /// `swift_task_deinitOnExecutorMainActorBackDeploy`, which over-releases and aborts the process - /// ("pointer being freed was not allocated") when a `@MainActor` class with non-trivial stored - /// properties is destroyed — it crashed `CaretGeometrySourceCacheTests` deterministically. - nonisolated deinit {} - - /// Debug-only counters (read by `FocusTracker` when `-cotabby-debug` is set) so a dump can show - /// whether the cache is actually hitting in the wild rather than us guessing. Not used for logic. - private(set) var runHits = 0 - private(set) var runMisses = 0 - private(set) var deepHits = 0 - private(set) var deepMisses = 0 - - var debugStats: String { - "run \(runHits)/\(runHits + runMisses) deep \(deepHits)/\(deepHits + deepMisses)" - } - - /// Returns the cached ordered text-run leaves for `key`, or nil when the focused field changed - /// (key mismatch) or nothing has been cached yet. - func textRunElements(for key: FieldKey) -> [AXUIElement]? { - guard runKey == key else { - runMisses += 1 - return nil - } - runHits += 1 - return textRunElements - } - - /// Replaces the cached leaves for `key`. Storing under a new key implicitly drops the previous - /// field's entry, so the cache never holds more than one field at a time. - func store(textRunElements elements: [AXUIElement], for key: FieldKey) { - runKey = key - textRunElements = elements - } - - /// Returns the cached deep geometry source leaf — the element whose zero-length selection range - /// produced the caret rect last time — so the resolver can read it directly instead of BFS-ing - /// the subtree again. Nil on field change or cold cache. - func deepSource(for key: FieldKey) -> AXUIElement? { - guard deepKey == key else { - deepMisses += 1 - return nil - } - deepHits += 1 - return deepSourceElement - } - - func store(deepSource element: AXUIElement, for key: FieldKey) { - deepKey = key - deepSourceElement = element - } - - func invalidate() { - runKey = nil - textRunElements = nil - deepKey = nil - deepSourceElement = nil - } -} diff --git a/Cotabby/Services/Focus/FocusSnapshotResolver.swift b/Cotabby/Services/Focus/FocusSnapshotResolver.swift index 5a94c15b..663f4264 100644 --- a/Cotabby/Services/Focus/FocusSnapshotResolver.swift +++ b/Cotabby/Services/Focus/FocusSnapshotResolver.swift @@ -10,25 +10,21 @@ import Logging @MainActor struct FocusSnapshotResolver { private let geometryResolver: AXTextGeometryResolver - /// Shared with `geometryResolver` so the deep-walk fast path and the run-walk fast path key off - /// the same per-field memo. Nil disables caching (tests, callers that inject a bare resolver). - private let caretGeometryCache: CaretGeometrySourceCache? + + /// Throttle window for the deep caret BFS. ~100ms keeps the walk off the per-keystroke hot path + /// in Chromium editors while staying short enough that caret lag during fast typing stays minor. + private static let deepWalkThrottleInterval: TimeInterval = 0.1 + + /// Carries deep-walk throttle state across the value-typed resolver's non-mutating polls. + private let deepWalkThrottle = DeepGeometryWalkThrottle() // MARK: - Debug AX tree dump (temporary — remove after caret placement is fixed) /// Set to true to print the AX tree every time focus changes. Check Xcode console. private static let dumpAXTree = false private static var lastDumpedElementID: String? - init( - geometryResolver: AXTextGeometryResolver? = nil, - caretGeometryCache: CaretGeometrySourceCache? = nil - ) { - let resolver = geometryResolver ?? AXTextGeometryResolver(cache: caretGeometryCache) - self.geometryResolver = resolver - // Adopt the resolver's own cache so the deep-walk fast path (this type) and the run-walk fast - // path (inside the resolver) always key off one memo. When a caller injects a custom resolver, - // its cache wins; the `caretGeometryCache` argument only seeds the default resolver. - self.caretGeometryCache = resolver.cache + init(geometryResolver: AXTextGeometryResolver? = nil) { + self.geometryResolver = geometryResolver ?? AXTextGeometryResolver() } /// Resolves the best editable candidate around the focused AX node and materializes a focus snapshot. @@ -58,11 +54,7 @@ struct FocusSnapshotResolver { } let candidates = candidateElements(around: focusedElement).map { - candidateSnapshot( - for: $0, - bundleIdentifier: bundleIdentifier, - focusChangeSequence: focusChangeSequence - ) + candidateSnapshot(for: $0, bundleIdentifier: bundleIdentifier) } let resolution = FocusCapabilityResolver.resolve( candidates: candidates.map(\.resolverCandidate)) @@ -126,18 +118,6 @@ struct FocusSnapshotResolver { ) } - // Populate the focused field's text-run cache once, for the resolved winner. The candidate - // probe above resolves caret geometry with the run cache read-only so a non-winning candidate - // can't evict the focused field's leaves on the same poll. `observedCharWidth` is non-nil only - // when the winner's caret came from the child-text-run path, so native / BoundsForRange fields - // that never use the run cache skip the walk entirely. - if resolvedCandidate.observedCharWidth != nil { - geometryResolver.cacheTextRunSources( - for: resolvedCandidate.element, - focusChangeSequence: focusChangeSequence - ) - } - // The input target and the geometry source don't need to be the same element. // Native AppKit apps give exact caret rects on the input target itself. Chrome's // AXTextArea, by contrast, answers BoundsForRange(loc-1, 1) with a multi-line union @@ -152,15 +132,26 @@ struct FocusSnapshotResolver { // 3. primary `.derived` // 4. deep `.derived` // 5. primary `.estimated` / unknown fallback - // The walk is skipped entirely when primary is already `.exact` to avoid wasted IPC. - let deepResult: CaretGeometryResult? = (resolvedCandidate.caretQuality == .exact) - ? nil - : resolveDeepGeometrySource( - focusedElement: focusedElement, - resolvedElement: resolvedCandidate.element, - cocoaAnchorFrame: resolvedCandidate.inputFrameRect, - focusChangeSequence: focusChangeSequence - ) + // The walk is skipped entirely when primary is already `.exact`, and otherwise throttled to + // one BFS per `deepWalkThrottleInterval` while focus stays in the same field. Chromium + // editors keep the focused element at `.derived`, so without the throttle the ~200-node walk + // ran on every keystroke and pinned a CPU core. Within the window we reuse the previous deep + // result, which can trail the live caret by up to one throttle interval of fast typing. + let deepResult: CaretGeometryResult? + if resolvedCandidate.caretQuality == .exact { + deepResult = nil + } else { + deepResult = deepWalkThrottle.result( + focusChangeSequence: focusChangeSequence, + interval: Self.deepWalkThrottleInterval + ) { + resolveDeepGeometrySource( + focusedElement: focusedElement, + resolvedElement: resolvedCandidate.element, + cocoaAnchorFrame: resolvedCandidate.inputFrameRect + ) + } + } guard let caret = Self.selectCaretGeometry( primaryRect: resolvedCandidate.caretRect, @@ -276,172 +267,9 @@ struct FocusSnapshotResolver { } } - for node in [focusedElement] + ancestors where shouldSearchEditableDescendants(from: node) { - for descendant in editableDescendantCandidates(from: node) { - append(descendant) - } - } - return ordered } - /// Chromium can report the page-level `AXWebArea` as focused while the actual compose box is - /// several levels below it. When that container owns the live text marker selection, we use it - /// as a bounded search root to recover the real editable input target instead of declaring the - /// whole focus unsupported. - private func shouldSearchEditableDescendants(from root: AXUIElement) -> Bool { - let role = AXHelper.stringValue(for: kAXRoleAttribute as CFString, on: root) ?? "Unknown" - let attributes = Set(AXHelper.attributeNames(on: root)) - let explicitEditableFlag = - attributes.contains("AXEditable") - ? AXHelper.boolValue(for: "AXEditable" as CFString, on: root) - : nil - - guard !AXHelper.hasStrongEditabilitySignal( - role: role, - explicitEditableFlag: explicitEditableFlag - ) else { - return false - } - - return attributes.contains("AXSelectedTextMarkerRange") - } - - /// Finds editable descendants under a browser document/container whose text marker range is - /// active. The score is intentionally simple: a real editable role/flag is required, then - /// geometry near the container's marker rect and live selection/value data decide ordering. - private func editableDescendantCandidates(from root: AXUIElement) -> [AXUIElement] { - let rootMarkerRect = AXHelper.textMarkerCaretRect(on: root) - var queue: [(element: AXUIElement, depth: Int)] = - AXHelper.childElements(of: root).map { ($0, 1) } - let maxDepth = 24 - let maxNodes = 1_500 - let maxResults = 12 - var visited = 0 - var seen = Set() - var scoredCandidates: [EditableDescendantCandidate] = [] - - while !queue.isEmpty, visited < maxNodes { - let (element, depth) = queue.removeFirst() - let identity = AXHelper.elementIdentity(for: element) - guard seen.insert(identity).inserted else { continue } - visited += 1 - - if let candidate = scoreEditableDescendant( - element, - depth: depth, - rootMarkerRect: rootMarkerRect - ) { - scoredCandidates.append(candidate) - } - - guard depth < maxDepth else { continue } - for child in AXHelper.childElements(of: element) { - queue.append((child, depth + 1)) - } - } - - let ordered = scoredCandidates - .sorted { lhs, rhs in - if lhs.score != rhs.score { - return lhs.score > rhs.score - } - return lhs.depth < rhs.depth - } - .prefix(maxResults) - .map(\.element) - - if !ordered.isEmpty { - CotabbyLogger.focus.debug( - "Recovered \(ordered.count) editable descendant candidate(s) from marker-owning container" - ) - } - - return Array(ordered) - } - - private func scoreEditableDescendant( - _ element: AXUIElement, - depth: Int, - rootMarkerRect: CGRect? - ) -> EditableDescendantCandidate? { - let role = AXHelper.stringValue(for: kAXRoleAttribute as CFString, on: element) ?? "Unknown" - let attributes = Set(AXHelper.attributeNames(on: element)) - let explicitEditableFlag = - attributes.contains("AXEditable") - ? AXHelper.boolValue(for: "AXEditable" as CFString, on: element) - : nil - - guard AXHelper.hasStrongEditabilitySignal( - role: role, - explicitEditableFlag: explicitEditableFlag - ), !AXHelper.isKnownReadOnlyRole(role) else { - return nil - } - - let selection = attributes.contains(kAXSelectedTextRangeAttribute as String) - ? AXHelper.rangeValue(for: kAXSelectedTextRangeAttribute as CFString, on: element) - : nil - let hasValue = attributes.contains(kAXValueAttribute as String) - && AXHelper.stringValue(for: kAXValueAttribute as CFString, on: element) != nil - let frame = attributes.contains("AXFrame") - ? AXHelper.rectValue(for: "AXFrame" as CFString, on: element) - : nil - let markerMatches = markerRect(rootMarkerRect, matchesEditableFrame: frame) - - var score = AXHelper.editabilityHintScore( - role: role, - explicitEditableFlag: explicitEditableFlag - ) - if role == kAXTextAreaRole as String { - score += 10 - } else if role == kAXTextFieldRole as String || role == "AXSearchField" { - score += 6 - } - if markerMatches { - score += 50 - } - if selection != nil { - score += 20 - } - if selection?.length == 0 { - score += 8 - } - if hasValue { - score += 5 - } - - // Without either marker geometry or a live selection, the node is just a generic editable - // descendant somewhere in the page. Keep the recovery path tied to the active browser - // selection so we don't accidentally pick an unrelated search field or hidden input. - guard markerMatches || selection != nil else { - return nil - } - - return EditableDescendantCandidate(element: element, score: score, depth: depth) - } - - private func markerRect(_ markerRect: CGRect?, matchesEditableFrame frame: CGRect?) -> Bool { - guard let markerRect, !markerRect.isEmpty, let frame, !frame.isEmpty else { - return false - } - - let cocoaFrame = AXHelper.cocoaRect(fromAccessibilityRect: frame) - let cocoaMarkerRect = AXHelper.validatedCocoaTextRect( - fromAccessibilityRect: markerRect, - anchorFrame: cocoaFrame - ) - let expandedCocoaFrame = cocoaFrame.insetBy(dx: -24, dy: -24) - if expandedCocoaFrame.intersects(cocoaMarkerRect) - || expandedCocoaFrame.contains(CGPoint(x: cocoaMarkerRect.midX, y: cocoaMarkerRect.midY)) { - return true - } - - let expandedRawFrame = frame.insetBy(dx: -24, dy: -24) - return expandedRawFrame.intersects(markerRect) - || expandedRawFrame.contains(CGPoint(x: markerRect.midX, y: markerRect.midY)) - } - /// Chooses the caret geometry to ship from the primary candidate and the optional deep-tree /// result, following a fixed precedence (see the call site). Pulled out of `resolveSnapshot` /// so that method stays under the cyclomatic-complexity limit; returns `nil` when neither @@ -502,13 +330,11 @@ struct FocusSnapshotResolver { private func resolveDeepGeometrySource( focusedElement: AXUIElement, resolvedElement: AXUIElement, - cocoaAnchorFrame: CGRect?, - focusChangeSequence: UInt64 + cocoaAnchorFrame: CGRect? ) -> CaretGeometryResult? { if let result = findDeepGeometrySource( from: resolvedElement, - cocoaAnchorFrame: cocoaAnchorFrame, - focusChangeSequence: focusChangeSequence + cocoaAnchorFrame: cocoaAnchorFrame ) { return result } @@ -522,8 +348,7 @@ struct FocusSnapshotResolver { return findDeepGeometrySource( from: focusedElement, - cocoaAnchorFrame: cocoaAnchorFrame, - focusChangeSequence: focusChangeSequence + cocoaAnchorFrame: cocoaAnchorFrame ) } @@ -534,31 +359,14 @@ struct FocusSnapshotResolver { /// We only read position from these nodes; the input target (where we type) stays unchanged. private func findDeepGeometrySource( from root: AXUIElement, - cocoaAnchorFrame: CGRect?, - focusChangeSequence: UInt64 + cocoaAnchorFrame: CGRect? ) -> CaretGeometryResult? { - let fieldKey = CaretGeometrySourceCache.FieldKey( - containerIdentifier: AXHelper.elementIdentity(for: root), - focusChangeSequence: focusChangeSequence - ) - - // Fast path: the leaf that held the caret last keystroke almost always still does, so try it - // directly before BFS-ing the subtree. A line change moves the active zero-length selection - // to a different leaf, so the cached one yields nil here and we fall through to a re-walk. - if let cache = caretGeometryCache, - let cached = cache.deepSource(for: fieldKey), - let result = caretGeometry( - at: cached, cocoaAnchorFrame: cocoaAnchorFrame, focusChangeSequence: focusChangeSequence - ) { - return result - } - var queue: [(element: AXUIElement, depth: Int)] = [(root, 0)] let maxDepth = 10 let maxNodes = 200 var visited = 0 var seen = Set() - var best: DeepGeometryCandidate? + var bestResult: (result: CaretGeometryResult, depth: Int)? while !queue.isEmpty, visited < maxNodes { let (element, depth) = queue.removeFirst() @@ -567,10 +375,37 @@ struct FocusSnapshotResolver { guard seen.insert(identity).inserted else { continue } visited += 1 - if let result = caretGeometry( - at: element, cocoaAnchorFrame: cocoaAnchorFrame, focusChangeSequence: focusChangeSequence - ), shouldPreferDeepResult(result, at: depth, over: best.map { ($0.result, $0.depth) }) { - best = DeepGeometryCandidate(result: result, depth: depth, element: element) + // Look for any node with an active caret (zero-length selection). + // Don't filter by role — Chrome uses AXStaticText for editable text runs. + if let range = AXHelper.rangeValue( + for: kAXSelectedTextRangeAttribute as CFString, on: element + ), range.length == 0 { + let paramAttrs = Set(AXHelper.parameterizedAttributeNames(on: element)) + let attrs = Set(AXHelper.attributeNames(on: element)) + let textValue = + attrs.contains(kAXValueAttribute as String) + ? AXHelper.stringValue(for: kAXValueAttribute as CFString, on: element) + : nil + let result = geometryResolver.resolveCaretRect( + for: element, + selection: range, + supportsBoundsForRange: paramAttrs.contains( + kAXBoundsForRangeParameterizedAttribute as String + ), + supportsFrame: attrs.contains("AXFrame"), + cocoaAnchorFrame: cocoaAnchorFrame, + textValue: textValue + ) + + if let result, result.quality == .exact || result.quality == .derived { + if shouldPreferDeepResult( + result, + at: depth, + over: bestResult + ) { + bestResult = (result, depth) + } + } } guard depth < maxDepth else { continue } @@ -579,63 +414,11 @@ struct FocusSnapshotResolver { } } - if let cache = caretGeometryCache, let best { - cache.store(deepSource: best.element, for: fieldKey) - } - return best?.result - } - - /// The winning deep-walk leaf plus the metadata `shouldPreferDeepResult` ranks it by, and the - /// element reference so the result can be cached as the field's deep geometry source. - private struct DeepGeometryCandidate { - let result: CaretGeometryResult - let depth: Int - let element: AXUIElement - } - - /// Resolves caret geometry from a single candidate leaf, returning a result only when the leaf - /// holds an active caret (zero-length selection) and produces exact/derived geometry. Shared by - /// the deep-walk BFS and its cached fast path so both apply identical acceptance rules. - /// Don't filter by role — Chrome exposes editable text runs as `AXStaticText`. - private func caretGeometry( - at element: AXUIElement, - cocoaAnchorFrame: CGRect?, - focusChangeSequence: UInt64 - ) -> CaretGeometryResult? { - guard let range = AXHelper.rangeValue( - for: kAXSelectedTextRangeAttribute as CFString, on: element - ), range.length == 0 else { - return nil - } - - let paramAttrs = Set(AXHelper.parameterizedAttributeNames(on: element)) - let attrs = Set(AXHelper.attributeNames(on: element)) - let textValue = - attrs.contains(kAXValueAttribute as String) - ? AXHelper.stringValue(for: kAXValueAttribute as CFString, on: element) - : nil - let result = geometryResolver.resolveCaretRect( - for: element, - selection: range, - supportsBoundsForRange: paramAttrs.contains( - kAXBoundsForRangeParameterizedAttribute as String - ), - supportsFrame: attrs.contains("AXFrame"), - cocoaAnchorFrame: cocoaAnchorFrame, - textValue: textValue, - focusChangeSequence: focusChangeSequence - ) - - guard let result, result.quality == .exact || result.quality == .derived else { - return nil - } - return result + return bestResult?.result } - /// Prefers exact marker/range geometry before depth. Browser AX wrappers can expose - /// superficially "valid" derived rectangles, but the Cotypist-style Chrome path is a live - /// zero-length selection on a text-run node; once we find exact geometry, a deeper estimate - /// should not displace it. + /// Prefers deeper descendants because browser AX wrappers can expose superficially "valid" + /// geometry on shallow nodes while the real caret anchor lives lower in the text-run leaves. private func shouldPreferDeepResult( _ candidate: CaretGeometryResult, at depth: Int, @@ -645,13 +428,12 @@ struct FocusSnapshotResolver { return true } - let candidateQualityScore = deepResultQualityScore(candidate.quality) - let bestQualityScore = deepResultQualityScore(best.result.quality) - if candidateQualityScore != bestQualityScore { - return candidateQualityScore > bestQualityScore + if depth != best.depth { + return depth > best.depth } - return depth > best.depth + return deepResultQualityScore(candidate.quality) + > deepResultQualityScore(best.result.quality) } private func deepResultQualityScore(_ quality: CaretGeometryQuality) -> Int { @@ -666,11 +448,8 @@ struct FocusSnapshotResolver { } /// Extracts the AX properties Cotabby needs from one candidate element near the current focus. - private func candidateSnapshot( - for element: AXUIElement, - bundleIdentifier: String, - focusChangeSequence: UInt64 - ) -> AXFocusCandidate { + private func candidateSnapshot(for element: AXUIElement, bundleIdentifier: String) + -> AXFocusCandidate { let role = AXHelper.stringValue(for: kAXRoleAttribute as CFString, on: element) ?? "Unknown" let subrole = AXHelper.stringValue(for: kAXSubroleAttribute as CFString, on: element) let supportedAttributes = Set(AXHelper.attributeNames(on: element)) @@ -728,8 +507,7 @@ struct FocusSnapshotResolver { kAXBoundsForRangeParameterizedAttribute as String), supportsFrame: supportedAttributes.contains("AXFrame"), cocoaAnchorFrame: inputFrameRect, - textValue: textValue, - focusChangeSequence: focusChangeSequence + textValue: textValue ) } let caretRect = caretResult?.rect @@ -886,6 +664,48 @@ struct FocusSnapshotResolver { } } +/// Throttles the deep-tree caret BFS so it runs at most once per `interval` while focus stays on one +/// field. `findDeepGeometrySource` walks up to ~200 AX nodes with several synchronous IPC round-trips +/// each; in Chromium editors (e.g. Gmail) the focused element reports only `.derived` primary +/// geometry, so the walk fired on every keystroke and pinned a CPU core. Reusing the prior deep +/// result inside the window keeps caret-source selection identical while collapsing the +/// per-keystroke AX traffic. +/// +/// Keyed on `FocusTracker`'s `focusChangeSequence` rather than the AX element: Chrome recycles AX +/// node handles, so an element-identity key would miss on nearly every poll and defeat the throttle, +/// whereas the sequence is derived from the field frame and stays stable across keystrokes in one +/// field. A changed sequence is a real field switch and forces an immediate fresh walk. +/// +/// A reference type so it can carry state across the value-typed resolver's non-mutating +/// `resolveSnapshot`. The resolver is constructed once and retained by `FocusTracker`. +@MainActor +final class DeepGeometryWalkThrottle { + private var lastSequence: UInt64? + private var lastWalkAt: Date? + private var cachedResult: CaretGeometryResult? + + /// Runs `walk` only when the throttle window has elapsed or the focused field changed; otherwise + /// returns the previous deep result. `now` is injectable for tests. + func result( + focusChangeSequence: UInt64, + interval: TimeInterval, + now: Date = Date(), + walk: () -> CaretGeometryResult? + ) -> CaretGeometryResult? { + if focusChangeSequence == lastSequence, + let lastWalkAt, + now.timeIntervalSince(lastWalkAt) < interval { + return cachedResult + } + + let result = walk() + lastSequence = focusChangeSequence + lastWalkAt = now + cachedResult = result + return result + } +} + /// AX data read from one candidate element near the current focus. /// This keeps candidate search state local to the resolver instead of leaking it into the tracker. private struct AXFocusCandidate { @@ -902,9 +722,3 @@ private struct AXFocusCandidate { let isSecure: Bool let resolverCandidate: FocusCapabilityCandidate } - -private struct EditableDescendantCandidate { - let element: AXUIElement - let score: Int - let depth: Int -} diff --git a/Cotabby/Services/Focus/FocusTracker.swift b/Cotabby/Services/Focus/FocusTracker.swift index 624065b5..481002f7 100644 --- a/Cotabby/Services/Focus/FocusTracker.swift +++ b/Cotabby/Services/Focus/FocusTracker.swift @@ -1,5 +1,4 @@ import AppKit -import ApplicationServices import Foundation import Logging @@ -25,9 +24,6 @@ final class FocusTracker { private let permissionProvider: @MainActor () -> Bool private let ignoredBundleIdentifier: String? private let snapshotResolver: FocusSnapshotResolver - /// Retained only so debug instrumentation can read cache hit/miss counts; resolution itself goes - /// through `snapshotResolver`, which shares this same instance. - private let caretGeometryCache: CaretGeometrySourceCache? private var timer: Timer? private var pollSequence = 0 @@ -52,17 +48,7 @@ final class FocusTracker { self.ignoredBundleIdentifier = ignoredBundleIdentifier // Default resolver construction must happen inside the actor-isolated initializer body. // Swift evaluates default parameter expressions before entering the `@MainActor` context. - // The caret-geometry cache is owned here so its lifetime matches the tracker's; it memoizes - // the focused field's text-run leaves so per-keystroke caret resolution can re-read them - // instead of re-walking the AX tree. - if let snapshotResolver { - self.snapshotResolver = snapshotResolver - self.caretGeometryCache = nil - } else { - let cache = CaretGeometrySourceCache() - self.snapshotResolver = FocusSnapshotResolver(caretGeometryCache: cache) - self.caretGeometryCache = cache - } + self.snapshotResolver = snapshotResolver ?? FocusSnapshotResolver() } /// Starts periodic AX polling and immediately captures an initial snapshot. @@ -171,14 +157,6 @@ final class FocusTracker { guard let focusedElement = AXHelper.focusedElement() else { let frontmost = NSWorkspace.shared.frontmostApplication - if let frontmost { - AXChromeFocusProbe.dumpIfNeeded( - application: frontmost, - systemFocusedElement: nil, - snapshot: nil, - reason: "system-wide AXFocusedUIElement was nil" - ) - } return inactiveCapture( applicationName: frontmost?.localizedName ?? "No active application", bundleIdentifier: frontmost?.bundleIdentifier, @@ -220,12 +198,6 @@ final class FocusTracker { ) guard let context = firstPassSnapshot.context else { - AXChromeFocusProbe.dumpIfNeeded( - application: application, - systemFocusedElement: focusedElement, - snapshot: firstPassSnapshot, - reason: "resolver produced no focused input context" - ) return FocusCaptureResult( snapshot: firstPassSnapshot, didChangeFocusedInput: clearFocusedInputSignatureIfNeeded() @@ -245,12 +217,6 @@ final class FocusTracker { application: application, focusChangeSequence: focusChangeSequence ) - AXChromeFocusProbe.dumpIfNeeded( - application: application, - systemFocusedElement: focusedElement, - snapshot: finalSnapshot, - reason: "focused input snapshot changed" - ) return FocusCaptureResult(snapshot: finalSnapshot, didChangeFocusedInput: true) } @@ -268,7 +234,7 @@ final class FocusTracker { } let millis = Double((ContinuousClock.now - start).components.attoseconds) / 1e15 let source = snapshot.context?.caretSource ?? snapshot.capability.shortLabel - let stats = caretGeometryCache?.debugStats ?? "no-cache" + let stats = "no-cache" let line = "Resolve timing: app=\(application.localizedName ?? "?") " + "resolveMs=\(String(format: "%.1f", millis)) caret=\(source) cache=[\(stats)]" CotabbyLogger.focus.debug("\(line)") @@ -311,383 +277,6 @@ private struct FocusCaptureResult { let didChangeFocusedInput: Bool } -/// Temporary Chrome-only AX probe for the Gmail/Claude focus investigation. -/// -/// The normal resolver intentionally starts from the focused element and walks a small local -/// neighborhood. This probe exists to test a different question: when Chromium reports toolbar focus -/// or no global focus, does the live editor selection still exist elsewhere under Chrome's app/window -/// AX tree? It requires both `-cotabby-debug` and `-cotabby-ax-probe`, is throttled, and logs -/// bounded structural metadata rather than unbounded page text. -@MainActor -private enum AXChromeFocusProbe { - private static let supportedBundleIdentifiers: Set = [ - "com.google.Chrome", - "company.thebrowser.Browser", - "com.brave.Browser" - ] - private static let probeArgument = "-cotabby-ax-probe" - private static let searchArgument = "-cotabby-ax-probe-search" - private static let maxAncestorDepth = 12 - private static let maxScanDepth = 28 - private static let maxScanNodes = 2_500 - private static let maxHighlightLines = 90 - private static let minRepeatInterval: TimeInterval = 3 - - private static var lastSignature: String? - private static var lastDumpAt: Date? - - static func dumpIfNeeded( - application: NSRunningApplication, - systemFocusedElement: AXUIElement?, - snapshot: FocusSnapshot?, - reason: String - ) { - guard CotabbyDebugOptions.isEnabled else { return } - guard ProcessInfo.processInfo.arguments.contains(probeArgument) else { return } - guard - let bundleIdentifier = application.bundleIdentifier, - supportedBundleIdentifiers.contains(bundleIdentifier) - else { - return - } - - let appElement = AXHelper.applicationElement(processIdentifier: application.processIdentifier) - let appFocusedElement = AXHelper.focusedElement(inApplication: appElement) - let focusedWindow = AXHelper.uiElementValue(for: kAXFocusedWindowAttribute as CFString, on: appElement) - let signature = [ - bundleIdentifier, - reason, - snapshot?.capability.summary ?? "no-snapshot", - systemFocusedElement.map(AXHelper.elementIdentity) ?? "system-nil", - appFocusedElement.map(AXHelper.elementIdentity) ?? "app-nil", - focusedWindow.map(AXHelper.elementIdentity) ?? "window-nil", - contextSignature(snapshot?.context) - ].joined(separator: "|") - - let now = Date() - if signature == lastSignature, - let lastDumpAt, - now.timeIntervalSince(lastDumpAt) < minRepeatInterval { - return - } - - lastSignature = signature - lastDumpAt = now - - let searchHint = configuredSearchHint() - var lines: [String] = [] - lines.append("========== CHROME AX FOCUS PROBE ==========") - lines.append("reason=\(reason)") - lines.append("app=\(application.localizedName ?? "Unknown") bundle=\(bundleIdentifier) pid=\(application.processIdentifier)") - lines.append("snapshot=\(snapshotSummary(snapshot))") - if let searchHint { - lines.append("searchHint=\"\(sanitize(searchHint, limit: 80))\"") - } - - lines.append("-- system AXFocusedUIElement --") - appendElementBlock(systemFocusedElement, searchHint: searchHint, to: &lines) - lines.append("-- app AXFocusedUIElement --") - appendElementBlock(appFocusedElement, searchHint: searchHint, to: &lines) - lines.append("-- app AXFocusedWindow --") - appendElementBlock(focusedWindow, searchHint: searchHint, to: &lines) - - appendAncestorBlock( - title: "-- system focus ancestors --", - root: systemFocusedElement, - searchHint: searchHint, - to: &lines - ) - appendAncestorBlock( - title: "-- app focus ancestors --", - root: appFocusedElement, - searchHint: searchHint, - to: &lines - ) - - let scanRoots = uniqueRoots([ - ("system-focus", systemFocusedElement), - ("app-focus", appFocusedElement), - ("focused-window", focusedWindow), - ("app-root", appElement) - ]) - for root in scanRoots { - appendDeepScanBlock(root: root, searchHint: searchHint, to: &lines) - } - - lines.append("========== END CHROME AX FOCUS PROBE ==========") - CotabbyLogger.focus.debug("\(lines.joined(separator: "\n"))") - } - - private static func contextSignature(_ context: FocusedInputSnapshot?) -> String { - guard let context else { return "context-nil" } - return [ - context.role, - context.subrole ?? "n/a", - "\(context.selection.location)+\(context.selection.length)", - "\(context.precedingText.count)+\(context.trailingText.count)", - formatRect(context.caretRect), - context.inputFrameRect.map(formatRect) ?? "input-nil" - ].joined(separator: "/") - } - - private static func snapshotSummary(_ snapshot: FocusSnapshot?) -> String { - guard let snapshot else { return "nil" } - guard let context = snapshot.context else { - return "\(snapshot.capability.shortLabel) reason=\"\(snapshot.capability.summary)\"" - } - - let textLength = context.precedingText.count + context.trailingText.count - return "\(snapshot.capability.shortLabel) role=\(context.role)/\(context.subrole ?? "n/a") " - + "selection=\(context.selection.location)+\(context.selection.length) " - + "textLength=\(textLength) caret=\(context.caretQuality.label):\(context.caretSource) " - + "caretRect=\(formatRect(context.caretRect)) inputRect=\(context.inputFrameRect.map(formatRect) ?? "nil")" - } - - private static func appendElementBlock( - _ element: AXUIElement?, - searchHint: String?, - to lines: inout [String] - ) { - guard let element else { - lines.append(" nil") - return - } - - lines.append(" \(describe(element, searchHint: searchHint))") - } - - private static func appendAncestorBlock( - title: String, - root: AXUIElement?, - searchHint: String?, - to lines: inout [String] - ) { - lines.append(title) - guard let root else { - lines.append(" nil") - return - } - - var current: AXUIElement? = root - for depth in 0..() - var visited = 0 - var highlights: [String] = [] - - while !queue.isEmpty, visited < maxScanNodes { - let item = queue.removeFirst() - let identity = AXHelper.elementIdentity(for: item.element) - guard seen.insert(identity).inserted else { continue } - visited += 1 - - if let highlight = highlightLine(for: item.element, path: item.path, searchHint: searchHint) { - highlights.append(highlight) - if highlights.count >= maxHighlightLines { - break - } - } - - guard item.depth < maxScanDepth else { continue } - for child in AXHelper.childElements(of: item.element) { - let path = "\(item.path) > \(rolePathComponent(child))" - queue.append((child, path, item.depth + 1)) - } - } - - lines.append(" visited=\(visited) highlights=\(highlights.count)") - if highlights.isEmpty { - lines.append(" no selected/text-marker/value/web nodes found within scan budget") - } else { - lines.append(contentsOf: highlights.map { " \($0)" }) - } - } - - private static func highlightLine( - for element: AXUIElement, - path: String, - searchHint: String? - ) -> String? { - let attributes = Set(AXHelper.attributeNames(on: element)) - let role = AXHelper.stringValue(for: kAXRoleAttribute as CFString, on: element) ?? "Unknown" - let value = attributes.contains(kAXValueAttribute as String) - ? AXHelper.stringValue(for: kAXValueAttribute as CFString, on: element) - : nil - let hasSelection = attributes.contains(kAXSelectedTextRangeAttribute as String) - let hasMarker = attributes.contains("AXSelectedTextMarkerRange") - let matchesHint = searchHint.flatMap { hint in - value?.range(of: hint, options: [.caseInsensitive, .diacriticInsensitive]) == nil ? nil : true - } ?? false - let isImportantRole = [ - kAXTextFieldRole as String, - kAXTextAreaRole as String, - kAXStaticTextRole as String, - "AXWebArea", - "AXSearchField" - ].contains(role) - - guard hasSelection || hasMarker || matchesHint || isImportantRole else { - return nil - } - - var parts: [String] = [path, describe(element, searchHint: searchHint)] - if matchesHint { - parts.append("MATCHED_SEARCH_HINT") - } - return parts.joined(separator: " | ") - } - - private static func describe(_ element: AXUIElement, searchHint: String?) -> String { - let attributes = Set(AXHelper.attributeNames(on: element)) - let parameterizedAttributes = Set(AXHelper.parameterizedAttributeNames(on: element)) - let role = AXHelper.stringValue(for: kAXRoleAttribute as CFString, on: element) ?? "Unknown" - let subrole = AXHelper.stringValue(for: kAXSubroleAttribute as CFString, on: element) - let title = AXHelper.stringValue(for: kAXTitleAttribute as CFString, on: element) - let description = AXHelper.stringValue(for: kAXDescriptionAttribute as CFString, on: element) - let value = attributes.contains(kAXValueAttribute as String) - ? AXHelper.stringValue(for: kAXValueAttribute as CFString, on: element) - : nil - let selection = attributes.contains(kAXSelectedTextRangeAttribute as String) - ? AXHelper.rangeValue(for: kAXSelectedTextRangeAttribute as CFString, on: element) - : nil - let markerRect = attributes.contains("AXSelectedTextMarkerRange") - ? AXHelper.textMarkerCaretRect(on: element) - : nil - let frame = attributes.contains("AXFrame") - ? AXHelper.rectValue(for: "AXFrame" as CFString, on: element) - : nil - let children = AXHelper.childElements(of: element).count - - var parts: [String] = ["role=\(role)/\(subrole ?? "n/a")"] - parts.append("id=\(AXHelper.elementIdentity(for: element))") - if let title, !title.isEmpty { - parts.append("title=\"\(sanitize(title, limit: 60))\"") - } - if let description, !description.isEmpty { - parts.append("description=\"\(sanitize(description, limit: 60))\"") - } - if let frame { - parts.append("frame=\(formatRect(frame))") - parts.append("cocoaFrame=\(formatRect(AXHelper.cocoaRect(fromAccessibilityRect: frame)))") - } - if let selection { - parts.append("selection=\(selection.location)+\(selection.length)") - } - if attributes.contains("AXSelectedTextMarkerRange") { - parts.append("selectedMarker=true") - } - if let markerRect, !markerRect.isEmpty { - parts.append("markerRect=\(formatRect(markerRect))") - } - if parameterizedAttributes.contains(kAXBoundsForRangeParameterizedAttribute as String) { - parts.append("boundsForRange=true") - } - if let editable = AXHelper.boolValue(for: "AXEditable" as CFString, on: element) { - parts.append("editable=\(editable)") - } - if let numberOfCharacters = AXHelper.copyAttributeValue("AXNumberOfCharacters" as CFString, on: element) as? NSNumber { - parts.append("characters=\(numberOfCharacters.intValue)") - } - if let value { - parts.append("valueLen=\(value.count)") - if valueMatchesSearchHint(value, searchHint: searchHint) { - parts.append("valuePreview=\"\(sanitize(value, limit: 80))\"") - } - } - if children > 0 { - parts.append("children=\(children)") - } - - return parts.joined(separator: " ") - } - - private static func valueMatchesSearchHint(_ value: String, searchHint: String?) -> Bool { - guard let searchHint, !searchHint.isEmpty else { - return false - } - - return value.range(of: searchHint, options: [.caseInsensitive, .diacriticInsensitive]) != nil - } - - private static func rolePathComponent(_ element: AXUIElement) -> String { - let role = AXHelper.stringValue(for: kAXRoleAttribute as CFString, on: element) ?? "?" - let subrole = AXHelper.stringValue(for: kAXSubroleAttribute as CFString, on: element) - if let subrole { - return "\(role)(\(subrole))" - } - - return role - } - - private static func uniqueRoots(_ roots: [(String, AXUIElement?)]) -> [ProbeRoot] { - var seen = Set() - var result: [ProbeRoot] = [] - for (label, element) in roots { - guard let element else { continue } - let identity = AXHelper.elementIdentity(for: element) - guard seen.insert(identity).inserted else { continue } - result.append(ProbeRoot(label: label, element: element)) - } - return result - } - - private static func configuredSearchHint() -> String? { - let arguments = ProcessInfo.processInfo.arguments - guard let index = arguments.firstIndex(of: searchArgument) else { - return nil - } - - let valueIndex = arguments.index(after: index) - guard arguments.indices.contains(valueIndex) else { - return nil - } - - let value = arguments[valueIndex].trimmingCharacters(in: .whitespacesAndNewlines) - return value.isEmpty ? nil : value - } - - private static func sanitize(_ text: String, limit: Int) -> String { - let compact = text - .replacingOccurrences(of: "\n", with: "\\n") - .replacingOccurrences(of: "\t", with: "\\t") - if compact.count <= limit { - return compact - } - - return "\(compact.prefix(limit))..." - } - - private static func formatRect(_ rect: CGRect) -> String { - String( - format: "(x=%.0f,y=%.0f,w=%.0f,h=%.0f)", - rect.origin.x, - rect.origin.y, - rect.width, - rect.height - ) - } - - private struct ProbeRoot { - let label: String - let element: AXUIElement - } -} - /// Stable-enough identity for one focused input as observed by polling. /// /// Text, selection, and caret position are deliberately excluded. Those can change inside the same diff --git a/Cotabby/Services/Visual/ScreenshotContextGenerator.swift b/Cotabby/Services/Visual/ScreenshotContextGenerator.swift index 260d456f..c7cea282 100644 --- a/Cotabby/Services/Visual/ScreenshotContextGenerator.swift +++ b/Cotabby/Services/Visual/ScreenshotContextGenerator.swift @@ -63,13 +63,17 @@ final class ScreenshotContextGenerator { do { extractedText = try await textExtractor.extractText(from: screenshot.image).text } catch ScreenTextExtractionError.noRecognizedText { - if let titleExcerpt = fallbackExcerpt(from: screenshot.windowTitle) { - return titleExcerpt - } else { + guard let windowTitle = screenshot.windowTitle, + hasMeaningfulSignal(windowTitle) + else { throw ScreenshotContextGenerationError.unavailable( "The screenshot did not contain enough visible text to build prompt context." ) } + + return VisualContextExcerpt( + text: boundedSummaryText(normalizeRecognizedText(windowTitle)) + ) } catch let error as ScreenTextExtractionError { throw ScreenshotContextGenerationError.unavailable(error.localizedDescription) } catch { @@ -88,11 +92,6 @@ final class ScreenshotContextGenerator { CotabbyLogger.app.debug("OCR extracted \(normalizedText.count) chars from screenshot") guard hasMeaningfulSignal(normalizedText) else { - if let titleExcerpt = fallbackExcerpt(from: screenshot.windowTitle) { - CotabbyLogger.app.debug("OCR text was too sparse; using window title as visual context fallback") - return titleExcerpt - } - throw ScreenshotContextGenerationError.unavailable( "The screenshot did not contain enough visible text to build prompt context." ) @@ -128,20 +127,6 @@ final class ScreenshotContextGenerator { ) } - /// A tiny or visually blank crop can still identify the surrounding document through the window - /// title. This keeps visual context best-effort instead of making a sparse OCR result look fatal. - private func fallbackExcerpt(from windowTitle: String?) -> VisualContextExcerpt? { - guard let windowTitle, - hasMeaningfulSignal(windowTitle) - else { - return nil - } - - return VisualContextExcerpt( - text: boundedSummaryText(normalizeRecognizedText(windowTitle)) - ) - } - private func captureScreenshot( for context: FocusedInputSnapshot, onStatusChange: (@Sendable (VisualContextStatus) async -> Void)? diff --git a/Cotabby/Services/Visual/VisualContextCoordinator.swift b/Cotabby/Services/Visual/VisualContextCoordinator.swift index 546ba14c..7fb5742a 100644 --- a/Cotabby/Services/Visual/VisualContextCoordinator.swift +++ b/Cotabby/Services/Visual/VisualContextCoordinator.swift @@ -158,7 +158,7 @@ final class VisualContextCoordinator { CotabbyLogger.app.debug("Visual context generation cancelled") return } catch let error as ScreenshotContextGenerationError { - logGenerationError(error) + CotabbyLogger.app.warning("Visual context generation error: \(error.localizedDescription)") setStatus(errorStatus(for: error), for: session.sessionID) } catch { CotabbyLogger.app.error("Visual context generation failed: \(error.localizedDescription)") @@ -243,17 +243,6 @@ final class VisualContextCoordinator { } } - /// `.unavailable` means the optional screenshot/OCR add-on had no useful signal. That should be - /// visible in diagnostics, but it is not the same severity as a broken capture or OCR pipeline. - private func logGenerationError(_ error: ScreenshotContextGenerationError) { - switch error { - case .unavailable(let message): - CotabbyLogger.app.debug("Visual context unavailable: \(message)") - case .failed(let message): - CotabbyLogger.app.warning("Visual context generation error: \(message)") - } - } - private func publishState() { onStateChange?(status, latestExcerpt) } diff --git a/Cotabby/Services/Visual/WindowScreenshotService.swift b/Cotabby/Services/Visual/WindowScreenshotService.swift index c439951c..d4b15a37 100644 --- a/Cotabby/Services/Visual/WindowScreenshotService.swift +++ b/Cotabby/Services/Visual/WindowScreenshotService.swift @@ -60,17 +60,13 @@ struct WindowScreenshotService { } let shareableContent = try await currentShareableContent() - let processWindows = shareableContent.windows.filter { - $0.owningApplication?.processID == processIdentifier && $0.isOnScreen - } - let substantialWindows = processWindows.filter { isSubstantialWindow($0) } let matchingWindow = - bestWindow(for: context, in: substantialWindows) - ?? substantialWindows.first(where: { $0.isActive }) - ?? substantialWindows.first - ?? bestWindow(for: context, in: processWindows) - ?? processWindows.first(where: { $0.isActive }) - ?? processWindows.first + shareableContent.windows.first(where: { + $0.owningApplication?.processID == processIdentifier && $0.isActive && $0.isOnScreen + }) + ?? shareableContent.windows.first(where: { + $0.owningApplication?.processID == processIdentifier && $0.isOnScreen + }) guard let matchingWindow else { CotabbyLogger.app.debug("No visible window for pid \(processIdentifier)") @@ -103,108 +99,10 @@ struct WindowScreenshotService { configuration.height = max(Int((localSourceRect.height * outputScale).rounded(.up)), 1) configuration.showsCursor = false - let inputFrameCG = context.inputFrameRect.map { convertBetweenAppKitAndCG(rect: $0) } - let caretRectCG = convertBetweenAppKitAndCG(rect: context.caretRect) - let windowFrameDescription = formatRect(matchingWindow.frame) - let inputDescription = formatOptionalRect(inputFrameCG) - let caretDescription = formatRect(caretRectCG) - let sourceDescription = formatRect(sourceRect) - let localSourceDescription = formatRect(localSourceRect) - let outputDescription = "\(configuration.width)x\(configuration.height)" - let captureGeometryMessage = [ - "Capture geometry:", - "windows=\(processWindows.count)", - "substantial=\(substantialWindows.count)", - "selected=\"\(windowTitle)\"", - "active=\(matchingWindow.isActive)", - "window=\(windowFrameDescription)", - "input=\(inputDescription)", - "caret=\(caretDescription)", - "source=\(sourceDescription)", - "local=\(localSourceDescription)", - "output=\(outputDescription)" - ].joined(separator: " ") - CotabbyLogger.app.debug("\(captureGeometryMessage)") - if !isSubstantialWindow(matchingWindow) { - let warningMessage = "Capture fell back to tiny window: title=\"\(windowTitle)\" frame=\(windowFrameDescription)" - CotabbyLogger.app.warning("\(warningMessage)") - } - let image = try await captureImage(filter: filter, configuration: configuration) return CapturedWindowScreenshot(image: image, windowTitle: matchingWindow.title) } - /// Browsers expose tiny toolbar/popover surfaces through ScreenCaptureKit. Those surfaces can - /// intersect stale or toolbar-shaped AX geometry, but they cannot contain useful page OCR context. - private func isSubstantialWindow(_ window: SCWindow) -> Bool { - window.frame.width >= 240 && window.frame.height >= 240 - } - - /// Chrome and Electron can expose multiple ScreenCaptureKit windows for one process, including - /// tiny auxiliary surfaces. The Accessibility caret/input rect is our strongest signal for - /// which window actually owns the focused editor, so prefer a window containing that geometry - /// before falling back to "active" ordering. - private func bestWindow( - for context: FocusedInputSnapshot, - in windows: [SCWindow] - ) -> SCWindow? { - let inputFrameCG = context.inputFrameRect.map { convertBetweenAppKitAndCG(rect: $0) } - let caretRectCG = convertBetweenAppKitAndCG(rect: context.caretRect) - let anchorPoint = CGPoint( - x: (inputFrameCG ?? caretRectCG).midX, - y: (inputFrameCG ?? caretRectCG).midY - ) - - return windows - .filter { window in - window.frame.contains(anchorPoint) - || inputFrameCG.map { window.frame.intersects($0) } == true - || window.frame.intersects(caretRectCG) - } - .sorted { lhs, rhs in - if lhs.isActive != rhs.isActive { - return lhs.isActive - } - - return visibleAreaScore(lhs, inputFrameCG: inputFrameCG, caretRectCG: caretRectCG) - > visibleAreaScore(rhs, inputFrameCG: inputFrameCG, caretRectCG: caretRectCG) - } - .first - } - - private func visibleAreaScore( - _ window: SCWindow, - inputFrameCG: CGRect?, - caretRectCG: CGRect - ) -> CGFloat { - let inputArea = inputFrameCG.map { intersectionArea(window.frame, $0) } ?? 0 - let caretArea = intersectionArea(window.frame, caretRectCG) - return max(inputArea, caretArea) - } - - private func intersectionArea(_ lhs: CGRect, _ rhs: CGRect) -> CGFloat { - let intersection = lhs.intersection(rhs) - guard !intersection.isNull, !intersection.isEmpty else { - return 0 - } - - return intersection.width * intersection.height - } - - private func formatOptionalRect(_ rect: CGRect?) -> String { - rect.map(formatRect) ?? "nil" - } - - private func formatRect(_ rect: CGRect) -> String { - String( - format: "(x=%.0f,y=%.0f,w=%.0f,h=%.0f)", - rect.origin.x, - rect.origin.y, - rect.width, - rect.height - ) - } - private func snapshotRect( around context: FocusedInputSnapshot, windowFrame: CGRect, diff --git a/Cotabby/Support/AXHelper.swift b/Cotabby/Support/AXHelper.swift index 82680d61..1b0f62e3 100644 --- a/Cotabby/Support/AXHelper.swift +++ b/Cotabby/Support/AXHelper.swift @@ -213,33 +213,6 @@ enum AXHelper { return unsafeBitCast(element, to: AXUIElement.self) } - /// Creates the AX application root for one running process. - /// - /// Browser focus bugs are often easier to diagnose from the app root than from the system-wide - /// focused element because Chromium can briefly report stale toolbar focus through the global API. - static func applicationElement(processIdentifier: pid_t) -> AXUIElement { - AXUIElementCreateApplication(processIdentifier) - } - - /// Reads an AXUIElement-valued attribute, such as `AXFocusedUIElement` or `AXFocusedWindow`. - static func uiElementValue(for attribute: CFString, on element: AXUIElement) -> AXUIElement? { - guard let value = copyAttributeValue(attribute, on: element) else { - return nil - } - - guard CFGetTypeID(value) == AXUIElementGetTypeID() else { - return nil - } - - // Same Core Foundation bridging rule as `focusedElement()`. - return unsafeBitCast(value, to: AXUIElement.self) - } - - /// Returns the focused element reported by a specific app's AX root. - static func focusedElement(inApplication applicationElement: AXUIElement) -> AXUIElement? { - uiElementValue(for: kAXFocusedUIElementAttribute as CFString, on: applicationElement) - } - /// Returns the running application that owns the given AX element. /// /// This matters for accessory apps (Raycast, Spotlight, Alfred) that show non-activating diff --git a/Cotabby/UI/DownloadableModelCatalogView.swift b/Cotabby/UI/DownloadableModelCatalogView.swift index b0765ac5..04f96f04 100644 --- a/Cotabby/UI/DownloadableModelCatalogView.swift +++ b/Cotabby/UI/DownloadableModelCatalogView.swift @@ -140,13 +140,8 @@ private struct DownloadableModelRow: View { .foregroundStyle(.blue) .frame(width: 40, alignment: .trailing) } else { - // A static glyph, not an indeterminate `ProgressView`. Indeterminate spinners - // animate continuously via a detached display-link/NSAnimation loop that can - // keep running (and pegging CPU) even after this window closes, and onboarding - // shows several of these at once. The "…" reads as "starting" without spinning. - Image(systemName: "ellipsis") - .font(.system(size: 11, weight: .medium)) - .foregroundStyle(.blue) + ProgressView() + .controlSize(.small) .frame(width: 40) } // Plain SF Symbol button keeps the row compact and matches @@ -212,11 +207,14 @@ private struct DownloadableModelRow: View { @ViewBuilder private var downloadProgressBar: some View { - // Always a determinate bar: it sits at 0 while the progress fraction is still unknown and - // fills as bytes arrive. An indeterminate linear `ProgressView` animates a moving stripe - // forever, which keeps a CoreAnimation loop pinned even when the window is closed. - ProgressView(value: state.progressFraction ?? 0, total: 1) - .progressViewStyle(.linear) - .tint(.blue) + if let progress = state.progressFraction { + ProgressView(value: progress, total: 1) + .progressViewStyle(.linear) + .tint(.blue) + } else { + ProgressView() + .progressViewStyle(.linear) + .tint(.blue) + } } } diff --git a/Cotabby/UI/HuggingFaceModelBrowserView.swift b/Cotabby/UI/HuggingFaceModelBrowserView.swift index 9823beb0..5bb88b3d 100644 --- a/Cotabby/UI/HuggingFaceModelBrowserView.swift +++ b/Cotabby/UI/HuggingFaceModelBrowserView.swift @@ -44,11 +44,13 @@ struct HuggingFaceModelBrowserView: View { EmptyView() case .searching: - // Plain label rather than an indeterminate spinner: the spinner animates continuously - // and can leak its animation loop after the window closes. The "…" already reads as busy. - Text("Searching…") - .font(.caption) - .foregroundStyle(.secondary) + HStack(spacing: 6) { + ProgressView() + .controlSize(.small) + Text("Searching…") + .font(.caption) + .foregroundStyle(.secondary) + } case .noResults: Text("No GGUF models found.") @@ -83,8 +85,12 @@ struct HuggingFaceModelBrowserView: View { searchService.loadMore() } label: { if searchService.isLoadingMore { - Text("Loading…") - .font(.caption) + HStack(spacing: 6) { + ProgressView() + .controlSize(.small) + Text("Loading…") + .font(.caption) + } } else { Text("Load More") .font(.caption) @@ -104,11 +110,14 @@ struct HuggingFaceModelBrowserView: View { if isRepoSelected(repoId) { switch searchService.detailState { case .loading: - // Plain label rather than an indeterminate spinner; see `.searching` above. - Text("Loading files…") - .font(.caption) - .foregroundStyle(.secondary) - .padding(.leading, 16) + HStack(spacing: 6) { + ProgressView() + .controlSize(.small) + Text("Loading files…") + .font(.caption) + .foregroundStyle(.secondary) + } + .padding(.leading, 16) .padding(.top, 4) case .failed(let message): @@ -226,10 +235,12 @@ private struct HFFileRow: View { actionButton } - if state.isDownloading { - // Determinate bar pinned at 0 until the fraction is known. An indeterminate linear - // `ProgressView` animates forever and leaks that animation loop past window close. - ProgressView(value: state.progressFraction ?? 0, total: 1) + if state.isDownloading, let progress = state.progressFraction { + ProgressView(value: progress, total: 1) + .progressViewStyle(.linear) + .tint(.blue) + } else if state.isDownloading { + ProgressView() .progressViewStyle(.linear) .tint(.blue) } @@ -263,10 +274,8 @@ private struct HFFileRow: View { .foregroundStyle(.blue) .frame(width: 40, alignment: .trailing) } else { - // Static glyph instead of an indeterminate spinner; see DownloadableModelCatalogView. - Image(systemName: "ellipsis") - .font(.system(size: 11, weight: .medium)) - .foregroundStyle(.blue) + ProgressView() + .controlSize(.small) .frame(width: 40) } Button { diff --git a/CotabbyTests/AXTextGeometryResolverTests.swift b/CotabbyTests/AXTextGeometryResolverTests.swift index b8c93702..6fd8458a 100644 --- a/CotabbyTests/AXTextGeometryResolverTests.swift +++ b/CotabbyTests/AXTextGeometryResolverTests.swift @@ -146,66 +146,4 @@ final class AXTextGeometryResolverTests: XCTestCase { XCTAssertTrue(resolver.rectIsNearAnchor(rect, anchor: nil)) XCTAssertTrue(resolver.rectIsNearAnchor(rect, anchor: .zero)) } - - // MARK: - Browser range-union guards - - func test_looksLikeOversizedSingleCharacterRange_rejectsWrappedRunUnion() { - let anchor = CGRect(x: 1206, y: 1177, width: 568, height: 416) - let chromeWrappedRun = CGRect(x: 1206, y: 1557, width: 533, height: 34) - - XCTAssertTrue( - resolver.looksLikeOversizedSingleCharacterRange(chromeWrappedRun, anchor: anchor) - ) - } - - func test_looksLikeOversizedSingleCharacterRange_acceptsNormalCharacterBounds() { - let anchor = CGRect(x: 1206, y: 1177, width: 568, height: 416) - let normalCharacterBounds = CGRect(x: 1206, y: 1537, width: 4, height: 15) - - XCTAssertFalse( - resolver.looksLikeOversizedSingleCharacterRange(normalCharacterBounds, anchor: anchor) - ) - } - - func test_looksLikeStaleLineStartRange_rejectsLeftEdgeRangeInsideLogicalLine() { - let anchor = CGRect(x: 576, y: 1164, width: 1258, height: 581) - let staleOutlookRange = CGRect(x: 574, y: 1673, width: 2, height: 18) - - XCTAssertTrue( - resolver.looksLikeStaleLineStartRange( - staleOutlookRange, - anchor: anchor, - text: "Hello from Outlook", - selection: NSRange(location: 8, length: 0) - ) - ) - } - - func test_looksLikeStaleLineStartRange_acceptsTrueLogicalLineStart() { - let anchor = CGRect(x: 576, y: 1164, width: 1258, height: 581) - let lineStartRange = CGRect(x: 576, y: 1673, width: 2, height: 18) - - XCTAssertFalse( - resolver.looksLikeStaleLineStartRange( - lineStartRange, - anchor: anchor, - text: "Hello\n", - selection: NSRange(location: 6, length: 0) - ) - ) - } - - func test_looksLikeStaleLineStartRange_acceptsRangeAwayFromLeftEdge() { - let anchor = CGRect(x: 576, y: 1164, width: 1258, height: 581) - let normalRange = CGRect(x: 650, y: 1673, width: 8, height: 18) - - XCTAssertFalse( - resolver.looksLikeStaleLineStartRange( - normalRange, - anchor: anchor, - text: "Hello from Outlook", - selection: NSRange(location: 8, length: 0) - ) - ) - } } diff --git a/CotabbyTests/CaretGeometrySourceCacheTests.swift b/CotabbyTests/CaretGeometrySourceCacheTests.swift deleted file mode 100644 index 23bc8577..00000000 --- a/CotabbyTests/CaretGeometrySourceCacheTests.swift +++ /dev/null @@ -1,76 +0,0 @@ -import ApplicationServices -import XCTest -@testable import Cotabby - -/// Tests for `CaretGeometrySourceCache` key matching and invalidation. -/// -/// The cache stores live `AXUIElement` references, so these tests use a real system-wide element as -/// an opaque stand-in — the behavior under test is the per-field keying and eviction, not the AX -/// content of the elements themselves. -@MainActor -final class CaretGeometrySourceCacheTests: XCTestCase { - private let element = AXUIElementCreateSystemWide() - - private func key(_ identifier: String, _ sequence: UInt64) -> CaretGeometrySourceCache.FieldKey { - CaretGeometrySourceCache.FieldKey(containerIdentifier: identifier, focusChangeSequence: sequence) - } - - func testColdCacheReturnsNil() { - let cache = CaretGeometrySourceCache() - XCTAssertNil(cache.textRunElements(for: key("field", 1))) - XCTAssertNil(cache.deepSource(for: key("field", 1))) - } - - func testRunElementsHitOnMatchingKey() { - let cache = CaretGeometrySourceCache() - cache.store(textRunElements: [element], for: key("field", 1)) - XCTAssertEqual(cache.textRunElements(for: key("field", 1))?.count, 1) - } - - func testRunElementsMissOnDifferentSequence() { - let cache = CaretGeometrySourceCache() - cache.store(textRunElements: [element], for: key("field", 1)) - // A focus change bumps the sequence, which must invalidate the previous field's entry even - // though the container identifier (a CFHash) could collide across recycled nodes. - XCTAssertNil(cache.textRunElements(for: key("field", 2))) - } - - func testStoringNewKeyEvictsPrevious() { - let cache = CaretGeometrySourceCache() - cache.store(textRunElements: [element], for: key("a", 1)) - cache.store(textRunElements: [element], for: key("b", 1)) - XCTAssertNil(cache.textRunElements(for: key("a", 1))) - XCTAssertEqual(cache.textRunElements(for: key("b", 1))?.count, 1) - } - - func testStoringDeepSourceNewKeyEvictsPrevious() { - let cache = CaretGeometrySourceCache() - cache.store(deepSource: element, for: key("a", 1)) - cache.store(deepSource: element, for: key("b", 1)) - // The deep-source slot shares the run slot's one-entry contract: a new key drops the old one. - XCTAssertNil(cache.deepSource(for: key("a", 1))) - XCTAssertNotNil(cache.deepSource(for: key("b", 1))) - } - - func testRunAndDeepEntriesAreIndependent() { - let cache = CaretGeometrySourceCache() - cache.store(textRunElements: [element], for: key("field", 1)) - // Caching runs must not imply a deep-source entry, and vice versa. - XCTAssertNil(cache.deepSource(for: key("field", 1))) - - cache.store(deepSource: element, for: key("field", 1)) - XCTAssertNotNil(cache.deepSource(for: key("field", 1))) - XCTAssertEqual(cache.textRunElements(for: key("field", 1))?.count, 1) - } - - func testInvalidateClearsBoth() { - let cache = CaretGeometrySourceCache() - cache.store(textRunElements: [element], for: key("field", 1)) - cache.store(deepSource: element, for: key("field", 1)) - - cache.invalidate() - - XCTAssertNil(cache.textRunElements(for: key("field", 1))) - XCTAssertNil(cache.deepSource(for: key("field", 1))) - } -} diff --git a/CotabbyTests/FocusSnapshotResolverSelectionTests.swift b/CotabbyTests/FocusSnapshotResolverSelectionTests.swift new file mode 100644 index 00000000..64e21ae3 --- /dev/null +++ b/CotabbyTests/FocusSnapshotResolverSelectionTests.swift @@ -0,0 +1,91 @@ +import CoreGraphics +import XCTest +@testable import Cotabby + +/// Tests the pure caret-geometry trust policy used by `FocusSnapshotResolver`. +/// +/// These tests intentionally avoid live Accessibility objects. The regression we are guarding +/// against is not whether AX can produce a rect; it is whether Cotabby trusts a descendant rect over +/// the focused input's own usable rect. +final class FocusSnapshotResolverSelectionTests: XCTestCase { + private let primaryRect = CGRect(x: 10, y: 20, width: 2, height: 16) + private let deepRect = CGRect(x: 100, y: 120, width: 2, height: 16) + + func testShouldSearchDeepOnlyForWeakPrimaryGeometry() { + XCTAssertFalse(CaretGeometrySelector.shouldSearchDeep( + primaryRect: primaryRect, + primaryQuality: .exact + )) + XCTAssertFalse(CaretGeometrySelector.shouldSearchDeep( + primaryRect: primaryRect, + primaryQuality: .derived + )) + XCTAssertTrue(CaretGeometrySelector.shouldSearchDeep( + primaryRect: primaryRect, + primaryQuality: .estimated + )) + XCTAssertTrue(CaretGeometrySelector.shouldSearchDeep( + primaryRect: primaryRect, + primaryQuality: nil + )) + XCTAssertTrue(CaretGeometrySelector.shouldSearchDeep( + primaryRect: nil, + primaryQuality: .derived + )) + } + + func testPrimaryExactWinsOverDeepExact() throws { + let selected = try XCTUnwrap(CaretGeometrySelector.select( + primaryRect: primaryRect, + primaryQuality: .exact, + primaryObservedCharWidth: 7, + deepResult: CaretGeometryResult(rect: deepRect, quality: .exact, observedCharWidth: 4) + )) + + XCTAssertEqual(selected.rect, primaryRect) + XCTAssertEqual(selected.quality, .exact) + XCTAssertEqual(selected.source, "exact primary") + XCTAssertEqual(selected.observedCharWidth, 7) + } + + func testPrimaryDerivedWinsOverDeepExact() throws { + let selected = try XCTUnwrap(CaretGeometrySelector.select( + primaryRect: primaryRect, + primaryQuality: .derived, + primaryObservedCharWidth: 8, + deepResult: CaretGeometryResult(rect: deepRect, quality: .exact, observedCharWidth: 3) + )) + + XCTAssertEqual(selected.rect, primaryRect) + XCTAssertEqual(selected.quality, .derived) + XCTAssertEqual(selected.source, "derived primary") + XCTAssertEqual(selected.observedCharWidth, 8) + } + + func testDeepExactWinsWhenPrimaryIsOnlyEstimated() throws { + let selected = try XCTUnwrap(CaretGeometrySelector.select( + primaryRect: primaryRect, + primaryQuality: .estimated, + primaryObservedCharWidth: nil, + deepResult: CaretGeometryResult(rect: deepRect, quality: .exact, observedCharWidth: 5) + )) + + XCTAssertEqual(selected.rect, deepRect) + XCTAssertEqual(selected.quality, .exact) + XCTAssertEqual(selected.source, "exact deep") + XCTAssertEqual(selected.observedCharWidth, 5) + } + + func testPrimaryFallbackStillWorksWithoutDeepGeometry() throws { + let selected = try XCTUnwrap(CaretGeometrySelector.select( + primaryRect: primaryRect, + primaryQuality: .estimated, + primaryObservedCharWidth: nil, + deepResult: nil + )) + + XCTAssertEqual(selected.rect, primaryRect) + XCTAssertEqual(selected.quality, .estimated) + XCTAssertEqual(selected.source, "estimated primary-fallback") + } +}