diff --git a/Cotabby.xcodeproj/project.pbxproj b/Cotabby.xcodeproj/project.pbxproj index cfc7030c..300e6ad9 100644 --- a/Cotabby.xcodeproj/project.pbxproj +++ b/Cotabby.xcodeproj/project.pbxproj @@ -330,6 +330,7 @@ 84A4CA05AF6885AE4FA4C13A /* SettingsAttentionEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A02336442BB735EE2E8D064 /* SettingsAttentionEvaluator.swift */; }; 862146ABDADC022A3BE74E00 /* CurrencyEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0537986794554F5FABE6EFF3 /* CurrencyEvaluator.swift */; }; 865C569A9BC95B08F440D199 /* SystemResourceSampler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 27A5D63F390E9B7A7FE343FE /* SystemResourceSampler.swift */; }; + 86AC625B4DD14EF807002FA2 /* WebContentFieldDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */; }; 87806DE08881D11F2608A13D /* MarkerSelectionSynthesizerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 52BAFA2F989C3C4F7FB892B5 /* MarkerSelectionSynthesizerTests.swift */; }; 8865B95FE84198D70390DF80 /* ClipboardContentDistillerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD89799BB82AF7A92559AEB /* ClipboardContentDistillerTests.swift */; }; 88BCD795A14E1C9308F7BB31 /* SuggestionAvailabilityEvaluatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C05B0439348261163B37C508 /* SuggestionAvailabilityEvaluatorTests.swift */; }; @@ -501,6 +502,7 @@ D603EC842D9D7A1A81899050 /* PermissionModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D82FFC568527700EC17C07D /* PermissionModels.swift */; }; D648DD70AD847F67B77CE052 /* OnboardingTemplateRecommenderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01B72736E416910878E8E493 /* OnboardingTemplateRecommenderTests.swift */; }; D6AD25168F108DA8D60E76EF /* SpellingDictionaryPicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF3A73EB848780061FC162C0 /* SpellingDictionaryPicker.swift */; }; + D75626719A38F535D475C675 /* WebContentFieldDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */; }; D800277BE3296DE2FB8198A8 /* ChromiumAccessibilityEnabler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8896D976C7F116EBA0F3969F /* ChromiumAccessibilityEnabler.swift */; }; D80D98F6C7B86AAF8C831865 /* he-100k.txt in Resources */ = {isa = PBXBuildFile; fileRef = 7C9BB65FA5FC42B89766B037 /* he-100k.txt */; }; D81AC2294A64E7A95B29E081 /* fr.txt in Resources */ = {isa = PBXBuildFile; fileRef = 5DCA3E46621815A19300365F /* fr.txt */; }; @@ -531,6 +533,7 @@ E46F50AEDA8FE13B02E3FA8D /* AXHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC70775535A3428991025AB8 /* AXHelper.swift */; }; E51FA12B690428CA431328FC /* WritingPaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D48B95B6665109B6C6A63B42 /* WritingPaneView.swift */; }; E54F5F03E16859D5A1E3437A /* MacroController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4638C74239D1DE2DC4D87975 /* MacroController.swift */; }; + E5CB34ED76BAE87E8A858112 /* WebContentFieldDetectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 210F9AD332273FE2EB3A9A01 /* WebContentFieldDetectorTests.swift */; }; E6EE3C13FA31F261CD734C69 /* DownloadOutcomeClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3DE1975F3B5F4A70478DBF41 /* DownloadOutcomeClassifier.swift */; }; E853B9C7AF93FA595DC417B2 /* EmojiVariantResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A8414BEB7E34F57607E37FE /* EmojiVariantResolver.swift */; }; E912D4617AE1376061DF1F00 /* LanguageSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4793D4EA5D36D7E5CC216C27 /* LanguageSupportTests.swift */; }; @@ -640,6 +643,7 @@ 1ED1EA9282E0AC7592E60889 /* SuggestionCoordinator+Prediction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SuggestionCoordinator+Prediction.swift"; sourceTree = ""; }; 1F761083EA5465023D82B5F4 /* BrowserDomainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowserDomainTests.swift; sourceTree = ""; }; 1F7ED2D9D71CFAD9A30977E9 /* ArithmeticEvaluator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ArithmeticEvaluator.swift; sourceTree = ""; }; + 210F9AD332273FE2EB3A9A01 /* WebContentFieldDetectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebContentFieldDetectorTests.swift; sourceTree = ""; }; 21CB3008986BE7FD2A4D9132 /* WelcomeCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WelcomeCoordinator.swift; sourceTree = ""; }; 224438039A86E5619294EAF7 /* EmojiUsageStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmojiUsageStoreTests.swift; sourceTree = ""; }; 22544F4B756E3E4144497D17 /* SuggestionCoordinator+Input.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SuggestionCoordinator+Input.swift"; sourceTree = ""; }; @@ -847,6 +851,7 @@ C1C5DE0F3FF63545000E2453 /* DisplayCoordinateConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DisplayCoordinateConverterTests.swift; sourceTree = ""; }; C375227649689775275AA4B3 /* SuggestionCoordinatorAcceptanceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionCoordinatorAcceptanceTests.swift; sourceTree = ""; }; C379D77029D6E88C8C1B9AF7 /* emoji.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = emoji.json; sourceTree = ""; }; + C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebContentFieldDetector.swift; sourceTree = ""; }; C648EBB10D7F8E0B904DEC91 /* de.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = de.txt; sourceTree = ""; }; C71031E8DB171047318B92FC /* SyntheticReplacePlannerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyntheticReplacePlannerTests.swift; sourceTree = ""; }; C727BF6FF8ACAAED30B0329F /* TypoGateTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypoGateTests.swift; sourceTree = ""; }; @@ -1312,6 +1317,7 @@ D2D0FE44138BCA8B2EE05AFE /* TypoCaseTransferTests.swift */, C727BF6FF8ACAAED30B0329F /* TypoGateTests.swift */, 050D929E13BE52E6282B64D2 /* VisualContextStartCoalescerTests.swift */, + 210F9AD332273FE2EB3A9A01 /* WebContentFieldDetectorTests.swift */, 1E0513E3B23937B099A3CFF2 /* WordCountFormatterTests.swift */, ); path = CotabbyTests; @@ -1506,6 +1512,7 @@ 08CE63B8725EBD71A4C024E1 /* TypoCaseTransfer.swift */, B8412FE2BAC406421248A03B /* TypoGate.swift */, 2F01FAC4F57EB08471521196 /* VisualContextStartCoalescer.swift */, + C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */, 815F2ABAF6AB75DA3AFBBCEF /* WordCountFormatter.swift */, ); path = Support; @@ -1911,6 +1918,7 @@ 0A15FAD03A93D57372D267E0 /* VisualContextCoordinator.swift in Sources */, F77DB394E9D6C6C482131BF9 /* VisualContextModels.swift in Sources */, 641A9FAF3009A3E2AA06D74B /* VisualContextStartCoalescer.swift in Sources */, + 86AC625B4DD14EF807002FA2 /* WebContentFieldDetector.swift in Sources */, 3AB45217DFC86AFC98C374D6 /* WelcomeCoordinator.swift in Sources */, 70D6F9480DA4104AD5669569 /* WelcomePermissionStepView.swift in Sources */, 4FEA3AC7ABB1982063BC0041 /* WelcomeTemplateStepView.swift in Sources */, @@ -2131,6 +2139,7 @@ E9E4CC657771DF9F4C56183C /* VisualContextCoordinator.swift in Sources */, 4190F8A76196B16ED94D0A55 /* VisualContextModels.swift in Sources */, 19CB55B62977376E9AE8D428 /* VisualContextStartCoalescer.swift in Sources */, + D75626719A38F535D475C675 /* WebContentFieldDetector.swift in Sources */, 4AC255BE2D0CCC67B8882C7A /* WelcomeCoordinator.swift in Sources */, 344B9BF352C97CFA830853D6 /* WelcomePermissionStepView.swift in Sources */, 286B7022E2A2774275004447 /* WelcomeTemplateStepView.swift in Sources */, @@ -2240,6 +2249,7 @@ 5C6B59C2E56A3C4260591095 /* TypoCaseTransferTests.swift in Sources */, BE688EC1957B4AE004063EFE /* TypoGateTests.swift in Sources */, D5CAF3B590E5EC2AFC72E57A /* VisualContextStartCoalescerTests.swift in Sources */, + E5CB34ED76BAE87E8A858112 /* WebContentFieldDetectorTests.swift in Sources */, 6AE0B46FB52D189D94E1F79A /* WordCountFormatterTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/Cotabby/App/Coordinators/SuggestionCoordinator+Acceptance.swift b/Cotabby/App/Coordinators/SuggestionCoordinator+Acceptance.swift index 4b815e45..9a7d6207 100644 --- a/Cotabby/App/Coordinators/SuggestionCoordinator+Acceptance.swift +++ b/Cotabby/App/Coordinators/SuggestionCoordinator+Acceptance.swift @@ -629,14 +629,21 @@ extension SuggestionCoordinator { /// Repairs untrustworthy caret anchors with a hidden-text-layout estimate before presentation. /// - /// Two repair modes, by resolver quality: + /// The repair's authority is scoped by where the AX geometry came from, not just its quality: /// - `.estimated` (AXFrame-only hosts): the AX guess has no real position at all, so a - /// passing estimate always replaces it. + /// passing estimate always replaces it. Applies to web and native hosts alike, because + /// there is no real measurement to protect. + /// - `.derived` in a native host: never repaired, the estimator does not run. Native AX + /// per-character bounds come from the app's real layout manager and reflect rendering the + /// uniform hidden layout cannot model (Notes renders a 23pt title line above 16pt body + /// lines, plus paragraph spacing, so the layout "disagrees" within a few lines while AX is + /// exactly right). A vertical mismatch there indicts the estimate, not AX. /// - `.derived` with measured run frames (Gmail/Outlook child-run hosts): the rect's Y is a /// real rendered line, so it is always kept and the estimator does not run at all; the /// layout estimate cannot beat measurement (and is blind to blank lines those hosts /// collapse out of the AX text). - /// - `.derived` without run frames (previous-character bounds): the estimate (calibrated + /// - `.derived` web content without run frames (previous-character bounds through a web + /// engine's AX bridge, which has known wrong-line pathologies): the estimate (calibrated /// with whatever the host revealed) only overrides the AX rect when the two disagree /// vertically by more than `verticallyAgrees` tolerates; on agreement the AX rect is kept. /// @@ -659,19 +666,34 @@ extension SuggestionCoordinator { ) -> LayoutRepairedAnchor { let quality = context.caretQuality guard quality == .estimated || quality == .derived else { - return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: nil) + return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: nil, skipReason: nil) } - // Run-measured derived rects are kept unconditionally — run frames carry the host's real - // line positions, including blank lines some hosts omit from the AX text — so skip the - // estimator entirely instead of computing a diagnostic it can never act on. This path runs + // Derived rects carry a real AX measurement, so whether the estimate may second-guess + // them depends on who produced the measurement. Both bypasses skip the estimator + // entirely rather than computing a diagnostic they can never act on: this path runs // inside the accept keystroke's handling window, where every spent millisecond of layout // work on a large flat prefix is pure risk during a rapid Tab burst. - if quality == .derived, context.observedContentEdges != nil { - return LayoutRepairedAnchor(rect: fallbackRect, quality: .derived, outcome: nil) + if quality == .derived { + // Native hosts: per-character bounds come from the app's own layout manager and are + // ground truth, while the uniform hidden layout cannot model rich text (Notes' taller + // title line and paragraph spacing put it a line off within a few paragraphs). Only + // web engines' AX bridges have the wrong-line pathologies the override exists for. + if !context.isWebContentField { + return LayoutRepairedAnchor( + rect: fallbackRect, quality: .derived, outcome: nil, skipReason: .nativeHostGeometry + ) + } + // Run-measured derived rects are kept unconditionally: run frames carry the host's + // real line positions, including blank lines some hosts omit from the AX text. + if context.observedContentEdges != nil { + return LayoutRepairedAnchor( + rect: fallbackRect, quality: .derived, outcome: nil, skipReason: .runMeasuredGeometry + ) + } } - // A `.derived` rect's height is a real rendered line box (child-run frame height); the + // A `.derived` rect's height is a real rendered line box (previous-character bounds); the // `.estimated` AXFrame fallback's height is the whole field, which the estimator's // sanitizer would discard anyway — pass it for derived geometry only. let observedLineHeight: CGFloat? = quality == .derived ? context.caretRect.height : nil @@ -694,11 +716,13 @@ extension SuggestionCoordinator { case .estimate(let estimate): if quality == .derived, verticallyAgrees(estimate: estimate, axRect: fallbackRect) { // Same line: keep the AX rect, whose X carries the host's real glyph positions. - return LayoutRepairedAnchor(rect: fallbackRect, quality: .derived, outcome: outcome) + return LayoutRepairedAnchor(rect: fallbackRect, quality: .derived, outcome: outcome, skipReason: nil) } - return LayoutRepairedAnchor(rect: estimate.caretRect, quality: .layoutEstimated, outcome: outcome) + return LayoutRepairedAnchor( + rect: estimate.caretRect, quality: .layoutEstimated, outcome: outcome, skipReason: nil + ) case .rejected: - return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: outcome) + return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: outcome, skipReason: nil) } } @@ -720,6 +744,20 @@ extension SuggestionCoordinator { let rect: CGRect let quality: CaretGeometryQuality let outcome: TextLayoutCaretEstimator.Outcome? + /// Why a derived rect bypassed the estimator without running it (nil when the estimator + /// ran, or when repair was out of scope for the quality entirely). Logged so "the caret + /// came from trusted AX and repair stood down" is distinguishable from "this field never + /// triggered repair" when diagnosing a misplaced overlay from the JSONL stream. + let skipReason: LayoutRepairSkipReason? + } + + /// Why `layoutRepairedAnchor` kept a derived AX rect without running the estimator at all. + /// Raw values feed the structured log stream. + enum LayoutRepairSkipReason: String { + /// Native (non-web) host: AX-derived geometry is ground truth for the trust policy. + case nativeHostGeometry = "native_host_geometry" + /// The rect was measured from child text-run frames, which outrank any estimate. + case runMeasuredGeometry = "run_measured_geometry" } /// Mirrors the repair outcome into the structured JSONL stream so a misplaced overlay can be @@ -734,7 +772,9 @@ extension SuggestionCoordinator { fallbackRect: CGRect, context: FocusedInputContext ) { - guard let outcome = anchor.outcome else { + // Exact/layout-estimated presentations never reach repair and carry nothing to log; bail + // before building metadata so the common path pays nothing per present. + guard anchor.outcome != nil || anchor.skipReason != nil else { return } var metadata: Logger.Metadata = [ @@ -750,13 +790,24 @@ extension SuggestionCoordinator { if let fieldFrame = context.inputFrameRect { metadata["field_top_y"] = .stringConvertible(Double(fieldFrame.maxY)) } + guard let outcome = anchor.outcome else { + // A trusted-AX bypass stood the estimator down without running it. Logged (unlike the + // qualities repair never applies to) so the stream can distinguish "repair deferred to + // trusted AX" from "this presentation never reached repair". + if let skipReason = anchor.skipReason { + metadata["repair_outcome"] = .string("skipped") + metadata["skip_reason"] = .string(skipReason.rawValue) + CotabbyLogger.suggestion.debug( + "Kept the AX caret; its geometry source outranks the layout estimate.", + metadata: metadata + ) + } + return + } switch outcome { case .estimate(let estimate): let substituted = anchor.quality == .layoutEstimated - let keptReason = context.observedContentEdges != nil - ? "kept_ax_run_measured" - : "kept_ax_agreement" - metadata["repair_outcome"] = .string(substituted ? "substituted" : keptReason) + metadata["repair_outcome"] = .string(substituted ? "substituted" : "kept_ax_agreement") metadata["line_index"] = .stringConvertible(estimate.lineIndex) metadata["multi_line_field"] = .stringConvertible(estimate.isMultiLineField) metadata["ax_mid_y_delta"] = .stringConvertible( diff --git a/Cotabby/Models/FocusModels.swift b/Cotabby/Models/FocusModels.swift index 7f219e8a..204a0a42 100644 --- a/Cotabby/Models/FocusModels.swift +++ b/Cotabby/Models/FocusModels.swift @@ -178,6 +178,13 @@ struct FocusedInputSnapshot: Equatable { /// call sites compiling unchanged. let isIntegratedTerminal: Bool + /// True when the resolved field's text is rendered by a web engine rather than a native text + /// view (see `WebContentFieldDetector`). The caret layout repair keys its trust policy on + /// this: web-engine caret bounds have known wrong-line pathologies the hidden-layout estimate + /// may repair, while native AX bounds are ground truth the estimate must never override. + /// The initializer default keeps existing call sites compiling unchanged. + let isWebContentField: Bool + /// Monotonic counter that increments every time polling observes a focused-input identity /// change. /// @@ -224,6 +231,7 @@ struct FocusedInputSnapshot: Equatable { selection: NSRange, isSecure: Bool, isIntegratedTerminal: Bool = false, + isWebContentField: Bool = false, focusChangeSequence: UInt64 = 0, focusedURLString: String? = nil, resolvedFieldStyle: ResolvedFieldStyle? = nil @@ -245,6 +253,7 @@ struct FocusedInputSnapshot: Equatable { self.selection = selection self.isSecure = isSecure self.isIntegratedTerminal = isIntegratedTerminal + self.isWebContentField = isWebContentField self.focusChangeSequence = focusChangeSequence self.focusedURLString = focusedURLString self.resolvedFieldStyle = resolvedFieldStyle diff --git a/Cotabby/Models/SuggestionModels.swift b/Cotabby/Models/SuggestionModels.swift index 12883a25..1aa66814 100644 --- a/Cotabby/Models/SuggestionModels.swift +++ b/Cotabby/Models/SuggestionModels.swift @@ -204,6 +204,10 @@ struct FocusedInputContext: Equatable, Sendable { let trailingText: String let selection: NSRange let isSecure: Bool + /// Whether the field's text is rendered by a web engine (see `WebContentFieldDetector`), + /// carried through so the presentation-time caret repair can scope estimator authority to + /// hosts whose AX caret geometry actually needs repairing. + let isWebContentField: Bool /// The host field's own text font/color, carried through so the overlay can match it. let resolvedFieldStyle: ResolvedFieldStyle? /// Carries the immutable focus-observation identity across debounce/generation boundaries. @@ -229,6 +233,7 @@ struct FocusedInputContext: Equatable, Sendable { trailingText = snapshot.trailingText selection = snapshot.selection isSecure = snapshot.isSecure + isWebContentField = snapshot.isWebContentField resolvedFieldStyle = snapshot.resolvedFieldStyle focusChangeSequence = snapshot.focusChangeSequence self.generation = generation diff --git a/Cotabby/Services/Focus/FocusSnapshotResolver.swift b/Cotabby/Services/Focus/FocusSnapshotResolver.swift index 388c6aaa..7b6632dc 100644 --- a/Cotabby/Services/Focus/FocusSnapshotResolver.swift +++ b/Cotabby/Services/Focus/FocusSnapshotResolver.swift @@ -226,6 +226,13 @@ struct FocusSnapshotResolver { domClassList: AXHelper.stringArrayValue( for: "AXDOMClassList" as CFString, on: focusedElement) ?? [] ) + // Web-vs-native classification for the caret-geometry trust policy. The DOM-attribute + // signal was computed in `candidateSnapshot` from the attribute list it already fetched, + // so this adds no AX round-trip to the focus poll. + let isWebContentField = WebContentFieldDetector.isWebContentField( + bundleIdentifier: bundleIdentifier, + vendsDOMAttributes: resolvedCandidate.vendsDOMAttributes + ) let context = FocusedInputSnapshot( applicationName: applicationName, bundleIdentifier: bundleIdentifier, @@ -244,6 +251,7 @@ struct FocusSnapshotResolver { selection: contextWindow.selection, isSecure: resolvedCandidate.isSecure, isIntegratedTerminal: isIntegratedTerminal, + isWebContentField: isWebContentField, focusChangeSequence: focusChangeSequence, focusedURLString: focusedURLString, resolvedFieldStyle: resolvedFieldStyle @@ -710,6 +718,9 @@ struct FocusSnapshotResolver { let caretRect = caretResult?.rect let caretQuality = caretResult?.quality let isSecure = isSecureElement(element: element, role: role, subrole: subrole) + // Recorded from the already-fetched attribute list (no extra AX call) so snapshot + // assembly can classify the field as web-rendered without touching the element again. + let vendsDOMAttributes = WebContentFieldDetector.vendsDOMAttributes(supportedAttributes) let elementIdentifier = AXHelper.elementIdentifier( for: element, bundleIdentifier: bundleIdentifier) let resolverCandidate = FocusCapabilityCandidate( @@ -739,6 +750,7 @@ struct FocusSnapshotResolver { caretSourceDetail: caretResult?.sourceDetail, inputFrameRect: inputFrameRect, isSecure: isSecure, + vendsDOMAttributes: vendsDOMAttributes, resolverCandidate: resolverCandidate ) } @@ -873,5 +885,8 @@ private struct AXFocusCandidate { let caretSourceDetail: String? let inputFrameRect: CGRect? let isSecure: Bool + /// Whether the element advertises DOM-reflection attributes, marking it as web-engine + /// content (see `WebContentFieldDetector`). + let vendsDOMAttributes: Bool let resolverCandidate: FocusCapabilityCandidate } diff --git a/Cotabby/Support/WebContentFieldDetector.swift b/Cotabby/Support/WebContentFieldDetector.swift new file mode 100644 index 00000000..f8e2cb40 --- /dev/null +++ b/Cotabby/Support/WebContentFieldDetector.swift @@ -0,0 +1,58 @@ +import Foundation + +/// File overview: +/// Classifies whether a focused text field's content is rendered by a web engine +/// (Chromium/WebKit/Gecko) rather than a native macOS text view. The caret layout repair uses +/// this to decide how much authority the hidden-text-layout estimate has over AX-measured +/// geometry: web engines have known wrong-line caret-bounds pathologies the estimate exists to +/// repair, while native AX bounds come from the app's real layout manager and outrank any +/// re-layout Cotabby could compute. +/// +/// Why this distinction is trustworthy: probing real hosts showed both sides concretely. +/// Apple Notes (native TextKit) answers per-character `AXBoundsForRange` with rects that match +/// its rendered lines exactly, including the taller title line (23pt) above 16pt body lines, +/// which a uniform hidden layout cannot model; its geometry must win. Gmail in Chrome answers +/// the same queries through the renderer's lossy AX bridge, which maps carets into neighboring +/// visual lines around blank lines; there the estimate must be allowed to override. +/// +/// Two independent signals, either of which marks a field as web content: +/// - The element vends DOM-reflection attributes (`AXDOMIdentifier`/`AXDOMClassList`). +/// Chromium and WebKit attach these to web-content nodes only; native AppKit elements never +/// vend them. This catches web fields in apps no bundle list could anticipate (opaque +/// Electron bundle ids like Cursor's `com.todesktop.*`, or a WKWebView embedded in a native +/// app). +/// - The host bundle is a known browser or Electron editor (`BrowserAppDetector`). This covers +/// browser-chrome fields (e.g. the omnibox) that are not DOM-backed but still speak the +/// browser toolkit's AX dialect rather than AppKit's, and engines whose DOM reflection is +/// absent (Gecko). +/// +/// Defaulting unknown hosts to "native" is deliberate: the failure mode for a misclassified +/// native field is keeping pre-repair behavior (never worse than before the estimator existed), +/// while misclassifying a web field merely forgoes a repair. +enum WebContentFieldDetector { + /// Attribute names only web-engine accessibility nodes vend. Checked against the element's + /// advertised attribute list, which the focus resolver already fetches, so this costs no + /// extra AX round-trip. + private static let domReflectionAttributes: Set = [ + "AXDOMIdentifier", + "AXDOMClassList" + ] + + /// Whether the element's advertised attribute names mark it as a web-content node. + static func vendsDOMAttributes(_ attributeNames: Set) -> Bool { + !domReflectionAttributes.isDisjoint(with: attributeNames) + } + + /// Whether the focused field should be treated as web-rendered content for caret-geometry + /// trust decisions. + static func isWebContentField( + bundleIdentifier: String?, + vendsDOMAttributes: Bool + ) -> Bool { + if vendsDOMAttributes { + return true + } + return BrowserAppDetector.isBrowser(bundleIdentifier: bundleIdentifier) + || BrowserAppDetector.isElectronEditor(bundleIdentifier: bundleIdentifier) + } +} diff --git a/CotabbyTests/CotabbyTestFixtures.swift b/CotabbyTests/CotabbyTestFixtures.swift index 35eb88a4..7db66f1b 100644 --- a/CotabbyTests/CotabbyTestFixtures.swift +++ b/CotabbyTests/CotabbyTestFixtures.swift @@ -27,6 +27,7 @@ enum CotabbyTestFixtures { selection: NSRange? = nil, isSecure: Bool = false, isIntegratedTerminal: Bool = false, + isWebContentField: Bool = false, focusChangeSequence: UInt64 = 1 ) -> FocusedInputSnapshot { let resolvedSelection = selection @@ -50,6 +51,7 @@ enum CotabbyTestFixtures { selection: resolvedSelection, isSecure: isSecure, isIntegratedTerminal: isIntegratedTerminal, + isWebContentField: isWebContentField, focusChangeSequence: focusChangeSequence ) } @@ -68,6 +70,7 @@ enum CotabbyTestFixtures { trailingText: String = "", selection: NSRange? = nil, isSecure: Bool = false, + isWebContentField: Bool = false, focusChangeSequence: UInt64 = 1, generation: UInt64 = 1 ) -> FocusedInputContext { @@ -86,6 +89,7 @@ enum CotabbyTestFixtures { trailingText: trailingText, selection: selection, isSecure: isSecure, + isWebContentField: isWebContentField, focusChangeSequence: focusChangeSequence ), generation: generation diff --git a/CotabbyTests/SuggestionCaretLayoutRepairTests.swift b/CotabbyTests/SuggestionCaretLayoutRepairTests.swift index a65ffb03..c6be57bf 100644 --- a/CotabbyTests/SuggestionCaretLayoutRepairTests.swift +++ b/CotabbyTests/SuggestionCaretLayoutRepairTests.swift @@ -6,6 +6,12 @@ import XCTest /// resolver quality is `.estimated`, the overlay anchor is recomputed from the hidden text layout /// and the geometry quality upgraded to `.layoutEstimated`. Every rejection must keep today's /// behavior bit-for-bit (the passed rect and `.estimated` survive untouched). +/// +/// For `.derived` geometry the rule is additionally scoped by provenance: only web-content hosts +/// (whose AX caret bounds have known wrong-line pathologies) expose their derived rects to the +/// estimate's line-mismatch override. Native hosts' derived rects are AX ground truth and bypass +/// the estimator entirely; the `.estimated` substitution stays provenance-independent because +/// there is no real measurement to protect. @MainActor final class SuggestionCaretLayoutRepairTests: XCTestCase { /// Deliberately far outside any field frame so a substitution is unmistakable. @@ -95,7 +101,7 @@ final class SuggestionCaretLayoutRepairTests: XCTestCase { XCTAssertEqual(anchor.outcome, .rejected(.prefixTruncated)) } - // MARK: - Derived geometry (line-mismatch gate) + // MARK: - Derived geometry (web hosts: line-mismatch gate) func test_layoutRepair_derivedAgreementKeepsAXRect() { // The estimate and the AX rect land on the same line: AX wins, because its X carries the @@ -106,7 +112,8 @@ final class SuggestionCaretLayoutRepairTests: XCTestCase { caretRect: axRect, inputFrameRect: frame, caretQuality: .derived, - precedingText: "Hello" + precedingText: "Hello", + isWebContentField: true ) let anchor = SuggestionCoordinator.layoutRepairedAnchor( @@ -123,16 +130,18 @@ final class SuggestionCaretLayoutRepairTests: XCTestCase { } } - func test_layoutRepair_derivedLineMismatchSubstitutesEstimate() { + func test_layoutRepair_derivedLineMismatchSubstitutesEstimateForWebContent() { // The AX rect sits three line boxes below where the text layout puts the caret — the - // Gmail-class blank-line drift this gate exists for. + // Gmail-class blank-line drift this gate exists for. Web content only: the host's AX + // bridge, not its layout, is the suspect there. let frame = CGRect(x: 0, y: 0, width: 300, height: 120) let axRect = CGRect(x: 50, y: 52, width: 2, height: 16) let context = CotabbyTestFixtures.focusedInputContext( caretRect: axRect, inputFrameRect: frame, caretQuality: .derived, - precedingText: "Hello" + precedingText: "Hello", + isWebContentField: true ) let anchor = SuggestionCoordinator.layoutRepairedAnchor( @@ -150,6 +159,62 @@ final class SuggestionCaretLayoutRepairTests: XCTestCase { XCTAssertEqual(anchor.rect.height, axRect.height, accuracy: 0.01) } + // MARK: - Derived geometry (native hosts: AX is ground truth) + + func test_layoutRepair_nativeDerivedLineMismatchKeepsAXRect() { + // Identical geometry to the web mismatch test, but the field is native. Native rich-text + // views render what a uniform hidden layout cannot model (Apple Notes draws a 23pt title + // line above 16pt body lines, plus paragraph spacing), so a vertical disagreement there + // means the estimate is wrong while the AX prev-character bounds are exactly right. + // Substituting here was the post-#670 Notes regression: ghost text drifted off the real + // caret line. The estimator must not even run (nil outcome): it would burn a TextKit + // layout inside the accept keystroke's handling window to compute nothing actionable. + let frame = CGRect(x: 0, y: 0, width: 300, height: 120) + let axRect = CGRect(x: 50, y: 52, width: 2, height: 16) + let context = CotabbyTestFixtures.focusedInputContext( + caretRect: axRect, + inputFrameRect: frame, + caretQuality: .derived, + precedingText: "Hello", + isWebContentField: false + ) + + let anchor = SuggestionCoordinator.layoutRepairedAnchor( + for: context, + fallbackRect: axRect, + pendingInsertion: "", + isRightToLeft: false + ) + + XCTAssertEqual(anchor.quality, .derived) + XCTAssertEqual(anchor.rect, axRect) + XCTAssertNil(anchor.outcome) + XCTAssertEqual(anchor.skipReason, .nativeHostGeometry) + } + + func test_layoutRepair_nativeEstimatedStillSubstitutes() { + // `.estimated` means AX offered no real caret position at all (only the field frame), so + // there is no native measurement for the trust policy to protect: the layout estimate + // applies regardless of provenance. + let frame = CGRect(x: 0, y: 0, width: 240, height: 32) + let context = CotabbyTestFixtures.focusedInputContext( + inputFrameRect: frame, + caretQuality: .estimated, + precedingText: "Hello", + isWebContentField: false + ) + + let anchor = SuggestionCoordinator.layoutRepairedAnchor( + for: context, + fallbackRect: fallbackRect, + pendingInsertion: "", + isRightToLeft: false + ) + + XCTAssertEqual(anchor.quality, .layoutEstimated) + XCTAssertTrue(frame.insetBy(dx: -1, dy: -1).contains(anchor.rect)) + } + func test_layoutRepair_runMeasuredDerivedKeepsAXEvenOnLineMismatch() { // Same wrong-looking vertical gap as the mismatch test, but this rect came from measured // child-run frames (content edges present). Run frames carry the host's real line @@ -164,7 +229,8 @@ final class SuggestionCaretLayoutRepairTests: XCTestCase { inputFrameRect: frame, caretQuality: .derived, observedContentEdges: ObservedContentEdges(leftX: 4, topY: 116), - precedingText: "Hello" + precedingText: "Hello", + isWebContentField: true ) let anchor = SuggestionCoordinator.layoutRepairedAnchor( @@ -177,6 +243,7 @@ final class SuggestionCaretLayoutRepairTests: XCTestCase { XCTAssertEqual(anchor.quality, .derived) XCTAssertEqual(anchor.rect, axRect) XCTAssertNil(anchor.outcome) + XCTAssertEqual(anchor.skipReason, .runMeasuredGeometry) } func test_layoutRepair_derivedKeepsAXRectWhenEstimatorRejects() { @@ -184,7 +251,8 @@ final class SuggestionCaretLayoutRepairTests: XCTestCase { let context = CotabbyTestFixtures.focusedInputContext( caretRect: axRect, caretQuality: .derived, - precedingText: "column\tvalue" + precedingText: "column\tvalue", + isWebContentField: true ) let anchor = SuggestionCoordinator.layoutRepairedAnchor( diff --git a/CotabbyTests/WebContentFieldDetectorTests.swift b/CotabbyTests/WebContentFieldDetectorTests.swift new file mode 100644 index 00000000..73e9a2bd --- /dev/null +++ b/CotabbyTests/WebContentFieldDetectorTests.swift @@ -0,0 +1,110 @@ +import XCTest +@testable import Cotabby + +/// Locks the web-vs-native field classification the caret-geometry trust policy depends on. +/// A false negative here merely forgoes a repair (pre-estimator behavior); a false positive +/// exposes trustworthy native AX geometry to estimator overrides, which is the Notes-class +/// regression the detector exists to prevent. +final class WebContentFieldDetectorTests: XCTestCase { + // MARK: - DOM-attribute signal + + func test_domIdentifierMarksElementAsWebContent() { + XCTAssertTrue(WebContentFieldDetector.vendsDOMAttributes(["AXRole", "AXDOMIdentifier"])) + } + + func test_domClassListMarksElementAsWebContent() { + XCTAssertTrue(WebContentFieldDetector.vendsDOMAttributes(["AXDOMClassList"])) + } + + func test_nativeAttributeSetDoesNotMarkElementAsWebContent() { + // The attribute surface Apple Notes' body text area actually advertises (probed live): + // a rich native text element, no DOM reflection. + XCTAssertFalse( + WebContentFieldDetector.vendsDOMAttributes( + ["AXRole", "AXValue", "AXSelectedTextRange", "AXFrame", "AXNumberOfCharacters"] + ) + ) + } + + // MARK: - Combined classification + + func test_unknownElectronBundleWithDOMAttributesIsWebContent() { + // Cursor ships under opaque per-build `com.todesktop.*` bundle ids no allowlist can + // track; the element-level DOM signal is what catches it. + XCTAssertTrue( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: "com.todesktop.230313mzl4w4u92", + vendsDOMAttributes: true + ) + ) + } + + func test_browserChromeFieldWithoutDOMAttributesIsWebContent() { + // The omnibox is not DOM-backed but still speaks the browser toolkit's AX dialect, not + // AppKit's, so it stays inside the estimator's jurisdiction. + XCTAssertTrue( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: "com.google.Chrome", + vendsDOMAttributes: false + ) + ) + } + + func test_safariIsWebContentByBundle() { + XCTAssertTrue( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: "com.apple.Safari", + vendsDOMAttributes: false + ) + ) + } + + func test_electronEditorBundleIsWebContent() { + XCTAssertTrue( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: "com.microsoft.VSCode", + vendsDOMAttributes: false + ) + ) + } + + func test_nativeAppIsNotWebContent() { + XCTAssertFalse( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: "com.apple.Notes", + vendsDOMAttributes: false + ) + ) + } + + func test_unknownBundleDefaultsToNative() { + // Unknown hosts default to the conservative side: keeping pre-repair behavior can never + // be worse than before the estimator existed. + XCTAssertFalse( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: "com.example.SomeNativeApp", + vendsDOMAttributes: false + ) + ) + } + + func test_nilBundleDefaultsToNative() { + XCTAssertFalse( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: nil, + vendsDOMAttributes: false + ) + ) + } + + func test_embeddedWebViewInNativeAppIsWebContent() { + // A WKWebView-hosted field inside a non-browser app: the bundle says native, the element + // says web. The element wins, because the text is rendered by the web engine. + XCTAssertTrue( + WebContentFieldDetector.isWebContentField( + bundleIdentifier: "com.example.SomeNativeApp", + vendsDOMAttributes: true + ) + ) + } +}