Skip to content

Trust native AX caret geometry; scope the layout-estimate override to web content #676

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
FuJacob merged 1 commit into from
Jun 12, 2026
Merged

Trust native AX caret geometry; scope the layout-estimate override to web content #676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Cotabby.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@
84A4CA05AF6885AE4FA4C13A /* SettingsAttentionEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A02336442BB735EE2E8D064 /* SettingsAttentionEvaluator.swift */; };
862146ABDADC022A3BE74E00 /* CurrencyEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0537986794554F5FABE6EFF3 /* CurrencyEvaluator.swift */; };
865C569A9BC95B08F440D199 /* SystemResourceSampler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 27A5D63F390E9B7A7FE343FE /* SystemResourceSampler.swift */; };
86AC625B4DD14EF807002FA2 /* WebContentFieldDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */; };
87806DE08881D11F2608A13D /* MarkerSelectionSynthesizerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 52BAFA2F989C3C4F7FB892B5 /* MarkerSelectionSynthesizerTests.swift */; };
8865B95FE84198D70390DF80 /* ClipboardContentDistillerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD89799BB82AF7A92559AEB /* ClipboardContentDistillerTests.swift */; };
88BCD795A14E1C9308F7BB31 /* SuggestionAvailabilityEvaluatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C05B0439348261163B37C508 /* SuggestionAvailabilityEvaluatorTests.swift */; };
Expand Down Expand Up @@ -501,6 +502,7 @@
D603EC842D9D7A1A81899050 /* PermissionModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D82FFC568527700EC17C07D /* PermissionModels.swift */; };
D648DD70AD847F67B77CE052 /* OnboardingTemplateRecommenderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01B72736E416910878E8E493 /* OnboardingTemplateRecommenderTests.swift */; };
D6AD25168F108DA8D60E76EF /* SpellingDictionaryPicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF3A73EB848780061FC162C0 /* SpellingDictionaryPicker.swift */; };
D75626719A38F535D475C675 /* WebContentFieldDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */; };
D800277BE3296DE2FB8198A8 /* ChromiumAccessibilityEnabler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8896D976C7F116EBA0F3969F /* ChromiumAccessibilityEnabler.swift */; };
D80D98F6C7B86AAF8C831865 /* he-100k.txt in Resources */ = {isa = PBXBuildFile; fileRef = 7C9BB65FA5FC42B89766B037 /* he-100k.txt */; };
D81AC2294A64E7A95B29E081 /* fr.txt in Resources */ = {isa = PBXBuildFile; fileRef = 5DCA3E46621815A19300365F /* fr.txt */; };
Expand Down Expand Up @@ -531,6 +533,7 @@
E46F50AEDA8FE13B02E3FA8D /* AXHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC70775535A3428991025AB8 /* AXHelper.swift */; };
E51FA12B690428CA431328FC /* WritingPaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D48B95B6665109B6C6A63B42 /* WritingPaneView.swift */; };
E54F5F03E16859D5A1E3437A /* MacroController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4638C74239D1DE2DC4D87975 /* MacroController.swift */; };
E5CB34ED76BAE87E8A858112 /* WebContentFieldDetectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 210F9AD332273FE2EB3A9A01 /* WebContentFieldDetectorTests.swift */; };
E6EE3C13FA31F261CD734C69 /* DownloadOutcomeClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3DE1975F3B5F4A70478DBF41 /* DownloadOutcomeClassifier.swift */; };
E853B9C7AF93FA595DC417B2 /* EmojiVariantResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A8414BEB7E34F57607E37FE /* EmojiVariantResolver.swift */; };
E912D4617AE1376061DF1F00 /* LanguageSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4793D4EA5D36D7E5CC216C27 /* LanguageSupportTests.swift */; };
Expand Down Expand Up @@ -640,6 +643,7 @@
1ED1EA9282E0AC7592E60889 /* SuggestionCoordinator+Prediction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SuggestionCoordinator+Prediction.swift"; sourceTree = "<group>"; };
1F761083EA5465023D82B5F4 /* BrowserDomainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowserDomainTests.swift; sourceTree = "<group>"; };
1F7ED2D9D71CFAD9A30977E9 /* ArithmeticEvaluator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ArithmeticEvaluator.swift; sourceTree = "<group>"; };
210F9AD332273FE2EB3A9A01 /* WebContentFieldDetectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebContentFieldDetectorTests.swift; sourceTree = "<group>"; };
21CB3008986BE7FD2A4D9132 /* WelcomeCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WelcomeCoordinator.swift; sourceTree = "<group>"; };
224438039A86E5619294EAF7 /* EmojiUsageStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmojiUsageStoreTests.swift; sourceTree = "<group>"; };
22544F4B756E3E4144497D17 /* SuggestionCoordinator+Input.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SuggestionCoordinator+Input.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -847,6 +851,7 @@
C1C5DE0F3FF63545000E2453 /* DisplayCoordinateConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DisplayCoordinateConverterTests.swift; sourceTree = "<group>"; };
C375227649689775275AA4B3 /* SuggestionCoordinatorAcceptanceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionCoordinatorAcceptanceTests.swift; sourceTree = "<group>"; };
C379D77029D6E88C8C1B9AF7 /* emoji.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = emoji.json; sourceTree = "<group>"; };
C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebContentFieldDetector.swift; sourceTree = "<group>"; };
C648EBB10D7F8E0B904DEC91 /* de.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = de.txt; sourceTree = "<group>"; };
C71031E8DB171047318B92FC /* SyntheticReplacePlannerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyntheticReplacePlannerTests.swift; sourceTree = "<group>"; };
C727BF6FF8ACAAED30B0329F /* TypoGateTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypoGateTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -1312,6 +1317,7 @@
D2D0FE44138BCA8B2EE05AFE /* TypoCaseTransferTests.swift */,
C727BF6FF8ACAAED30B0329F /* TypoGateTests.swift */,
050D929E13BE52E6282B64D2 /* VisualContextStartCoalescerTests.swift */,
210F9AD332273FE2EB3A9A01 /* WebContentFieldDetectorTests.swift */,
1E0513E3B23937B099A3CFF2 /* WordCountFormatterTests.swift */,
);
path = CotabbyTests;
Expand Down Expand Up @@ -1506,6 +1512,7 @@
08CE63B8725EBD71A4C024E1 /* TypoCaseTransfer.swift */,
B8412FE2BAC406421248A03B /* TypoGate.swift */,
2F01FAC4F57EB08471521196 /* VisualContextStartCoalescer.swift */,
C3A35FAA742408D002B75920 /* WebContentFieldDetector.swift */,
815F2ABAF6AB75DA3AFBBCEF /* WordCountFormatter.swift */,
);
path = Support;
Expand Down Expand Up @@ -1911,6 +1918,7 @@
0A15FAD03A93D57372D267E0 /* VisualContextCoordinator.swift in Sources */,
F77DB394E9D6C6C482131BF9 /* VisualContextModels.swift in Sources */,
641A9FAF3009A3E2AA06D74B /* VisualContextStartCoalescer.swift in Sources */,
86AC625B4DD14EF807002FA2 /* WebContentFieldDetector.swift in Sources */,
3AB45217DFC86AFC98C374D6 /* WelcomeCoordinator.swift in Sources */,
70D6F9480DA4104AD5669569 /* WelcomePermissionStepView.swift in Sources */,
4FEA3AC7ABB1982063BC0041 /* WelcomeTemplateStepView.swift in Sources */,
Expand Down Expand Up @@ -2131,6 +2139,7 @@
E9E4CC657771DF9F4C56183C /* VisualContextCoordinator.swift in Sources */,
4190F8A76196B16ED94D0A55 /* VisualContextModels.swift in Sources */,
19CB55B62977376E9AE8D428 /* VisualContextStartCoalescer.swift in Sources */,
D75626719A38F535D475C675 /* WebContentFieldDetector.swift in Sources */,
4AC255BE2D0CCC67B8882C7A /* WelcomeCoordinator.swift in Sources */,
344B9BF352C97CFA830853D6 /* WelcomePermissionStepView.swift in Sources */,
286B7022E2A2774275004447 /* WelcomeTemplateStepView.swift in Sources */,
Expand Down Expand Up @@ -2240,6 +2249,7 @@
5C6B59C2E56A3C4260591095 /* TypoCaseTransferTests.swift in Sources */,
BE688EC1957B4AE004063EFE /* TypoGateTests.swift in Sources */,
D5CAF3B590E5EC2AFC72E57A /* VisualContextStartCoalescerTests.swift in Sources */,
E5CB34ED76BAE87E8A858112 /* WebContentFieldDetectorTests.swift in Sources */,
6AE0B46FB52D189D94E1F79A /* WordCountFormatterTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
87 changes: 69 additions & 18 deletions Cotabby/App/Coordinators/SuggestionCoordinator+Acceptance.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -629,14 +629,21 @@ extension SuggestionCoordinator {

/// Repairs untrustworthy caret anchors with a hidden-text-layout estimate before presentation.
///
/// Two repair modes, by resolver quality:
/// The repair's authority is scoped by where the AX geometry came from, not just its quality:
/// - `.estimated` (AXFrame-only hosts): the AX guess has no real position at all, so a
/// passing estimate always replaces it.
/// passing estimate always replaces it. Applies to web and native hosts alike, because
/// there is no real measurement to protect.
/// - `.derived` in a native host: never repaired, the estimator does not run. Native AX
/// per-character bounds come from the app's real layout manager and reflect rendering the
/// uniform hidden layout cannot model (Notes renders a 23pt title line above 16pt body
/// lines, plus paragraph spacing, so the layout "disagrees" within a few lines while AX is
/// exactly right). A vertical mismatch there indicts the estimate, not AX.
/// - `.derived` with measured run frames (Gmail/Outlook child-run hosts): the rect's Y is a
/// real rendered line, so it is always kept and the estimator does not run at all; the
/// layout estimate cannot beat measurement (and is blind to blank lines those hosts
/// collapse out of the AX text).
/// - `.derived` without run frames (previous-character bounds): the estimate (calibrated
/// - `.derived` web content without run frames (previous-character bounds through a web
/// engine's AX bridge, which has known wrong-line pathologies): the estimate (calibrated
/// with whatever the host revealed) only overrides the AX rect when the two disagree
/// vertically by more than `verticallyAgrees` tolerates; on agreement the AX rect is kept.
///
Expand All @@ -659,19 +666,34 @@ extension SuggestionCoordinator {
) -> LayoutRepairedAnchor {
let quality = context.caretQuality
guard quality == .estimated || quality == .derived else {
return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: nil)
return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: nil, skipReason: nil)
}

// Run-measured derived rects are kept unconditionally — run frames carry the host's real
// line positions, including blank lines some hosts omit from the AX text — so skip the
// estimator entirely instead of computing a diagnostic it can never act on. This path runs
// Derived rects carry a real AX measurement, so whether the estimate may second-guess
// them depends on who produced the measurement. Both bypasses skip the estimator
// entirely rather than computing a diagnostic they can never act on: this path runs
// inside the accept keystroke's handling window, where every spent millisecond of layout
// work on a large flat prefix is pure risk during a rapid Tab burst.
if quality == .derived, context.observedContentEdges != nil {
return LayoutRepairedAnchor(rect: fallbackRect, quality: .derived, outcome: nil)
if quality == .derived {
// Native hosts: per-character bounds come from the app's own layout manager and are
// ground truth, while the uniform hidden layout cannot model rich text (Notes' taller
// title line and paragraph spacing put it a line off within a few paragraphs). Only
// web engines' AX bridges have the wrong-line pathologies the override exists for.
if !context.isWebContentField {
return LayoutRepairedAnchor(
rect: fallbackRect, quality: .derived, outcome: nil, skipReason: .nativeHostGeometry
)
}
// Run-measured derived rects are kept unconditionally: run frames carry the host's
// real line positions, including blank lines some hosts omit from the AX text.
if context.observedContentEdges != nil {
return LayoutRepairedAnchor(
rect: fallbackRect, quality: .derived, outcome: nil, skipReason: .runMeasuredGeometry
)
}
}

// A `.derived` rect's height is a real rendered line box (child-run frame height); the
// A `.derived` rect's height is a real rendered line box (previous-character bounds); the
// `.estimated` AXFrame fallback's height is the whole field, which the estimator's
// sanitizer would discard anyway — pass it for derived geometry only.
let observedLineHeight: CGFloat? = quality == .derived ? context.caretRect.height : nil
Expand All @@ -694,11 +716,13 @@ extension SuggestionCoordinator {
case .estimate(let estimate):
if quality == .derived, verticallyAgrees(estimate: estimate, axRect: fallbackRect) {
// Same line: keep the AX rect, whose X carries the host's real glyph positions.
return LayoutRepairedAnchor(rect: fallbackRect, quality: .derived, outcome: outcome)
return LayoutRepairedAnchor(rect: fallbackRect, quality: .derived, outcome: outcome, skipReason: nil)
}
return LayoutRepairedAnchor(rect: estimate.caretRect, quality: .layoutEstimated, outcome: outcome)
return LayoutRepairedAnchor(
rect: estimate.caretRect, quality: .layoutEstimated, outcome: outcome, skipReason: nil
)
case .rejected:
return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: outcome)
return LayoutRepairedAnchor(rect: fallbackRect, quality: quality, outcome: outcome, skipReason: nil)
}
}

Expand All @@ -720,6 +744,20 @@ extension SuggestionCoordinator {
let rect: CGRect
let quality: CaretGeometryQuality
let outcome: TextLayoutCaretEstimator.Outcome?
/// Why a derived rect bypassed the estimator without running it (nil when the estimator
/// ran, or when repair was out of scope for the quality entirely). Logged so "the caret
/// came from trusted AX and repair stood down" is distinguishable from "this field never
/// triggered repair" when diagnosing a misplaced overlay from the JSONL stream.
let skipReason: LayoutRepairSkipReason?
}

/// Why `layoutRepairedAnchor` kept a derived AX rect without running the estimator at all.
/// Raw values feed the structured log stream.
enum LayoutRepairSkipReason: String {
/// Native (non-web) host: AX-derived geometry is ground truth for the trust policy.
case nativeHostGeometry = "native_host_geometry"
/// The rect was measured from child text-run frames, which outrank any estimate.
case runMeasuredGeometry = "run_measured_geometry"
}

/// Mirrors the repair outcome into the structured JSONL stream so a misplaced overlay can be
Expand All @@ -734,7 +772,9 @@ extension SuggestionCoordinator {
fallbackRect: CGRect,
context: FocusedInputContext
) {
guard let outcome = anchor.outcome else {
// Exact/layout-estimated presentations never reach repair and carry nothing to log; bail
// before building metadata so the common path pays nothing per present.
guard anchor.outcome != nil || anchor.skipReason != nil else {
return
}
var metadata: Logger.Metadata = [
Expand All @@ -750,13 +790,24 @@ extension SuggestionCoordinator {
if let fieldFrame = context.inputFrameRect {
metadata["field_top_y"] = .stringConvertible(Double(fieldFrame.maxY))
}
guard let outcome = anchor.outcome else {
// A trusted-AX bypass stood the estimator down without running it. Logged (unlike the
// qualities repair never applies to) so the stream can distinguish "repair deferred to
// trusted AX" from "this presentation never reached repair".
if let skipReason = anchor.skipReason {
metadata["repair_outcome"] = .string("skipped")
metadata["skip_reason"] = .string(skipReason.rawValue)
CotabbyLogger.suggestion.debug(
"Kept the AX caret; its geometry source outranks the layout estimate.",
metadata: metadata
)
}
return
}
switch outcome {
case .estimate(let estimate):
let substituted = anchor.quality == .layoutEstimated
let keptReason = context.observedContentEdges != nil
? "kept_ax_run_measured"
: "kept_ax_agreement"
metadata["repair_outcome"] = .string(substituted ? "substituted" : keptReason)
metadata["repair_outcome"] = .string(substituted ? "substituted" : "kept_ax_agreement")
metadata["line_index"] = .stringConvertible(estimate.lineIndex)
metadata["multi_line_field"] = .stringConvertible(estimate.isMultiLineField)
metadata["ax_mid_y_delta"] = .stringConvertible(
Expand Down
9 changes: 9 additions & 0 deletions Cotabby/Models/FocusModels.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,13 @@ struct FocusedInputSnapshot: Equatable {
/// call sites compiling unchanged.
let isIntegratedTerminal: Bool

/// True when the resolved field's text is rendered by a web engine rather than a native text
/// view (see `WebContentFieldDetector`). The caret layout repair keys its trust policy on
/// this: web-engine caret bounds have known wrong-line pathologies the hidden-layout estimate
/// may repair, while native AX bounds are ground truth the estimate must never override.
/// The initializer default keeps existing call sites compiling unchanged.
let isWebContentField: Bool

/// Monotonic counter that increments every time polling observes a focused-input identity
/// change.
///
Expand Down Expand Up @@ -224,6 +231,7 @@ struct FocusedInputSnapshot: Equatable {
selection: NSRange,
isSecure: Bool,
isIntegratedTerminal: Bool = false,
isWebContentField: Bool = false,
focusChangeSequence: UInt64 = 0,
focusedURLString: String? = nil,
resolvedFieldStyle: ResolvedFieldStyle? = nil
Expand All @@ -245,6 +253,7 @@ struct FocusedInputSnapshot: Equatable {
self.selection = selection
self.isSecure = isSecure
self.isIntegratedTerminal = isIntegratedTerminal
self.isWebContentField = isWebContentField
self.focusChangeSequence = focusChangeSequence
self.focusedURLString = focusedURLString
self.resolvedFieldStyle = resolvedFieldStyle
Expand Down
Loading