Accept one word at a time in space-less scripts

Cotabby/Support/SuggestionSessionReconciler.swift

@@ -247,6 +247,17 @@ enum SuggestionSessionReconciler {
             index = remainingText.index(after: index)
         }
 
+        // Space-less scripts (CJK, Japanese, Korean, Thai, ...) put no whitespace between words, so the
+        // whitespace scan above swallows an entire run as a single "word". When the token begins with
+        // such a script, segment it with ICU word breaking and accept only the first word, so one Tab
+        // advances by a single word the way it does in space-delimited text. Space-delimited scripts
+        // never enter this branch, so Latin / Cyrillic / Arabic acceptance stays byte-for-byte unchanged.
+        if tokenStart < index,
+           remainingText[tokenStart].beginsSpacelessScriptWord,
+           let wordEnd = firstSegmentedWordEnd(in: remainingText, from: tokenStart, notPast: index) {
+            index = wordEnd
+        }
+
         if !autoAcceptTrailingPunctuation,
            let wordEnd = wordEndTrimmingTrailingPunctuation(in: remainingText, from: tokenStart, to: index) {
             index = wordEnd
@@ -255,6 +266,26 @@ enum SuggestionSessionReconciler {
         return String(remainingText[..<index])
     }
 
+    /// The index just past the first ICU word in `text[from..<limit]`, or nil when segmentation finds
+    /// no word there. Only the space-less-script branch of `nextAcceptanceChunk` calls this; the result
+    /// is clamped to `limit` so it can never extend past the non-whitespace token the caller already
+    /// bounded. `.substringNotRequired` skips materializing the word string we don't need.
+    private static func firstSegmentedWordEnd(
+        in text: String,
+        from start: String.Index,
+        notPast limit: String.Index
+    ) -> String.Index? {
+        var wordEnd: String.Index?
+        text.enumerateSubstrings(in: start..<limit, options: [.byWords, .substringNotRequired]) { _, range, _, stop in
+            wordEnd = range.upperBound
+            stop = true
+        }
+        guard let wordEnd, wordEnd > start else {
+            return nil
+        }
+        return min(wordEnd, limit)
+    }
+
     /// Accepts a full phrase up to the next sentence terminator (`.`, `!`, `?`, `\n`) or the end
     /// of the buffered suggestion tail. Composes over `nextAcceptanceChunk` so word-boundary,
     /// internal-punctuation, and leading-whitespace policy stay identical across the seams of a
@@ -490,6 +521,34 @@ private extension String {
 }
 
 private extension Character {
+    /// True when the character begins a word of a space-less script (Han, Hiragana, Katakana, Hangul,
+    /// Thai, Lao, Khmer, Myanmar, ...). These scripts write words without separating spaces, so the
+    /// whitespace-run acceptance rule would over-accept a whole run; `nextAcceptanceChunk` switches to
+    /// ICU word segmentation when a token begins with one of them. Detection is by leading scalar so it
+    /// stays a cheap, allocation-free check on the common (space-delimited) path.
+    var beginsSpacelessScriptWord: Bool {
+        guard let scalar = unicodeScalars.first else {
+            return false
+        }
+        switch scalar.value {
+        case 0x3040...0x30FF,   // Hiragana + Katakana
+             0x3400...0x4DBF,   // CJK Unified Ideographs Extension A
+             0x4E00...0x9FFF,   // CJK Unified Ideographs
+             0xF900...0xFAFF,   // CJK Compatibility Ideographs
+             0xAC00...0xD7A3,   // Hangul syllables
+             0x1100...0x11FF,   // Hangul Jamo
+             0x0E00...0x0E7F,   // Thai
+             0x0E80...0x0EFF,   // Lao
+             0x1780...0x17FF,   // Khmer
+             0x1000...0x109F,   // Myanmar
+             0x20000...0x2A6DF, // CJK Unified Ideographs Extension B
+             0x30000...0x3134F: // CJK Unified Ideographs Extension G
+            return true
+        default:
+            return false
+        }
+    }
+
     /// Alphanumerics form the core of a "word"; everything else trailing a word is punctuation that
     /// can be peeled into its own acceptance part when auto-accept is disabled.
     var isAcceptanceWordCharacter: Bool {

CotabbyTests/SuggestionSessionReconcilerTests.swift

@@ -131,6 +131,61 @@ final class SuggestionSessionReconcilerTests: XCTestCase {
         )
     }
 
+    // MARK: - Space-less-script word acceptance
+
+    func test_nextAcceptanceChunk_latinAcceptanceIsUnchangedBySpacelessBranch() {
+        // Regression guard: the space-less branch must never alter space-delimited acceptance.
+        XCTAssertEqual(SuggestionSessionReconciler.nextAcceptanceChunk(from: "hello world"), "hello")
+        XCTAssertEqual(SuggestionSessionReconciler.nextAcceptanceChunk(from: "don't stop now"), "don't")
+        XCTAssertEqual(SuggestionSessionReconciler.nextAcceptanceChunk(from: "U.S.A today"), "U.S.A")
+        XCTAssertEqual(SuggestionSessionReconciler.nextAcceptanceChunk(from: "1.5 times"), "1.5")
+        XCTAssertEqual(SuggestionSessionReconciler.nextAcceptanceChunk(from: "café René"), "café")
+        // A space-less script appearing later in the tail must not pull the first Latin token early.
+        XCTAssertEqual(SuggestionSessionReconciler.nextAcceptanceChunk(from: "world 你好"), "world")
+    }
+
+    func test_nextAcceptanceChunk_segmentsChineseBelowWholeLength() {
+        // ICU word segmentation may be per-character or dictionary-based depending on the OS, so this
+        // asserts the robust property (accept one segment, not the whole run) rather than a pinned word.
+        let run = "你好世界"
+        let chunk = SuggestionSessionReconciler.nextAcceptanceChunk(from: run)
+        XCTAssertFalse(chunk.isEmpty)
+        XCTAssertTrue(run.hasPrefix(chunk))
+        XCTAssertLessThan(chunk.count, run.count, "a space-less Chinese run must segment, not accept the whole run")
+    }
+
+    func test_nextAcceptanceChunk_segmentsJapaneseRunBelowWholeLength() {
+        let run = "今日はいい天気です"
+        let chunk = SuggestionSessionReconciler.nextAcceptanceChunk(from: run)
+        XCTAssertFalse(chunk.isEmpty)
+        XCTAssertTrue(run.hasPrefix(chunk))
+        XCTAssertLessThan(chunk.count, run.count, "a space-less Japanese run must segment, not accept whole")
+    }
+
+    func test_nextAcceptanceChunk_segmentsThaiRunBelowWholeLength() {
+        let run = "สวัสดีครับ"
+        let chunk = SuggestionSessionReconciler.nextAcceptanceChunk(from: run)
+        XCTAssertFalse(chunk.isEmpty)
+        XCTAssertTrue(run.hasPrefix(chunk))
+        XCTAssertLessThan(chunk.count, run.count, "a space-less Thai run must segment, not accept whole")
+    }
+
+    func test_nextAcceptanceChunk_chineseAcceptanceStaysWithinRunBeforeSpace() {
+        let chunk = SuggestionSessionReconciler.nextAcceptanceChunk(from: "你好 world")
+        XCTAssertFalse(chunk.isEmpty)
+        XCTAssertTrue("你好".hasPrefix(chunk), "acceptance must stay within the CJK run and not cross the space")
+        XCTAssertFalse(chunk.contains(" "))
+    }
+
+    func test_nextAcceptanceChunk_keepsLeadingWhitespaceBeforeSpacelessWord() {
+        let chunk = SuggestionSessionReconciler.nextAcceptanceChunk(from: " 你好世界")
+        XCTAssertTrue(chunk.hasPrefix(" "), "leading whitespace is preserved before the segmented word")
+        let afterSpace = String(chunk.dropFirst())
+        XCTAssertFalse(afterSpace.isEmpty)
+        XCTAssertTrue("你好世界".hasPrefix(afterSpace))
+        XCTAssertLessThan(afterSpace.count, 4, "only the first segment is accepted, not the whole run")
+    }
+
     // MARK: - Phrase chunker
 
     func test_nextAcceptancePhrase_returnsEmptyForEmptyTail() {