From 2a7019aab8844f9a06878964493b08755400a490 Mon Sep 17 00:00:00 2001 From: ginlimeb-ship-it Date: Sat, 31 Oct 2026 13:04:16 +0900 Subject: [PATCH] Fix Korean and other rewriting IME input from the on-screen keyboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The on-screen keyboard handler sent each newly typed character and then reset the text field to the sentinel. IMEs that build a character by rewriting the one before it, like the iOS Korean keyboard (ㅎ -> 하 -> 한), lost everything after the first jamo because the character they wanted to rewrite was already gone. Instead, keep the text field contents while typing and send a diff against what was already sent: backspaces for removed characters, then the new text. Marked text from other IMEs (Japanese, Chinese) is held back until it's committed, since it can contain Latin letters. Marked Hangul is sent right away and fixed up by the diff as it changes. The field is reset to the sentinel only at whitespace, punctuation, Return, an empty field, or when the text gets long. Keyboard text events are now sent in order on a serial queue. Before, each key went through a concurrent global queue, so a backspace could be sent after the text meant to replace it. Return commits any marked text and sends it before the Return key. Co-Authored-By: Claude Opus 5.5 --- Limelight/Input/StreamView.m | 224 +++++++++++++++++++++++++++-------- 1 file changed, 176 insertions(+), 48 deletions(-) diff --git a/Limelight/Input/StreamView.m b/Limelight/Input/StreamView.m index 7dedeb35e..f2dfab970 100644 --- a/Limelight/Input/StreamView.m +++ b/Limelight/Input/StreamView.m @@ -24,6 +24,14 @@ @implementation StreamView { BOOL isInputingText; NSMutableSet* keysDown; + // Text (after the sentinel) that has already been sent to the host from the + // on-screen keyboard. Edits are sent as a diff against this, so IMEs that + // compose by rewriting the previous character (like Korean) work. + NSString* sentKeyboardText; + + // Serial queue so backspaces and the text that replaces them can't be reordered + dispatch_queue_t keyboardTextQueue; + float streamAspectRatio; // iOS 13.4 mouse support @@ -55,6 +63,8 @@ - (void) setupStreamView:(ControllerSupport*)controllerSupport TemporarySettings* settings = [[[DataManager alloc] init] getSettings]; keysDown = [[NSMutableSet alloc] init]; + sentKeyboardText = @""; + keyboardTextQueue = dispatch_queue_create("com.moonlight.keyboard-text", DISPATCH_QUEUE_SERIAL); keyInputField = [[KeyboardInputField alloc] initWithFrame:CGRectZero]; [keyInputField setKeyboardType:UIKeyboardTypeDefault]; [keyInputField setAutocorrectionType:UITextAutocorrectionTypeNo]; @@ -361,6 +371,7 @@ - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event { // Prepare the textbox used to capture keyboard events. keyInputField.delegate = self; keyInputField.text = @"0"; + sentKeyboardText = @""; #if !TARGET_OS_TV // Prepare the toolbar above the keyboard for more options UIToolbar *customToolbarView = [[UIToolbar alloc] initWithFrame:CGRectMake(0, 0, self.bounds.size.width, 44)]; @@ -773,9 +784,20 @@ - (BOOL)gestureRecognizerShouldBegin:(UIGestureRecognizer *)gestureRecognizer { - (BOOL)textFieldShouldReturn:(UITextField *)textField { // This method is called when the "Return" key is pressed. - LiSendKeyboardEvent(0x0d, KEY_ACTION_DOWN, 0); - usleep(50 * 1000); - LiSendKeyboardEvent(0x0d, KEY_ACTION_UP, 0); + + // Commit any in-progress IME composition and send it before the Return key + if (textField.markedTextRange != nil) { + [textField unmarkText]; + } + [self onKeyboardPressed:textField]; + + dispatch_async(keyboardTextQueue, ^{ + LiSendKeyboardEvent(0x0d, KEY_ACTION_DOWN, 0); + usleep(50 * 1000); + LiSendKeyboardEvent(0x0d, KEY_ACTION_UP, 0); + }); + + [self resetKeyboardTextField:textField]; return NO; } @@ -786,47 +808,148 @@ - (void)textFieldDidEndEditing:(UITextField *)textField { [keysDown removeAllObjects]; } -- (void)onKeyboardPressed:(UITextField *)textField { - NSString* inputText = textField.text; - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ - // If the text became empty, we know the user pressed the backspace key. - if ([inputText isEqual:@""]) { - LiSendKeyboardEvent(0x08, KEY_ACTION_DOWN, 0); - usleep(50 * 1000); - LiSendKeyboardEvent(0x08, KEY_ACTION_UP, 0); - } else { - // Character 0 will be our known sentinel value - - // Check if any characters exist which can't be represented in a basic key event - for (int i = 1; i < [inputText length]; i++) { - struct KeyEvent event = [KeyboardSupport translateKeyEvent:[inputText characterAtIndex:i] withModifierFlags:0]; - if (event.keycode == 0) { - // We found an unknown key, so send the entire string as UTF-8 - const char* utf8String = [inputText UTF8String]; - - // Skip the first character which is our sentinel - LiSendUtf8TextEvent(utf8String + 1, (int)strlen(utf8String) - 1); - return; - } - } - - // We didn't find any unknown characters, so send them all as basic key events - for (int i = 1; i < [inputText length]; i++) { - struct KeyEvent event = [KeyboardSupport translateKeyEvent:[inputText characterAtIndex:i] withModifierFlags:0]; - assert(event.keycode != 0); - [self sendLowLevelEvent:event]; - } +// Returns YES if the string is non-empty and made up only of Hangul syllables and jamo +static BOOL isHangul(NSString* string) { + if (string.length == 0) { + return NO; + } + for (NSUInteger i = 0; i < string.length; i++) { + unichar c = [string characterAtIndex:i]; + if (!((c >= 0x1100 && c <= 0x11FF) || // Hangul Jamo + (c >= 0x3130 && c <= 0x318F) || // Hangul Compatibility Jamo + (c >= 0xA960 && c <= 0xA97F) || // Hangul Jamo Extended-A + (c >= 0xAC00 && c <= 0xD7A3) || // Hangul Syllables + (c >= 0xD7B0 && c <= 0xD7FF))) { // Hangul Jamo Extended-B + return NO; } - }); - - // Reset text field back to known state + } + return YES; +} + +// Splits a string into user-perceived characters (a Hangul syllable or an emoji is one character) +static NSArray* composedCharacters(NSString* string) { + NSMutableArray* characters = [[NSMutableArray alloc] init]; + [string enumerateSubstringsInRange:NSMakeRange(0, string.length) + options:NSStringEnumerationByComposedCharacterSequences + usingBlock:^(NSString* substring, NSRange substringRange, NSRange enclosingRange, BOOL* stop) { + [characters addObject:substring]; + }]; + return characters; +} + +- (void)resetKeyboardTextField:(UITextField *)textField { + // Reset text field back to known state. Character 0 is our known sentinel value. textField.text = @"0"; + sentKeyboardText = @""; // Move the insertion point back to the end of the text box UITextRange *textRange = [textField textRangeFromPosition:textField.endOfDocument toPosition:textField.endOfDocument]; [textField setSelectedTextRange:textRange]; } +// Must be called on keyboardTextQueue +- (void)sendKeyboardText:(NSString*)text { + if (text.length == 0) { + return; + } + + // Check if any characters exist which can't be represented in a basic key event + for (NSUInteger i = 0; i < text.length; i++) { + struct KeyEvent event = [KeyboardSupport translateKeyEvent:[text characterAtIndex:i] withModifierFlags:0]; + if (event.keycode == 0) { + // We found an unknown key, so send the entire string as UTF-8 + const char* utf8String = [text UTF8String]; + LiSendUtf8TextEvent(utf8String, (int)strlen(utf8String)); + return; + } + } + + // We didn't find any unknown characters, so send them all as basic key events + for (NSUInteger i = 0; i < text.length; i++) { + struct KeyEvent event = [KeyboardSupport translateKeyEvent:[text characterAtIndex:i] withModifierFlags:0]; + assert(event.keycode != 0); + [self sendLowLevelEventNow:event]; + } +} + +- (void)onKeyboardPressed:(UITextField *)textField { + NSString* inputText = textField.text ?: @""; + + // While an IME is composing, the marked text isn't final yet. For most IMEs + // (Japanese, Chinese) it can contain Latin letters that must not reach the host, + // so only consider the committed text before it and send the rest once it's + // committed. Hangul being composed is real text that only gets replaced, so + // send it right away and let the diff below fix it up as it changes. + UITextRange* markedTextRange = textField.markedTextRange; + if (markedTextRange != nil && !isHangul([textField textInRange:markedTextRange])) { + NSInteger committedLength = [textField offsetFromPosition:textField.beginningOfDocument toPosition:markedTextRange.start]; + if (committedLength <= 0 || committedLength > (NSInteger)inputText.length) { + return; + } + inputText = [inputText substringToIndex:committedLength]; + } + + // If the text became empty, the user pressed backspace past our sentinel. + // Normally the field held only the sentinel, so that's one backspace on the host. + // If it still held sent text (holding backspace deletes a whole word at once), + // delete that text on the host instead. + if (inputText.length == 0) { + NSUInteger backspaces = MAX(composedCharacters(sentKeyboardText).count, (NSUInteger)1); + dispatch_async(keyboardTextQueue, ^{ + for (NSUInteger i = 0; i < backspaces; i++) { + LiSendKeyboardEvent(0x08, KEY_ACTION_DOWN, 0); + usleep(50 * 1000); + LiSendKeyboardEvent(0x08, KEY_ACTION_UP, 0); + } + }); + [self resetKeyboardTextField:textField]; + return; + } + + // Strip our sentinel + NSString* currentText = [inputText hasPrefix:@"0"] ? [inputText substringFromIndex:1] : inputText; + + // Some IMEs (like Korean) compose by replacing the characters they just typed + // (ㅎ -> 하 -> 한), so we can't just send what was appended. Instead, find what + // changed since the last update and send backspaces for the removed characters + // followed by the new ones. + NSArray* sentCharacters = composedCharacters(sentKeyboardText); + NSArray* currentCharacters = composedCharacters(currentText); + NSUInteger commonLength = 0; + while (commonLength < sentCharacters.count && + commonLength < currentCharacters.count && + [sentCharacters[commonLength] isEqualToString:currentCharacters[commonLength]]) { + commonLength++; + } + + NSUInteger backspaces = sentCharacters.count - commonLength; + NSString* addedText = [[currentCharacters subarrayWithRange:NSMakeRange(commonLength, currentCharacters.count - commonLength)] componentsJoinedByString:@""]; + sentKeyboardText = currentText; + + if (backspaces > 0 || addedText.length > 0) { + dispatch_async(keyboardTextQueue, ^{ + for (NSUInteger i = 0; i < backspaces; i++) { + LiSendKeyboardEvent(0x08, KEY_ACTION_DOWN, 0); + usleep(50 * 1000); + LiSendKeyboardEvent(0x08, KEY_ACTION_UP, 0); + } + [self sendKeyboardText:addedText]; + }); + } + + // Reset the text field once the user reaches a point where no IME could still + // be composing (whitespace or punctuation), or if the text gets long. + // Resetting after every character breaks IMEs that rewrite previous characters. + if (markedTextRange == nil && currentText.length > 0) { + unichar lastCharacter = [currentText characterAtIndex:currentText.length - 1]; + if (currentText.length >= 256 || + [[NSCharacterSet whitespaceAndNewlineCharacterSet] characterIsMember:lastCharacter] || + [[NSCharacterSet punctuationCharacterSet] characterIsMember:lastCharacter]) { + [self resetKeyboardTextField:textField]; + } + } +} + - (void)specialCharPressed:(UIKeyCommand *)cmd { struct KeyEvent event = [KeyboardSupport translateKeyEvent:0x20 withModifierFlags:[cmd modifierFlags]]; event.keycode = [[dictCodes valueForKey:[cmd input]] intValue]; @@ -840,21 +963,26 @@ - (void)keyPressed:(UIKeyCommand *)cmd { - (void)sendLowLevelEvent:(struct KeyEvent)event { dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ - // When we want to send a modified key (like uppercase letters) we need to send the - // modifier ("shift") seperately from the key itself. - if (event.modifier != 0) { - LiSendKeyboardEvent(event.modifierKeycode, KEY_ACTION_DOWN, event.modifier); - } - // Let the host know these are not (necessarily) normalized to US English scancodes - LiSendKeyboardEvent2(event.keycode, KEY_ACTION_DOWN, event.modifier, SS_KBE_FLAG_NON_NORMALIZED); - usleep(50 * 1000); - LiSendKeyboardEvent2(event.keycode, KEY_ACTION_UP, event.modifier, SS_KBE_FLAG_NON_NORMALIZED); - if (event.modifier != 0) { - LiSendKeyboardEvent(event.modifierKeycode, KEY_ACTION_UP, event.modifier); - } + [self sendLowLevelEventNow:event]; }); } +// Sends the key event synchronously on the calling thread +- (void)sendLowLevelEventNow:(struct KeyEvent)event { + // When we want to send a modified key (like uppercase letters) we need to send the + // modifier ("shift") seperately from the key itself. + if (event.modifier != 0) { + LiSendKeyboardEvent(event.modifierKeycode, KEY_ACTION_DOWN, event.modifier); + } + // Let the host know these are not (necessarily) normalized to US English scancodes + LiSendKeyboardEvent2(event.keycode, KEY_ACTION_DOWN, event.modifier, SS_KBE_FLAG_NON_NORMALIZED); + usleep(50 * 1000); + LiSendKeyboardEvent2(event.keycode, KEY_ACTION_UP, event.modifier, SS_KBE_FLAG_NON_NORMALIZED); + if (event.modifier != 0) { + LiSendKeyboardEvent(event.modifierKeycode, KEY_ACTION_UP, event.modifier); + } +} + - (BOOL)canBecomeFirstResponder { return YES; }