add passthrough touch support

Limelight/Database/DataManager.h

@@ -28,6 +28,7 @@
                        enableHdr:(BOOL)enableHdr
                   btMouseSupport:(BOOL)btMouseSupport
                absoluteTouchMode:(BOOL)absoluteTouchMode
+            passthroughTouchMode:(BOOL)passthroughTouchMode
                     statsOverlay:(BOOL)statsOverlay;
 
 - (NSArray*) getHosts;

Limelight/Database/DataManager.m

@@ -67,6 +67,7 @@ - (void) saveSettingsWithBitrate:(NSInteger)bitrate
                        enableHdr:(BOOL)enableHdr
                   btMouseSupport:(BOOL)btMouseSupport
                absoluteTouchMode:(BOOL)absoluteTouchMode
+            passthroughTouchMode:(BOOL)passthroughTouchMode
                     statsOverlay:(BOOL)statsOverlay {
 
     [_managedObjectContext performBlockAndWait:^{
@@ -86,6 +87,7 @@ - (void) saveSettingsWithBitrate:(NSInteger)bitrate
         settingsToSave.enableHdr = enableHdr;
         settingsToSave.btMouseSupport = btMouseSupport;
         settingsToSave.absoluteTouchMode = absoluteTouchMode;
+        settingsToSave.passthroughTouchMode = passthroughTouchMode;
         settingsToSave.statsOverlay = statsOverlay;
 
         [self saveData];

Limelight/Database/TemporarySettings.h

@@ -33,6 +33,7 @@
 @property (nonatomic) BOOL enableHdr;
 @property (nonatomic) BOOL btMouseSupport;
 @property (nonatomic) BOOL absoluteTouchMode;
+@property (nonatomic) BOOL passthroughTouchMode;
 @property (nonatomic) BOOL statsOverlay;
 
 - (id) initFromSettings:(Settings*)settings;

Limelight/Database/TemporarySettings.m

@@ -84,6 +84,7 @@ - (id) initFromSettings:(Settings*)settings {
     self.onscreenControls = settings.onscreenControls;
     self.btMouseSupport = settings.btMouseSupport;
     self.absoluteTouchMode = settings.absoluteTouchMode;
+    self.passthroughTouchMode = settings.passthroughTouchMode;
     self.statsOverlay = settings.statsOverlay;
 #endif
     self.uniqueId = settings.uniqueId;

Limelight/Input/PassthroughTouchHandler.h

@@ -0,0 +1,19 @@
+//
+//  PassthroughTouchHandler.h
+//  Moonlight
+//
+//  Created by ZigZagT on 5/20/2024.
+//  Copyright © 2024 Moonlight Game Streaming Project. All rights reserved.
+//
+
+#import "StreamView.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface PassthroughTouchHandler : UIResponder
+
+-(id)initWithView:(StreamView*)view;
+
+@end
+
+NS_ASSUME_NONNULL_END

Limelight/Input/PassthroughTouchHandler.m

@@ -0,0 +1,92 @@
+//
+//  PassthroughTouchHandler.m
+//  Moonlight
+//
+//  Created by ZigZagT on 5/20/2024.
+//  Copyright © 2024 Moonlight Game Streaming Project. All rights reserved.
+//
+
+#import "PassthroughTouchHandler.h"
+
+#include <Limelight.h>
+
+@implementation PassthroughTouchHandler {
+    StreamView* view;
+}
+
+- (id)initWithView:(StreamView*)view {
+    self = [self init];
+    self->view = view;
+    return self;
+}
+
+
+- (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
+    for (UITouch* touch in touches) {
+        CGPoint location = [self->view adjustCoordinatesForVideoArea:[touch locationInView:self->view]];
+        CGSize videoSize = [self->view getVideoAreaSize];
+        LiSendTouchEvent(
+            LI_TOUCH_EVENT_DOWN,
+            (uint32_t) (uint64_t) touch,
+            location.x / videoSize.width,
+            location.y / videoSize.height,
+            0,
+            0,
+            0,
+            LI_ROT_UNKNOWN
+        );
+    }
+}
+
+- (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
+    for (UITouch* touch in touches) {
+        CGPoint location = [self->view adjustCoordinatesForVideoArea:[touch locationInView:self->view]];
+        CGSize videoSize = [self->view getVideoAreaSize];
+        LiSendTouchEvent(
+            LI_TOUCH_EVENT_MOVE,
+            (uint32_t) (uint64_t) touch,
+            location.x / videoSize.width,
+            location.y / videoSize.height,
+            0,
+            0,
+            0,
+            LI_ROT_UNKNOWN
+        );
+    }
+}
+
+- (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
+    for (UITouch* touch in touches) {
+        CGPoint location = [self->view adjustCoordinatesForVideoArea:[touch locationInView:self->view]];
+        CGSize videoSize = [self->view getVideoAreaSize];
+        LiSendTouchEvent(
+            LI_TOUCH_EVENT_UP,
+            (uint32_t) (uint64_t) touch,
+            location.x / videoSize.width,
+            location.y / videoSize.height,
+            0,
+            0,
+            0,
+            LI_ROT_UNKNOWN
+        );
+    }
+}
+
+- (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event {
+    for (UITouch* touch in touches) {
+        CGPoint location = [self->view adjustCoordinatesForVideoArea:[touch locationInView:self->view]];
+        CGSize videoSize = [self->view getVideoAreaSize];
+        LiSendTouchEvent(
+            LI_TOUCH_EVENT_CANCEL,
+            (uint32_t) (uint64_t) touch,
+            location.x / videoSize.width,
+            location.y / videoSize.height,
+            0,
+            0,
+            0,
+            LI_ROT_UNKNOWN
+        );
+    }
+}
+
+@end

Limelight/Input/StreamView.h

@@ -34,4 +34,8 @@
 - (void) updateCursorLocation:(CGPoint)location isMouse:(BOOL)isMouse;
 #endif
 
+- (CGSize) getVideoAreaSize;
+- (CGPoint) adjustCoordinatesForVideoArea:(CGPoint)point;
+- (void) cancelAllTouchEvents;
+
 @end

Limelight/Input/StreamView.m

@@ -13,13 +13,15 @@
 #import "KeyboardSupport.h"
 #import "RelativeTouchHandler.h"
 #import "AbsoluteTouchHandler.h"
+#import "PassthroughTouchHandler.h"
 #import "KeyboardInputField.h"
 
 static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
 
 @implementation StreamView {
     OnScreenControls* onScreenControls;
 
+    int keyboardGestureNFingers;
     KeyboardInputField* keyInputField;
     BOOL isInputingText;
     NSMutableSet* keysDown;
@@ -51,6 +53,7 @@ - (void) setupStreamView:(ControllerSupport*)controllerSupport
                   config:(StreamConfiguration*)streamConfig {
     self->interactionDelegate = interactionDelegate;
     self->streamAspectRatio = (float)streamConfig.width / (float)streamConfig.height;
+    self->keyboardGestureNFingers = 3;
 
     TemporarySettings* settings = [[[DataManager alloc] init] getSettings];
 
@@ -68,7 +71,15 @@ - (void) setupStreamView:(ControllerSupport*)controllerSupport
 #else
     // iOS uses RelativeTouchHandler or AbsoluteTouchHandler depending on user preference
     if (settings.absoluteTouchMode) {
-        self->touchHandler = [[AbsoluteTouchHandler alloc] initWithView:self];
+        if (settings.passthroughTouchMode) {
+            self->touchHandler = [[PassthroughTouchHandler alloc] initWithView:self];
+            // "Touch gestures for Windows" uses up to four fingers according to Microsoft support articles.
+            // Register keyboard gesture with 5 fingers can avoid conflict.
+            // Also the popup keyboarde became less useful since Windows has a built in on-screen keyboard.
+            self->keyboardGestureNFingers = 5;
+        } else {
+            self->touchHandler = [[AbsoluteTouchHandler alloc] initWithView:self];
+        }
     }
     else {
         self->touchHandler = [[RelativeTouchHandler alloc] initWithView:self];
@@ -318,6 +329,21 @@ - (void)sendStylusHoverEvent:(UIHoverGestureRecognizer*)gesture API_AVAILABLE(io
 
 #endif
 
+- (void)cancelAllTouchEvents {
+    // Use this to cancel touch events when the intention is meant for local device.
+    // i.e. open notification center, end streaming by swiping on edge
+    LiSendTouchEvent(
+        LI_TOUCH_EVENT_CANCEL_ALL,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        LI_ROT_UNKNOWN
+    );
+}
+
 - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
     if ([self handleMouseButtonEvent:BUTTON_ACTION_PRESS
                           forTouches:touches
@@ -351,7 +377,7 @@ - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
         // is triggered.
         [touchHandler touchesBegan:touches withEvent:event];
 
-        if ([[event allTouches] count] == 3) {
+        if ([[event allTouches] count] == self->keyboardGestureNFingers) {
             if (isInputingText) {
                 Log(LOG_D, @"Closing the keyboard");
                 [keyInputField resignFirstResponder];

Limelight/Limelight.xcdatamodeld/Moonlight v1.10.xcdatamodel/contents

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22225" systemVersion="23A344" minimumToolsVersion="Xcode 7.3" sourceLanguage="Objective-C" userDefinedModelVersionIdentifier="">
+<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22757" systemVersion="23E224" minimumToolsVersion="Xcode 7.3" sourceLanguage="Objective-C" userDefinedModelVersionIdentifier="">
     <entity name="App" representedClassName="App" syncable="YES" codeGenerationType="class">
         <attribute name="hdrSupported" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="hidden" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
@@ -32,6 +32,7 @@
         <attribute name="multiController" attributeType="Boolean" defaultValueString="YES" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="onscreenControls" attributeType="Integer 32" defaultValueString="1" usesScalarValueType="NO" syncable="YES"/>
         <attribute name="optimizeGames" attributeType="Boolean" defaultValueString="YES" usesScalarValueType="YES" syncable="YES"/>
+        <attribute name="passthroughTouchMode" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="playAudioOnPC" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="preferredCodec" attributeType="Integer 32" defaultValueString="YES" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="statsOverlay" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>

Limelight/ViewControllers/SettingsViewController.h

@@ -16,6 +16,7 @@
 @property (strong, nonatomic) IBOutlet UISegmentedControl *resolutionSelector;
 @property (strong, nonatomic) IBOutlet UIView *resolutionDisplayView;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *touchModeSelector;
+@property (strong, nonatomic) IBOutlet UISegmentedControl *passthroughTouchModeSelector;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *onscreenControlSelector;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *optimizeSettingsSelector;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *multiControllerSelector;

Limelight/ViewControllers/SettingsViewController.m

@@ -242,6 +242,8 @@ - (void)viewDidLoad {
 
     [self.touchModeSelector setSelectedSegmentIndex:currentSettings.absoluteTouchMode ? 1 : 0];
     [self.touchModeSelector addTarget:self action:@selector(touchModeChanged) forControlEvents:UIControlEventValueChanged];
+    [self.passthroughTouchModeSelector setSelectedSegmentIndex:currentSettings.passthroughTouchMode ? 1 : 0];
+    [self.passthroughTouchModeSelector setEnabled:currentSettings.absoluteTouchMode];
     [self.statsOverlaySelector setSelectedSegmentIndex:currentSettings.statsOverlay ? 1 : 0];
     [self.btMouseSelector setSelectedSegmentIndex:currentSettings.btMouseSupport ? 1 : 0];
     [self.optimizeSettingsSelector setSelectedSegmentIndex:currentSettings.optimizeGames ? 1 : 0];
@@ -268,6 +270,7 @@ - (void)viewDidLoad {
 - (void) touchModeChanged {
     // Disable on-screen controls in absolute touch mode
     [self.onscreenControlSelector setEnabled:[self.touchModeSelector selectedSegmentIndex] == 0];
+    [self.passthroughTouchModeSelector setEnabled:[self.touchModeSelector selectedSegmentIndex] == 1];
 }
 
 - (void) updateBitrate {
@@ -536,6 +539,7 @@ - (void) saveSettings {
     BOOL btMouseSupport = [self.btMouseSelector selectedSegmentIndex] == 1;
     BOOL useFramePacing = [self.framePacingSelector selectedSegmentIndex] == 1;
     BOOL absoluteTouchMode = [self.touchModeSelector selectedSegmentIndex] == 1;
+    BOOL passthroughTouchMode = [self.passthroughTouchModeSelector selectedSegmentIndex] == 1;
     BOOL statsOverlay = [self.statsOverlaySelector selectedSegmentIndex] == 1;
     BOOL enableHdr = [self.hdrSelector selectedSegmentIndex] == 1;
     [dataMan saveSettingsWithBitrate:_bitrate
@@ -553,6 +557,7 @@ - (void) saveSettings {
                            enableHdr:enableHdr
                       btMouseSupport:btMouseSupport
                    absoluteTouchMode:absoluteTouchMode
+                passthroughTouchMode:passthroughTouchMode
                         statsOverlay:statsOverlay];
 }
 

Limelight/ViewControllers/StreamFrameViewController.m

@@ -186,11 +186,15 @@ - (void)viewDidLoad
                                                object: nil];
 #endif
 
-    // Only enable scroll and zoom in absolute touch mode
-    if (_settings.absoluteTouchMode) {
+    // Only enable scroll and zoom in absolute touch mode with exception of passthrough touch mode
+    // Windows has its own pinch / pan / scroll handling which conflicts with UIScrollView gestures.
+    if (_settings.absoluteTouchMode && !_settings.passthroughTouchMode) {
         _scrollView = [[UIScrollView alloc] initWithFrame:self.view.frame];
+        _scrollView.delaysContentTouches = NO;
 #if !TARGET_OS_TV
         [_scrollView.panGestureRecognizer setMinimumNumberOfTouches:2];
+        [_scrollView.panGestureRecognizer setDelaysTouchesBegan: NO];
+        [_scrollView.panGestureRecognizer setDelaysTouchesEnded: NO];
 #endif
         [_scrollView setShowsHorizontalScrollIndicator:NO];
         [_scrollView setShowsVerticalScrollIndicator:NO];
@@ -306,6 +310,7 @@ - (void)updateOverlayText:(NSString*)text {
 }
 
 - (void) returnToMainFrame {
+    [self->_streamView cancelAllTouchEvents];
     // Reset display mode back to default
     [self updatePreferredDisplayMode:NO];
 
@@ -317,6 +322,7 @@ - (void) returnToMainFrame {
 
 // This will fire if the user opens control center or gets a low battery message
 - (void)applicationWillResignActive:(NSNotification *)notification {
+    [self->_streamView cancelAllTouchEvents];
     if (_inactivityTimer != nil) {
         [_inactivityTimer invalidate];
     }

Moonlight.xcodeproj/project.pbxproj

@@ -136,6 +136,7 @@
 		FBD349621A0089F6002D2A60 /* DataManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FBD349611A0089F6002D2A60 /* DataManager.m */; };
 		FBDE86E019F7A837001C18A8 /* UIComputerView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86DF19F7A837001C18A8 /* UIComputerView.m */; };
 		FBDE86E619F82297001C18A8 /* UIAppView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86E519F82297001C18A8 /* UIAppView.m */; };
+		FD10B0BF2BF1729A009AC503 /* PassthroughTouchHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = FD10B0BE2BF1729A009AC503 /* PassthroughTouchHandler.m */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -398,6 +399,8 @@
 		FBDE86DF19F7A837001C18A8 /* UIComputerView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = UIComputerView.m; sourceTree = "<group>"; };
 		FBDE86E419F82297001C18A8 /* UIAppView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = UIAppView.h; sourceTree = "<group>"; };
 		FBDE86E519F82297001C18A8 /* UIAppView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = UIAppView.m; sourceTree = "<group>"; };
+		FD10B0BD2BF1729A009AC503 /* PassthroughTouchHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PassthroughTouchHandler.h; sourceTree = "<group>"; };
+		FD10B0BE2BF1729A009AC503 /* PassthroughTouchHandler.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PassthroughTouchHandler.m; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -580,6 +583,8 @@
 		FB89460919F646E200339C8A /* Input */ = {
 			isa = PBXGroup;
 			children = (
+				FD10B0BD2BF1729A009AC503 /* PassthroughTouchHandler.h */,
+				FD10B0BE2BF1729A009AC503 /* PassthroughTouchHandler.m */,
 				FB89460A19F646E200339C8A /* ControllerSupport.h */,
 				FB89460B19F646E200339C8A /* ControllerSupport.m */,
 				FB89460C19F646E200339C8A /* StreamView.h */,
@@ -1084,6 +1089,7 @@
 				FBD3495319FF36FB002D2A60 /* SWRevealViewController.m in Sources */,
 				FB1D59971BBCCB6400F482CA /* ComputerScrollView.m in Sources */,
 				FBD3495019FF2174002D2A60 /* SettingsViewController.m in Sources */,
+				FD10B0BF2BF1729A009AC503 /* PassthroughTouchHandler.m in Sources */,
 				FB89462C19F646E200339C8A /* HttpManager.m in Sources */,
 				FB89462D19F646E200339C8A /* MDNSManager.m in Sources */,
 				FB89462B19F646E200339C8A /* StreamView.m in Sources */,