add passthrough touch support

moonlight-stream · May 21, 2024 · 2e73fb1 · 2e73fb1
1 parent 557765f
commit 2e73fb1
Show file tree

Hide file tree

Showing 13 changed files with 99 additions and 9 deletions.
diff --git a/Limelight/Database/DataManager.h b/Limelight/Database/DataManager.h
@@ -28,6 +28,7 @@
                        enableHdr:(BOOL)enableHdr
                   btMouseSupport:(BOOL)btMouseSupport
                absoluteTouchMode:(BOOL)absoluteTouchMode
+            passthroughTouchMode:(BOOL)passthroughTouchMode
                     statsOverlay:(BOOL)statsOverlay;
 
 - (NSArray*) getHosts;

diff --git a/Limelight/Database/DataManager.m b/Limelight/Database/DataManager.m
@@ -67,6 +67,7 @@ - (void) saveSettingsWithBitrate:(NSInteger)bitrate
                        enableHdr:(BOOL)enableHdr
                   btMouseSupport:(BOOL)btMouseSupport
                absoluteTouchMode:(BOOL)absoluteTouchMode
+            passthroughTouchMode:(BOOL)passthroughTouchMode
                     statsOverlay:(BOOL)statsOverlay {
 
     [_managedObjectContext performBlockAndWait:^{
@@ -86,6 +87,7 @@ - (void) saveSettingsWithBitrate:(NSInteger)bitrate
         settingsToSave.enableHdr = enableHdr;
         settingsToSave.btMouseSupport = btMouseSupport;
         settingsToSave.absoluteTouchMode = absoluteTouchMode;
+        settingsToSave.passthroughTouchMode = passthroughTouchMode;
         settingsToSave.statsOverlay = statsOverlay;
 
         [self saveData];

diff --git a/Limelight/Database/TemporarySettings.h b/Limelight/Database/TemporarySettings.h
@@ -33,6 +33,7 @@
 @property (nonatomic) BOOL enableHdr;
 @property (nonatomic) BOOL btMouseSupport;
 @property (nonatomic) BOOL absoluteTouchMode;
+@property (nonatomic) BOOL passthroughTouchMode;
 @property (nonatomic) BOOL statsOverlay;
 
 - (id) initFromSettings:(Settings*)settings;

diff --git a/Limelight/Database/TemporarySettings.m b/Limelight/Database/TemporarySettings.m
@@ -84,6 +84,7 @@ - (id) initFromSettings:(Settings*)settings {
     self.onscreenControls = settings.onscreenControls;
     self.btMouseSupport = settings.btMouseSupport;
     self.absoluteTouchMode = settings.absoluteTouchMode;
+    self.passthroughTouchMode = settings.passthroughTouchMode;
     self.statsOverlay = settings.statsOverlay;
 #endif
     self.uniqueId = settings.uniqueId;

diff --git a/Limelight/Input/StreamView.h b/Limelight/Input/StreamView.h
@@ -34,4 +34,8 @@
 - (void) updateCursorLocation:(CGPoint)location isMouse:(BOOL)isMouse;
 #endif
 
+- (CGSize) getVideoAreaSize;
+- (CGPoint) adjustCoordinatesForVideoArea:(CGPoint)point;
+- (void) cancelAllTouchEvents;
+
 @end
diff --git a/Limelight/Input/StreamView.m b/Limelight/Input/StreamView.m
@@ -13,13 +13,15 @@
 #import "KeyboardSupport.h"
 #import "RelativeTouchHandler.h"
 #import "AbsoluteTouchHandler.h"
+#import "PassthroughTouchHandler.h"
 #import "KeyboardInputField.h"
 
 static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
 
 @implementation StreamView {
     OnScreenControls* onScreenControls;
 
+    int keyboardGestureNFingers;
     KeyboardInputField* keyInputField;
     BOOL isInputingText;
     NSMutableSet* keysDown;
@@ -51,6 +53,7 @@ - (void) setupStreamView:(ControllerSupport*)controllerSupport
                   config:(StreamConfiguration*)streamConfig {
     self->interactionDelegate = interactionDelegate;
     self->streamAspectRatio = (float)streamConfig.width / (float)streamConfig.height;
+    self->keyboardGestureNFingers = 3;
 
     TemporarySettings* settings = [[[DataManager alloc] init] getSettings];
 
@@ -68,7 +71,15 @@ - (void) setupStreamView:(ControllerSupport*)controllerSupport
 #else
     // iOS uses RelativeTouchHandler or AbsoluteTouchHandler depending on user preference
     if (settings.absoluteTouchMode) {
-        self->touchHandler = [[AbsoluteTouchHandler alloc] initWithView:self];
+        if (settings.passthroughTouchMode) {
+            self->touchHandler = [[PassthroughTouchHandler alloc] initWithView:self];
+            // "Touch gestures for Windows" uses up to four fingers according to Microsoft support articles.
+            // Register keyboard gesture with 5 fingers can avoid conflict.
+            // Also the popup keyboarde became less useful since Windows has a built in on-screen keyboard.
+            self->keyboardGestureNFingers = 5;
+        } else {
+            self->touchHandler = [[AbsoluteTouchHandler alloc] initWithView:self];
+        }
     }
     else {
         self->touchHandler = [[RelativeTouchHandler alloc] initWithView:self];
@@ -318,6 +329,21 @@ - (void)sendStylusHoverEvent:(UIHoverGestureRecognizer*)gesture API_AVAILABLE(io
 
 #endif
 
+- (void)cancelAllTouchEvents {
+    // Use this to cancel touch events when the intention is meant for local device.
+    // i.e. open notification center, end streaming by swiping on edge
+    LiSendTouchEvent(
+        LI_TOUCH_EVENT_CANCEL_ALL,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        LI_ROT_UNKNOWN
+    );
+}
+
 - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
     if ([self handleMouseButtonEvent:BUTTON_ACTION_PRESS
                           forTouches:touches
@@ -351,7 +377,7 @@ - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
         // is triggered.
         [touchHandler touchesBegan:touches withEvent:event];
 
-        if ([[event allTouches] count] == 3) {
+        if ([[event allTouches] count] == self->keyboardGestureNFingers) {
             if (isInputingText) {
                 Log(LOG_D, @"Closing the keyboard");
                 [keyInputField resignFirstResponder];

diff --git a/Limelight/Limelight.xcdatamodeld/Moonlight v1.10.xcdatamodel/contents b/Limelight/Limelight.xcdatamodeld/Moonlight v1.10.xcdatamodel/contents
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22225" systemVersion="23A344" minimumToolsVersion="Xcode 7.3" sourceLanguage="Objective-C" userDefinedModelVersionIdentifier="">
+<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22757" systemVersion="23E224" minimumToolsVersion="Xcode 7.3" sourceLanguage="Objective-C" userDefinedModelVersionIdentifier="">
     <entity name="App" representedClassName="App" syncable="YES" codeGenerationType="class">
         <attribute name="hdrSupported" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="hidden" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
@@ -32,6 +32,7 @@
         <attribute name="multiController" attributeType="Boolean" defaultValueString="YES" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="onscreenControls" attributeType="Integer 32" defaultValueString="1" usesScalarValueType="NO" syncable="YES"/>
         <attribute name="optimizeGames" attributeType="Boolean" defaultValueString="YES" usesScalarValueType="YES" syncable="YES"/>
+        <attribute name="passthroughTouchMode" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="playAudioOnPC" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="preferredCodec" attributeType="Integer 32" defaultValueString="YES" usesScalarValueType="YES" syncable="YES"/>
         <attribute name="statsOverlay" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES" syncable="YES"/>

diff --git a/Limelight/ViewControllers/SettingsViewController.h b/Limelight/ViewControllers/SettingsViewController.h
@@ -16,6 +16,7 @@
 @property (strong, nonatomic) IBOutlet UISegmentedControl *resolutionSelector;
 @property (strong, nonatomic) IBOutlet UIView *resolutionDisplayView;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *touchModeSelector;
+@property (strong, nonatomic) IBOutlet UISegmentedControl *passthroughTouchModeSelector;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *onscreenControlSelector;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *optimizeSettingsSelector;
 @property (strong, nonatomic) IBOutlet UISegmentedControl *multiControllerSelector;

diff --git a/Limelight/ViewControllers/SettingsViewController.m b/Limelight/ViewControllers/SettingsViewController.m
@@ -242,6 +242,8 @@ - (void)viewDidLoad {
 
     [self.touchModeSelector setSelectedSegmentIndex:currentSettings.absoluteTouchMode ? 1 : 0];
     [self.touchModeSelector addTarget:self action:@selector(touchModeChanged) forControlEvents:UIControlEventValueChanged];
+    [self.passthroughTouchModeSelector setSelectedSegmentIndex:currentSettings.passthroughTouchMode ? 1 : 0];
+    [self.passthroughTouchModeSelector setEnabled:currentSettings.absoluteTouchMode];
     [self.statsOverlaySelector setSelectedSegmentIndex:currentSettings.statsOverlay ? 1 : 0];
     [self.btMouseSelector setSelectedSegmentIndex:currentSettings.btMouseSupport ? 1 : 0];
     [self.optimizeSettingsSelector setSelectedSegmentIndex:currentSettings.optimizeGames ? 1 : 0];
@@ -268,6 +270,7 @@ - (void)viewDidLoad {
 - (void) touchModeChanged {
     // Disable on-screen controls in absolute touch mode
     [self.onscreenControlSelector setEnabled:[self.touchModeSelector selectedSegmentIndex] == 0];
+    [self.passthroughTouchModeSelector setEnabled:[self.touchModeSelector selectedSegmentIndex] == 1];
 }
 
 - (void) updateBitrate {
@@ -536,6 +539,7 @@ - (void) saveSettings {
     BOOL btMouseSupport = [self.btMouseSelector selectedSegmentIndex] == 1;
     BOOL useFramePacing = [self.framePacingSelector selectedSegmentIndex] == 1;
     BOOL absoluteTouchMode = [self.touchModeSelector selectedSegmentIndex] == 1;
+    BOOL passthroughTouchMode = [self.passthroughTouchModeSelector selectedSegmentIndex] == 1;
     BOOL statsOverlay = [self.statsOverlaySelector selectedSegmentIndex] == 1;
     BOOL enableHdr = [self.hdrSelector selectedSegmentIndex] == 1;
     [dataMan saveSettingsWithBitrate:_bitrate
@@ -553,6 +557,7 @@ - (void) saveSettings {
                            enableHdr:enableHdr
                       btMouseSupport:btMouseSupport
                    absoluteTouchMode:absoluteTouchMode
+                passthroughTouchMode:passthroughTouchMode
                         statsOverlay:statsOverlay];
 }
 

diff --git a/Limelight/ViewControllers/StreamFrameViewController.m b/Limelight/ViewControllers/StreamFrameViewController.m
@@ -186,11 +186,15 @@ - (void)viewDidLoad
                                                object: nil];
 #endif
 
-    // Only enable scroll and zoom in absolute touch mode
-    if (_settings.absoluteTouchMode) {
+    // Only enable scroll and zoom in absolute touch mode with exception of passthrough touch mode
+    // Windows has its own pinch / pan / scroll handling which conflicts with UIScrollView gestures.
+    if (_settings.absoluteTouchMode && !_settings.passthroughTouchMode) {
         _scrollView = [[UIScrollView alloc] initWithFrame:self.view.frame];
+        _scrollView.delaysContentTouches = NO;
 #if !TARGET_OS_TV
         [_scrollView.panGestureRecognizer setMinimumNumberOfTouches:2];
+        [_scrollView.panGestureRecognizer setDelaysTouchesBegan: NO];
+        [_scrollView.panGestureRecognizer setDelaysTouchesEnded: NO];
 #endif
         [_scrollView setShowsHorizontalScrollIndicator:NO];
         [_scrollView setShowsVerticalScrollIndicator:NO];
@@ -306,6 +310,7 @@ - (void)updateOverlayText:(NSString*)text {
 }
 
 - (void) returnToMainFrame {
+    [self->_streamView cancelAllTouchEvents];
     // Reset display mode back to default
     [self updatePreferredDisplayMode:NO];
 
@@ -317,6 +322,7 @@ - (void) returnToMainFrame {
 
 // This will fire if the user opens control center or gets a low battery message
 - (void)applicationWillResignActive:(NSNotification *)notification {
+    [self->_streamView cancelAllTouchEvents];
     if (_inactivityTimer != nil) {
         [_inactivityTimer invalidate];
     }

diff --git a/Moonlight.xcodeproj/project.pbxproj b/Moonlight.xcodeproj/project.pbxproj
@@ -136,6 +136,7 @@
 		FBD349621A0089F6002D2A60 /* DataManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FBD349611A0089F6002D2A60 /* DataManager.m */; };
 		FBDE86E019F7A837001C18A8 /* UIComputerView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86DF19F7A837001C18A8 /* UIComputerView.m */; };
 		FBDE86E619F82297001C18A8 /* UIAppView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86E519F82297001C18A8 /* UIAppView.m */; };
+		FD10B0BF2BF1729A009AC503 /* PassthroughTouchHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = FD10B0BE2BF1729A009AC503 /* PassthroughTouchHandler.m */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -398,6 +399,8 @@
 		FBDE86DF19F7A837001C18A8 /* UIComputerView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = UIComputerView.m; sourceTree = "<group>"; };
 		FBDE86E419F82297001C18A8 /* UIAppView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = UIAppView.h; sourceTree = "<group>"; };
 		FBDE86E519F82297001C18A8 /* UIAppView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = UIAppView.m; sourceTree = "<group>"; };
+		FD10B0BD2BF1729A009AC503 /* PassthroughTouchHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PassthroughTouchHandler.h; sourceTree = "<group>"; };
+		FD10B0BE2BF1729A009AC503 /* PassthroughTouchHandler.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PassthroughTouchHandler.m; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -580,6 +583,8 @@
 		FB89460919F646E200339C8A /* Input */ = {
 			isa = PBXGroup;
 			children = (
+				FD10B0BD2BF1729A009AC503 /* PassthroughTouchHandler.h */,
+				FD10B0BE2BF1729A009AC503 /* PassthroughTouchHandler.m */,
 				FB89460A19F646E200339C8A /* ControllerSupport.h */,
 				FB89460B19F646E200339C8A /* ControllerSupport.m */,
 				FB89460C19F646E200339C8A /* StreamView.h */,
@@ -1084,6 +1089,7 @@
 				FBD3495319FF36FB002D2A60 /* SWRevealViewController.m in Sources */,
 				FB1D59971BBCCB6400F482CA /* ComputerScrollView.m in Sources */,
 				FBD3495019FF2174002D2A60 /* SettingsViewController.m in Sources */,
+				FD10B0BF2BF1729A009AC503 /* PassthroughTouchHandler.m in Sources */,
 				FB89462C19F646E200339C8A /* HttpManager.m in Sources */,
 				FB89462D19F646E200339C8A /* MDNSManager.m in Sources */,
 				FB89462B19F646E200339C8A /* StreamView.m in Sources */,

diff --git a/iPad.storyboard b/iPad.storyboard
@@ -1,9 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="22155" targetRuntime="iOS.CocoaTouch.iPad" propertyAccessControl="none" useAutolayout="YES" colorMatched="YES" initialViewController="EVd-wq-ego">
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="32700.99.1234" targetRuntime="iOS.CocoaTouch.iPad" propertyAccessControl="none" useAutolayout="YES" colorMatched="YES" initialViewController="EVd-wq-ego">
     <device id="ipad11_0rounded" orientation="portrait" layout="fullscreen" appearance="dark"/>
     <dependencies>
         <deployment identifier="iOS"/>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="22131"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="22684"/>
         <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
     </dependencies>
     <scenes>
@@ -159,6 +159,23 @@
                                 <color key="tintColor" red="0.6716768741607666" green="0.61711704730987549" blue="0.99902987480163574" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
                                 <color key="selectedSegmentTintColor" red="0.6716768741607666" green="0.61711704730987549" blue="0.99902987480163574" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
                             </segmentedControl>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Passthrough Touch Events" textAlignment="natural" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="hZG-fD-Dq3" userLabel="Passthrough Touch Events">
+                                <rect key="frame" x="16" y="968" width="400" height="21"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                <color key="textColor" red="0.93902439019999995" green="0.9625305918" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                            <segmentedControl opaque="NO" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="plain" selectedSegmentIndex="0" translatesAutoresizingMaskIntoConstraints="NO" id="aJK-RV-OTP" userLabel="Passthrough Touch Mode Selector">
+                                <rect key="frame" x="16" y="997" width="459" height="28"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <segments>
+                                    <segment title="No"/>
+                                    <segment title="Yes"/>
+                                </segments>
+                                <color key="tintColor" red="0.6716768742" green="0.61711704730000005" blue="0.99902987480000005" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                                <color key="selectedSegmentTintColor" red="0.6716768742" green="0.61711704730000005" blue="0.99902987480000005" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                            </segmentedControl>
                             <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Optimize Game Settings" textAlignment="natural" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="WSI-9V-G6c">
                                 <rect key="frame" x="16" y="388" width="186" height="21"/>
                                 <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
@@ -329,6 +346,7 @@
                         <outlet property="multiControllerSelector" destination="KlC-fG-wEi" id="giM-F7-So5"/>
                         <outlet property="onscreenControlSelector" destination="qSU-wh-tqA" id="j8d-fB-Z2c"/>
                         <outlet property="optimizeSettingsSelector" destination="nCO-OT-dV1" id="FB0-Rt-C44"/>
+                        <outlet property="passthroughTouchModeSelector" destination="aJK-RV-OTP" id="jCt-y6-5Jn"/>
                         <outlet property="resolutionDisplayView" destination="9cG-JD-glb" id="TT1-hy-MUC"/>
                         <outlet property="resolutionSelector" destination="ckc-Dm-8ex" id="rl6-rx-wd3"/>
                         <outlet property="scrollView" destination="WRy-3f-gEP" id="V8B-oF-27B"/>