Merge pull request #7893 from utmapp/ios-touch-gestures

display(iOS): fix touch gestures and add drag to scroll mode

Platform/iOS/Display/VMDisplayMetalViewController+Private.h

@@ -34,6 +34,10 @@ NS_ASSUME_NONNULL_BEGIN
 
 // cursor handling
 @property (nonatomic) CGPoint lastTwoPanOrigin;
+@property (nonatomic) CGPoint lastLongPressOrigin;
+@property (nonatomic) CGPoint lastTapLocation;
+@property (nonatomic) NSTimeInterval lastTapTime;
+@property (nonatomic) BOOL isTouchScrolling;
 @property (nonatomic) BOOL mouseLeftDown;
 @property (nonatomic) BOOL mouseRightDown;
 @property (nonatomic) BOOL mouseMiddleDown;

Platform/iOS/Display/VMDisplayMetalViewController+Touch.h

@@ -23,6 +23,9 @@ typedef NS_ENUM(NSInteger, VMGestureType) {
     VMGestureTypeRightClick,
     VMGestureTypeMoveScreen,
     VMGestureTypeMouseWheel,
+    VMGestureTypeMiddleClick,
+    VMGestureTypeRightDrag,
+    VMGestureTypeMiddleDrag,
     VMGestureTypeMax
 };
 
@@ -30,6 +33,7 @@ typedef NS_ENUM(NSInteger, VMMouseType) {
     VMMouseTypeRelative,
     VMMouseTypeAbsolute,
     VMMouseTypeAbsoluteHideCursor,
+    VMMouseTypeAbsoluteScroll,
     VMMouseTypeMax
 };
 

Platform/iOS/Display/VMDisplayMetalViewController+Touch.m

@@ -30,6 +30,9 @@
 const CGFloat kScrollSpeedReduction = 100.0f;
 const CGFloat kCursorResistance = 50.0f;
 const CGFloat kScrollResistance = 10.0f;
+const CGFloat kLongPressDragThreshold = 10.0f;
+const CGFloat kRepeatTapDistance = 24.0f;
+const NSTimeInterval kRepeatTapInterval = 0.5;
 
 @implementation VMDisplayMetalViewController (Gestures)
 
@@ -165,6 +168,11 @@ const CGFloat kScrollResistance = 10.0f;
     return [self gestureTypeForSetting:@"GestureTwoScroll"];
 }
 
+/// A two finger swipe must be told apart from a two finger pan unless both scroll
+- (BOOL)isTwoFingerSwipeExclusive {
+    return self.twoFingerScrollType != VMGestureTypeNone && self.twoFingerPanType != VMGestureTypeMouseWheel;
+}
+
 - (VMGestureType)threeFingerPanType {
     return [self gestureTypeForSetting:@"GestureThreePan"];
 }
@@ -287,6 +295,10 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 - (void)scrollWithInertia:(UIPanGestureRecognizer *)sender {
     CGPoint location = [sender locationInView:sender.view];
     CGPoint velocity = [sender velocityInView:sender.view];
+    if (sender == self.pan) { // one finger drags the content along with it, the opposite of turning a wheel
+        location.y = -location.y;
+        velocity.y = -velocity.y;
+    }
     if (sender.state == UIGestureRecognizerStateBegan) {
         [self.scroll startMovement:location];
     }
@@ -301,6 +313,8 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 - (IBAction)gesturePan:(UIPanGestureRecognizer *)sender {
     if (self.serverModeCursor) {  // otherwise we handle in touchesMoved
         [self moveMouseWithInertia:sender];
+    } else if (self.isTouchScrolling) {
+        [self scrollWithInertia:sender];
     }
 }
 
@@ -328,7 +342,9 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
             [self moveScreen:sender];
             break;
         case VMGestureTypeDragCursor:
-            [self dragCursor:sender.state primary:YES secondary:NO middle:NO];
+        case VMGestureTypeRightDrag:
+        case VMGestureTypeMiddleDrag:
+            [self dragCursor:sender.state gestureType:self.twoFingerPanType];
             [self moveMouseWithInertia:sender];
             break;
         case VMGestureTypeMouseWheel:
@@ -345,7 +361,9 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
             [self moveScreen:sender];
             break;
         case VMGestureTypeDragCursor:
-            [self dragCursor:sender.state primary:YES secondary:NO middle:NO];
+        case VMGestureTypeRightDrag:
+        case VMGestureTypeMiddleDrag:
+            [self dragCursor:sender.state gestureType:self.threeFingerPanType];
             [self moveMouseWithInertia:sender];
             break;
         case VMGestureTypeMouseWheel:
@@ -431,7 +449,7 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
             self.mouseMiddleDown = YES;
         }
         [self.vmInput sendMouseButton:button mask:self.mouseButtonDown pressed:YES];
-    } else if (state == UIGestureRecognizerStateEnded) {
+    } else if (state == UIGestureRecognizerStateEnded || state == UIGestureRecognizerStateCancelled) {
         self.mouseLeftDown = NO;
         self.mouseRightDown = NO;
         self.mouseMiddleDown = NO;
@@ -439,26 +457,118 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
     }
 }
 
+- (CSInputButton)buttonForGestureType:(VMGestureType)type {
+    switch (type) {
+        case VMGestureTypeDragCursor:
+            return kCSInputButtonLeft;
+        case VMGestureTypeRightClick:
+        case VMGestureTypeRightDrag:
+            return kCSInputButtonRight;
+        case VMGestureTypeMiddleClick:
+        case VMGestureTypeMiddleDrag:
+            return kCSInputButtonMiddle;
+        default:
+            return kCSInputButtonNone;
+    }
+}
+
+- (void)dragCursor:(UIGestureRecognizerState)state gestureType:(VMGestureType)type {
+    CSInputButton button = [self buttonForGestureType:type];
+    [self dragCursor:state
+             primary:button == kCSInputButtonLeft
+           secondary:button == kCSInputButtonRight
+              middle:button == kCSInputButtonMiddle];
+}
+
+/// A finger never lands on the same spot twice and the guest only counts clicks on the same spot as a double click
+- (BOOL)isRepeatTapAtLocation:(CGPoint)location {
+    CGFloat distance = hypot(location.x - self.lastTapLocation.x, location.y - self.lastTapLocation.y);
+    NSTimeInterval elapsed = NSProcessInfo.processInfo.systemUptime - self.lastTapTime;
+    return elapsed < kRepeatTapInterval && distance < kRepeatTapDistance;
+}
+
 - (IBAction)gestureTap:(UITapGestureRecognizer *)sender {
-    if (sender.state == UIGestureRecognizerStateEnded &&
-        self.serverModeCursor) { // otherwise we handle in touchesBegan
-        [self mouseClick:kCSInputButtonLeft location:[sender locationInView:sender.view]];
+    if (sender.state != UIGestureRecognizerStateEnded) {
+        return;
+    }
+    CGPoint location = [sender locationInView:sender.view];
+    if (self.isTouchScrolling) {
+        if ([self isRepeatTapAtLocation:location]) {
+            location = self.lastTapLocation;
+        }
+        self.lastTapLocation = location;
+        self.lastTapTime = NSProcessInfo.processInfo.systemUptime;
+        [self mouseClick:kCSInputButtonLeft location:location];
+    } else if (self.serverModeCursor) { // otherwise we handle in touchesBegan
+        [self mouseClick:kCSInputButtonLeft location:location];
     }
 }
 
 - (IBAction)gestureTwoTap:(UITapGestureRecognizer *)sender {
-    if (sender.state == UIGestureRecognizerStateEnded &&
-        self.twoFingerTapType == VMGestureTypeRightClick) {
-        [self mouseClick:kCSInputButtonRight location:[sender locationInView:sender.view]];
+    CSInputButton button = [self buttonForGestureType:self.twoFingerTapType];
+    if (sender.state == UIGestureRecognizerStateEnded && button != kCSInputButtonNone) {
+        [self mouseClick:button location:[sender locationInView:sender.view]];
+    }
+}
+
+- (void)showLongPressIndicatorAtLocation:(CGPoint)location {
+    UIView *indicator = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 80, 80)];
+    indicator.center = location;
+    indicator.userInteractionEnabled = NO;
+    indicator.layer.cornerRadius = 40;
+    indicator.layer.borderWidth = 3;
+    indicator.layer.borderColor = UIColor.whiteColor.CGColor;
+    indicator.layer.shadowOpacity = 0.5f;
+    indicator.layer.shadowRadius = 3;
+    indicator.layer.shadowOffset = CGSizeZero;
+    [self.mtkView addSubview:indicator];
+    [UIView animateWithDuration:0.3 delay:0 options:UIViewAnimationOptionCurveEaseOut animations:^{
+        indicator.alpha = 0;
+        indicator.transform = CGAffineTransformMakeScale(1.4f, 1.4f);
+    } completion:^(BOOL finished) {
+        [indicator removeFromSuperview];
+    }];
+}
+
+/// A touch does not hold the button when it scrolls so a long press that moves becomes the drag
+- (void)touchScrollLongPress:(UILongPressGestureRecognizer *)sender {
+    CGPoint location = [sender locationInView:sender.view];
+    if (sender.state == UIGestureRecognizerStateBegan) {
+        self.lastLongPressOrigin = location;
+        [self.cursor startMovement:location];
+        [self.cursor updateMovement:location];
+        [self showLongPressIndicatorAtLocation:location];
+        if (self.longPressType == VMGestureTypeDragCursor) {
+            [self dragCursor:sender.state gestureType:VMGestureTypeDragCursor];
+        }
+    } else if (sender.state == UIGestureRecognizerStateChanged) {
+        CGFloat distance = hypot(location.x - self.lastLongPressOrigin.x, location.y - self.lastLongPressOrigin.y);
+        if (!self.mouseLeftDown && distance >= kLongPressDragThreshold) {
+            [self dragCursor:UIGestureRecognizerStateBegan gestureType:VMGestureTypeDragCursor];
+        }
+        if (self.mouseLeftDown) {
+            [self.cursor updateMovement:location];
+        }
+    } else if (self.mouseLeftDown) {
+        [self dragCursor:UIGestureRecognizerStateEnded gestureType:VMGestureTypeDragCursor];
+    } else if (sender.state == UIGestureRecognizerStateEnded) {
+        CSInputButton button = [self buttonForGestureType:self.longPressType];
+        if (button != kCSInputButtonNone) {
+            [self mouseClick:button location:self.lastLongPressOrigin];
+        }
     }
 }
 
 - (IBAction)gestureLongPress:(UILongPressGestureRecognizer *)sender {
-    if (sender.state == UIGestureRecognizerStateEnded &&
-        self.longPressType == VMGestureTypeRightClick) {
-        [self mouseClick:kCSInputButtonRight location:[sender locationInView:sender.view]];
+    if (self.isTouchScrolling) {
+        [self touchScrollLongPress:sender];
     } else if (self.longPressType == VMGestureTypeDragCursor) {
-        [self dragCursor:sender.state primary:YES secondary:NO middle:NO];
+        [self dragCursor:sender.state gestureType:self.longPressType];
+    } else if (sender.state == UIGestureRecognizerStateEnded) {
+        CSInputButton button = [self buttonForGestureType:self.longPressType];
+        if (button != kCSInputButtonNone) {
+            [self mouseClick:button location:[sender locationInView:sender.view]];
+        }
     }
 }
 
@@ -499,10 +609,13 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 - (IBAction)gestureSwipeScroll:(UISwipeGestureRecognizer *)sender {
     if (sender.state == UIGestureRecognizerStateEnded &&
         self.twoFingerScrollType == VMGestureTypeMouseWheel) {
+        // same direction as the pan that may be scrolling along with the swipe
+        CSInputScroll up = self.isInvertScroll ? kCSInputScrollDown : kCSInputScrollUp;
+        CSInputScroll down = self.isInvertScroll ? kCSInputScrollUp : kCSInputScrollDown;
         if (sender == self.swipeScrollUp) {
-            [self.vmInput sendMouseScroll:kCSInputScrollUp buttonMask:self.mouseButtonDown dy:0];
+            [self.vmInput sendMouseScroll:up buttonMask:self.mouseButtonDown dy:0];
         } else if (sender == self.swipeScrollDown) {
-            [self.vmInput sendMouseScroll:kCSInputScrollDown buttonMask:self.mouseButtonDown dy:0];
+            [self.vmInput sendMouseScroll:down buttonMask:self.mouseButtonDown dy:0];
         } else {
             NSAssert(0, @"Invalid call to gestureSwipeScroll");
         }
@@ -512,12 +625,6 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 #pragma mark - UIGestureRecognizerDelegate
 
 - (BOOL)gestureRecognizer:(UIGestureRecognizer *)gestureRecognizer shouldRequireFailureOfGestureRecognizer:(UIGestureRecognizer *)otherGestureRecognizer {
-    if (gestureRecognizer == self.twoPan && otherGestureRecognizer == self.swipeUp) {
-        return YES;
-    }
-    if (gestureRecognizer == self.twoPan && otherGestureRecognizer == self.swipeDown) {
-        return YES;
-    }
     if (gestureRecognizer == self.twoTap && otherGestureRecognizer == self.swipeDown) {
         return YES;
     }
@@ -527,32 +634,17 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
     if (gestureRecognizer == self.tap && otherGestureRecognizer == self.twoTap) {
         return YES;
     }
-    if (gestureRecognizer == self.longPress && otherGestureRecognizer == self.tap) {
-        return YES;
-    }
-    if (gestureRecognizer == self.longPress && otherGestureRecognizer == self.twoTap) {
-        return YES;
-    }
-    if (gestureRecognizer == self.pinch && otherGestureRecognizer == self.swipeDown) {
-        return YES;
-    }
-    if (gestureRecognizer == self.pinch && otherGestureRecognizer == self.swipeUp) {
-        return YES;
-    }
-    if (gestureRecognizer == self.pan && otherGestureRecognizer == self.swipeUp) {
-        return YES;
-    }
-    if (gestureRecognizer == self.pan && otherGestureRecognizer == self.swipeDown) {
+    if (gestureRecognizer == self.tap && otherGestureRecognizer == self.longPress) {
         return YES;
     }
+    // a swipe can take half a second to fail so a pan only waits for one that matches the same number of touches
     if (gestureRecognizer == self.threePan && otherGestureRecognizer == self.swipeUp) {
         return YES;
     }
     if (gestureRecognizer == self.threePan && otherGestureRecognizer == self.swipeDown) {
         return YES;
     }
-    // only if we do not disable two finger swipe
-    if (self.twoFingerScrollType != VMGestureTypeNone) {
+    if (self.isTwoFingerSwipeExclusive) {
         if (gestureRecognizer == self.twoPan && otherGestureRecognizer == self.swipeScrollUp) {
             return YES;
         }
@@ -567,6 +659,18 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 #endif
 }
 
+- (BOOL)gestureRecognizerShouldBegin:(UIGestureRecognizer *)gestureRecognizer {
+    if (gestureRecognizer == self.pinch) {
+        // before iOS 18 a pinch begins with two of three fingers and takes them from the three finger gestures
+        return self.threePan.numberOfTouches < 3;
+    }
+    if (gestureRecognizer == self.longPress) {
+        // otherwise a disabled long press would swallow the tap waiting on it
+        return self.longPressType != VMGestureTypeNone;
+    }
+    return YES;
+}
+
 - (BOOL)gestureRecognizer:(UIGestureRecognizer *)gestureRecognizer shouldRecognizeSimultaneouslyWithGestureRecognizer:(UIGestureRecognizer *)otherGestureRecognizer {
     if (gestureRecognizer == self.twoPan && otherGestureRecognizer == self.pinch) {
         if (self.twoFingerPanType == VMGestureTypeMoveScreen) {
@@ -575,9 +679,8 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
             return NO;
         }
     } else if (gestureRecognizer == self.pan && otherGestureRecognizer == self.longPress) {
-        return YES;
-    } else if (self.twoFingerScrollType == VMGestureTypeNone && otherGestureRecognizer == self.twoPan) {
-        // if two finger swipe is disabled, we can also recognize two finger pans
+        return !self.isTouchScrolling; // there the pan scrolls and the long press moves the cursor itself
+    } else if (!self.isTwoFingerSwipeExclusive && otherGestureRecognizer == self.twoPan) {
         if (gestureRecognizer == self.swipeScrollUp) {
             return YES;
         } else if (gestureRecognizer == self.swipeScrollDown) {
@@ -658,9 +761,24 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
                 type = VMMouseTypeRelative;
             }
 #endif
+            if (event.allTouches.count == touches.count) { // a touch joining later does not change the gesture
+                self.isTouchScrolling = (type == VMMouseTypeAbsoluteScroll);
+            }
             if ([self switchMouseType:type]) {
+                self.isTouchScrolling = NO; // the cursor mode changes under this touch
                 [self dragCursor:UIGestureRecognizerStateEnded primary:YES secondary:YES middle:YES]; // reset drag
             } else if (!self.vmInput.serverModeCursor) { // start click for client mode
+                if (self.isTouchScrolling) { // clicks come from the gestures instead
+                    if (event.allTouches.count == 1) {
+                        CGPoint pos = [touch locationInView:self.mtkView];
+                        if (![self isRepeatTapAtLocation:pos]) { // else leave the cursor for the double click
+                            [self.cursor startMovement:pos];
+                            [self.cursor updateMovement:pos];
+                        }
+                        [self.scroll startMovement:pos]; // stop a fling like a touch screen does
+                    }
+                    break;
+                }
                 BOOL primary = YES;
                 BOOL secondary = NO;
                 BOOL middle = NO;
@@ -691,7 +809,7 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 
 - (void)touchesMoved:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
     // move cursor in client mode, in server mode we handle in gesturePan
-    if (!self.delegate.qemuInputLegacy && !self.vmInput.serverModeCursor) {
+    if (!self.delegate.qemuInputLegacy && !self.vmInput.serverModeCursor && !self.isTouchScrolling) {
         for (UITouch *touch in touches) {
             [self.cursor updateMovement:[touch locationInView:self.mtkView]];
             break; // handle single touch
@@ -702,7 +820,7 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 
 - (void)touchesCancelled:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
     // release click in client mode, in server mode we handle in gesturePan
-    if (!self.delegate.qemuInputLegacy && !self.vmInput.serverModeCursor) {
+    if (!self.delegate.qemuInputLegacy && !self.vmInput.serverModeCursor && !self.isTouchScrolling) {
         [self dragCursor:UIGestureRecognizerStateEnded primary:YES secondary:YES middle:YES];
     }
     [super touchesCancelled:touches withEvent:event];
@@ -710,7 +828,7 @@ static CGRect CGRectClipToBounds(CGRect rect1, CGRect rect2) {
 
 - (void)touchesEnded:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
     // release click in client mode, in server mode we handle in gesturePan
-    if (!self.delegate.qemuInputLegacy && !self.vmInput.serverModeCursor) {
+    if (!self.delegate.qemuInputLegacy && !self.vmInput.serverModeCursor && !self.isTouchScrolling) {
         [self dragCursor:UIGestureRecognizerStateEnded primary:YES secondary:YES middle:YES];
     }
     [super touchesEnded:touches withEvent:event];

Platform/iOS/Settings.bundle/Root.plist

@@ -259,12 +259,14 @@
 				<string>Disabled</string>
 				<string>Click &amp; Hold</string>
 				<string>Right Click</string>
+				<string>Middle Click</string>
 			</array>
 			<key>Values</key>
 			<array>
 				<integer>0</integer>
 				<integer>1</integer>
 				<integer>2</integer>
+				<integer>5</integer>
 			</array>
 		</dict>
 		<dict>
@@ -282,11 +284,13 @@
 			<array>
 				<string>Disabled</string>
 				<string>Right Click</string>
+				<string>Middle Click</string>
 			</array>
 			<key>Values</key>
 			<array>
 				<integer>0</integer>
 				<integer>2</integer>
+				<integer>5</integer>
 			</array>
 		</dict>
 		<dict>
@@ -306,6 +310,8 @@
 				<string>Move Screen</string>
 				<string>Click &amp; Hold</string>
 				<string>Mouse Wheel</string>
+				<string>Right Click &amp; Hold</string>
+				<string>Middle Click &amp; Hold</string>
 			</array>
 			<key>Values</key>
 			<array>
@@ -313,6 +319,8 @@
 				<integer>3</integer>
 				<integer>1</integer>
 				<integer>4</integer>
+				<integer>6</integer>
+				<integer>7</integer>
 			</array>
 		</dict>
 		<dict>
@@ -354,6 +362,8 @@
 				<string>Move Screen</string>
 				<string>Click &amp; Hold</string>
 				<string>Mouse Wheel</string>
+				<string>Right Click &amp; Hold</string>
+				<string>Middle Click &amp; Hold</string>
 			</array>
 			<key>Values</key>
 			<array>
@@ -361,6 +371,8 @@
 				<integer>3</integer>
 				<integer>1</integer>
 				<integer>4</integer>
+				<integer>6</integer>
+				<integer>7</integer>
 			</array>
 		</dict>
 		<dict>
@@ -385,12 +397,14 @@
 				<string>Drag cursor</string>
 				<string>Touch mode (always show cursor)</string>
 				<string>Touch mode (try hiding cursor)</string>
+				<string>Touch mode (drag to scroll)</string>
 			</array>
 			<key>Values</key>
 			<array>
 				<integer>0</integer>
 				<integer>1</integer>
 				<integer>2</integer>
+				<integer>3</integer>
 			</array>
 		</dict>
 		<dict>

Platform/iOS/VMWindowView.swift

@@ -57,6 +57,7 @@ struct VMWindowView: View {
                     case .display(_, _):
                         VMDisplayHostedView(vm: session.vm, device: device, state: $state)
                             .prefersPersistentSystemOverlaysHidden()
+                            .defersSystemGesturesOnAllEdges()
                     case .serial(_, _):
                         VMDisplayHostedView(vm: session.vm, device: device, state: $state)
                             .prefersPersistentSystemOverlaysHidden()
@@ -312,6 +313,19 @@ fileprivate struct VMToolbarOrnamentModifier: ViewModifier {
 #endif
 
 private extension View {
+    /// A touch that starts at a screen edge is otherwise held back for the system and never reaches the guest
+    func defersSystemGesturesOnAllEdges() -> some View {
+        #if os(visionOS)
+        return self
+        #else
+        if #available(iOS 16, *) {
+            return self.defersSystemGestures(on: .all)
+        } else {
+            return self
+        }
+        #endif
+    }
+
     func prefersPersistentSystemOverlaysHidden() -> some View {
         if #available(iOS 16, *) {
             return self.persistentSystemOverlays(.hidden)