servo: Merge #8232 - Correct event dispatching for multiple simultaneous touch points (from mbrubeck:glutin-touch); r=glennw

Instead of just converting the mouse into a single "touch" input, Servo can now listen for multi-touch events from Glutin, maintain a list of active touch points, and dispatch events for all of them.

r? @glennw (for the compositor changes) and @jdm (for the DOM changes)

Source-Repo: https://github.com/servo/servo
Source-Revision: 3fdaa6e3f32f6996c416e75119177b98d404adb2
This commit is contained in:
Matt Brubeck 2015-11-03 22:16:21 +05:01
parent 91208857f8
commit 4b6e081f7b
7 changed files with 227 additions and 102 deletions

View File

@ -35,8 +35,9 @@ use msg::constellation_msg::{NavigationDirection, PipelineId, WindowSizeData};
use pipeline::CompositionPipeline;
use profile_traits::mem::{self, ReportKind, Reporter, ReporterRequest};
use profile_traits::time::{self, ProfilerCategory, profile};
use script_traits::CompositorEvent::{TouchDownEvent, TouchMoveEvent, TouchUpEvent};
use script_traits::CompositorEvent::TouchEvent;
use script_traits::{ConstellationControlMsg, LayoutControlMsg, MouseButton};
use script_traits::{TouchEventType, TouchId};
use scrolling::ScrollingTimerProxy;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::collections::{HashMap, HashSet};
@ -72,23 +73,6 @@ enum ReadyState {
ReadyToSaveImage,
}
/// The states of the touch input state machine.
///
/// TODO: Currently Add support for "flinging" (scrolling inertia), pinch zooming, better
/// support for multiple touch points.
enum TouchState {
/// Not tracking any touch point
Nothing,
/// A touchstart event was dispatched to the page, but the response wasn't received yet.
WaitingForScript,
/// Script is consuming the current touch point; don't perform default actions.
DefaultPrevented,
/// A single touch point is active and may perform click or pan default actions.
Touching,
/// A single touch point is active and has started panning.
Panning,
}
/// NB: Never block on the constellation, because sometimes the constellation blocks on us.
pub struct IOCompositor<Window: WindowMethods> {
/// The application window.
@ -201,6 +185,27 @@ pub struct IOCompositor<Window: WindowMethods> {
pending_subpages: HashSet<PipelineId>,
}
/// The states of the touch input state machine.
///
/// TODO: Currently Add support for "flinging" (scrolling inertia), pinch zooming.
enum TouchState {
/// Not tracking any touch point
Nothing,
/// A touchstart event was dispatched to the page, but the response wasn't received yet.
/// Contains the number of active touch points.
WaitingForScript(u32),
/// Script is consuming the current touch sequence; don't perform default actions.
/// Contains the number of active touch points.
DefaultPrevented(u32),
/// A single touch point is active and may perform click or pan default actions.
Touching,
/// A single touch point is active and has started panning.
Panning,
/// A multi-touch gesture is in progress. Contains the number of active touch points.
MultiTouch(u32),
}
pub struct ScrollEvent {
delta: TypedPoint2D<DevicePixel, f32>,
cursor: TypedPoint2D<DevicePixel, i32>,
@ -537,10 +542,14 @@ impl<Window: WindowMethods> IOCompositor<Window> {
(Msg::TouchEventProcessed(result), ShutdownState::NotShuttingDown) => {
match self.touch_gesture_state {
TouchState::WaitingForScript => {
TouchState::WaitingForScript(n) => {
self.touch_gesture_state = match result {
EventResult::DefaultAllowed => TouchState::Touching,
EventResult::DefaultPrevented => TouchState::DefaultPrevented,
EventResult::DefaultPrevented => TouchState::DefaultPrevented(n),
EventResult::DefaultAllowed => if n > 1 {
TouchState::MultiTouch(n)
} else {
TouchState::Touching
}
};
}
_ => {}
@ -1056,6 +1065,15 @@ impl<Window: WindowMethods> IOCompositor<Window> {
self.on_mouse_window_move_event_class(cursor);
}
WindowEvent::Touch(event_type, identifier, location) => {
match event_type {
TouchEventType::Down => self.on_touch_down(identifier, location),
TouchEventType::Move => self.on_touch_move(identifier, location),
TouchEventType::Up => self.on_touch_up(identifier, location),
TouchEventType::Cancel => self.on_touch_cancel(identifier, location),
}
}
WindowEvent::Scroll(delta, cursor) => {
self.on_scroll_window_event(delta, cursor);
}
@ -1127,8 +1145,8 @@ impl<Window: WindowMethods> IOCompositor<Window> {
if opts::get().convert_mouse_to_touch {
match mouse_window_event {
MouseWindowEvent::Click(_, _) => {}
MouseWindowEvent::MouseDown(_, p) => self.on_touch_down(0, p),
MouseWindowEvent::MouseUp(_, p) => self.on_touch_up(0, p),
MouseWindowEvent::MouseDown(_, p) => self.on_touch_down(TouchId(0), p),
MouseWindowEvent::MouseUp(_, p) => self.on_touch_up(TouchId(0), p),
}
return
}
@ -1146,7 +1164,7 @@ impl<Window: WindowMethods> IOCompositor<Window> {
fn on_mouse_window_move_event_class(&mut self, cursor: TypedPoint2D<DevicePixel, f32>) {
if opts::get().convert_mouse_to_touch {
self.on_touch_move(0, cursor);
self.on_touch_move(TouchId(0), cursor);
return
}
@ -1156,31 +1174,40 @@ impl<Window: WindowMethods> IOCompositor<Window> {
}
}
fn on_touch_down(&mut self, identifier: i32, point: TypedPoint2D<DevicePixel, f32>) {
match self.touch_gesture_state {
fn on_touch_down(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
self.touch_gesture_state = match self.touch_gesture_state {
TouchState::Nothing => {
// TODO: Don't wait for script if we know the page has no touch event listeners.
self.first_touch_point = Some(point);
self.last_touch_point = Some(point);
self.touch_gesture_state = TouchState::WaitingForScript;
TouchState::WaitingForScript(1)
}
TouchState::WaitingForScript => {
// TODO: Queue events while waiting for script?
TouchState::Touching |
TouchState::Panning => {
// Was a single-touch sequence; now a multi-touch sequence:
TouchState::MultiTouch(2)
}
TouchState::DefaultPrevented => {}
TouchState::Touching => {}
TouchState::Panning => {}
}
TouchState::WaitingForScript(n) => {
TouchState::WaitingForScript(n + 1)
}
TouchState::DefaultPrevented(n) => {
TouchState::DefaultPrevented(n + 1)
}
TouchState::MultiTouch(n) => {
TouchState::MultiTouch(n + 1)
}
};
if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
result.layer.send_event(self, TouchDownEvent(identifier, result.point.to_untyped()));
result.layer.send_event(self, TouchEvent(TouchEventType::Down, identifier,
result.point.to_untyped()));
}
}
fn on_touch_move(&mut self, identifier: i32, point: TypedPoint2D<DevicePixel, f32>) {
fn on_touch_move(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
match self.touch_gesture_state {
TouchState::Nothing => warn!("Got unexpected touch move event"),
TouchState::WaitingForScript => {
TouchState::WaitingForScript(_) => {
// TODO: Queue events while waiting for script?
}
TouchState::Touching => {
@ -1209,40 +1236,67 @@ impl<Window: WindowMethods> IOCompositor<Window> {
None => warn!("last_touch_point not set")
}
}
TouchState::DefaultPrevented => {
TouchState::DefaultPrevented(_) => {
// Send the event to script.
if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
result.layer.send_event(self,
TouchMoveEvent(identifier, result.point.to_untyped()));
result.layer.send_event(self, TouchEvent(TouchEventType::Move, identifier,
result.point.to_untyped()));
}
}
TouchState::MultiTouch(_) => {
// TODO: Pinch zooming.
}
}
self.last_touch_point = Some(point);
}
fn on_touch_up(&mut self, identifier: i32, point: TypedPoint2D<DevicePixel, f32>) {
// TODO: Track the number of active touch points, and don't reset stuff until it is zero.
self.first_touch_point = None;
self.last_touch_point = None;
fn on_touch_up(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
// Send the event to script.
if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
result.layer.send_event(self, TouchUpEvent(identifier, result.point.to_untyped()));
result.layer.send_event(self, TouchEvent(TouchEventType::Up, identifier,
result.point.to_untyped()));
}
match self.touch_gesture_state {
TouchState::Nothing => warn!("Got unexpected touch up event"),
TouchState::WaitingForScript => {}
self.touch_gesture_state = match self.touch_gesture_state {
TouchState::Touching => {
// TODO: If the duration exceeds some threshold, send a contextmenu event instead.
// TODO: Don't send a click if preventDefault is called on the touchend event.
self.simulate_mouse_click(point);
TouchState::Nothing
}
TouchState::Panning => {}
TouchState::DefaultPrevented => {}
TouchState::Nothing |
TouchState::Panning |
TouchState::WaitingForScript(1) |
TouchState::DefaultPrevented(1) |
TouchState::MultiTouch(1) => {
TouchState::Nothing
}
TouchState::WaitingForScript(n) => TouchState::WaitingForScript(n - 1),
TouchState::DefaultPrevented(n) => TouchState::DefaultPrevented(n - 1),
TouchState::MultiTouch(n) => TouchState::MultiTouch(n - 1),
};
}
fn on_touch_cancel(&mut self, identifier: TouchId, point: TypedPoint2D<DevicePixel, f32>) {
// Send the event to script.
if let Some(result) = self.find_topmost_layer_at_point(point / self.scene.scale) {
result.layer.send_event(self, TouchEvent(TouchEventType::Cancel, identifier,
result.point.to_untyped()));
}
self.touch_gesture_state = TouchState::Nothing;
self.touch_gesture_state = match self.touch_gesture_state {
TouchState::Nothing |
TouchState::Touching |
TouchState::Panning |
TouchState::WaitingForScript(1) |
TouchState::DefaultPrevented(1) |
TouchState::MultiTouch(1) => {
TouchState::Nothing
}
TouchState::WaitingForScript(n) => TouchState::WaitingForScript(n - 1),
TouchState::DefaultPrevented(n) => TouchState::DefaultPrevented(n - 1),
TouchState::MultiTouch(n) => TouchState::MultiTouch(n - 1),
};
}
/// http://w3c.github.io/touch-events/#mouse-events

View File

@ -13,7 +13,7 @@ use layers::geometry::DevicePixel;
use layers::platform::surface::NativeDisplay;
use msg::constellation_msg::{Key, KeyModifiers, KeyState};
use net_traits::net_error_list::NetError;
use script_traits::MouseButton;
use script_traits::{MouseButton, TouchEventType, TouchId};
use std::fmt::{Debug, Error, Formatter};
use std::rc::Rc;
use url::Url;
@ -27,6 +27,7 @@ pub enum MouseWindowEvent {
MouseUp(MouseButton, TypedPoint2D<DevicePixel, f32>),
}
#[derive(Clone)]
pub enum WindowNavigateMsg {
Forward,
@ -59,6 +60,8 @@ pub enum WindowEvent {
MouseWindowEventClass(MouseWindowEvent),
/// Sent when a mouse move.
MouseWindowMoveEventClass(TypedPoint2D<DevicePixel, f32>),
/// Touch event: type, identifier, point
Touch(TouchEventType, TouchId, TypedPoint2D<DevicePixel, f32>),
/// Sent when the user scrolls. The first point is the delta and the second point is the
/// origin.
Scroll(TypedPoint2D<DevicePixel, f32>, TypedPoint2D<DevicePixel, i32>),
@ -88,6 +91,7 @@ impl Debug for WindowEvent {
WindowEvent::LoadUrl(..) => write!(f, "LoadUrl"),
WindowEvent::MouseWindowEventClass(..) => write!(f, "Mouse"),
WindowEvent::MouseWindowMoveEventClass(..) => write!(f, "MouseMove"),
WindowEvent::Touch(..) => write!(f, "Touch"),
WindowEvent::Scroll(..) => write!(f, "Scroll"),
WindowEvent::Zoom(..) => write!(f, "Zoom"),
WindowEvent::PinchZoom(..) => write!(f, "PinchZoom"),

View File

@ -13,6 +13,7 @@ use dom::bindings::codegen::Bindings::EventTargetBinding::EventTargetMethods;
use dom::bindings::codegen::Bindings::NodeBinding::NodeMethods;
use dom::bindings::codegen::Bindings::NodeFilterBinding::NodeFilter;
use dom::bindings::codegen::Bindings::PerformanceBinding::PerformanceMethods;
use dom::bindings::codegen::Bindings::TouchBinding::TouchMethods;
use dom::bindings::codegen::Bindings::WindowBinding::WindowMethods;
use dom::bindings::codegen::InheritTypes::{ElementTypeId, HTMLElementTypeId, NodeTypeId};
use dom::bindings::codegen::UnionTypes::NodeOrString;
@ -84,7 +85,7 @@ use net_traits::CookieSource::NonHTTP;
use net_traits::{AsyncResponseTarget, PendingAsyncLoad};
use num::ToPrimitive;
use script_task::{MainThreadScriptMsg, Runnable};
use script_traits::{MouseButton, UntrustedNodeAddress};
use script_traits::{MouseButton, TouchEventType, TouchId, UntrustedNodeAddress};
use selectors::states::*;
use std::ascii::AsciiExt;
use std::borrow::ToOwned;
@ -175,8 +176,10 @@ pub struct Document {
/// This field is set to the document itself for inert documents.
/// https://html.spec.whatwg.org/multipage/#appropriate-template-contents-owner-document
appropriate_template_contents_owner_document: MutNullableHeap<JS<Document>>,
// The collection of ElementStates that have been changed since the last restyle.
/// The collection of ElementStates that have been changed since the last restyle.
element_state_changes: DOMRefCell<HashMap<JS<Element>, ElementState>>,
/// http://w3c.github.io/touch-events/#dfn-active-touch-point
active_touch_points: DOMRefCell<Vec<JS<Touch>>>,
}
impl PartialEq for Document {
@ -727,9 +730,17 @@ impl Document {
pub fn handle_touch_event(&self,
js_runtime: *mut JSRuntime,
identifier: i32,
point: Point2D<f32>,
event_name: String) -> bool {
event_type: TouchEventType,
TouchId(identifier): TouchId,
point: Point2D<f32>)
-> bool {
let event_name = match event_type {
TouchEventType::Down => "touchstart",
TouchEventType::Move => "touchmove",
TouchEventType::Up => "touchend",
TouchEventType::Cancel => "touchcancel",
};
let node = match self.hit_test(&point) {
Some(node_address) => node::from_untrusted_node_address(js_runtime, node_address),
None => return false
@ -744,7 +755,7 @@ impl Document {
}
},
};
let target = el.upcast::<EventTarget>();
let target = Root::upcast::<EventTarget>(el);
let window = &*self.window;
let client_x = Finite::wrap(point.x as f64);
@ -752,26 +763,58 @@ impl Document {
let page_x = Finite::wrap(point.x as f64 + window.PageXOffset() as f64);
let page_y = Finite::wrap(point.y as f64 + window.PageYOffset() as f64);
let touch = Touch::new(window, identifier, target,
let touch = Touch::new(window, identifier, target.r(),
client_x, client_y, // TODO: Get real screen coordinates?
client_x, client_y,
page_x, page_y);
match event_type {
TouchEventType::Down => {
// Add a new touch point
self.active_touch_points.borrow_mut().push(JS::from_rooted(&touch));
}
TouchEventType::Move => {
// Replace an existing touch point
let mut active_touch_points = self.active_touch_points.borrow_mut();
match active_touch_points.iter_mut().find(|t| t.Identifier() == identifier) {
Some(t) => *t = JS::from_rooted(&touch),
None => warn!("Got a touchmove event for a non-active touch point")
}
}
TouchEventType::Up |
TouchEventType::Cancel => {
// Remove an existing touch point
let mut active_touch_points = self.active_touch_points.borrow_mut();
match active_touch_points.iter().position(|t| t.Identifier() == identifier) {
Some(i) => { active_touch_points.swap_remove(i); }
None => warn!("Got a touchend event for a non-active touch point")
}
}
}
let mut touches = RootedVec::new();
touches.push(JS::from_rooted(&touch));
let touches = TouchList::new(window, touches.r());
touches.extend(self.active_touch_points.borrow().iter().cloned());
let mut changed_touches = RootedVec::new();
changed_touches.push(JS::from_rooted(&touch));
let mut target_touches = RootedVec::new();
target_touches.extend(self.active_touch_points.borrow().iter().filter(
|t| t.Target() == target).cloned());
let event = TouchEvent::new(window,
event_name,
event_name.to_owned(),
EventBubbles::Bubbles,
EventCancelable::Cancelable,
Some(window),
0i32,
&touches, &touches, &touches,
&TouchList::new(window, touches.r()),
&TouchList::new(window, changed_touches.r()),
&TouchList::new(window, target_touches.r()),
// FIXME: modifier keys
false, false, false, false);
let event = event.upcast::<Event>();
let result = event.fire(target);
let result = event.fire(target.r());
window.reflow(ReflowGoal::ForDisplay,
ReflowQueryType::NoQuery,
@ -1268,6 +1311,7 @@ impl Document {
base_element: Default::default(),
appropriate_template_contents_owner_document: Default::default(),
element_state_changes: DOMRefCell::new(HashMap::new()),
active_touch_points: DOMRefCell::new(Vec::new()),
}
}

View File

@ -78,12 +78,12 @@ use profile_traits::mem::{self, OpaqueSender, Report, ReportKind, ReportsChan};
use profile_traits::time::{self, ProfilerCategory, profile};
use script_traits::CompositorEvent::{ClickEvent, ResizeEvent};
use script_traits::CompositorEvent::{KeyEvent, MouseMoveEvent};
use script_traits::CompositorEvent::{MouseDownEvent, MouseUpEvent};
use script_traits::CompositorEvent::{TouchDownEvent, TouchMoveEvent, TouchUpEvent};
use script_traits::CompositorEvent::{MouseDownEvent, MouseUpEvent, TouchEvent};
use script_traits::{CompositorEvent, ConstellationControlMsg};
use script_traits::{InitialScriptState, MouseButton, NewLayoutInfo};
use script_traits::{OpaqueScriptLayoutChannel, ScriptState, ScriptTaskFactory};
use script_traits::{TimerEvent, TimerEventChan, TimerEventRequest, TimerSource};
use script_traits::{TouchEventType, TouchId};
use std::any::Any;
use std::borrow::ToOwned;
use std::cell::{Cell, RefCell};
@ -1763,27 +1763,27 @@ impl ScriptTask {
std_mem::swap(&mut *self.mouse_over_targets.borrow_mut(), &mut *mouse_over_targets);
}
TouchDownEvent(identifier, point) => {
let default_action_allowed =
self.handle_touch_event(pipeline_id, identifier, point, "touchstart");
if default_action_allowed {
// TODO: Wait to see if preventDefault is called on the first touchmove event.
self.compositor.borrow_mut().send(ScriptToCompositorMsg::TouchEventProcessed(
EventResult::DefaultAllowed)).unwrap();
} else {
self.compositor.borrow_mut().send(ScriptToCompositorMsg::TouchEventProcessed(
EventResult::DefaultPrevented)).unwrap();
TouchEvent(event_type, identifier, point) => {
let handled = self.handle_touch_event(pipeline_id, event_type, identifier, point);
match event_type {
TouchEventType::Down => {
if handled {
// TODO: Wait to see if preventDefault is called on the first touchmove event.
self.compositor.borrow_mut()
.send(ScriptToCompositorMsg::TouchEventProcessed(
EventResult::DefaultAllowed)).unwrap();
} else {
self.compositor.borrow_mut()
.send(ScriptToCompositorMsg::TouchEventProcessed(
EventResult::DefaultPrevented)).unwrap();
}
}
_ => {
// TODO: Calling preventDefault on a touchup event should prevent clicks.
}
}
}
TouchMoveEvent(identifier, point) => {
self.handle_touch_event(pipeline_id, identifier, point, "touchmove");
}
TouchUpEvent(identifier, point) => {
self.handle_touch_event(pipeline_id, identifier, point, "touchend");
}
KeyEvent(key, state, modifiers) => {
let page = get_page(&self.root_page(), pipeline_id);
let document = page.document();
@ -1805,13 +1805,13 @@ impl ScriptTask {
fn handle_touch_event(&self,
pipeline_id: PipelineId,
identifier: i32,
point: Point2D<f32>,
event_name: &str) -> bool {
event_type: TouchEventType,
identifier: TouchId,
point: Point2D<f32>)
-> bool {
let page = get_page(&self.root_page(), pipeline_id);
let document = page.document();
document.r().handle_touch_event(self.js_runtime.rt(), identifier, point,
event_name.to_owned())
document.r().handle_touch_event(self.js_runtime.rt(), event_type, identifier, point)
}
/// https://html.spec.whatwg.org/multipage/#navigating-across-documents

View File

@ -158,6 +158,25 @@ pub enum MouseButton {
Right,
}
/// The type of input represented by a multi-touch event.
#[derive(Clone, Copy, Debug)]
pub enum TouchEventType {
/// A new touch point came in contact with the screen.
Down,
/// An existing touch point changed location.
Move,
/// A touch point was removed from the screen.
Up,
/// The system stopped tracking a touch point.
Cancel,
}
/// An opaque identifier for a touch point.
///
/// http://w3c.github.io/touch-events/#widl-Touch-identifier
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct TouchId(pub i32);
/// Events from the compositor that the script task needs to know about
pub enum CompositorEvent {
/// The window was resized.
@ -170,12 +189,8 @@ pub enum CompositorEvent {
MouseUpEvent(MouseButton, Point2D<f32>),
/// The mouse was moved over a point.
MouseMoveEvent(Point2D<f32>),
/// A touch began at a point.
TouchDownEvent(i32, Point2D<f32>),
/// A touch was moved over a point.
TouchMoveEvent(i32, Point2D<f32>),
/// A touch ended at a point.
TouchUpEvent(i32, Point2D<f32>),
/// A touch event was generated with a touch ID and location.
TouchEvent(TouchEventType, TouchId, Point2D<f32>),
/// A key was pressed.
KeyEvent(Key, KeyState, KeyModifiers),
}

View File

@ -272,12 +272,6 @@ impl DebugOptions {
pub fn new(debug_string: &str) -> Result<DebugOptions, &str> {
let mut debug_options = DebugOptions::default();
// FIXME: Glutin currently converts touch input to mouse events on Android.
// Convert it back to touch events.
if cfg!(target_os = "android") {
debug_options.convert_mouse_to_touch = true;
}
for option in debug_string.split(',') {
match option {
"help" => debug_options.help = true,

View File

@ -16,6 +16,7 @@ use gleam::gl;
use glutin;
#[cfg(feature = "window")]
use glutin::{Api, ElementState, Event, GlRequest, MouseButton, VirtualKeyCode, MouseScrollDelta};
use glutin::{TouchPhase};
use layers::geometry::DevicePixel;
use layers::platform::surface::NativeDisplay;
#[cfg(feature = "window")]
@ -157,6 +158,8 @@ impl Window {
}
fn handle_window_event(&self, event: glutin::Event) -> bool {
use script_traits::{TouchEventType, TouchId};
match event {
Event::KeyboardInput(element_state, _scan_code, virtual_key_code) => {
if virtual_key_code.is_some() {
@ -206,6 +209,17 @@ impl Window {
};
self.scroll_window(dx, dy);
},
Event::Touch(touch) => {
let phase = match touch.phase {
TouchPhase::Started => TouchEventType::Down,
TouchPhase::Moved => TouchEventType::Move,
TouchPhase::Ended => TouchEventType::Up,
TouchPhase::Cancelled => TouchEventType::Cancel,
};
let id = TouchId(touch.id as i32);
let point = Point2D::typed(touch.location.0 as f32, touch.location.1 as f32);
self.event_queue.borrow_mut().push(WindowEvent::Touch(phase, id, point));
}
Event::Refresh => {
self.event_queue.borrow_mut().push(WindowEvent::Refresh);
}