Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-07 09:54:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1564549 - Implement move and explore by touch natively. r=geckoview-reviewers,yzen,snorp

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D45599

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Eitan Isaacson 2019-09-23 23:42:49 +00:00
parent c484dec9b9
commit 0f147877cd
10 changed files with 131 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
accessible/android/AccessibleWrap.cpp
View File

@ -6,17 +6,22 @@
#include "AccessibleWrap.h"
#include "Accessible-inl.h"
#include "AccEvent.h"
#include "AndroidInputType.h"
#include "DocAccessibleWrap.h"
#include "IDSet.h"
#include "JavaBuiltins.h"
#include "SessionAccessibility.h"
#include "TraversalRule.h"
#include "Pivot.h"
#include "nsAccessibilityService.h"
#include "nsEventShell.h"
#include "nsPersistentProperties.h"
#include "nsIAccessibleAnnouncementEvent.h"
#include "nsIStringBundle.h"
#include "nsAccUtils.h"
#include "nsTextEquivUtils.h"
#include "RootAccessible.h"
#include "mozilla/a11y/PDocAccessibleChild.h"
#include "mozilla/jni/GeckoBundleUtils.h"
@ -79,6 +84,27 @@ nsresult AccessibleWrap::HandleAccEvent(AccEvent* aEvent) {
}
break;
}
case nsIAccessibleEvent::EVENT_TEXT_CARET_MOVED: {
if (accessible != aEvent->Document() && !aEvent->IsFromUserInput()) {
AccCaretMoveEvent* caretEvent = downcast_accEvent(aEvent);
if (IsHyperText()) {
DOMPoint point =
AsHyperText()->OffsetToDOMPoint(caretEvent->GetCaretOffset());
if (Accessible* newPos =
doc->GetAccessibleOrContainer(point.node)) {
static_cast<AccessibleWrap*>(newPos)->Pivot(
java::SessionAccessibility::HTML_GRANULARITY_DEFAULT, true,
true);
}
}
}
break;
}
case nsIAccessibleEvent::EVENT_SCROLLING_START: {
accessible->Pivot(java::SessionAccessibility::HTML_GRANULARITY_DEFAULT,
true, true);
break;
}
default:
break;
}
@ -123,15 +149,11 @@ nsresult AccessibleWrap::HandleAccEvent(AccEvent* aEvent) {
RefPtr<AccessibleWrap> newPosition =
static_cast<AccessibleWrap*>(vcEvent->NewAccessible());
auto oldPosition = static_cast<AccessibleWrap*>(vcEvent->OldAccessible());
if (sessionAcc && newPosition) {
if (oldPosition != newPosition) {
if (vcEvent->Reason() == nsIAccessiblePivot::REASON_POINT) {
sessionAcc->SendHoverEnterEvent(newPosition);
} else {
sessionAcc->SendAccessibilityFocusedEvent(newPosition);
}
if (vcEvent->Reason() == nsIAccessiblePivot::REASON_POINT) {
sessionAcc->SendHoverEnterEvent(newPosition);
} else {
sessionAcc->SendAccessibilityFocusedEvent(newPosition);
}
if (vcEvent->BoundaryType() != nsIAccessiblePivot::NO_BOUNDARY) {
@ -248,6 +270,37 @@ bool AccessibleWrap::GetSelectionBounds(int32_t* aStartOffset,
return false;
}
void AccessibleWrap::Pivot(int32_t aGranularity, bool aForward,
bool aInclusive) {
a11y::Pivot pivot(RootAccessible());
TraversalRule rule(aGranularity);
Accessible* result = aForward ? pivot.Next(this, rule, aInclusive)
: pivot.Prev(this, rule, aInclusive);
if (result && (result != this || aInclusive)) {
PivotMoveReason reason = aForward ? nsIAccessiblePivot::REASON_NEXT
: nsIAccessiblePivot::REASON_PREV;
RefPtr<AccEvent> event = new AccVCChangeEvent(
result->Document(), this, -1, -1, result, -1, -1, reason,
nsIAccessiblePivot::NO_BOUNDARY, eFromUserInput);
nsEventShell::FireEvent(event);
}
}
void AccessibleWrap::ExploreByTouch(float aX, float aY) {
a11y::Pivot pivot(RootAccessible());
TraversalRule rule;
Accessible* result = pivot.AtPoint(aX, aY, rule);
if (result && result != this) {
RefPtr<AccEvent> event =
new AccVCChangeEvent(result->Document(), this, -1, -1, result, -1, -1,
nsIAccessiblePivot::REASON_POINT,
nsIAccessiblePivot::NO_BOUNDARY, eFromUserInput);
nsEventShell::FireEvent(event);
}
}
uint32_t AccessibleWrap::GetFlags(role aRole, uint64_t aState,
uint8_t aActionCount) {
uint32_t flags = 0;

4
accessible/android/AccessibleWrap.h
View File

@ -35,6 +35,10 @@ class AccessibleWrap : public Accessible {
virtual bool GetSelectionBounds(int32_t* aStartOffset, int32_t* aEndOffset);
virtual void Pivot(int32_t aGranularity, bool aForward, bool aInclusive);
virtual void ExploreByTouch(float aX, float aY);
mozilla::java::GeckoBundle::LocalRef ToBundle(bool aSmall = false);
mozilla::java::GeckoBundle::LocalRef ToBundle(

12
accessible/android/Platform.cpp
View File

@ -130,13 +130,11 @@ void a11y::ProxyVirtualCursorChangeEvent(
return;
}
if (aOldPosition != aNewPosition) {
if (aReason == nsIAccessiblePivot::REASON_POINT) {
sessionAcc->SendHoverEnterEvent(WrapperFor(aNewPosition));
} else {
RefPtr<AccessibleWrap> wrapperForNewPosition = WrapperFor(aNewPosition);
sessionAcc->SendAccessibilityFocusedEvent(wrapperForNewPosition);
}
if (aReason == nsIAccessiblePivot::REASON_POINT) {
sessionAcc->SendHoverEnterEvent(WrapperFor(aNewPosition));
} else {
RefPtr<AccessibleWrap> wrapperForNewPosition = WrapperFor(aNewPosition);
sessionAcc->SendAccessibilityFocusedEvent(wrapperForNewPosition);
}
if (aBoundaryType != nsIAccessiblePivot::NO_BOUNDARY) {

14
accessible/android/ProxyAccessibleWrap.cpp
View File

@ -4,8 +4,11 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "ProxyAccessibleWrap.h"
#include "nsPersistentProperties.h"
#include "mozilla/a11y/DocAccessiblePlatformExtParent.h"
using namespace mozilla::a11y;
ProxyAccessibleWrap::ProxyAccessibleWrap(ProxyAccessible* aProxy)
@ -106,6 +109,17 @@ bool ProxyAccessibleWrap::GetSelectionBounds(int32_t* aStartOffset,
return Proxy()->SelectionBoundsAt(0, unused, aStartOffset, aEndOffset);
}
void ProxyAccessibleWrap::Pivot(int32_t aGranularity, bool aForward,
bool aInclusive) {
Unused << Proxy()->Document()->GetPlatformExtension()->SendPivot(
Proxy()->ID(), aGranularity, aForward, aInclusive);
}
void ProxyAccessibleWrap::ExploreByTouch(float aX, float aY) {
Unused << Proxy()->Document()->GetPlatformExtension()->SendExploreByTouch(
Proxy()->ID(), aX, aY);
}
role ProxyAccessibleWrap::WrapperRole() { return Proxy()->Role(); }
AccessibleWrap* ProxyAccessibleWrap::WrapperParent() {

5
accessible/android/ProxyAccessibleWrap.h
View File

@ -59,6 +59,11 @@ class ProxyAccessibleWrap : public AccessibleWrap {
virtual bool GetSelectionBounds(int32_t* aStartOffset,
int32_t* aEndOffset) override;
virtual void Pivot(int32_t aGranularity, bool aForward,
bool aInclusive) override;
virtual void ExploreByTouch(float aX, float aY) override;
virtual void WrapperDOMNodeID(nsString& aDOMNodeID) override;
private:

9
accessible/android/SessionAccessibility.cpp
View File

@ -115,6 +115,15 @@ void SessionAccessibility::Click(int32_t aID) {
FORWARD_ACTION_TO_ACCESSIBLE(DoAction, 0);
}
void SessionAccessibility::Pivot(int32_t aID, int32_t aGranularity,
bool aForward, bool aInclusive) {
FORWARD_ACTION_TO_ACCESSIBLE(Pivot, aGranularity, aForward, aInclusive);
}
void SessionAccessibility::ExploreByTouch(int32_t aID, float aX, float aY) {
FORWARD_ACTION_TO_ACCESSIBLE(ExploreByTouch, aX, aY);
}
SessionAccessibility* SessionAccessibility::GetInstanceFor(
ProxyAccessible* aAccessible) {
auto tab =

2
accessible/android/SessionAccessibility.h
View File

@ -54,6 +54,8 @@ class SessionAccessibility final
jni::Object::LocalRef GetNodeInfo(int32_t aID);
void SetText(int32_t aID, jni::String::Param aText);
void Click(int32_t aID);
void Pivot(int32_t aID, int32_t aGranularity, bool aForward, bool aInclusive);
void ExploreByTouch(int32_t aID, float aX, float aY);
void StartNativeAccessibility();
// Event methods

6
accessible/ipc/extension/android/DocAccessiblePlatformExtChild.cpp
View File

@ -15,7 +15,7 @@ namespace a11y {
mozilla::ipc::IPCResult DocAccessiblePlatformExtChild::RecvPivot(
uint64_t aID, int32_t aGranularity, bool aForward, bool aInclusive) {
if (auto acc = IdToAccessibleWrap(aID)) {
// XXX: Forward to appropriate wrapper method.
acc->Pivot(aGranularity, aForward, aInclusive);
}
return IPC_OK();
@ -66,7 +66,9 @@ mozilla::ipc::IPCResult DocAccessiblePlatformExtChild::RecvPaste(uint64_t aID) {
mozilla::ipc::IPCResult DocAccessiblePlatformExtChild::RecvExploreByTouch(
uint64_t aID, float aX, float aY) {
// XXX: Forward to appropriate wrapper method.
if (auto acc = IdToAccessibleWrap(aID)) {
acc->ExploreByTouch(aX, aY);
}
return IPC_OK();
}

2
accessible/ipc/extension/android/PDocAccessiblePlatformExt.ipdl
View File

@ -27,7 +27,7 @@ child:
async Paste(int32_t aID);
async ExploreByTouch(int32_t aID, float aX, float aY);
async ExploreByTouch(uint64_t aID, float aX, float aY);
};
}
}

66
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
View File

@ -22,6 +22,7 @@ import android.os.Bundle;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.support.annotation.UiThread;
import android.text.TextUtils;
import android.util.Log;
import android.util.SparseArray;
import android.view.InputDevice;
@ -200,24 +201,11 @@ public class SessionAccessibility {
switch (action) {
case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS:
if (mAccessibilityFocusedNode == virtualViewId) {
mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityClearCursor", null);
}
sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED, virtualViewId, CLASSNAME_UNKNOWN, null);
return true;
case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS:
if (virtualViewId == View.NO_ID) {
sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, View.NO_ID, CLASSNAME_WEBVIEW, null);
} else {
if (mFocusedNode == virtualViewId && mHoveredOnNode != virtualViewId) {
// If we are sending accessibility focus to the focused node, sync up the state with Gecko.
// XXX: This is a stopgap for now until we remove the JS layer and manipulate the Gecko a11y virtual cursor directly
// with the given virtualViewId
mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityCursorToFocused", null);
} else {
sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, virtualViewId, CLASSNAME_UNKNOWN, null);
}
}
sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, virtualViewId,
virtualViewId == View.NO_ID ? CLASSNAME_WEBVIEW : CLASSNAME_UNKNOWN, null);
return true;
case AccessibilityNodeInfo.ACTION_CLICK:
nativeProvider.click(virtualViewId);
@ -255,16 +243,16 @@ public class SessionAccessibility {
nativeProvider.click(virtualViewId);
return true;
case AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT:
requestViewFocus();
nativeProvider.pivot(virtualViewId, arguments != null ?
arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING) : "",
true, false);
return true;
case AccessibilityNodeInfo.ACTION_PREVIOUS_HTML_ELEMENT:
requestViewFocus();
if (arguments != null) {
data = new GeckoBundle(1);
data.putString("rule", arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING));
} else {
data = null;
}
mSession.getEventDispatcher().dispatch(action == AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT ?
"GeckoView:AccessibilityNext" : "GeckoView:AccessibilityPrevious", data);
nativeProvider.pivot(virtualViewId, arguments != null ?
arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING) : "",
false, false);
return true;
case AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY:
case AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY:
@ -496,11 +484,7 @@ public class SessionAccessibility {
// This is mostly here to let TalkBack know we are a legit "WebView".
bundle.putCharSequence(
"ACTION_ARGUMENT_HTML_ELEMENT_STRING_VALUES",
"ARTICLE,BUTTON,CHECKBOX,COMBOBOX,CONTROL," +
"FOCUSABLE,FRAME,GRAPHIC,H1,H2,H3,H4,H5,H6," +
"HEADING,LANDMARK,LINK,LIST,LIST_ITEM,MAIN," +
"MEDIA,RADIO,SECTION,TABLE,TEXT_FIELD," +
"UNVISITED_LINK,VISITED_LINK");
TextUtils.join(",", sHtmlGranularities));
}
@ -564,8 +548,6 @@ public class SessionAccessibility {
private int mAccessibilityFocusedNode = 0;
// The current node with focus
private int mFocusedNode = 0;
// A node with no accessibility focus that is currently being hovered.
private int mHoveredOnNode = 0;
// Viewport cache
final SparseArray<GeckoBundle> mViewportCache = new SparseArray<>();
// Focus cache
@ -702,9 +684,6 @@ public class SessionAccessibility {
final GeckoBundle ret = new GeckoBundle(2);
ret.putBoolean("touchEnabled", isTouchExplorationEnabled());
ret.putBoolean("enabled", isEnabled());
// "GeckoView:AccessibilitySettings" is dispatched to the Gecko thread.
EventDispatcher.getInstance().dispatch("GeckoView:AccessibilitySettings", ret);
// "GeckoView:AccessibilityEnabled" is dispatched to the UI thread.
EventDispatcher.getInstance().dispatch("GeckoView:AccessibilityEnabled", ret);
if (GeckoThread.isStateAtLeast(GeckoThread.State.PROFILE_READY)) {
@ -740,9 +719,10 @@ public class SessionAccessibility {
requestViewFocus();
final GeckoBundle data = new GeckoBundle(2);
data.putDoubleArray("coordinates", new double[] {event.getRawX(), event.getRawY()});
mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityExploreByTouch", data);
nativeProvider.exploreByTouch(
mAccessibilityFocusedNode != 0 ? mAccessibilityFocusedNode : View.NO_ID,
event.getRawX(), event.getRawY());
return true;
}
@ -826,12 +806,8 @@ public class SessionAccessibility {
mAccessibilityFocusedNode = 0;
}
break;
case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER:
mHoveredOnNode = sourceId;
break;
case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED:
mAccessibilityFocusedNode = sourceId;
mHoveredOnNode = 0;
break;
case AccessibilityEvent.TYPE_VIEW_FOCUSED:
mFocusedNode = sourceId;
@ -878,6 +854,16 @@ public class SessionAccessibility {
@WrapForJNI(dispatchTo = "gecko")
public native void click(int id);
public void pivot(final int id, final String granularity, final boolean forward, final boolean inclusive) {
pivotNative(id, java.util.Arrays.asList(sHtmlGranularities).indexOf(granularity), forward, inclusive);
}
@WrapForJNI(dispatchTo = "gecko", stubName = "Pivot")
public native void pivotNative(int id, int granularity, boolean forward, boolean inclusive);
@WrapForJNI(dispatchTo = "gecko")
public native void exploreByTouch(int id, float x, float y);
@WrapForJNI(calledFrom = "gecko", stubName = "SendEvent")
private void sendEventNative(final int eventType, final int sourceId, final int className, final GeckoBundle eventData) {
ThreadUtils.postToUiThread(new Runnable() {