!2497 merge master into master

支持屏幕朗读注入点击、双击、长按事件

Created-by: chenrui_wza
Commit-by: chenrui_wza
Merged-by: openharmony_ci
Description: ### 一、内容说明(相关的Issue)



### 二、建议测试周期和提测地址  
  建议测试完成时间:xxxx.xx.xx  
  投产上线时间:xxxx.xx.xx  
  提测地址:CI环境/压测环境  
  测试账号:  

### 三、变更内容
  * 3.1 关联PR列表

  * 3.2 数据库和部署说明  
    1. 常规更新 
    2. 重启unicorn
    3. 重启sidekiq
    4. 迁移任务:是否有迁移任务,没有写 "无"
    5. rake脚本:`bundle exec xxx RAILS_ENV = production`;没有写 "无"

  * 3.4 其他技术优化内容(做了什么,变更了什么)
    - 重构了 xxxx 代码
    - xxxx 算法优化


  * 3.5 废弃通知(什么字段、方法弃用?)



  * 3.6  后向不兼容变更(是否有无法向后兼容的变更?)


  
### 四、研发自测点(自测哪些?冒烟用例全部自测?)
  自测测试结论:


### 五、测试关注点(需要提醒QA重点关注的、可能会忽略的地方)
  检查点:

| 需求名称 | 是否影响xx公共模块 | 是否需要xx功能 | 需求升级是否依赖其他子产品 |
|------|------------|----------|---------------|
| xxx  | 否          | 需要       | 不需要           |
|      |            |          |               |

  接口测试:

  性能测试:

  并发测试:

  其他:



See merge request: openharmony/accessibility!2497
This commit is contained in:
openharmony_ci
2026-04-27 12:13:21 +08:00
28 changed files with 308 additions and 56 deletions
@@ -116,11 +116,12 @@ public:
* @param actionArguments The parameter for action type.
* @param requestId Matched the request and response. It needn't cared by ACE, transfer it by callback only.
* @param callback To transfer the node info to ASAC and it defined by ASAC.
* @param rect The element's boundary rectangle info.
* @return Return RET_OK if perform action successfully, otherwise refer to the RetError for the failure.
*/
virtual RetError ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId, const int32_t action,
const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback) override;
const sptr<IAccessibilityElementOperatorCallback> &callback, const Rect &rect) override;
/**
* @brief Get the result of Cursor Position through the proxy object.
@@ -252,6 +253,14 @@ private:
*/
bool SendTransactCmd(AccessibilityInterfaceCode code, MessageParcel &data,
MessageParcel &reply, MessageOption &option);
/**
* @brief Write action arguments to message parcel.
* @param data The message parcel to write to.
* @param actionArguments The action arguments to write.
* @return true: Write successfully; otherwise is not.
*/
bool WriteActionArguments(MessageParcel &data,
const std::map<std::string, std::string> &actionArguments);
static inline BrokerDelegator<AccessibleAbilityChannelProxy> delegator;
};
} // namespace Accessibility
@@ -111,11 +111,12 @@ public:
* @param actionArguments The parameter for action type.
* @param requestId Matched the request and response. It needn't cared by ACE, transfer it by callback only.
* @param callback To transfer the node info to ASAC and it defined by ASAC.
* @param rect The element's boundary rectangle info.
* @return Return RET_OK if perform action successfully, otherwise refer to the RetError for the failure.
*/
virtual RetError ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId, const int32_t action,
const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback) = 0;
const sptr<IAccessibilityElementOperatorCallback> &callback, const Rect &rect) = 0;
/**
* @brief Get the result of Cursor Position through the proxy object.
@@ -59,6 +59,26 @@ bool AccessibleAbilityChannelProxy::SendTransactCmd(AccessibilityInterfaceCode c
return true;
}
bool AccessibleAbilityChannelProxy::WriteActionArguments(MessageParcel &data,
const std::map<std::string, std::string> &actionArguments)
{
std::vector<std::string> actionArgumentsKey {};
std::vector<std::string> actionArgumentsValue {};
for (auto iter = actionArguments.begin(); iter != actionArguments.end(); iter++) {
actionArgumentsKey.push_back(iter->first);
actionArgumentsValue.push_back(iter->second);
}
if (!data.WriteStringVector(actionArgumentsKey)) {
HILOG_ERROR("actionArgumentsKey write error");
return false;
}
if (!data.WriteStringVector(actionArgumentsValue)) {
HILOG_ERROR("actionArgumentsValue write error");
return false;
}
return true;
}
RetError AccessibleAbilityChannelProxy::SearchElementInfoByAccessibilityId(const ElementBasicInfo elementBasicInfo,
const int32_t requestId, const sptr<IAccessibilityElementOperatorCallback> &callback,
const int32_t mode, bool isFilter, bool systemApi)
@@ -384,7 +404,7 @@ RetError AccessibleAbilityChannelProxy::UnholdRunningLock()
RetError AccessibleAbilityChannelProxy::ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback)
const sptr<IAccessibilityElementOperatorCallback> &callback, const Rect &rect)
{
HILOG_DEBUG();
@@ -407,22 +427,9 @@ RetError AccessibleAbilityChannelProxy::ExecuteAction(const int32_t accessibilit
HILOG_ERROR("action write error: %{public}d, ", action);
return RET_ERR_IPC_FAILED;
}
std::vector<std::string> actionArgumentsKey {};
std::vector<std::string> actionArgumentsValue {};
for (auto iter = actionArguments.begin(); iter != actionArguments.end(); iter++) {
actionArgumentsKey.push_back(iter->first);
actionArgumentsValue.push_back(iter->second);
}
if (!data.WriteStringVector(actionArgumentsKey)) {
HILOG_ERROR("actionArgumentsKey write error");
if (!WriteActionArguments(data, actionArguments)) {
return RET_ERR_IPC_FAILED;
}
if (!data.WriteStringVector(actionArgumentsValue)) {
HILOG_ERROR("actionArgumentsValue write error");
return RET_ERR_IPC_FAILED;
}
if (!data.WriteInt32(requestId)) {
HILOG_ERROR("requestId write error: %{public}d, ", requestId);
return RET_ERR_IPC_FAILED;
@@ -431,7 +438,11 @@ RetError AccessibleAbilityChannelProxy::ExecuteAction(const int32_t accessibilit
HILOG_ERROR("callback write error");
return RET_ERR_IPC_FAILED;
}
RectParcel rectParcel(rect);
if (!data.WriteParcelable(&rectParcel)) {
HILOG_ERROR("rect write error");
return RET_ERR_IPC_FAILED;
}
if (!SendTransactCmd(AccessibilityInterfaceCode::PERFORM_ACTION,
data, reply, option)) {
HILOG_ERROR("fail to perform accessibility action");
@@ -321,7 +321,15 @@ ErrCode AccessibleAbilityChannelStub::HandleExecuteAction(MessageParcel &data, M
return ERR_INVALID_VALUE;
}
RetError result = ExecuteAction(accessibilityWindowId, elementId, action, actionArguments, requestId, callback);
sptr<RectParcel> rectParcel = data.ReadStrongParcelable<RectParcel>();
if (rectParcel == nullptr) {
HILOG_ERROR("Read RectParcel failed");
return ERR_INVALID_VALUE;
}
Rect rect = *rectParcel;
RetError result = ExecuteAction(accessibilityWindowId, elementId, action,
actionArguments, requestId, callback, rect);
HILOG_DEBUG("ExecuteAction ret = %{public}d", result);
reply.WriteInt32(result);
return NO_ERROR;
@@ -66,7 +66,7 @@ public:
}
RetError ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId, const int32_t action,
const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback) override
const sptr<IAccessibilityElementOperatorCallback> &callback, const Rect &rect) override
{
return RET_OK;
}
@@ -92,7 +92,7 @@ public:
* @return RET_OK: execute action successfully; otherwise refer to the RetError for the failure.
*/
RetError ExecuteAction(int32_t accessibilityWindowId,
int64_t elementId, int32_t action, const std::map<std::string, std::string> &actionArguments);
int64_t elementId, int32_t action, const std::map<std::string, std::string> &actionArguments, const Rect &rect);
/**
* @brief Package status to send server data.
@@ -160,7 +160,7 @@ RetError AccessibleAbilityChannelClient::GetCursorPosition(
}
RetError AccessibleAbilityChannelClient::ExecuteAction(int32_t accessibilityWindowId,
int64_t elementId, int32_t action, const std::map<std::string, std::string> &actionArguments)
int64_t elementId, int32_t action, const std::map<std::string, std::string> &actionArguments, const Rect &rect)
{
#ifdef OHOS_BUILD_ENABLE_HITRACE
HITRACE_METER_NAME(HITRACE_TAG_ACCESSIBILITY_MANAGER, "ExecuteAction");
@@ -172,7 +172,7 @@ RetError AccessibleAbilityChannelClient::ExecuteAction(int32_t accessibilityWind
if (action == ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS &&
accessibilityFocusedElementId_ != INVALID_WINDOW_ID && accessibilityFocusedWindowId_ != INVALID_WINDOW_ID) {
ExecuteAction(accessibilityFocusedWindowId_, accessibilityFocusedElementId_,
ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS, actionArguments);
ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS, actionArguments, rect);
}
int32_t requestId = GenerateRequestId();
@@ -185,7 +185,7 @@ RetError AccessibleAbilityChannelClient::ExecuteAction(int32_t accessibilityWind
ffrt::future<void> promiseFuture = elementOperator->promise_.get_future();
RetError ret = proxy_->ExecuteAction(accessibilityWindowId,
elementId, action, actionArguments, requestId, elementOperator);
elementId, action, actionArguments, requestId, elementOperator, rect);
if (ret != RET_OK) {
HILOG_ERROR("ExecuteAction failed. action[%{public}d], ret[%{public}d]", action, ret);
return ret;
@@ -1157,9 +1157,10 @@ RetError AccessibleAbilityClientImpl::ExecuteAction(const AccessibilityElementIn
windowId = elementInfo.GetWindowId();
}
int64_t elementId = elementInfo.GetAccessibilityId();
const Rect rect = elementInfo.GetRectInScreen();
HILOG_DEBUG("windowId[%{public}d], elementId[%{public}" PRId64 "], action[%{public}d", windowId, elementId, action);
Accessibility::RetError ret = channelClient_->ExecuteAction(windowId, elementId, action,
const_cast<std::map<std::string, std::string> &>(actionArguments));
const_cast<std::map<std::string, std::string> &>(actionArguments), rect);
#ifdef ACCESSIBILITY_EMULATOR_DEFINED
reporter.setResult(ret);
#endif // ACCESSIBILITY_EMULATOR_DEFINED
@@ -42,9 +42,9 @@ public:
MOCK_METHOD6(FocusMoveSearch, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t direction, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback, bool systemApi));
MOCK_METHOD6(ExecuteAction, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
MOCK_METHOD7(ExecuteAction, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string>& actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback));
const sptr<IAccessibilityElementOperatorCallback>& callback, const Rect& rect));
MOCK_METHOD1(EnableScreenCurtain, RetError(bool isEnable));
MOCK_METHOD0(HoldRunningLock, RetError());
MOCK_METHOD0(UnholdRunningLock, RetError());
@@ -44,9 +44,9 @@ public:
MOCK_METHOD6(FocusMoveSearch, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t direction, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback, bool systemApi));
MOCK_METHOD6(ExecuteAction, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
MOCK_METHOD7(ExecuteAction, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string>& actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback));
const sptr<IAccessibilityElementOperatorCallback>& callback, const Rect& rect));
MOCK_METHOD1(EnableScreenCurtain, RetError(bool isEnable));
MOCK_METHOD0(HoldRunningLock, RetError());
MOCK_METHOD0(UnholdRunningLock, RetError());
@@ -115,7 +115,7 @@ RetError AccessibleAbilityChannelProxy::UnholdRunningLock()
RetError AccessibleAbilityChannelProxy::ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string>& actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback)
const sptr<IAccessibilityElementOperatorCallback>& callback, const Rect &rect)
{
GTEST_LOG_(INFO) << "MOCK AccessibleAbilityChannelProxy ExecuteAction";
(void)accessibilityWindowId;
@@ -124,6 +124,7 @@ RetError AccessibleAbilityChannelProxy::ExecuteAction(const int32_t accessibilit
(void)actionArguments;
(void)requestId;
(void)callback;
(void)rect;
return RET_OK;
}
@@ -185,10 +185,11 @@ HWTEST_F(AccessibleAbilityChannelClientTest, SendSimulateGesture_002, TestSize.L
HWTEST_F(AccessibleAbilityChannelClientTest, ExecuteAction_001, TestSize.Level1)
{
GTEST_LOG_(INFO) << "ExecuteAction_001 start";
EXPECT_CALL(*stub_, ExecuteAction(_, _, _, _, _, _)).Times(1).WillOnce(Return(RET_ERR_FAILED));
EXPECT_CALL(*stub_, ExecuteAction(_, _, _, _, _, _, _)).Times(1).WillOnce(Return(RET_ERR_FAILED));
std::map<std::string, std::string> actionArguments;
Rect rect;
EXPECT_EQ(instance_->ExecuteAction(ACCESSIBILITY_WINDOW_ID,
ELEMENT_ID, ActionType::ACCESSIBILITY_ACTION_SELECT, actionArguments), RET_ERR_FAILED);
ELEMENT_ID, ActionType::ACCESSIBILITY_ACTION_SELECT, actionArguments, rect), RET_ERR_FAILED);
GTEST_LOG_(INFO) << "ExecuteAction_001 end";
}
@@ -205,7 +206,7 @@ HWTEST_F(AccessibleAbilityChannelClientTest, ExecuteAction_002, TestSize.Level1)
ASSERT_TRUE(client);
std::map<std::string, std::string> actionArguments;
EXPECT_EQ(client->ExecuteAction(ACCESSIBILITY_WINDOW_ID,
ELEMENT_ID, ActionType::ACCESSIBILITY_ACTION_SELECT, actionArguments), RET_ERR_SAMGR);
ELEMENT_ID, ActionType::ACCESSIBILITY_ACTION_SELECT, actionArguments, Rect()), RET_ERR_SAMGR);
GTEST_LOG_(INFO) << "ExecuteAction_002 end";
}
@@ -217,10 +218,11 @@ HWTEST_F(AccessibleAbilityChannelClientTest, ExecuteAction_002, TestSize.Level1)
HWTEST_F(AccessibleAbilityChannelClientTest, ExecuteAction_003, TestSize.Level1)
{
GTEST_LOG_(INFO) << "ExecuteAction_003 start";
EXPECT_CALL(*stub_, ExecuteAction(_, _, _, _, _, _)).Times(1).WillOnce(Return(RET_OK));
EXPECT_CALL(*stub_, ExecuteAction(_, _, _, _, _, _, _)).Times(1).WillOnce(Return(RET_OK));
std::map<std::string, std::string> actionArguments;
Rect rect;
EXPECT_EQ(instance_->ExecuteAction(ACCESSIBILITY_WINDOW_ID, ELEMENT_ID,
ActionType::ACCESSIBILITY_ACTION_SELECT, actionArguments), RET_ERR_TIME_OUT);
ActionType::ACCESSIBILITY_ACTION_SELECT, actionArguments, rect), RET_ERR_TIME_OUT);
GTEST_LOG_(INFO) << "ExecuteAction_003 end";
}
@@ -36,6 +36,7 @@ const std::string ACTION_ARGU_MOVE_UNIT = "textMoveUnit";
const std::string ACTION_ARGU_SPAN_ID = "spanId";
const std::string ACTION_ARGU_CLICK_ENHANCE_DATA = "enhanceData";
const std::string ACTION_ARGU_CLICK_TIMESTAMP = "timeStamp";
const std::string ACTION_ARGU_INJECT_ACTION_TYPE = "injectActionType";
const std::string ACTION_ARGU_CUSTOM_ACTION = "customActions";
// HtmlItemType
@@ -40,6 +40,7 @@ extern const std::string ACTION_ARGU_MOVE_UNIT;
extern const std::string ACTION_ARGU_SPAN_ID;
extern const std::string ACTION_ARGU_CLICK_ENHANCE_DATA;
extern const std::string ACTION_ARGU_CLICK_TIMESTAMP;
extern const std::string ACTION_ARGU_INJECT_ACTION_TYPE;
extern const std::string ACTION_ARGU_CUSTOM_ACTION;
// HtmlItemType
@@ -191,6 +191,7 @@ enum ActionType : int32_t {
ACCESSIBILITY_ACTION_NOTIFICATIONCENTER = 0x00080000,
ACCESSIBILITY_ACTION_CONTROLCENTER = 0x00008000,
ACCESSIBILITY_ACTION_CUSTOM = 0x20000000,
ACCESSIBILITY_ACTION_INJECT_ACTION = 0x40000000,
ACCESSIBILITY_ACTION_TYPE_MASK = 0x1FFFFFFF,
};
@@ -604,6 +605,13 @@ enum AccessibilityEventType : int32_t {
TYPE_FOCUS_INVISIBLE = 71
};
enum InjectActionType : int32_t {
INJECT_ACTION_TYPE_INVALID = -1,
INJECT_ACTION_TYPE_CLICK = 1,
INJECT_ACTION_TYPE_DOUBLE_CLICK = 2,
INJECT_ACTION_TYPE_LONG_CLICK = 3,
};
enum AccessibilityAction : int32_t {
ERROR = -1,
ACCESSIBILITY_FOCUS = 0,
@@ -627,7 +635,8 @@ enum AccessibilityAction : int32_t {
NOTIFICATION_CENTER = 18,
CONTROL_CENTER = 19,
SPAN_CLICK = 20,
CUSTOM_ACTION = 21
INJECT_ACTION = 21,
CUSTOM_ACTION = 22
};
const std::vector<std::string> ACTION_NAMES = {
@@ -652,7 +661,8 @@ const std::vector<std::string> ACTION_NAMES = {
"notificationCenter", //AccessibilityAction.NOTIFICATION_CENTER=18
"controlCenter", //AccessibilityAction.CONTROL_CENTER=19
"spanClick", //AccessibilityAction.SPAN_CLICK=20
"customActions" //AccessibilityAction.CUSTOM_ACTION=21
"injectAction", //AccessibilityAction.INJECT_ACTION=21
"customActions" //AccessibilityAction.CUSTOM_ACTION=22
};
enum class AnnounceType : uint32_t {
@@ -17,7 +17,7 @@ import ExtensionContext from 'application.ExtensionContext';
import Want from '@ohos.app.ability.Want';
import { AsyncCallback, BusinessError, Callback } from '@ohos.base';
import accessibility from '@ohos.accessibility';
import { AccessibilityAction, FocusMoveResultCode } from '@ohos.accessibility';
import { AccessibilityAction, FocusMoveResultCode, InjectActionType } from '@ohos.accessibility';
export interface FocusMoveResult {
target: Array<AccessibilityElement>;
@@ -536,7 +536,8 @@ export class Parameter {
selectTextInForWard?: boolean = false;
offset?: string = '';
spanId?: string = '';
scrollType?: string = '';
scrollType?: string = '';
injectActionType?: InjectActionType;
}
export type FocusDirection = 'up' | 'down' | 'left' | 'right' | 'forward' | 'backward';
@@ -471,7 +471,8 @@ std::string ConvertActionTypeToString(ActionType type)
{ActionType::ACCESSIBILITY_ACTION_SPAN_CLICK, "spanClick"},
{ActionType::ACCESSIBILITY_ACTION_CUSTOM, "customActions"},
{ActionType::ACCESSIBILITY_ACTION_NEXT_HTML_ITEM, "nextHtmlItem"},
{ActionType::ACCESSIBILITY_ACTION_PREVIOUS_HTML_ITEM, "previousHtmlItem"}
{ActionType::ACCESSIBILITY_ACTION_PREVIOUS_HTML_ITEM, "previousHtmlItem"},
{ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION, "injectAction"}
};
if (triggerActionTable.find(type) == triggerActionTable.end()) {
@@ -532,6 +533,7 @@ int32_t ConvertOperationTypeToTarget(ActionType type)
{ActionType::ACCESSIBILITY_ACTION_SET_CURSOR_POSITION, AccessibilityAction::SET_CURSOR_POSITION},
{ActionType::ACCESSIBILITY_ACTION_SET_TEXT, AccessibilityAction::SET_TEXT},
{ActionType::ACCESSIBILITY_ACTION_SPAN_CLICK, AccessibilityAction::SPAN_CLICK},
{ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION, AccessibilityAction::INJECT_ACTION},
{ActionType::ACCESSIBILITY_ACTION_CUSTOM, AccessibilityAction::CUSTOM_ACTION}
};
@@ -40,7 +40,8 @@ export enum AccessibilityAction {
NOTIFICATION_CENTER = 18,
CONTROL_CENTER = 19,
SPAN_CLICK = 20,
CUSTOM_ACTION = 21
INJECT_ACTION = 21,
CUSTOM_ACTION = 22
}
export enum AccessibilityEventType {
@@ -130,7 +131,7 @@ namespace accessibility {
export type Action = 'accessibilityFocus' | 'clearAccessibilityFocus' | 'focus' | 'clearFocus' | 'clearSelection' |
'click' | 'longClick' | 'cut' | 'copy' | 'paste' | 'select' | 'setText' | 'delete' |
'scrollForward' | 'scrollBackward' | 'setSelection' | 'setCursorPosition' | 'home' |
'back' | 'recentTask' | 'notificationCenter' | 'controlCenter' | 'common' | 'customActions';
'back' | 'recentTask' | 'notificationCenter' | 'controlCenter' | 'common' | 'injectAction' | 'customActions';
export type WindowUpdateType = 'add' | 'remove' | 'bounds' | 'active' | 'focus';
@@ -546,3 +547,9 @@ export enum FocusMoveResultCode {
DOUBLE_CHECK_CHILD_PROPERTY_AND_GET_LAST = 7,
SEARCH_FAILURE_IN_SCROLL = 8
}
export enum InjectActionType {
CLICK = 1,
DOUBLE_CLICK = 2,
LONG_CLICK = 3
}
+15 -1
View File
@@ -325,7 +325,9 @@ ActionType ANIUtils::ConvertStringToAccessibleOperationType(const std::string &t
{"recentTask", ActionType::ACCESSIBILITY_ACTION_RECENTTASK},
{"notificationCenter", ActionType::ACCESSIBILITY_ACTION_NOTIFICATIONCENTER},
{"controlCenter", ActionType::ACCESSIBILITY_ACTION_CONTROLCENTER},
{"customActions", ActionType::ACCESSIBILITY_ACTION_CUSTOM}};
{"injectAction", ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION},
{"customActions", ActionType::ACCESSIBILITY_ACTION_CUSTOM}
};
if (accessibleOperationTypeTable.find(type) == accessibleOperationTypeTable.end()) {
HILOG_WARN("invalid key[%{public}s]", type.c_str());
@@ -1339,6 +1341,18 @@ void ANIUtils::ConvertActionArgsJSToANI(ani_env *env, ani_object obj,
args.insert(std::pair<std::string, std::string>("customActions", str.c_str()));
}
break;
case ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION: {
if (env->Object_GetFieldByName_Ref(obj, "injectActionType", &fiedNameValue) == ANI_OK) {
int32_t injectActionType = 0;
if (env->EnumItem_GetValue_Int(static_cast<ani_enum_item>(fiedNameValue),
&injectActionType) != ANI_OK) {
HILOG_ERROR("Failed to get injectActionType enum value");
break;
}
args.insert(std::pair<std::string, std::string>("injectActionType", std::to_string(injectActionType)));
}
break;
}
case ActionType::ACCESSIBILITY_ACTION_SCROLL_FORWARD:
case ActionType::ACCESSIBILITY_ACTION_SCROLL_BACKWARD:
SetScrollTypeParam(env, obj, args);
@@ -819,7 +819,8 @@ std::string ConvertOperationTypeToString(ActionType type)
{ActionType::ACCESSIBILITY_ACTION_SPAN_CLICK, "spanClick"},
{ActionType::ACCESSIBILITY_ACTION_CUSTOM, "customActions"},
{ActionType::ACCESSIBILITY_ACTION_NEXT_HTML_ITEM, "nextHtmlItem"},
{ActionType::ACCESSIBILITY_ACTION_PREVIOUS_HTML_ITEM, "previousHtmlItem"}
{ActionType::ACCESSIBILITY_ACTION_PREVIOUS_HTML_ITEM, "previousHtmlItem"},
{ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION, "injectAction"}
};
if (triggerActionTable.find(type) == triggerActionTable.end()) {
@@ -932,7 +933,9 @@ ActionType ConvertStringToAccessibleOperationType(const std::string &type)
{"spanClick", ActionType::ACCESSIBILITY_ACTION_SPAN_CLICK},
{"customActions", ActionType::ACCESSIBILITY_ACTION_CUSTOM},
{"nextHtmlItem", ActionType::ACCESSIBILITY_ACTION_NEXT_HTML_ITEM},
{"previousHtmlItem", ActionType::ACCESSIBILITY_ACTION_PREVIOUS_HTML_ITEM}};
{"previousHtmlItem", ActionType::ACCESSIBILITY_ACTION_PREVIOUS_HTML_ITEM},
{"injectAction", ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION}
};
if (accessibleOperationTypeTable.find(type) == accessibleOperationTypeTable.end()) {
HILOG_WARN("invalid key[%{public}s]", type.c_str());
@@ -1034,6 +1037,13 @@ bool ConvertActionArgsJSToNAPI(
args.insert(std::pair<std::string, std::string>("customActions", str.c_str()));
}
break;
case ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION:
napi_create_string_utf8(env, "injectActionType", NAPI_AUTO_LENGTH, &propertyNameValue);
str = ConvertStringJSToNAPI(env, object, propertyNameValue, hasProperty);
if (hasProperty) {
args.insert(std::pair<std::string, std::string>("injectActionType", str.c_str()));
}
break;
case ActionType::ACCESSIBILITY_ACTION_SCROLL_FORWARD:
ret = SetScrollTypeParam(env, object, args);
break;
@@ -1165,7 +1175,21 @@ std::string ConvertStringJSToNAPI(napi_env env, napi_value object, napi_value pr
if (hasProperty) {
napi_value itemValue = nullptr;
napi_get_property(env, object, propertyNameValue, &itemValue);
str = GetStringFromNAPI(env, itemValue);
napi_status status;
napi_valuetype valuetype = napi_null;
status = napi_typeof(env, itemValue, &valuetype);
if (status != napi_ok) {
HILOG_ERROR("ConvertStringJSToNAPI napi_typeof error and status is %{public}d", status);
return str;
}
if (valuetype == napi_string) {
str = GetStringFromNAPI(env, itemValue);
} else if (valuetype == napi_number) {
int32_t num;
napi_get_value_int32(env, itemValue, &num);
str = std::to_string(num);
}
}
return str;
}
@@ -194,6 +194,19 @@ static napi_value CreateAccessibilityEventType(napi_env env)
return objValue;
}
static napi_value CreateInjectActionType(napi_env env)
{
napi_value objValue = nullptr;
napi_create_object(env, &objValue);
napi_set_named_property(env, objValue, "CLICK",
CreateIntObject(env, InjectActionType::INJECT_ACTION_TYPE_CLICK));
napi_set_named_property(env, objValue, "DOUBLE_CLICK",
CreateIntObject(env, InjectActionType::INJECT_ACTION_TYPE_DOUBLE_CLICK));
napi_set_named_property(env, objValue, "LONG_CLICK",
CreateIntObject(env, InjectActionType::INJECT_ACTION_TYPE_LONG_CLICK));
return objValue;
}
static napi_value CreateAccessibilityAction(napi_env env)
{
napi_value objValue = nullptr;
@@ -226,6 +239,7 @@ static napi_value CreateAccessibilityAction(napi_env env)
napi_set_named_property(env, objValue, "CONTROL_CENTER",
CreateIntObject(env, AccessibilityAction::CONTROL_CENTER));
napi_set_named_property(env, objValue, "SPAN_CLICK", CreateIntObject(env, AccessibilityAction::SPAN_CLICK));
napi_set_named_property(env, objValue, "INJECT_ACTION", CreateIntObject(env, AccessibilityAction::INJECT_ACTION));
return objValue;
}
@@ -271,6 +285,7 @@ static napi_value Init(napi_env env, napi_value exports)
napi_set_named_property(env, exports, "AccessibilityEventType", CreateAccessibilityEventType(env));
napi_set_named_property(env, exports, "AccessibilityAction", CreateAccessibilityAction(env));
napi_set_named_property(env, exports, "InjectActionType", CreateInjectActionType(env));
auto &instance = OHOS::AccessibilityConfig::AccessibilityConfig::GetInstance();
(void)instance.InitializeContext();
@@ -59,7 +59,7 @@ public:
RetError ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId, const int32_t action,
const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback) override;
const sptr<IAccessibilityElementOperatorCallback> &callback, const Rect &rect) override;
RetError GetWindow(const int32_t windowId, AccessibilityWindowInfo &windowInfo) override;
@@ -107,6 +107,11 @@ private:
RetError ExecuteActionAsync(const int32_t accessibilityWindowId, const int64_t elementId, const int32_t action,
const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback);
RetError HandleInjectAction(const std::map<std::string, std::string> &actionArguments, const Rect &rect);
void InjectEventToInput(int32_t xPos, int32_t yPos, InjectActionType injectActionType,
int32_t displayWidth, int32_t displayHeight);
void CalculateCenterPosition(const Rect &rect, int32_t &xPos, int32_t &yPos,
int32_t displayWidth, int32_t displayHeight);
std::string clientName_ = "";
int32_t accountId_ = -1;
std::shared_ptr<AppExecFwk::EventHandler> eventHandler_ = nullptr;
@@ -18,8 +18,10 @@
#include "accessibility_window_connection.h"
#include "accessibility_window_manager.h"
#include "accessible_ability_connection.h"
#include "display_manager.h"
#include "hilog_wrapper.h"
#include <cinttypes>
#include <unordered_map>
#ifdef OHOS_BUILD_ENABLE_POWER_MANAGER
#include "accessibility_power_manager.h"
#endif
@@ -30,6 +32,7 @@ namespace Accessibility {
namespace {
constexpr int32_t WINDOW_ID_INVALID = -1;
constexpr int64_t ELEMENT_ID_INVALID = -1;
const int32_t LONG_PRESS_EVENT_INTERVAL = 500;
MMI::InputManager* inputManager_ = MMI::InputManager::GetInstance();
std::map<int32_t, std::pair<bool, std::pair<int32_t, int32_t>>> accessibleKeyCodeTable = {
{ActionType::ACCESSIBILITY_ACTION_HOME,
@@ -42,6 +45,24 @@ namespace {
{true, {MMI::KeyEvent::KEYCODE_CALL_NOTIFICATION_CENTER, MMI::KeyEvent::KEYCODE_CALL_NOTIFICATION_CENTER}}},
{ActionType::ACCESSIBILITY_ACTION_CONTROLCENTER,
{true, {MMI::KeyEvent::KEYCODE_CALL_CONTROL_CENTER, MMI::KeyEvent::KEYCODE_CALL_CONTROL_CENTER}}}};
InjectActionType StringToInjectAction(const std::string& injectActionType)
{
static const std::unordered_map<std::string, InjectActionType> actionMap = {
{"click", INJECT_ACTION_TYPE_CLICK},
{"doubleClick", INJECT_ACTION_TYPE_DOUBLE_CLICK},
{"longClick", INJECT_ACTION_TYPE_LONG_CLICK},
{"1", INJECT_ACTION_TYPE_CLICK},
{"2", INJECT_ACTION_TYPE_DOUBLE_CLICK},
{"3", INJECT_ACTION_TYPE_LONG_CLICK}
};
auto it = actionMap.find(injectActionType);
if (it != actionMap.end()) {
return it->second;
}
return INJECT_ACTION_TYPE_INVALID;
}
} // namespace
AccessibleAbilityChannel::AccessibleAbilityChannel(
@@ -464,7 +485,7 @@ RetError AccessibleAbilityChannel::UnholdRunningLock()
RetError AccessibleAbilityChannel::ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback)
const sptr<IAccessibilityElementOperatorCallback> &callback, const Rect &rect)
{
HILOG_DEBUG("ExecuteAction elementId:%{public}" PRId64 " winId:%{public}d, action:%{public}d, requestId:%{public}d",
elementId, accessibilityWindowId, action, requestId);
@@ -493,9 +514,125 @@ RetError AccessibleAbilityChannel::ExecuteAction(const int32_t accessibilityWind
callback->SetExecuteActionResult(true, requestId);
return RET_OK;
}
if (action == ActionType::ACCESSIBILITY_ACTION_INJECT_ACTION) {
RetError ret = HandleInjectAction(actionArguments, rect);
if (ret != RET_OK) {
HILOG_ERROR("Handle Inject Click failed!");
callback->SetExecuteActionResult(false, requestId);
return RET_ERR_FAILED;
}
callback->SetExecuteActionResult(true, requestId);
return RET_OK;
}
return ExecuteActionAsync(accessibilityWindowId, elementId, action, actionArguments, requestId, callback);
}
RetError AccessibleAbilityChannel::HandleInjectAction(const std::map<std::string, std::string> &actionArguments,
const Rect &rect)
{
HILOG_DEBUG();
auto it = actionArguments.find("injectActionType");
if (it == actionArguments.end()) {
HILOG_ERROR("HandleInjectAction injectActionType not found in actionArguments");
return RET_ERR_INVALID_PARAM;
}
std::string clickTypeStr = it->second;
HILOG_INFO("HandleInjectAction clickType: %{public}s", clickTypeStr.c_str());
InjectActionType injectActionType = StringToInjectAction(clickTypeStr);
if (injectActionType == INJECT_ACTION_TYPE_INVALID) {
HILOG_ERROR("HandleInjectAction invalid clickType: %{public}s", clickTypeStr.c_str());
return RET_ERR_INVALID_PARAM;
}
int32_t xPos = 0;
int32_t yPos = 0;
uint64_t displayId = Rosen::DisplayManager::GetInstance().GetDefaultDisplayId();
sptr<Rosen::Display> display = Rosen::DisplayManager::GetInstance().GetDisplayById(displayId);
if (!display) {
HILOG_ERROR("Get display failed");
return RET_ERR_FAILED;
}
int32_t displayWidth = display->GetWidth();
int32_t displayHeight = display->GetHeight();
HILOG_INFO("HandleInjectAction displayWidth: %{public}d, displayHeight: %{public}d", displayWidth, displayHeight);
CalculateCenterPosition(rect, xPos, yPos, displayWidth, displayHeight);
InjectEventToInput(xPos, yPos, injectActionType, displayWidth, displayHeight);
return RET_OK;
}
void AccessibleAbilityChannel::CalculateCenterPosition(const Rect &rect, int32_t &xPos, int32_t &yPos,
int32_t displayWidth, int32_t displayHeight)
{
int32_t leftTopXPos = rect.GetLeftTopXScreenPostion();
int32_t rightBottomXPos = rect.GetRightBottomXScreenPostion();
int32_t leftTopYPos = rect.GetLeftTopYScreenPostion();
int32_t rightBottomYPos = rect.GetRightBottomYScreenPostion();
HILOG_DEBUG("CalculateCenterPosition leftTopXPos: %{public}d, rightBottomXPos: %{public}d, "
"leftTopYPos: %{public}d, rightBottomYPos: %{public}d",
leftTopXPos, rightBottomXPos, leftTopYPos, rightBottomYPos);
leftTopXPos = leftTopXPos > 0 ? leftTopXPos : 0;
leftTopYPos = leftTopYPos > 0 ? leftTopYPos : 0;
rightBottomXPos = (displayWidth > 0 && displayWidth < rightBottomXPos) ?
displayWidth : rightBottomXPos;
rightBottomYPos = (displayHeight > 0 && displayHeight < rightBottomYPos) ?
displayHeight : rightBottomYPos;
xPos = leftTopXPos + (rightBottomXPos - leftTopXPos) / DIVISOR_TWO;
yPos = leftTopYPos + (rightBottomYPos - leftTopYPos) / DIVISOR_TWO;
HILOG_DEBUG("CalculateCenterPosition xPos: %{public}d, yPos: %{public}d", xPos, yPos);
}
void AccessibleAbilityChannel::InjectEventToInput(int32_t xPos, int32_t yPos, InjectActionType injectActionType,
int32_t displayWidth, int32_t displayHeight)
{
HILOG_INFO("InjectEventToInput injectActionType: %{public}d, position: [%{public}d, %{public}d]",
injectActionType, xPos, yPos);
if (xPos < 0 || yPos < 0 || (displayWidth > 0 && xPos >= displayWidth) ||
(displayHeight > 0 && yPos >= displayHeight)) {
HILOG_ERROR("InjectEventToInput invalid position: [%{public}d, %{public}d], "
"displaySize: [%{public}d, %{public}d]", xPos, yPos, displayWidth, displayHeight);
return;
}
auto injectTouchEvent = [](int32_t x, int32_t y, int32_t action) {
std::shared_ptr<MMI::PointerEvent> pointerEvent = MMI::PointerEvent::Create();
pointerEvent->SetSourceType(MMI::PointerEvent::SOURCE_TYPE_TOUCHSCREEN);
pointerEvent->SetPointerAction(action);
pointerEvent->AddFlag(MMI::InputEvent::EVENT_FLAG_ACCESSIBILITY);
pointerEvent->AddFlag(MMI::InputEvent::EVENT_FLAG_NO_INTERCEPT);
MMI::PointerEvent::PointerItem item;
item.SetDisplayX(x);
item.SetDisplayY(y);
item.SetRawDisplayX(x);
item.SetRawDisplayY(y);
item.SetPointerId(1);
pointerEvent->AddPointerItem(item);
pointerEvent->SetPointerId(1);
MMI::InputManager::GetInstance()->SimulateInputEvent(pointerEvent);
};
if (injectActionType == INJECT_ACTION_TYPE_CLICK) {
injectTouchEvent(xPos, yPos, MMI::PointerEvent::POINTER_ACTION_DOWN);
injectTouchEvent(xPos, yPos, MMI::PointerEvent::POINTER_ACTION_UP);
} else if (injectActionType == INJECT_ACTION_TYPE_DOUBLE_CLICK) {
for (int i = 0; i < DIVISOR_TWO; i++) {
injectTouchEvent(xPos, yPos, MMI::PointerEvent::POINTER_ACTION_DOWN);
injectTouchEvent(xPos, yPos, MMI::PointerEvent::POINTER_ACTION_UP);
}
} else if (injectActionType == INJECT_ACTION_TYPE_LONG_CLICK) {
injectTouchEvent(xPos, yPos, MMI::PointerEvent::POINTER_ACTION_DOWN);
std::this_thread::sleep_for(std::chrono::milliseconds(LONG_PRESS_EVENT_INTERVAL));
injectTouchEvent(xPos, yPos, MMI::PointerEvent::POINTER_ACTION_UP);
}
}
RetError AccessibleAbilityChannel::ExecuteActionAsync(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback> &callback)
@@ -45,9 +45,9 @@ public:
MOCK_METHOD6(
FocusMoveSearch, RetError(const int32_t accessibilityWindowId, const int64_t elementId, const int32_t direction,
const int32_t requestId, const sptr<IAccessibilityElementOperatorCallback>& callback, bool systemApi));
MOCK_METHOD6(ExecuteAction, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
MOCK_METHOD7(ExecuteAction, RetError(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback));
const sptr<IAccessibilityElementOperatorCallback>& callback, const Rect& rect));
MOCK_METHOD2(GetWindows, RetError(std::vector<AccessibilityWindowInfo> &windows, bool systemApi));
MOCK_METHOD3(GetWindowsByDisplayId, RetError(const uint64_t displayId,
std::vector<AccessibilityWindowInfo> &windows, bool systemApi));
@@ -121,13 +121,14 @@ RetError AccessibleAbilityChannelProxy::UnholdRunningLock()
RetError AccessibleAbilityChannelProxy::ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback)
const sptr<IAccessibilityElementOperatorCallback>& callback, const Rect &rect)
{
(void)accessibilityWindowId;
(void)elementId;
(void)requestId;
(void)callback;
(void)actionArguments;
(void)rect;
return RET_OK;
}
@@ -115,7 +115,7 @@ RetError AccessibleAbilityChannel::FocusMoveSearch(const int32_t accessibilityWi
RetError AccessibleAbilityChannel::ExecuteAction(const int32_t accessibilityWindowId, const int64_t elementId,
const int32_t action, const std::map<std::string, std::string> &actionArguments, const int32_t requestId,
const sptr<IAccessibilityElementOperatorCallback>& callback)
const sptr<IAccessibilityElementOperatorCallback>& callback, const Rect &rect)
{
GTEST_LOG_(INFO) << "MOCK AccessibleAbilityChannel ExecuteAction";
(void)accessibilityWindowId;
@@ -124,6 +124,7 @@ RetError AccessibleAbilityChannel::ExecuteAction(const int32_t accessibilityWind
(void)actionArguments;
(void)requestId;
(void)callback;
(void)rect;
return RET_OK;
}
@@ -192,7 +192,7 @@ HWTEST_F(AccessibleAbilityChannelUnitTest,
{
GTEST_LOG_(INFO) << "AccessibleAbilityChannel_Unittest_ExecuteAction_001 start";
std::map<std::string, std::string> actionArguments;
EXPECT_EQ(channel_->ExecuteAction(WINDOW_ID, ELEMENT_ID, 1, actionArguments, 0, nullptr), RET_ERR_NULLPTR);
EXPECT_EQ(channel_->ExecuteAction(WINDOW_ID, ELEMENT_ID, 1, actionArguments, 0, nullptr, Rect()), RET_ERR_NULLPTR);
GTEST_LOG_(INFO) << "AccessibleAbilityChannel_Unittest_ExecuteAction_001 end";
}
@@ -418,7 +418,7 @@ HWTEST_F(AamsAccessibleAbilityChannelTest, AccessibleAbilityChannel_ModuleTest_E
ASSERT_TRUE(AccessibilityHelper::GetInstance().GetTestStub());
RetError result =
AccessibilityHelper::GetInstance().GetTestStub()->ExecuteAction(0, 4, 3, actionArguments, 0, nullptr);
AccessibilityHelper::GetInstance().GetTestStub()->ExecuteAction(0, 4, 3, actionArguments, 0, nullptr, Rect());
sleep(2);
GTEST_LOG_(INFO) << "Test result";
EXPECT_EQ(-1, proxy_->testChannelElementId_);
@@ -446,7 +446,7 @@ HWTEST_F(AamsAccessibleAbilityChannelTest, AccessibleAbilityChannel_ModuleTest_E
ASSERT_TRUE(AccessibilityHelper::GetInstance().GetTestStub());
RetError result = AccessibilityHelper::GetInstance().GetTestStub()->ExecuteAction(
ACTIVE_WINDOW_ID, 4, 3, actionArguments, 0, nullptr);
ACTIVE_WINDOW_ID, 4, 3, actionArguments, 0, nullptr, Rect());
sleep(2);
GTEST_LOG_(INFO) << "Test result";
EXPECT_EQ(-1, proxy_->testChannelElementId_);
@@ -624,7 +624,7 @@ HWTEST_F(AamsAccessibleAbilityChannelTest, AccessibleAbilityChannel_ModuleTest_E
ASSERT_TRUE(AccessibilityHelper::GetInstance().GetTestStub());
RetError result =
AccessibilityHelper::GetInstance().GetTestStub()->ExecuteAction(0, 4, 3, actionArguments, 0, nullptr);
AccessibilityHelper::GetInstance().GetTestStub()->ExecuteAction(0, 4, 3, actionArguments, 0, nullptr, Rect());
sleep(2);
GTEST_LOG_(INFO) << "Test result";
EXPECT_EQ(-1, proxy_->testChannelElementId_);