From b1da5a73b481dda4e54b8a34ea1ca5afdb1d82d5 Mon Sep 17 00:00:00 2001 From: mamingshuai Date: Wed, 2 Jun 2021 00:45:03 +0800 Subject: [PATCH] update OpenHarmony 2.0 Canary --- .gitattributes | 15 + .gitee/ISSUE_TEMPLATE.zh-CN.md | 11 - .gitee/PULL_REQUEST_TEMPLATE.zh-CN.md | 12 - .gitignore | 1 - BUILD.gn | 34 +- README.md | 29 +- README_zh.md | 1 - config/hits.json | 16 - config/user_config.xml | 129 +- extension/setup.py | 66 + extension/src/xdevice_extension/__init__.py | 25 + .../src/xdevice_extension/_core/__init__.py | 17 + .../src/xdevice_extension/_core/constants.py | 249 ++ .../_core/driver/__init__.py | 17 + .../xdevice_extension/_core/driver/drivers.py | 2156 +++++++++++++++++ .../xdevice_extension/_core/driver/kunpeng.py | 104 + .../xdevice_extension/_core/driver/parser.py | 766 ++++++ .../_core/environment/__init__.py | 17 + .../_core/environment/device.py | 387 +++ .../_core/environment/device_monitor.py | 120 + .../_core/environment/device_state.py | 120 + .../_core/environment/dmlib.py | 1187 +++++++++ .../_core/environment/emulator.py | 59 + .../_core/environment/manager_device.py | 370 +++ .../src/xdevice_extension/_core/exception.py | 97 + .../_core/executor/__init__.py | 17 + .../_core/executor/listener.py | 98 + .../_core/testkit/__init__.py | 17 + .../xdevice_extension/_core/testkit/kit.py | 960 ++++++++ .../src/xdevice_extension/_core/utils.py | 238 ++ figures/icon-caution.gif | Bin figures/icon-danger.gif | Bin figures/icon-note.gif | Bin figures/icon-notice.gif | Bin figures/icon-tip.gif | Bin figures/icon-warning.gif | Bin run.bat | 0 run.sh | 11 +- setup.py | 2 +- src/xdevice/__main__.py | 0 src/xdevice/_core/command/console.py | 0 src/xdevice/_core/config/config_manager.py | 0 src/xdevice/_core/config/resource_manager.py | 0 src/xdevice/_core/constants.py | 0 src/xdevice/_core/driver/device_test.py | 4 + src/xdevice/_core/driver/drivers_lite.py | 0 src/xdevice/_core/driver/parser_lite.py | 0 src/xdevice/_core/environment/device_lite.py | 5 +- src/xdevice/_core/environment/dmlib_lite.py | 20 +- src/xdevice/_core/environment/manager_env.py | 0 src/xdevice/_core/environment/manager_lite.py | 0 src/xdevice/_core/exception.py | 0 src/xdevice/_core/executor/concurrent.py | 0 src/xdevice/_core/executor/listener.py | 0 src/xdevice/_core/executor/request.py | 0 src/xdevice/_core/executor/scheduler.py | 0 src/xdevice/_core/executor/source.py | 2 + src/xdevice/_core/interface.py | 0 src/xdevice/_core/logger.py | 3 +- src/xdevice/_core/plugin.py | 0 src/xdevice/_core/report/__main__.py | 0 src/xdevice/_core/report/encrypt.py | 0 src/xdevice/_core/report/reporter_helper.py | 4 +- src/xdevice/_core/report/suite_reporter.py | 0 .../_core/resource/config/user_config.xml | 129 +- .../_core/resource/template/report.html | 3 +- src/xdevice/_core/testkit/json_parser.py | 0 src/xdevice/_core/testkit/kit_lite.py | 13 +- src/xdevice/_core/utils.py | 0 src/xdevice/variables.py | 4 + 70 files changed, 7303 insertions(+), 232 deletions(-) create mode 100644 .gitattributes delete mode 100644 .gitee/ISSUE_TEMPLATE.zh-CN.md delete mode 100644 .gitee/PULL_REQUEST_TEMPLATE.zh-CN.md mode change 100755 => 100644 README_zh.md delete mode 100644 config/hits.json mode change 100755 => 100644 config/user_config.xml create mode 100644 extension/setup.py create mode 100644 extension/src/xdevice_extension/__init__.py create mode 100644 extension/src/xdevice_extension/_core/__init__.py create mode 100644 extension/src/xdevice_extension/_core/constants.py create mode 100644 extension/src/xdevice_extension/_core/driver/__init__.py create mode 100644 extension/src/xdevice_extension/_core/driver/drivers.py create mode 100644 extension/src/xdevice_extension/_core/driver/kunpeng.py create mode 100644 extension/src/xdevice_extension/_core/driver/parser.py create mode 100644 extension/src/xdevice_extension/_core/environment/__init__.py create mode 100644 extension/src/xdevice_extension/_core/environment/device.py create mode 100644 extension/src/xdevice_extension/_core/environment/device_monitor.py create mode 100644 extension/src/xdevice_extension/_core/environment/device_state.py create mode 100644 extension/src/xdevice_extension/_core/environment/dmlib.py create mode 100644 extension/src/xdevice_extension/_core/environment/emulator.py create mode 100644 extension/src/xdevice_extension/_core/environment/manager_device.py create mode 100644 extension/src/xdevice_extension/_core/exception.py create mode 100644 extension/src/xdevice_extension/_core/executor/__init__.py create mode 100644 extension/src/xdevice_extension/_core/executor/listener.py create mode 100644 extension/src/xdevice_extension/_core/testkit/__init__.py create mode 100644 extension/src/xdevice_extension/_core/testkit/kit.py create mode 100644 extension/src/xdevice_extension/_core/utils.py mode change 100755 => 100644 figures/icon-caution.gif mode change 100755 => 100644 figures/icon-danger.gif mode change 100755 => 100644 figures/icon-note.gif mode change 100755 => 100644 figures/icon-notice.gif mode change 100755 => 100644 figures/icon-tip.gif mode change 100755 => 100644 figures/icon-warning.gif mode change 100755 => 100644 run.bat mode change 100755 => 100644 run.sh mode change 100755 => 100644 setup.py mode change 100755 => 100644 src/xdevice/__main__.py mode change 100755 => 100644 src/xdevice/_core/command/console.py mode change 100755 => 100644 src/xdevice/_core/config/config_manager.py mode change 100755 => 100644 src/xdevice/_core/config/resource_manager.py mode change 100755 => 100644 src/xdevice/_core/constants.py mode change 100755 => 100644 src/xdevice/_core/driver/device_test.py mode change 100755 => 100644 src/xdevice/_core/driver/drivers_lite.py mode change 100755 => 100644 src/xdevice/_core/driver/parser_lite.py mode change 100755 => 100644 src/xdevice/_core/environment/device_lite.py mode change 100755 => 100644 src/xdevice/_core/environment/dmlib_lite.py mode change 100755 => 100644 src/xdevice/_core/environment/manager_env.py mode change 100755 => 100644 src/xdevice/_core/environment/manager_lite.py mode change 100755 => 100644 src/xdevice/_core/exception.py mode change 100755 => 100644 src/xdevice/_core/executor/concurrent.py mode change 100755 => 100644 src/xdevice/_core/executor/listener.py mode change 100755 => 100644 src/xdevice/_core/executor/request.py mode change 100755 => 100644 src/xdevice/_core/executor/scheduler.py mode change 100755 => 100644 src/xdevice/_core/executor/source.py mode change 100755 => 100644 src/xdevice/_core/interface.py mode change 100755 => 100644 src/xdevice/_core/logger.py mode change 100755 => 100644 src/xdevice/_core/plugin.py mode change 100755 => 100644 src/xdevice/_core/report/__main__.py mode change 100755 => 100644 src/xdevice/_core/report/encrypt.py mode change 100755 => 100644 src/xdevice/_core/report/suite_reporter.py mode change 100755 => 100644 src/xdevice/_core/resource/config/user_config.xml mode change 100755 => 100644 src/xdevice/_core/testkit/json_parser.py mode change 100755 => 100644 src/xdevice/_core/testkit/kit_lite.py mode change 100755 => 100644 src/xdevice/_core/utils.py mode change 100755 => 100644 src/xdevice/variables.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..51c63e2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +*.tgz filter=lfs diff=lfs merge=lfs -text +*.trp filter=lfs diff=lfs merge=lfs -text +*.apk filter=lfs diff=lfs merge=lfs -text +*.jar filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.asm filter=lfs diff=lfs merge=lfs -text +*.8svn filter=lfs diff=lfs merge=lfs -text +*.9svn filter=lfs diff=lfs merge=lfs -text +*.dylib filter=lfs diff=lfs merge=lfs -text +*.exe filter=lfs diff=lfs merge=lfs -text +*.a filter=lfs diff=lfs merge=lfs -text +*.so filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.dll filter=lfs diff=lfs merge=lfs -text diff --git a/.gitee/ISSUE_TEMPLATE.zh-CN.md b/.gitee/ISSUE_TEMPLATE.zh-CN.md deleted file mode 100644 index 651e02c..0000000 --- a/.gitee/ISSUE_TEMPLATE.zh-CN.md +++ /dev/null @@ -1,11 +0,0 @@ -### 该问题是怎么引起的？ - - - -### 重现步骤 - - - -### 报错信息 - - diff --git a/.gitee/PULL_REQUEST_TEMPLATE.zh-CN.md b/.gitee/PULL_REQUEST_TEMPLATE.zh-CN.md deleted file mode 100644 index 86dd358..0000000 --- a/.gitee/PULL_REQUEST_TEMPLATE.zh-CN.md +++ /dev/null @@ -1,12 +0,0 @@ -### 相关的Issue - - -### 原因（目的、解决的问题等） - - -### 描述（做了什么，变更了什么） - - -### 测试用例（新增、改动、可能影响的功能） - - diff --git a/.gitignore b/.gitignore index 278e115..0d20b64 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ *.pyc -/extension/ \ No newline at end of file diff --git a/BUILD.gn b/BUILD.gn index dc9998e..78c2a56 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -1,17 +1,17 @@ -# Copyright (c) 2020 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import("//test/xts/tools/lite/build/suite_lite.gni") - -deploy_suite("xdevice") { - suite_name = "acts,hits,ssts" -} +# Copyright (c) 2020 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import("//test/xts/tools/lite/build/suite_lite.gni") + +deploy_suite("xdevice") { + suite_name = "acts,hits,ssts" +} diff --git a/README.md b/README.md index 589ee9b..c428f32 100755 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ XDevice consists of the following sub-modules: - **environment**: configures the test framework environment, enabling device discovery and device management. - **testkit**: provides test tools to implement JSON parsing, network file mounting, etc. - **resource**: provides the device connection configuration file and report template definitions. -- **adapter**: adapts the test framework to open-source software. ## Directory Structure @@ -117,10 +116,10 @@ The environment requirements for using this module are as follows: ``` help: - Use help to get information. + use help to get information. usage: - run: Display a list of supported run commands. - list: Display a list of supported devices and task records. + run: Display a list of supported run command. + list: Display a list of supported device and task record. Examples: help run help list @@ -136,15 +135,15 @@ The environment requirements for using this module are as follows: ``` list: - Display device list and task records. + This command is used to display device list and task record. usage: list list history list Introduction: - list: Display the device list. - list history: Display historical records of a series of tasks. - list : Display historical records of tasks with the specified IDs. + list: display device list + list history: display history record of a serial of tasks + list : display history record about task what contains specific id Examples: list list history @@ -162,8 +161,8 @@ The environment requirements for using this module are as follows: ``` run: - Execute the selected test cases. - The command execution process includes use case compilation, execution, and result collection. + This command is used to execute the selected testcases. + It includes a series of processes such as use case compilation, execution, and result collection. usage: run [-l TESTLIST [TESTLIST ...] | -tf TESTFILE [TESTFILE ...]] [-tc TESTCASE] [-c CONFIG] [-sn DEVICE_SN] [-rp REPORT_PATH [REPORT_PATH ...]] @@ -179,8 +178,8 @@ The environment requirements for using this module are as follows: action task Specify tests to run. positional arguments: - action Specify the action to do. - task Specify the task name, such as ssts, acts, and hits. + action Specify action + task Specify task name,such as "ssts", "acts", "hits" ``` >![](figures/icon-note.gif) **NOTE:** @@ -207,15 +206,15 @@ The environment requirements for using this module are as follows: Structure of the report directory (the default or the specified one) ├── result # Test case execution results of the module │ ├── module name.xml - │ ├── ... + │ ├── ... ... │ ├── log # Running logs of devices and tasks │ ├── device 1.log - │ ├── ... + │ ├── ... ... │ ├── task.log ├── summary_report.html # Visual report ├── summary_report.html # Statistical report - └── ... + └── ... ... ``` diff --git a/README_zh.md b/README_zh.md old mode 100755 new mode 100644 index 4e5bb8d..c3d427b --- a/README_zh.md +++ b/README_zh.md @@ -20,7 +20,6 @@ xdevice主要包括以下几个主要模块： - environment，测试框架的环境配置模块，提供设备发现，设备管理的功能。 - testkit，测试框架工具模块，提供json解析，网络文件挂载等操作。 - resource，测试框架资源模块，提供设备连接配置文件和报告模板定义。 -- adapter，测试框架适配开源软件的模块。 ## 目录 diff --git a/config/hits.json b/config/hits.json deleted file mode 100644 index 243cc6f..0000000 --- a/config/hits.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "description": "Config for hits test suites", - "kits": [ - { - "type": "QueryKit", - "server": "NfsServer", - "mount": [ - { - "source": "resource/tools/query.bin", - "target": "/test_root/tools" - } - ], - "query" : "/test_root/tools/query.bin" - } - ] -} diff --git a/config/user_config.xml b/config/user_config.xml old mode 100755 new mode 100644 index 5439585..61b89cb --- a/config/user_config.xml +++ b/config/user_config.xml @@ -1,66 +1,63 @@ - - - - - - - - cmd - 115200 - 8 - 1 - 20 - - - - deploy - 115200 - - - - - - cmd - 115200 - 8 - 1 - 1 - - - - - - - - - - - - - - - - - - - - - - INFO - + + + + + + + + cmd + 115200 + 8 + 1 + 20 + + + + deploy + 115200 + + + + + + cmd + 115200 + 8 + 1 + 1 + + + + + + + + + + + + + + + + + + + + + + INFO + \ No newline at end of file diff --git a/extension/setup.py b/extension/setup.py new file mode 100644 index 0000000..ecaff0e --- /dev/null +++ b/extension/setup.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from setuptools import setup + +INSTALL_REQUIRES = [ +] + + +def main(): + setup(name='xdevice-extension', + description='xdevice extension test framework', + url='', + package_dir={'': 'src'}, + packages=['xdevice_extension', + 'xdevice_extension._core', + 'xdevice_extension._core.driver', + 'xdevice_extension._core.environment', + 'xdevice_extension._core.executor', + 'xdevice_extension._core.testkit' + ], + package_data={ + }, + entry_points={ + 'driver': [ + 'drivers=xdevice_extension._core.driver.drivers', + 'kunpeng=xdevice_extension._core.driver.kunpeng' + ], + 'parser': [ + 'parser=xdevice_extension._core.driver.parser' + ], + 'device': [ + 'default=xdevice_extension._core.environment.device' + ], + 'manager': [ + 'device=xdevice_extension._core.environment.manager_device' + ], + 'listener': [ + 'listener=xdevice_extension._core.executor.listener' + ], + 'testkit': [ + 'kit=xdevice_extension._core.testkit.kit' + ] + }, + zip_safe=False, + install_requires=INSTALL_REQUIRES, + ) + + +if __name__ == "__main__": + main() diff --git a/extension/src/xdevice_extension/__init__.py b/extension/src/xdevice_extension/__init__.py new file mode 100644 index 0000000..770f4d7 --- /dev/null +++ b/extension/src/xdevice_extension/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from ._core.driver.drivers import RemoteTestRunner +from ._core.driver.drivers import RemoteDexRunner + +__all__ = [ + "RemoteTestRunner", + "RemoteDexRunner" +] diff --git a/extension/src/xdevice_extension/_core/__init__.py b/extension/src/xdevice_extension/_core/__init__.py new file mode 100644 index 0000000..eb8b49d --- /dev/null +++ b/extension/src/xdevice_extension/_core/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/extension/src/xdevice_extension/_core/constants.py b/extension/src/xdevice_extension/_core/constants.py new file mode 100644 index 0000000..ac4ee17 --- /dev/null +++ b/extension/src/xdevice_extension/_core/constants.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from dataclasses import dataclass + +__all__ = ["DeviceOsType", "ProductForm", "TestType", "TestExecType", + "DeviceTestType", "HostTestType", "HostDrivenTestType", + "SchedulerType", "ListenerType", "ToolCommandType", + "TEST_DRIVER_SET", "LogType", "ParserType", "CKit", "ComType", + "DeviceLabelType", "DeviceLiteKernel", "GTestConst", "ManagerType", + "CommonParserType", "DeviceConnectorType", "FilePermission"] + + +@dataclass +class DeviceOsType(object): + """ + DeviceOsType enumeration + """ + default = "default" + lite = "lite" + + +@dataclass +class DeviceConnectorType: + hdc = "usb-hdc" + + +@dataclass +class ProductForm(object): + """ + ProductForm enumeration + """ + phone = "phone" + car = "ivi" + television = "tv" + watch = "watch" + tablet = 'tablet' + + +@dataclass +class TestType(object): + """ + TestType enumeration + """ + unittest = "unittest" + mst = "moduletest" + systemtest = "systemtest" + perf = "performance" + sec = "security" + reli = "reliability" + dst = "distributedtest" + all = "ALL" + + +@dataclass +class ComType(object): + """ + ComType enumeration + """ + cmd_com = "cmd" + deploy_com = "deploy" + + +@dataclass +class DeviceLabelType(object): + """ + DeviceLabelType enumeration + """ + wifiiot = "wifiiot" + ipcamera = "ipcamera" + watch = "watch" + phone = "phone" + + +@dataclass +class DeviceLiteKernel(object): + """ + Lite device os enumeration + """ + linux_kernel = "linux" + lite_kernel = "lite" + + +TEST_TYPE_DICT = { + "UT": TestType.unittest, + "MST": TestType.mst, + "ST": TestType.systemtest, + "PERF": TestType.perf, + "SEC": TestType.sec, + "RELI": TestType.reli, + "DST": TestType.dst, + "ALL": TestType.all, +} + + +@dataclass +class TestExecType(object): + """ + TestExecType enumeration according to test execution method + """ + # A test running on the device + device_test = "device" + # A test running on the host (pc) + host_test = "host" + # A test running on the host that interacts with one or more devices. + host_driven_test = "hostdriven" + + +@dataclass +class DeviceTestType(object): + """ + DeviceTestType enumeration + """ + cpp_test = "CppTest" + dex_test = "DexTest" + dex_junit_test = "DexJUnitTest" + hap_test = "HapTest" + junit_test = "JUnitTest" + jsunit_test = "JSUnitTest" + ctest_lite = "CTestLite" + cpp_test_lite = "CppTestLite" + lite_cpp_test = "LiteUnitTest" + open_source_test = "OpenSourceTest" + build_only_test = "BuildOnlyTestLite" + + +@dataclass +class HostTestType(object): + """ + HostTestType enumeration + """ + host_gtest = "HostGTest" + host_junit_test = "HostJUnitTest" + + +@dataclass +class HostDrivenTestType(object): + """ + HostDrivenType enumeration + """ + device_test = "DeviceTest" + + +TEST_DRIVER_SET = { + DeviceTestType.cpp_test, + DeviceTestType.dex_test, + DeviceTestType.hap_test, + DeviceTestType.junit_test, + DeviceTestType.dex_junit_test, + DeviceTestType.jsunit_test, + DeviceTestType.cpp_test_lite, + DeviceTestType.ctest_lite, + DeviceTestType.lite_cpp_test, + HostDrivenTestType.device_test +} + + +@dataclass +class SchedulerType(object): + """ + SchedulerType enumeration + """ + # default scheduler + scheduler = "Scheduler" + + +@dataclass +class LogType: + tool = "Tool" + device = "Device" + + +@dataclass +class ListenerType: + log = "Log" + report = "Report" + upload = "Upload" + collect = "Collect" + collect_lite = "CollectLite" + + +@dataclass +class ParserType: + ctest_lite = "CTestLite" + cpp_test_lite = "CppTestLite" + cpp_test_list_lite = "CppTestListLite" + open_source_test = "OpenSourceTest" + build_only_test = "BuildOnlyTestLite" + + +@dataclass +class CommonParserType: + jsunit = "JSUnit" + cpptest = "CppTest" + cpptest_list = "CppTestList" + junit = "JUnit" + + +@dataclass +class ManagerType: + device = "device" + lite_device = "device_lite" + + +@dataclass +class ToolCommandType(object): + toolcmd_key_help = "help" + toolcmd_key_show = "show" + toolcmd_key_run = "run" + toolcmd_key_quit = "quit" + toolcmd_key_list = "list" + + +@dataclass +class CKit: + push = "PushKit" + command = "CommandKit" + config = "ConfigKit" + wifi = "WIFIKit" + propertycheck = 'PropertyCheckKit' + sts = 'STSKit' + shell = "ShellKit" + testbundle = "TestBundleKit" + appinstall = "AppInstallKit" + + +@dataclass +class GTestConst(object): + exec_para_filter = "--gtest_filter" + exec_para_level = "--gtest_testsize" +class FilePermission(object): + mode_777 = 0o777 + mode_755 = 0o755 + mode_644 = 0o644 diff --git a/extension/src/xdevice_extension/_core/driver/__init__.py b/extension/src/xdevice_extension/_core/driver/__init__.py new file mode 100644 index 0000000..eb8b49d --- /dev/null +++ b/extension/src/xdevice_extension/_core/driver/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/extension/src/xdevice_extension/_core/driver/drivers.py b/extension/src/xdevice_extension/_core/driver/drivers.py new file mode 100644 index 0000000..f9d6b06 --- /dev/null +++ b/extension/src/xdevice_extension/_core/driver/drivers.py @@ -0,0 +1,2156 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re +import time +import json +import shutil +import zipfile +import tempfile +import stat +from dataclasses import dataclass + +from xdevice import ParamError +from xdevice import ExecuteTerminate +from xdevice import IDriver +from xdevice import platform_logger +from xdevice import Plugin +from xdevice import get_plugin +from xdevice import JsonParser +from xdevice import ShellHandler +from xdevice import TestDescription +from xdevice import ResourceManager +from xdevice import get_device_log_file +from xdevice import check_result_report +from xdevice import get_kit_instances +from xdevice import get_config_value +from xdevice import do_module_kit_setup +from xdevice import do_module_kit_teardown + +from xdevice_extension._core.constants import DeviceTestType +from xdevice_extension._core.constants import DeviceConnectorType +from xdevice_extension._core.constants import CommonParserType +from xdevice_extension._core.environment.dmlib import DisplayOutputReceiver +from xdevice_extension._core.exception import ShellCommandUnresponsiveException +from xdevice_extension._core.exception import HapNotSupportTest +from xdevice_extension._core.exception import HdcCommandRejectedException +from xdevice_extension._core.exception import HdcError +from xdevice_extension._core.testkit.kit import junit_para_parse +from xdevice_extension._core.testkit.kit import junit_dex_para_parse +from xdevice_extension._core.testkit.kit import reset_junit_para +from xdevice_extension._core.utils import get_filename_extension +from xdevice_extension._core.executor.listener import CollectingTestListener +from xdevice_extension._core.testkit.kit import gtest_para_parse + +__all__ = ["CppTestDriver", "DexTestDriver", "HapTestDriver", + "JSUnitTestDriver", "JUnitTestDriver", "RemoteTestRunner", + "RemoteDexRunner", "disable_keyguard"] +LOG = platform_logger("Drivers") +DEFAULT_TEST_PATH = "/%s/%s/" % ("data", "test") +ON_DEVICE_TEST_DIR_LOCATION = "/%s/%s/%s/" % ("data", "local", "tmp") + +FAILED_RUN_TEST_ATTEMPTS = 3 +TIME_OUT = 900 * 1000 + + +@dataclass +class ZunitConst(object): + z_unit_app = "ohos.unittest.App" + output_dir = "OUTPUT_DIR=" + output_file = "OUTPUT_FILE=" + test_class = "TEST_CLASS=" + exec_class = "EXEC_CLASS=" + exec_method = "EXEC_METHOD=" + exec_level = "EXEC_LEVEL=" + jacoco_exec_file = "JACOCO_EXEC_FILE=" + jtest_status_filename = "jtest_status.txt" + remote_command_dir = "commandtmp" + + +def get_level_para_string(level_string): + level_list = list(set(level_string.split(","))) + level_para_string = "" + for item in level_list: + if not item.isdigit(): + continue + item = item.strip(" ") + level_para_string = "%sLevel%s," % (level_para_string, item) + level_para_string = level_para_string.strip(",") + return level_para_string + + +def get_execute_java_test_files(suite_file): + java_test_file = "" + test_info_file = "%s.info" % suite_file[:suite_file.rfind(".")] + if not os.path.exists(test_info_file): + return java_test_file + try: + test_info_file_open = os.open(test_info_file, os.O_RDWR, + stat.S_IWUSR | stat.S_IRUSR) + with os.fdopen(test_info_file_open, "r") as file_desc: + lines = file_desc.readlines() + for line in lines: + class_name, _ = line.split(',', 1) + class_name = class_name.strip() + if not class_name.endswith("Test"): + continue + java_test_file = "%s%s," % (java_test_file, class_name) + except(IOError, ValueError) as err_msg: + LOG.exception("Error to read info file: ", err_msg, exc_info=False) + if java_test_file != "": + java_test_file = java_test_file[:-1] + return java_test_file + + +def get_java_test_para(testcase, testlevel): + exec_class = "*" + exec_method = "*" + exec_level = "" + + if "" != testcase and "" == testlevel: + pos = testcase.rfind(".") + if pos != -1: + exec_class = testcase[0:pos] + exec_method = testcase[pos + 1:] + exec_level = "" + else: + exec_class = "*" + exec_method = testcase + exec_level = "" + elif "" == testcase and "" != testlevel: + exec_class = "*" + exec_method = "*" + exec_level = get_level_para_string(testlevel) + + return exec_class, exec_method, exec_level + + +def get_xml_output(config, json_config): + xml_output = config.testargs.get("xml-output") + if not xml_output: + if get_config_value('xml-output', json_config.get_driver(), False): + xml_output = get_config_value('xml-output', + json_config.get_driver(), False) + else: + xml_output = "false" + else: + xml_output = xml_output[0] + xml_output = str(xml_output).lower() + return xml_output + + +def get_result_savepath(testsuit_path, result_rootpath): + findkey = "%stests%s" % (os.sep, os.sep) + filedir, _ = os.path.split(testsuit_path) + pos = filedir.find(findkey) + if -1 != pos: + subpath = filedir[pos + len(findkey):] + pos1 = subpath.find(os.sep) + if -1 != pos1: + subpath = subpath[pos1 + len(os.sep):] + result_path = os.path.join(result_rootpath, "result", subpath) + else: + result_path = os.path.join(result_rootpath, "result") + else: + result_path = os.path.join(result_rootpath, "result") + + if not os.path.exists(result_path): + os.makedirs(result_path) + + LOG.info("result_savepath = %s" % result_path) + return result_path + + +# all testsuit common Unavailable test result xml +def _create_empty_result_file(filepath, filename, error_message): + error_message = str(error_message) + error_message = error_message.replace("\"", """) + error_message = error_message.replace("<", "<") + error_message = error_message.replace(">", ">") + error_message = error_message.replace("&", "&") + if filename.endswith(".hap"): + filename = filename.split(".")[0] + if not os.path.exists(filepath): + file_open = os.open(filepath, os.O_WRONLY | os.O_CREAT | os.O_APPEND, + 0o755) + with os.fdopen(file_open, "w") as file_desc: + time_stamp = time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime()) + file_desc.write('\n') + file_desc.write('\n' % time_stamp) + file_desc.write( + ' \n' % + (filename, error_message)) + file_desc.write(' \n') + file_desc.write('\n') + file_desc.flush() + return + + +class ResultManager(object): + def __init__(self, testsuit_path, result_rootpath, device, + device_testpath): + self.testsuite_path = testsuit_path + self.result_rootpath = result_rootpath + self.device = device + self.device_testpath = device_testpath + self.testsuite_name = os.path.basename(self.testsuite_path) + self.is_coverage = False + + def set_is_coverage(self, is_coverage): + self.is_coverage = is_coverage + + def get_test_results(self, error_message=""): + # Get test result files + filepath = self.obtain_test_result_file() + if not os.path.exists(filepath): + _create_empty_result_file(filepath, self.testsuite_name, + error_message) + + # Get coverage data files + if self.is_coverage: + self.obtain_coverage_data() + + return filepath + + def obtain_test_result_file(self): + result_savepath = get_result_savepath(self.testsuite_path, + self.result_rootpath) + if self.testsuite_path.endswith('.hap'): + filepath = os.path.join(result_savepath, "%s.xml" % str( + self.testsuite_name).split(".")[0]) + + remote_result_name = "" + if self.device.is_file_exist(os.path.join(self.device_testpath, + "testcase_result.xml")): + remote_result_name = "testcase_result.xml" + elif self.device.is_file_exist(os.path.join(self.device_testpath, + "report.xml")): + remote_result_name = "report.xml" + + if remote_result_name: + self.device.pull_file( + os.path.join(self.device_testpath, remote_result_name), + filepath) + else: + LOG.error("%s no report file", self.device_testpath) + + else: + filepath = os.path.join(result_savepath, "%s.xml" % + self.testsuite_name) + remote_result_file = os.path.join(self.device_testpath, + "%s.xml" % self.testsuite_name) + + if self.device.is_file_exist(remote_result_file): + self.device.pull_file(remote_result_file, result_savepath) + else: + LOG.error("%s not exists", remote_result_file) + return filepath + + def is_exist_target_in_device(self, path, target): + command = "ls -l %s | grep %s" % (path, target) + + check_result = False + stdout_info = self.device.execute_shell_command(command) + if stdout_info != "" and stdout_info.find(target) != -1: + check_result = True + return check_result + + def obtain_coverage_data(self): + java_cov_path = os.path.abspath( + os.path.join(self.result_rootpath, "..", "coverage/data/exec")) + dst_target_name = "%s.exec" % self.testsuite_name + src_target_name = "jacoco.exec" + if self.is_exist_target_in_device(self.device_testpath, + src_target_name): + if not os.path.exists(java_cov_path): + os.makedirs(java_cov_path) + self.device.pull_file( + os.path.join(self.device_testpath, src_target_name), + os.path.join(java_cov_path, dst_target_name)) + + cxx_cov_path = os.path.abspath( + os.path.join(self.result_rootpath, "..", "coverage/data/cxx", + self.testsuite_name)) + target_name = "obj" + if self.is_exist_target_in_device(self.device_testpath, target_name): + if not os.path.exists(cxx_cov_path): + os.makedirs(cxx_cov_path) + src_file = os.path.join(self.device_testpath, target_name) + self.device.pull_file(src_file, cxx_cov_path) + + +@Plugin(type=Plugin.DRIVER, id=DeviceTestType.cpp_test) +class CppTestDriver(IDriver): + """ + CppTestDriver is a Test that runs a native test package on given device. + """ + + def __init__(self): + self.result = "" + self.error_message = "" + self.config = None + self.rerun = True + self.rerun_all = True + self.runner = None + + def __check_environment__(self, device_options): + pass + + def __check_config__(self, config): + pass + + def __execute__(self, request): + try: + LOG.debug("Start execute xdevice extension CppTest") + + self.config = request.config + self.config.device = request.config.environment.devices[0] + + config_file = request.root.source.config_file + self.result = "%s.xml" % \ + os.path.join(request.config.report_path, + "result", request.root.source.test_name) + + hilog = get_device_log_file( + request.config.report_path, + request.config.device.__get_serial__(), + "device_hilog") + + hilog_open = os.open(hilog, os.O_WRONLY | os.O_CREAT | os.O_APPEND, + 0o755) + with os.fdopen(hilog_open, "a") as hilog_file_pipe: + self.config.device.start_catch_device_log(hilog_file_pipe) + self._run_cpp_test(config_file, listeners=request.listeners, + request=request) + hilog_file_pipe.flush() + + except Exception as exception: + self.error_message = exception + if not getattr(exception, "error_no", ""): + setattr(exception, "error_no", "03404") + LOG.exception(self.error_message, exc_info=False, error_no="03404") + raise exception + + finally: + self.config.device.stop_catch_device_log() + self.result = check_result_report( + request.config.report_path, self.result, self.error_message) + + def _run_cpp_test(self, config_file, listeners=None, request=None): + try: + if not os.path.exists(config_file): + LOG.error("Error: Test cases don't exit %s." % config_file, + error_no="00102") + raise ParamError( + "Error: Test cases don't exit %s." % config_file, + error_no="00102") + + json_config = JsonParser(config_file) + kits = get_kit_instances(json_config, self.config.resource_path, + self.config.testcases_path) + + for listener in listeners: + listener.device_sn = self.config.device.device_sn + + self._get_driver_config(json_config) + do_module_kit_setup(request, kits) + self.runner = RemoteCppTestRunner(self.config) + self.runner.suite_name = request.root.source.test_name + + if hasattr(self.config, "history_report_path") and \ + self.config.testargs.get("test"): + self._do_test_retry(listeners, self.config.testargs) + else: + gtest_para_parse(self.config.testargs, self.runner) + self._do_test_run(listeners) + + finally: + do_module_kit_teardown(request) + + def _do_test_retry(self, listener, testargs): + for test in testargs.get("test"): + test_item = test.split("#") + if len(test_item) != 2: + continue + self.runner.add_instrumentation_arg( + "gtest_filter", "%s.%s" % (test_item[0], test_item[1])) + self.runner.run(listener) + + def _do_test_run(self, listener): + test_to_run = self._collect_test_to_run() + LOG.debug("collected test count is: %s" % len(test_to_run)) + if not test_to_run: + self.runner.run(listener) + else: + self._run_with_rerun(listener, test_to_run) + + def _collect_test_to_run(self): + if self.rerun: + self.runner.add_instrumentation_arg("gtest_list_tests", True) + run_results = self.runner.dry_run() + self.runner.remove_instrumentation_arg("gtest_list_tests") + return run_results + return None + + def _run_tests(self, listener): + test_tracker = CollectingTestListener() + listener_copy = listener.copy() + listener_copy.append(test_tracker) + self.runner.run(listener_copy) + test_run = test_tracker.get_current_run_results() + return test_run + + def _run_with_rerun(self, listener, expected_tests): + LOG.debug("ready to run with rerun, expect run: %s" + % len(expected_tests)) + test_run = self._run_tests(listener) + LOG.debug("run with rerun, has run: %s" % len(expected_tests)) + if len(test_run) < len(expected_tests): + expected_tests = TestDescription.remove_test(expected_tests, + test_run) + if not expected_tests: + LOG.debug("No tests to re-run, all tests executed at least " + "once.") + if self.rerun_all: + self._rerun_all(expected_tests, listener) + else: + self._rerun_serially(expected_tests, listener) + + def _rerun_all(self, expected_tests, listener): + tests = [] + for test in expected_tests: + tests.append("%s.%s" % (test.class_name, test.test_name)) + self.runner.add_instrumentation_arg("gtest_filter", ":".join(tests)) + LOG.debug("ready to rerun file, expect run: %s" % len(expected_tests)) + test_run = self._run_tests(listener) + LOG.debug("rerun file, has run: %s" % len(test_run)) + if len(test_run) < len(expected_tests): + expected_tests = TestDescription.remove_test(expected_tests, + test_run) + if not expected_tests: + LOG.debug("Rerun textFile success") + self._rerun_serially(expected_tests, listener) + + def _rerun_serially(self, expected_tests, listener): + LOG.debug("rerun serially, expected run: %s" % len(expected_tests)) + for test in expected_tests: + self.runner.add_instrumentation_arg( + "gtest_filter", "%s.%s" % (test.class_name, test.test_name)) + self.runner.rerun(listener, test) + self.runner.remove_instrumentation_arg("gtest_filter") + + def _get_driver_config(self, json_config): + target_test_path = get_config_value('native-test-device-path', + json_config.get_driver(), False) + if target_test_path: + self.config.target_test_path = target_test_path + else: + self.config.target_test_path = DEFAULT_TEST_PATH + + self.config.module_name = get_config_value( + 'module-name', json_config.get_driver(), False) + + timeout_config = get_config_value('native-test-timeout', + json_config.get_driver(), False) + if timeout_config: + self.config.timeout = int(timeout_config) + else: + self.config.timeout = TIME_OUT + + rerun = get_config_value('rerun', json_config.get_driver(), False) + if isinstance(rerun, bool): + self.rerun = rerun + elif str(rerun).lower() == "false": + self.rerun = False + else: + self.rerun = True + + def __result__(self): + return self.result if os.path.exists(self.result) else "" + + +class RemoteCppTestRunner: + def __init__(self, config): + self.arg_list = {} + self.suite_name = None + self.config = config + self.rerun_attempt = FAILED_RUN_TEST_ATTEMPTS + + def dry_run(self): + parsers = get_plugin(Plugin.PARSER, CommonParserType.cpptest_list) + if parsers: + parsers = parsers[:1] + parser_instances = [] + for parser in parsers: + parser_instance = parser.__class__() + parser_instances.append(parser_instance) + handler = ShellHandler(parser_instances) + + command = "cd %s; chmod +x *; ./%s %s" \ + % (self.config.target_test_path, self.config.module_name, + self.get_args_command()) + + self.config.device.execute_shell_command( + command, timeout=self.config.timeout, receiver=handler, retry=0) + return parser_instances[0].tests + + def run(self, listener): + handler = self._get_shell_handler(listener) + command = "cd %s; chmod +x *; ./%s %s" \ + % (self.config.target_test_path, self.config.module_name, + self.get_args_command()) + + self.config.device.execute_shell_command( + command, timeout=self.config.timeout, receiver=handler, retry=0) + + def rerun(self, listener, test): + if self.rerun_attempt: + test_tracker = CollectingTestListener() + listener_copy = listener.copy() + listener_copy.append(test_tracker) + handler = self._get_shell_handler(listener_copy) + try: + command = "cd %s; chmod +x *; ./%s %s" \ + % (self.config.target_test_path, + self.config.module_name, + self.get_args_command()) + + self.config.device.execute_shell_command( + command, timeout=self.config.timeout, receiver=handler, + retry=0) + + except ShellCommandUnresponsiveException: + LOG.debug("Exception: ShellCommandUnresponsiveException") + finally: + if not len(test_tracker.get_current_run_results()): + LOG.debug("No test case is obtained finally") + self.rerun_attempt -= 1 + handler.parsers[0].mark_test_as_blocked(test) + else: + LOG.debug("not execute and mark as blocked finally") + handler = self._get_shell_handler(listener) + handler.parsers[0].mark_test_as_blocked(test) + + def add_instrumentation_arg(self, name, value): + if not name or not value: + return + self.arg_list[name] = value + + def remove_instrumentation_arg(self, name): + if not name: + return + if name in self.arg_list: + del self.arg_list[name] + + def get_args_command(self): + args_commands = "" + for key, value in self.arg_list.items(): + if key == "gtest_list_tests": + args_commands = "%s --%s" % (args_commands, key) + else: + args_commands = "%s --%s=%s" % (args_commands, key, value) + return args_commands + + def _get_shell_handler(self, listener): + parsers = get_plugin(Plugin.PARSER, CommonParserType.cpptest) + if parsers: + parsers = parsers[:1] + parser_instances = [] + for parser in parsers: + parser_instance = parser.__class__() + parser_instance.suite_name = self.suite_name + parser_instance.listeners = listener + parser_instances.append(parser_instance) + handler = ShellHandler(parser_instances) + return handler + + +@Plugin(type=Plugin.DRIVER, id=DeviceTestType.junit_test) +class JUnitTestDriver(IDriver): + """ + JUnitTestDriver is a Test that runs a native test package on given device. + """ + + def __init__(self): + self.result = "" + self.error_message = "" + self.kits = [] + self.config = None + self.rerun = True + self.runner = None + self.rerun_using_test_file = True + self.temp_file_list = [] + self.is_no_test = False + + def __check_environment__(self, device_options): + pass + + def __check_config__(self, config): + if hasattr(config, "devices") and len(config.devices) > 1: + for device in config.devices: + device_name = device.get("name") + if not device_name: + self.error_message = "JUnitTest Load Error(03100)" + raise ParamError("device name not set in config file", + error_no="03100") + + def __execute__(self, request): + try: + LOG.debug("Start execute xdevice extension JUnit Test") + + self.config = request.config + self.config.device = request.get_devices()[0] + self.config.devices = request.get_devices() + + config_file = request.get_config_file() + LOG.info("config file: %s", config_file) + self.result = os.path.join(request.get("report_path"), "result", + ".".join((request.get_test_name(), + "xml"))) + self.__check_config__(self.config) + + device_log_pipes = [] + try: + for device in self.config.devices: + device_name = device.get("name", "") + hilog = get_device_log_file( + request.config.report_path, device.__get_serial__(), + "device_hilog", device_name) + hilog_open = os.open(hilog, os.O_WRONLY | + os.O_CREAT | os.O_APPEND, 0o755) + hilog_pipe = os.fdopen(hilog_open, "a") + device.start_catch_device_log(hilog_pipe) + device_log_pipes.extend([hilog_pipe]) + + self._run_junit(config_file, listeners=request.listeners, + request=request) + finally: + for device_log_pipe in device_log_pipes: + device_log_pipe.flush() + device_log_pipe.close() + for device in self.config.devices: + device.stop_catch_device_log() + + except Exception as exception: + if not getattr(exception, "error_no", ""): + setattr(exception, "error_no", "03405") + LOG.exception(self.error_message, exc_info=False, error_no="03405") + raise exception + + finally: + if not self._is_ignore_report(): + self.result = check_result_report( + request.config.report_path, self.result, + self.error_message) + else: + LOG.debug("hide result and not generate report") + + def _run_junit(self, config_file, listeners, request): + try: + if not os.path.exists(config_file): + error_msg = "Error: Test cases %s don't exist." % config_file + LOG.error(error_msg, error_no="00102") + raise ParamError(error_msg, error_no="00102") + + for device in self.config.devices: + cmd = "target mount" \ + if device.usb_type == DeviceConnectorType.hdc \ + else "remount" + device.hdc_command(cmd) + json_config = JsonParser(config_file) + self.kits = get_kit_instances(json_config, + self.config.resource_path, + self.config.testcases_path) + + for listener in listeners: + listener.device_sn = self.config.device.device_sn + + self._get_driver_config(json_config) + do_module_kit_setup(request, self.kits) + self.runner = RemoteTestRunner(self.config) + self.runner.suite_name = request.get_test_name() + self.runner.suite_file = "%s.hap" % \ + get_filename_extension(config_file)[0] + + self._get_runner_config(json_config) + if hasattr(self.config, "history_report_path") and \ + self.config.testargs.get("test"): + self._do_test_retry(listeners, self.config.testargs) + else: + self._do_include_tests() + self.runner.junit_para = junit_para_parse( + self.config.device, self.config.testargs, "-s") + self._do_test_run(listeners) + + finally: + do_module_kit_teardown(request) + if self.runner and self.runner.junit_para and ( + self.runner.junit_para.find("testFile") != -1 + or self.runner.junit_para.find("notTestFile") != -1): + self._junit_clear() + + def _junit_clear(self): + self.config.device.execute_shell_command( + "rm -r /%s/%s/%s/%s" % ("data", "local", "tmp", "ajur")) + for temp_file in self.temp_file_list: + if os.path.exists(temp_file): + os.remove(temp_file) + self.temp_file_list.clear() + + def _get_driver_config(self, json_config): + package = get_config_value('package', json_config.get_driver(), False) + runner = "ohos.testkit.runner.Runner" + include_tests = get_config_value("include-tests", + json_config.get_driver(), True, []) + if not package: + raise ParamError("Can't find package in config file.") + self.config.package = package + self.config.runner = runner + self.config.include_tests = include_tests + + self.config.xml_output = get_xml_output(self.config, json_config) + + timeout_config = get_config_value('shell-timeout', + json_config.get_driver(), False) + if timeout_config: + self.config.timeout = int(timeout_config) + else: + self.config.timeout = TIME_OUT + + nohup = get_config_value('nohup', json_config.get_driver(), False) + if nohup and (nohup == "true" or nohup == "True"): + self.config.nohup = True + else: + self.config.nohup = False + + def _get_runner_config(self, json_config): + test_timeout = get_config_value('test-timeout', + json_config.get_driver(), False) + if test_timeout: + self.runner.add_instrumentation_arg("timeout_sec", + int(test_timeout)) + + def _do_test_retry(self, listener, testargs): + for test in testargs.get("test"): + test_item = test.split("#") + if len(test_item) != 2: + continue + self.runner.class_name = test_item[0] + self.runner.test_name = test_item[1] + self.runner.run(listener) + + def _do_test_run(self, listener): + if not self._check_package(): + LOG.error("%s is not supported test" % self.config.package) + raise HapNotSupportTest("%s is not supported test" % + self.config.package) + test_to_run = self._collect_test_to_run() + LOG.debug("collected test count is: %s" % len(test_to_run)) + if not test_to_run: + self.is_no_test = True + self.runner.run(listener) + else: + self._run_with_rerun(listener, test_to_run) + + def _check_package(self): + command = '''systemdumper -s 401 -a "-bundle %s"''' % \ + self.config.package + output = self.config.device.execute_shell_command(command) + LOG.debug("systemdumper output: %s" % output) + if output and "ohos.testkit.runner.EntryAbility" in output: + return True + else: + LOG.info("try hidumper command to check package") + command = '''hidumper -s 401 -a "-bundle %s"''' % \ + self.config.package + output = self.config.device.execute_shell_command(command) + LOG.debug("hidumper output: %s" % output) + if output and "ohos.testkit.runner.EntryAbility" in output: + return True + return False + + def _run_tests(self, listener): + test_tracker = CollectingTestListener() + listener_copy = listener.copy() + listener_copy.append(test_tracker) + self.runner.run(listener_copy) + test_run = test_tracker.get_current_run_results() + return test_run + + def _run_with_rerun(self, listener, expected_tests): + LOG.debug("ready to run with rerun, expect run: %s" + % len(expected_tests)) + test_run = self._run_tests(listener) + LOG.debug("run with rerun, has run: %s" % len(expected_tests)) + if len(test_run) < len(expected_tests): + expected_tests = TestDescription.remove_test( + expected_tests, test_run) + if not expected_tests: + LOG.debug("No tests to re-run, all tests executed at least " + "once.") + if self.rerun_using_test_file: + self._rerun_file(expected_tests, listener) + else: + self._rerun_serially(expected_tests, listener) + + def _make_test_file(self, expected_tests, test_file_path): + file_name = 'xdevice_testFile_%s.txt' % self.runner.suite_name + file_path = os.path.join(test_file_path, file_name) + try: + file_path_open = os.open(file_path, os.O_WRONLY | os.O_CREAT | + os.O_APPEND, 0o755) + with os.fdopen(file_path_open, "a") as file_desc: + for test in expected_tests: + file_desc.write("%s#%s" % (test.class_name, + test.test_name)) + file_desc.write("\n") + file_desc.flush() + except(IOError, ValueError) as err_msg: + LOG.exception("Error for make long command file: ", err_msg, + exc_info=False, error_no="03200") + return file_name, file_path + + def _rerun_file(self, expected_tests, listener): + test_file_path = tempfile.mkdtemp(prefix="test_file_", + dir=self.config.report_path) + file_name, file_path = self._make_test_file( + expected_tests, test_file_path) + self.config.device.push_file(file_path, ON_DEVICE_TEST_DIR_LOCATION) + file_path_on_device = ''.join((ON_DEVICE_TEST_DIR_LOCATION, file_name)) + self.runner.add_instrumentation_arg("testFile", file_path_on_device) + self.runner.junit_para = reset_junit_para(self.runner.junit_para, "-s") + LOG.debug("ready to rerun file, expect run: %s" % len(expected_tests)) + test_run = self._run_tests(listener) + LOG.debug("rerun file, has run: %s" % len(test_run)) + self.config.device.execute_shell_command("rm %s" % file_path_on_device) + if len(test_run) < len(expected_tests): + expected_tests = TestDescription.remove_test(expected_tests, + test_run) + if not expected_tests: + LOG.debug("Rerun textFile success") + self._rerun_serially(expected_tests, listener) + shutil.rmtree(test_file_path) + + def _rerun_serially(self, expected_tests, listener): + LOG.debug("rerun serially, expected run: %s" % len(expected_tests)) + self.runner.remove_instrumentation_arg("testFile") + for test in expected_tests: + self.runner.class_name = test.class_name + self.runner.test_name = test.test_name + self.runner.rerun(listener, test) + + def _collect_test_to_run(self): + if self.rerun and self.config.xml_output == "false" and \ + not self.config.nohup: + self.runner.set_test_collection(True) + tests = self._collect_test_and_retry() + self.runner.set_test_collection(False) + return tests + return None + + def _collect_test_and_retry(self): + collector = CollectingTestListener() + listener = [collector] + self.runner.run(listener) + run_results = collector.get_current_run_results() + return run_results + + def _do_include_tests(self): + """ + handler the include-tests parameters in json file of current module. + the main approach is to inject new dict into "testargs". + then leave it to the method of "junit_dex_para_parse" to processing. + """ + if not self.config.include_tests: + return + keys_list = [key.strip() for key in self.config.testargs.keys()] + if "test-file-include-filter" in keys_list: + return + test_filter = self._slice_include_tests() + if not test_filter: + LOG.error("invalid include-tests! please check json file.") + return + if "test" in keys_list or "class" in keys_list: + test_list = list() + if "test" in keys_list: + for element in self.config.testargs.get("test", []): + if self._filter_valid_test(element, test_filter): + test_list.append(element.strip()) + self.config.testargs.pop("test") + if "class" in keys_list: + for element in self.config.testargs.get("class", []): + if self._filter_valid_test(element, test_filter): + test_list.append(element.strip()) + self.config.testargs.pop("class") + else: + test_list = [ele.strip() for ele in self.config.include_tests] + if test_list: + import datetime + prefix = datetime.datetime.now().strftime('%Y%m%d%H%M%S%f') + + save_file = \ + os.path.join(self.config.report_path, "temp_%s.txt" % prefix) + save_file_open = os.open(save_file, os.O_WRONLY + | os.O_CREAT, 0o755) + with os.fdopen(save_file_open, "w") as save_handler: + for test in test_list: + save_handler.write("{}\n".format(test.strip())) + save_handler.flush() + self.temp_file_list.append(save_file) + include_tests_key = "test-file-include-filter" + self.config.testargs.update( + {include_tests_key: self.temp_file_list}) + LOG.debug("handle include-tests, write to %s, data length is %s" % + (self.temp_file_list[0], len(test_list))) + else: + msg = "there is any valid test after filter by 'include-tests'" + LOG.error(msg) + raise ParamError(msg) + + def _slice_include_tests(self): + test_filter = dict() + for include_test in self.config.include_tests: + include_test = include_test.strip() + if include_test: + # element like 'class#method' + if "#" in include_test: + test_list = test_filter.get("test_in", []) + test_list.append(include_test) + test_filter.update({"test_in": test_list}) + # element like 'class' + else: + class_list = test_filter.get("class_in", []) + class_list.append(include_test) + test_filter.update({"class_in": class_list}) + else: + LOG.warning("there is empty element in include-tests") + if len([ele for test in test_filter.values() for ele in test]) > 0: + return test_filter + + @classmethod + def _filter_valid_test(cls, element, test_filter): + element = element.strip() + # if element in the list which element like 'class#method' + if element in test_filter.get("test_in", []): + return element + # if element is the list which element like 'class' + element_items = element.split("#") + if element_items[0].strip() in test_filter.get("class_in", []): + return element + raise ParamError("{} not match 'include-tests'!".format(element)) + + def _is_ignore_report(self): + if self.config.task and not self.config.testlist and \ + not self.config.testfile: + if self.is_no_test and self.config.testargs.get("level", None): + return True + + def __result__(self): + return self.result if os.path.exists(self.result) else "" + + +class RemoteTestRunner: + def __init__(self, config): + self.arg_list = {} + self.suite_name = None + self.suite_file = None + self.class_name = None + self.test_name = None + self.junit_para = None + self.config = config + self.rerun_attempt = FAILED_RUN_TEST_ATTEMPTS + + def __check_environment__(self, device_options): + pass + + def run(self, listener): + handler = self._get_shell_handler(listener) + # execute test case + command = "aa start -p %s " \ + "-n ohos.testkit.runner.EntryAbility" \ + " -s unittest %s -s rawLog true %s %s" \ + % (self.config.package, self.config.runner, self.junit_para, + self.get_args_command()) + + try: + if self.config.nohup: + nohup_command = "nohup %s &" % command + result_value = self.config.device.execute_shell_cmd_background( + nohup_command, timeout=self.config.timeout) + elif self.config.xml_output == "true": + result_value = self.config.device.execute_shell_command( + command, timeout=self.config.timeout, + retry=0) + else: + self.config.device.execute_shell_command( + command, timeout=self.config.timeout, receiver=handler, + retry=0) + return + except ShellCommandUnresponsiveException: + LOG.debug("Exception: ShellCommandUnresponsiveException") + else: + self.config.target_test_path = "/%s/%s/%s/%s/%s/" % \ + ("data", "user", "0", + self.config.package, "cache") + result = ResultManager(self.suite_file, self.config.report_path, + self.config.device, + self.config.target_test_path) + result.get_test_results(result_value) + + def rerun(self, listener, test): + if self.rerun_attempt: + listener_copy = listener.copy() + test_tracker = CollectingTestListener() + listener_copy.append(test_tracker) + handler = self._get_shell_handler(listener_copy) + try: + command = "aa start -p %s " \ + "-n ohos.testkit.runner.EntryAbility" \ + " -s unittest %s -s rawLog true %s" \ + % (self.config.package, self.config.runner, + self.get_args_command()) + + self.config.device.execute_shell_command( + command, timeout=self.config.timeout, receiver=handler, + retry=0) + + except Exception as error: + LOG.error("rerun error %s, %s" % (error, error.__class__)) + finally: + if not len(test_tracker.get_current_run_results()): + LOG.debug("No test case is obtained finally") + self.rerun_attempt -= 1 + handler.parsers[0].mark_test_as_blocked(test) + else: + LOG.debug("not execute and mark as blocked finally") + handler = self._get_shell_handler(listener) + handler.parsers[0].mark_test_as_blocked(test) + + def set_test_collection(self, collect): + if collect: + self.add_instrumentation_arg("log", "true") + else: + self.remove_instrumentation_arg("log") + + def add_instrumentation_arg(self, name, value): + if not name or not value: + return + self.arg_list[name] = value + + def remove_instrumentation_arg(self, name): + if not name: + return + if name in self.arg_list: + del self.arg_list[name] + + def get_args_command(self): + args_commands = "" + for key, value in self.arg_list.items(): + args_commands = "%s -s %s %s" % (args_commands, key, value) + if self.class_name and self.test_name: + args_commands = "%s -s class %s#%s" % (args_commands, + self.class_name, + self.test_name) + elif self.class_name: + args_commands = "%s -s class %s" % (args_commands, self.class_name) + return args_commands + + def _get_shell_handler(self, listener): + parsers = get_plugin(Plugin.PARSER, CommonParserType.junit) + if parsers: + parsers = parsers[:1] + parser_instances = [] + for parser in parsers: + parser_instance = parser.__class__() + parser_instance.suite_name = self.suite_name + parser_instance.listeners = listener + parser_instances.append(parser_instance) + handler = ShellHandler(parser_instances) + return handler + + +@Plugin(type=Plugin.DRIVER, id=DeviceTestType.dex_junit_test) +class DexJunitDriver(IDriver): + """ + DexJunitDriver is a Test that runs a junit test package on given device. + """ + + def __init__(self): + self.result = "" + self.error_message = "" + self.config = None + self.rerun = True + self.runner = None + self.rerun_using_test_file = True + + def __check_environment__(self, device_options): + pass + + def __check_config__(self, config): + pass + + def __execute__(self, request): + try: + LOG.debug("Start execute xdevice extension Dex Junit Test") + + self.config = request.config + self.config.device = request.config.environment.devices[0] + + config_file = request.root.source.config_file + self.result = os.path.join(request.config.report_path, "result", + '.'.join((request.get_module_name(), + "xml"))) + + hilog = get_device_log_file( + request.config.report_path, + request.config.device.__get_serial__(), + "device_hilog") + hilog_open = os.open(hilog, os.O_WRONLY | os.O_CREAT | os.O_APPEND, + 0o755) + with os.fdopen(hilog_open, "a") as hilog_file_pipe: + self.config.device.start_catch_device_log(hilog_file_pipe) + self._run_dex_junit(config_file, listeners=request.listeners, + request=request) + hilog_file_pipe.flush() + + except Exception as exception: + self.error_message = exception + if not getattr(exception, "error_no", ""): + setattr(exception, "error_no", "03406") + LOG.exception(self.error_message, exc_info=False, error_no="03406") + raise exception + + finally: + self.config.device.stop_catch_device_log() + self.result = check_result_report( + request.config.report_path, self.result, self.error_message) + + def _run_dex_junit(self, config_file, listeners, request): + try: + if not os.path.exists(config_file): + LOG.error("Error: Test cases don't exit %s." % config_file) + raise ParamError( + "Error: Test cases don't exit %s." % config_file, + error_no="00102") + + json_config = JsonParser(config_file) + kits = get_kit_instances(json_config, self.config.resource_path, + self.config.testcases_path) + + for listener in listeners: + listener.device_sn = self.config.device.device_sn + + self._get_driver_config(json_config) + do_module_kit_setup(request, kits) + self.runner = RemoteDexRunner(self.config) + self.runner.suite_name = request.get_module_name() + self._get_runner_config(json_config) + + if hasattr(self.config, "history_report_path") and \ + self.config.testargs.get("test"): + self._do_test_retry(listeners, self.config.testargs) + else: + self.runner.junit_para = junit_dex_para_parse( + self.config.device, self.config.testargs) + self._do_test_run(listeners) + finally: + do_module_kit_teardown(request) + if self.runner and self.runner.junit_para and ( + self.runner.junit_para.find("testFile") != -1 + or self.runner.junit_para.find("notTestFile") != -1): + self._junit_clear() + + def _do_test_retry(self, listener, testargs): + for test in testargs.get("test"): + test_item = test.split("#") + if len(test_item) != 2: + continue + self.runner.class_name = test_item[0] + self.runner.test_name = test_item[1] + self.runner.run(listener) + + def _do_test_run(self, listener): + test_to_run = self._collect_test_to_run() + LOG.debug("collected test count is: %s" % len(test_to_run)) + if not test_to_run: + self.runner.run(listener) + else: + self._run_with_rerun(listener, test_to_run) + + def _run_tests(self, listener): + test_tracker = CollectingTestListener() + try: + listener_copy = listener.copy() + listener_copy.append(test_tracker) + self.runner.run(listener_copy) + except ShellCommandUnresponsiveException: + LOG.debug("run test exception: ShellCommandUnresponsiveException") + finally: + test_run = test_tracker.get_current_run_results() + return test_run + + def _run_with_rerun(self, listener, expected_tests): + LOG.debug("ready to run with rerun, expect run: %s" + % len(expected_tests)) + test_run = self._run_tests(listener) + LOG.debug("run with rerun, has run: %s" % len(expected_tests)) + if len(test_run) < len(expected_tests): + expected_tests = TestDescription.remove_test(expected_tests, + test_run) + if not expected_tests: + LOG.debug("No tests to re-run, all tests executed at least " + "once.") + if self.rerun_using_test_file: + self._rerun_file(expected_tests, listener) + else: + self._rerun_serially(expected_tests, listener) + + def _make_test_file(self, expected_tests, test_file_path): + file_name = 'xdevice_testFile_%s.txt' % self.runner.suite_name + file_path = os.path.join(test_file_path, file_name) + try: + file_path_open = os.open(file_path, os.O_WRONLY | os.O_CREAT | + os.O_APPEND, 0o755) + with os.fdopen(file_path_open, "a") as file_desc: + for test in expected_tests: + file_desc.write("%s#%s" % (test.class_name, + test.test_name)) + file_desc.write("\n") + file_desc.flush() + except(IOError, ValueError) as err_msg: + LOG.exception("Error for make long command file: ", err_msg, + exc_info=False, error_no="03200") + return file_name, file_path + + def _rerun_file(self, expected_tests, listener): + test_file_path = tempfile.mkdtemp(prefix="test_file_", + dir=self.config.report_path) + file_name, file_path = self._make_test_file( + expected_tests, test_file_path) + self.config.device.push_file(file_path, ON_DEVICE_TEST_DIR_LOCATION) + file_path_on_device = ''.join((ON_DEVICE_TEST_DIR_LOCATION, file_name)) + self.runner.add_instrumentation_arg("testFile", file_path_on_device) + self.runner.junit_para = "" + LOG.debug("ready to rerun file, expect run: %s" % len(expected_tests)) + test_run = self._run_tests(listener) + LOG.debug("rerun file, has run: %s" % len(test_run)) + self.config.device.execute_shell_command("rm %s" % file_path_on_device) + if len(test_run) < len(expected_tests): + expected_tests = TestDescription.remove_test(expected_tests, + test_run) + if not expected_tests: + LOG.debug("Rerun textFile success") + self._rerun_serially(expected_tests, listener) + shutil.rmtree(test_file_path) + + def _rerun_serially(self, expected_tests, listener): + LOG.debug("rerun serially, expected run: %s" % len(expected_tests)) + self.runner.remove_instrumentation_arg("testFile") + for test in expected_tests: + self.runner.class_name = test.class_name + self.runner.test_name = test.test_name + self.runner.rerun(listener, test) + + def _collect_test_to_run(self): + if self.rerun: + self.runner.set_test_collection(True) + tests = self._collect_test_and_retry() + self.runner.set_test_collection(False) + return tests + return None + + def _collect_test_and_retry(self): + collector = CollectingTestListener() + listener = [collector] + self.runner.run(listener) + run_results = collector.get_current_run_results() + return run_results + + def _junit_clear(self): + _lock_screen(self.config.device) + self.config.device.execute_shell_command( + "rm -r /%s/%s/%s/%s" % ("data", "local", "tmp", "ajur")) + + def _get_driver_config(self, json_config): + self.config.remote_path = get_config_value( + 'device-test-path', json_config.get_driver(), + default="/%s/%s" % ("data", "tmp"), is_list=False) + module_name = get_config_value( + 'module-name', json_config.get_driver(), False) + if module_name: + self.config.module_name = module_name + else: + raise ParamError("Can't find module_name.", error_no="03201") + + rerun = get_config_value('rerun', json_config.get_driver(), False) + if isinstance(rerun, bool): + self.rerun = rerun + elif rerun == "False" or rerun == "false": + self.rerun = False + + timeout_config = get_config_value('shell-timeout', + json_config.get_driver(), False) + if timeout_config: + self.config.timeout = int(timeout_config) + else: + self.config.timeout = TIME_OUT + + def _get_runner_config(self, json_config): + test_timeout = get_config_value('test-timeout', + json_config.get_driver(), False) + if test_timeout: + self.runner.add_instrumentation_arg("timeout_msec", + int(test_timeout)) + + def _query_runner_name(self, package): + stdout = self.config.device.hdc_command( + "shell pm list instrumentation") + packages = stdout.split("\n") + for item in packages: + if package in item: + return item.split("/")[1].split(" ")[0] + + def __result__(self): + return self.result if os.path.exists(self.result) else "" + + +class RemoteDexRunner: + def __init__(self, config): + self.arg_list = {} + self.suite_name = None + self.junit_para = "" + self.config = config + self.class_name = None + self.test_name = None + self.rerun_attempt = FAILED_RUN_TEST_ATTEMPTS + + def run(self, listener): + handler = self._get_shell_handler(listener) + command = "export BOOTCLASSPATH=$BOOTCLASSPATH:" \ + "{remote_path}/{module_name};cd {remote_path}; " \ + "app_process -cp {remote_path}/{module_name} / " \ + "ohos.testkit.runner.ZUnitRunner {junit_para}{arg_list}" \ + " --rawLog true --coverage false --debug false " \ + "--classpathToScan {remote_path}/{module_name}".format( + remote_path=self.config.remote_path, + module_name=self.config.module_name, + junit_para=self.junit_para, arg_list=self.get_args_command()) + + try: + self.config.device.execute_shell_command( + command, timeout=self.config.timeout, + receiver=handler, retry=0) + except ConnectionResetError: + if len(listener) == 1 and isinstance(listener[0], + CollectingTestListener): + LOG.info("try subprocess ") + listener[0].tests.clear() + command = ["shell", command] + result = self.config.device.hdc_command( + command, timeout=self.config.timeout, retry=0, + join_result=True) + handler.__read__(result) + handler.__done__() + LOG.info("get current testcase: %s " % + len(listener[0].get_current_run_results())) + + def rerun(self, listener, test): + if self.rerun_attempt: + listener_copy = listener.copy() + test_tracker = CollectingTestListener() + listener_copy.append(test_tracker) + handler = self._get_shell_handler(listener_copy) + try: + command = "export BOOTCLASSPATH=$BOOTCLASSPATH:" \ + "{remote_path}/{module_name};cd {remote_path}; " \ + "app_process -cp {remote_path}/{module_name} / " \ + "ohos.testkit.runner.ZUnitRunner {arg_list} " \ + "--rawLog true --coverage false " \ + "--debug false --classpathToScan " \ + "{remote_path}/{module_name}".format( + remote_path=self.config.remote_path, + module_name=self.config.module_name, + arg_list=self.get_args_command()) + self.config.device.execute_shell_command( + command, timeout=self.config.timeout, + receiver=handler, retry=0) + + except ShellCommandUnresponsiveException: + LOG.debug("Exception: ShellCommandUnresponsiveException") + finally: + if not len(test_tracker.get_current_run_results()): + LOG.debug("No test case is obtained finally") + self.rerun_attempt -= 1 + handler.parsers[0].mark_test_as_blocked(test) + else: + LOG.debug("not execute and mark as blocked finally") + handler = self._get_shell_handler(listener) + handler.parsers[0].mark_test_as_blocked(test) + + def set_test_collection(self, collect): + if collect: + self.add_instrumentation_arg("log", "true") + else: + self.remove_instrumentation_arg("log") + + def add_instrumentation_arg(self, name, value): + if not name or not value: + return + self.arg_list[name] = value + + def remove_instrumentation_arg(self, name): + if not name: + return + if name in self.arg_list: + del self.arg_list[name] + + def get_args_command(self): + args_commands = "" + for key, value in self.arg_list.items(): + args_commands = "%s --%s %s" % (args_commands, key, value) + if self.class_name and self.test_name: + args_commands = "%s --class %s#%s" % (args_commands, + self.class_name, + self.test_name) + elif self.class_name: + args_commands = "%s --class %s" % (args_commands, self.class_name) + return args_commands + + def _get_shell_handler(self, listener): + parsers = get_plugin(Plugin.PARSER, CommonParserType.junit) + if parsers: + parsers = parsers[:1] + parser_instances = [] + for parser in parsers: + parser_instance = parser.__class__() + parser_instance.suite_name = self.suite_name + parser_instance.listeners = listener + parser_instances.append(parser_instance) + handler = ShellHandler(parser_instances) + return handler + + +@Plugin(type=Plugin.DRIVER, id=DeviceTestType.dex_test) +class DexTestDriver(IDriver): + """ + DexTestDriver is a Test that runs a native test package on given device. + """ + # test driver config + config = None + result = "" + + def __check_environment__(self, device_options): + pass + + def __check_config__(self, config): + pass + + def __execute__(self, request): + try: + LOG.debug("Start execute xdevice extension DexTest") + + self.config = request.config + self.config.target_test_path = DEFAULT_TEST_PATH + self.config.device = request.config.environment.devices[0] + + suite_file = request.root.source.source_file + if not suite_file: + LOG.error("test source '%s' not exists" % + request.root.source.source_string, error_no="00110") + return + + LOG.debug("Testsuite FilePath: %s" % suite_file) + serial = request.config.device.__get_serial__() + device_log_file = get_device_log_file(request.config.report_path, + serial) + device_log_file_open = os.open(device_log_file, os.O_WRONLY | + os.O_CREAT | os.O_APPEND, 0o755) + with os.fdopen(device_log_file_open, "a") as file_pipe: + self.config.device.start_catch_device_log(file_pipe) + self._init_junit_test() + self._run_junit_test(suite_file) + file_pipe.flush() + finally: + self.config.device.stop_catch_device_log() + + def _init_junit_test(self): + cmd = "target mount" \ + if self.config.device.usb_type == DeviceConnectorType.hdc \ + else "remount" + self.config.device.hdc_command(cmd) + self.config.device.execute_shell_command( + "rm -rf %s" % self.config.target_test_path) + self.config.device.execute_shell_command( + "mkdir -p %s" % self.config.target_test_path) + self.config.device.execute_shell_command( + "mount -o rw,remount,rw /%s" % "system") + + def _run_junit_test(self, suite_file): + filename = os.path.basename(suite_file) + suitefile_target_test_path = self.config.target_test_path + junit_test_para = self._get_junit_test_para(filename, suite_file) + is_coverage_test = True if self.config.coverage == "coverage" else \ + False + + # push testsuite file + self.config.device.push_file(suite_file, self.config.target_test_path) + + # push resource files + resource_manager = ResourceManager() + resource_data_dic, resource_dir = \ + resource_manager.get_resource_data_dic(suite_file) + resource_manager.process_preparer_data(resource_data_dic, resource_dir, + self.config.device) + + # execute testcase + return_message = self._execute_suitefile_junittest( + filename, junit_test_para, suitefile_target_test_path) + result = ResultManager(suite_file, self.config.report_path, + self.config.device, + self.config.target_test_path) + result.set_is_coverage(is_coverage_test) + self.result = result.get_test_results(return_message) + + resource_manager.process_cleaner_data(resource_data_dic, resource_dir, + self.config.device) + + def _get_junit_test_para(self, filename, suite_file): + exec_info = get_java_test_para(self.config.testcase, + self.config.testlevel) + java_test_file = get_execute_java_test_files(suite_file) + junit_test_para = self._get_dex_test_para(filename, java_test_file, + exec_info) + return junit_test_para + + def _get_dex_test_para(self, filename, java_test_file, exec_info): + exec_class, exec_method, exec_level = exec_info + dex_test_para = "%s %s%s %s%s %s%s %s%s %s%s %s%s" % ( + ZunitConst.z_unit_app, ZunitConst.output_dir, + self.config.target_test_path, + ZunitConst.output_file, filename, + ZunitConst.test_class, java_test_file, + ZunitConst.exec_class, exec_class, + ZunitConst.exec_method, exec_method, + ZunitConst.exec_level, exec_level) + if self.config.coverage == "coverage": + dex_test_para = ''.join((dex_test_para, ' ', + ZunitConst.jacoco_exec_file, filename, + ".exec")) + return dex_test_para + + def _execute_suitefile_junittest(self, filename, testpara, + target_test_path): + return_message = self._execute_dexfile_junittest(filename, testpara, + target_test_path) + return return_message + + def _execute_dexfile_junittest(self, filename, testpara, target_test_path): + from xdevice import Variables + long_command_path = tempfile.mkdtemp(prefix="long_command_", + dir=self.config.report_path) + if self.config.coverage == "coverage": + if Variables.source_code_rootpath == "": + LOG.error("source_code_rootpath is empty.", error_no="03202") + strip_num = 0 + else: + build_variant_outpath = os.path.join( + Variables.source_code_rootpath, "out", + self.config.build_variant) + strip_num = len(build_variant_outpath.split(os.sep)) - 1 + + command = "cd %s; rm -rf %s.xml; chmod +x *; " \ + "export BOOTCLASSPATH=%s%s:$BOOTCLASSPATH;" \ + "export GCOV_PREFIX=%s; export GCOV_PREFIX_STRIP=%d;" \ + " app_process %s%s %s" \ + % ( + target_test_path, filename, target_test_path, + filename, + target_test_path, strip_num, + target_test_path, filename, testpara) + else: + command = "cd %s; rm -rf %s.xml; chmod +x *; " \ + "export BOOTCLASSPATH=%s%s:$BOOTCLASSPATH;" \ + " app_process %s%s %s" \ + % ( + target_test_path, filename, target_test_path, + filename, + target_test_path, filename, testpara) + + LOG.info("command: %s" % command) + sh_file_name, file_path = \ + self._make_long_command_file(command, long_command_path, filename) + remote_command_dir = os.path.join(target_test_path, + ZunitConst.remote_command_dir) + self.config.device.execute_shell_command( + "mkdir -p %s" % remote_command_dir) + cmd = "file send" \ + if self.config.device.usb_type == DeviceConnectorType.hdc \ + else "push" + self.config.device.hdc_command( + "%s %s %s" % (cmd, file_path, remote_command_dir)) + try: + display_receiver = DisplayOutputReceiver() + self.config.device.execute_shell_command( + "sh %s/%s" % (remote_command_dir, sh_file_name), + receiver=display_receiver, timeout=TIME_OUT) + return_message = display_receiver.output + if display_receiver.output: + time.sleep(1) + except (ExecuteTerminate, HdcCommandRejectedException, + ShellCommandUnresponsiveException, HdcError) as exception: + if not getattr("exception", "error_no", ""): + setattr(exception, "error_no", "03203") + return_message = str(exception.args) + shutil.rmtree(long_command_path) + return return_message + + @staticmethod + def _make_long_command_file(command, longcommand_path, filename): + sh_file_name = '%s.sh' % filename + file_path = os.path.join(longcommand_path, sh_file_name) + try: + file_path_open = os.open(file_path, os.O_WRONLY | os.O_CREAT | + os.O_APPEND, 0o755) + with os.fdopen(file_path_open, "a") as file_desc: + file_desc.write(command) + file_desc.flush() + except(IOError, ValueError) as err_msg: + LOG.exception("Error for make long command file: ", err_msg, + exc_info=False, error_no="03200") + return sh_file_name, file_path + + def __result__(self): + return self.result if os.path.exists(self.result) else "" + + +@Plugin(type=Plugin.DRIVER, id=DeviceTestType.hap_test) +class HapTestDriver(IDriver): + """ + HapTestDriver is a Test that runs a native test package on given device. + """ + # test driver config + config = None + instrument_hap_file_suffix = '_ad.hap' + result = "" + + def __init__(self): + self.ability_name = "" + self.package_name = "" + self.activity_name = "" + + def __check_environment__(self, device_options): + pass + + def __check_config__(self, config): + pass + + def __execute__(self, request): + try: + LOG.debug("Start execute xdevice extension HapTest") + + self.config = request.config + self.config.target_test_path = DEFAULT_TEST_PATH + self.config.device = request.config.environment.devices[0] + + suite_file = request.root.source.source_file + if not suite_file: + LOG.error("test source '%s' not exists" % + request.root.source.source_string, error_no="00110") + return + + LOG.debug("Testsuite FilePath: %s" % suite_file) + package_name, ability_name = self._get_package_and_ability_name( + suite_file) + self.package_name = package_name + self.ability_name = ability_name + self.activity_name = "%s.MainAbilityShellActivity" % \ + self.package_name + self.config.test_hap_out_path = \ + "/data/data/%s/files/test/result/" % self.package_name + self.config.test_suite_timeout = 300 * 1000 + + serial = request.config.device.__get_serial__() + device_log_file = get_device_log_file(request.config.report_path, + serial) + device_log_file_open = os.open( + device_log_file, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o755) + with os.fdopen(device_log_file_open, "a")as file_pipe: + self.config.device.start_catch_device_log(file_pipe) + self._init_junit_test() + self._run_junit_test(suite_file) + file_pipe.flush() + finally: + self.config.device.stop_catch_device_log() + + def _init_junit_test(self): + cmd = "target mount" \ + if self.config.device.usb_type == DeviceConnectorType.hdc \ + else "remount" + self.config.device.hdc_command(cmd) + self.config.device.execute_shell_command( + "rm -rf %s" % self.config.target_test_path) + self.config.device.execute_shell_command( + "mkdir -p %s" % self.config.target_test_path) + self.config.device.execute_shell_command( + "mount -o rw,remount,rw /%s" % "system") + + def _run_junit_test(self, suite_file): + filename = os.path.basename(suite_file) + suitefile_target_test_path = self.config.test_hap_out_path + junit_test_para = self._get_junit_test_para(filename, suite_file) + is_coverage_test = True if self.config.coverage == "coverage" else \ + False + + # push testsuite file + self.config.device.push_file(suite_file, self.config.target_test_path) + + resource_manager = ResourceManager() + resource_data_dic, resource_dir = \ + resource_manager.get_resource_data_dic(suite_file) + resource_manager.process_preparer_data(resource_data_dic, resource_dir, + self.config.device) + + # execute testcase + install_result = self._install_hap(filename) + result = ResultManager(suite_file, self.config.report_path, + self.config.device, + self.config.test_hap_out_path) + result.set_is_coverage(is_coverage_test) + if install_result: + return_message = self._execute_suitefile_junittest( + filename, junit_test_para, suitefile_target_test_path) + + self.result = result.get_test_results(return_message) + self._unistall_hap(self.package_name) + else: + self.result = result.get_test_results("Error: install hap failed.") + LOG.error("Error: install hap failed.", error_no="03204") + + resource_manager.process_cleaner_data(resource_data_dic, resource_dir, + self.config.device) + + def _get_junit_test_para(self, filename, suite_file): + if not filename.endswith(self.instrument_hap_file_suffix): + exec_class, exec_method, exec_level = get_java_test_para( + self.config.testcase, self.config.testlevel) + java_test_file = get_execute_java_test_files(suite_file) + junit_test_para = self._get_hap_test_para(java_test_file, + exec_class, exec_method, + exec_level) + else: + junit_test_para = get_execute_java_test_files(suite_file) + return junit_test_para + + @staticmethod + def _get_hap_test_para(java_test_file, exec_class, exec_method, + exec_level): + hap_test_para = "%s%s#%s%s#%s%s#%s%s" % ( + ZunitConst.test_class, java_test_file, + ZunitConst.exec_class, exec_class, + ZunitConst.exec_method, exec_method, + ZunitConst.exec_level, exec_level) + return hap_test_para + + def _execute_suitefile_junittest(self, filename, testpara, + target_test_path): + return_message = self._execute_hapfile_junittest(filename, testpara, + target_test_path) + return return_message + + def _execute_hapfile_junittest(self, filename, testpara, target_test_path): + _unlock_screen(self.config.device) + _unlock_device(self.config.device) + + try: + if not filename.endswith(self.instrument_hap_file_suffix): + return_message = self.start_hap_activity(testpara, filename) + LOG.info("HAP Testcase is executing, please wait a moment...") + if "Error" not in return_message: + self._check_hap_finished(target_test_path) + else: + return_message = self.start_instrument_hap_activity(testpara) + except (ExecuteTerminate, HdcCommandRejectedException, + ShellCommandUnresponsiveException, HdcError) as exception: + return_message = str(exception.args) + if not getattr(exception, "error_no", ""): + setattr(exception, "error_no", "03203") + + _lock_screen(self.config.device) + return return_message + + def _init_hap_device(self): + self.config.device.execute_shell_command( + "rm -rf %s" % self.config.test_hap_out_path) + self.config.device.execute_shell_command( + "mkdir -p %s" % self.config.test_hap_out_path) + + def _install_hap(self, filename): + message = self.config.device.execute_shell_command( + "bm install -p %s" % os.path.join(self.config.target_test_path, + filename)) + message = str(message).rstrip() + if message == "" or "Success" in message: + return_code = True + if message != "": + LOG.info(message) + else: + return_code = False + if message != "": + LOG.warning(message) + + _sleep_according_to_result(return_code) + return return_code + + def start_hap_activity(self, testpara, filename): + execute_para = testpara + if self.config.coverage == "coverage": + execute_para = ''.join((execute_para, ' ', + ZunitConst.jacoco_exec_file, filename, + ".exec")) + try: + display_receiver = DisplayOutputReceiver() + self.config.device.execute_shell_command( + "am start -S -n %s/%s --es param '%s'" % + (self.package_name, self.activity_name, + execute_para), receiver=display_receiver, + timeout=self.config.test_suite_timeout) + _sleep_according_to_result(display_receiver.output) + return_message = display_receiver.output + + except (ExecuteTerminate, HdcCommandRejectedException, + ShellCommandUnresponsiveException, HdcError) as exception: + return_message = exception.args + if not getattr(exception, "error_no", ""): + setattr(exception, "error_no", "03203") + return return_message + + def start_instrument_hap_activity(self, testpara): + from xdevice import Variables + try: + display_receiver = DisplayOutputReceiver() + if self.config.coverage != "coverage": + self.config.device.execute_shell_command( + "aa start -p %s -n %s -s AbilityTestCase %s -w %s" % + (self.package_name, self.ability_name, testpara, + str(self.config.test_suite_timeout)), + receiver=display_receiver, + timeout=self.config.test_suite_timeout) + else: + build_variant_outpath = os.path.join( + Variables.source_code_rootpath, "out", + self.config.build_variant) + strip_num = len(build_variant_outpath.split(os.sep)) - 1 + self.config.device.execute_shell_command( + "cd %s; export GCOV_PREFIX=%s; " + "export GCOV_PREFIX_STRIP=%d; " + "aa start -p %s -n %s -s AbilityTestCase %s -w %s" % + (self.config.target_test_path, + self.config.target_test_path, + strip_num, self.package_name, self.ability_name, testpara, + str(self.config.test_suite_timeout)), + receiver=display_receiver, + timeout=self.config.test_suite_timeout) + _sleep_according_to_result(display_receiver.output) + return_message = display_receiver.output + except (ExecuteTerminate, HdcCommandRejectedException, + ShellCommandUnresponsiveException, HdcError) as exception: + return_message = exception.args + if not getattr(exception, "error_no", ""): + setattr(exception, "error_no", "03203") + return return_message + + def _check_hap_finished(self, target_test_path): + run_timeout = True + sleep_duration = 3 + target_file = os.path.join(target_test_path, + ZunitConst.jtest_status_filename) + for _ in range( + int(self.config.test_suite_timeout / (1000 * sleep_duration))): + check_value = self.config.device.is_file_exist(target_file) + LOG.info("%s state: %s", self.config.device.device_sn, + self.config.device.test_device_state.value) + if not check_value: + time.sleep(sleep_duration) + continue + run_timeout = False + break + if run_timeout: + return_code = False + LOG.error("HAP Testcase executed timeout or exception, please " + "check detail information from system log", + error_no="03205") + else: + return_code = True + LOG.info("HAP Testcase executed finished") + return return_code + + def _unistall_hap(self, package_name): + return_message = self.config.device.execute_shell_command( + "pm uninstall %s" % package_name) + _sleep_according_to_result(return_message) + return return_message + + @staticmethod + def _get_package_and_ability_name(hap_filepath): + package_name = "" + ability_name = "" + + if os.path.exists(hap_filepath): + filename = os.path.basename(hap_filepath) + + # unzip the hap file + hap_bak_path = os.path.abspath( + os.path.join(os.path.dirname(hap_filepath), + "%s.bak" % filename)) + try: + with zipfile.ZipFile(hap_filepath) as zf_desc: + zf_desc.extractall(path=hap_bak_path) + except RuntimeError as error: + LOG.error(error, error_no="03206") + + # verify config.json file + app_profile_path = os.path.join(hap_bak_path, + "config.json") + if not os.path.exists(app_profile_path): + LOG.info("file %s not exists" % app_profile_path) + return package_name, ability_name + + if os.path.isdir(app_profile_path): + LOG.info("%s is a folder, and not a file" % app_profile_path) + return package_name, ability_name + + # get package_name and ability_name value. + app_profile_path_open = os.open(app_profile_path, os.O_RDONLY, + stat.S_IWUSR | stat.S_IRUSR) + with os.fdopen(app_profile_path_open, 'r') as load_f: + load_dict = json.load(load_f) + profile_list = load_dict.values() + for profile in profile_list: + package_name = profile.get("package") + if not package_name: + continue + abilities = profile.get("abilities") + for abilitie in abilities: + abilities_name = abilitie.get("name") + if abilities_name.startswith("."): + ability_name = ''.join( + (package_name, + abilities_name[abilities_name.find("."):])) + else: + ability_name = abilities_name + break + break + + # delete hap_bak_path + if os.path.exists(hap_bak_path): + shutil.rmtree(hap_bak_path) + else: + LOG.info("file %s not exists" % hap_filepath) + + return package_name, ability_name + + def __result__(self): + return self.result if os.path.exists(self.result) else "" + + +@Plugin(type=Plugin.DRIVER, id=DeviceTestType.jsunit_test) +class JSUnitTestDriver(IDriver): + """ + JSUnitTestDriver is a Test that runs a native test package on given device. + """ + + def __init__(self): + self.timeout = 80 * 1000 + self.start_time = None + self.result = "" + self.error_message = "" + self.kits = [] + self.config = None + + def __check_environment__(self, device_options): + pass + + def __check_config__(self, config): + pass + + def __execute__(self, request): + try: + LOG.debug("Start execute xdevice extension JSUnit Test") + self.result = os.path.join(request.config.report_path, + "result", + '.'.join((request.get_module_name(), + "xml"))) + self.config = request.config + self.config.device = request.config.environment.devices[0] + + config_file = request.root.source.config_file + suite_file = request.root.source.source_file + + if not suite_file: + raise ParamError( + "test source '%s' not exists" % + request.root.source.source_string, error_no="00110") + + LOG.debug("Test case file path: %s" % suite_file) + self._run_jsunit(config_file, request) + self.generate_console_output(request) + except Exception as exception: + self.error_message = exception + if not getattr(exception, "error_no", ""): + setattr(exception, "error_no", "03409") + LOG.exception(self.error_message, exc_info=True, error_no="03409") + raise exception + finally: + self.config.device.stop_catch_device_log() + self.result = check_result_report( + request.config.report_path, self.result, self.error_message) + + def generate_console_output(self, request): + report_name = request.get_module_name() + parsers = get_plugin( + Plugin.PARSER, CommonParserType.jsunit) + if parsers: + parsers = parsers[:1] + for listener in request.listeners: + listener.device_sn = self.config.device.device_sn + parser_instances = [] + + for parser in parsers: + parser_instance = parser.__class__() + parser_instance.suites_name = "{}_suites".format(report_name) + parser_instance.suite_name = report_name + parser_instance.listeners = request.listeners + parser_instances.append(parser_instance) + handler = ShellHandler(parser_instances) + + from xdevice_extension._core import utils + command = "hdc_std -t %s shell hilog -x " % self.config.device.\ + device_sn + + output = utils.start_standing_subprocess(command, return_result=True) + LOG.debug("start to parsing hilog") + handler.__read__(output) + handler.__done__() + + def read_device_log(self, device_log_file, result_message): + device_log_file_open = os.open(device_log_file, os.O_RDONLY, + stat.S_IWUSR | stat.S_IRUSR) + with os.fdopen(device_log_file_open, "r") as file_read_pipe: + while True: + data = file_read_pipe.readline() + result_message += data + report_name = "" + if re.match(r'.*\[create report\]*', data): + _, index = re.match(r'.*\[create report\]*', data).span() + report_name = data[index:].split(".")[0] + if result_message.find("[create report]") != -1 or \ + int(time.time()) - int(self.start_time) > \ + self.timeout: + break + return result_message, report_name + + def _run_jsunit(self, config_file, request): + try: + if not os.path.exists(config_file): + LOG.error("Error: Test cases don't exist %s." % config_file) + raise ParamError( + "Error: Test cases don't exist %s." % config_file, + error_no="00102") + + json_config = JsonParser(config_file) + self.kits = get_kit_instances(json_config, + self.config.resource_path, + self.config.testcases_path) + + package, ability_name = self._get_driver_config(json_config) + self.config.device.hdc_command("target mount") + do_module_kit_setup(request, self.kits) + + # execute test case + self.config.device.hdc_command("shell hilog -r") + + command = "aa start -d 123 -a %s -b %s" \ + % (ability_name, package) + self.start_time = time.time() + result_value = self.config.device.execute_shell_command( + command, timeout=self.timeout) + if "success" in str(result_value).lower(): + LOG.info("execute %s's testcase success. result value=%s" + % (package, result_value)) + time.sleep(60) + else: + LOG.info("execute %s's testcase failed. result value=%s" + % (package, result_value)) + + finally: + do_module_kit_teardown(request) + + def _jsunit_clear(self): + self.config.device.execute_shell_command( + "rm -r /%s/%s/%s/%s" % ("data", "local", "tmp", "ajur")) + + def _get_driver_config(self, json_config): + package = get_config_value('package', json_config.get_driver(), False) + default_ability = "{}.MainAbility".format(package) + ability_name = get_config_value('abilityName', json_config. + get_driver(), False, default_ability) + self.xml_output = get_xml_output(self.config, json_config) + timeout_config = get_config_value('native-test-timeout', + json_config.get_driver(), False) + if timeout_config: + self.timeout = int(timeout_config) + + if not package: + raise ParamError("Can't find package in config file.", + error_no="03201") + return package, ability_name + + def __result__(self): + return self.result if os.path.exists(self.result) else "" + + +def disable_keyguard(device): + _unlock_screen(device) + _unlock_device(device) + + +def _unlock_screen(device): + device.execute_shell_command("svc power stayon true") + time.sleep(1) + + +def _unlock_device(device): + device.execute_shell_command("input keyevent 82") + time.sleep(1) + device.execute_shell_command("wm dismiss-keyguard") + time.sleep(1) + + +def _lock_screen(device): + time.sleep(1) + + +def _sleep_according_to_result(result): + if result: + time.sleep(1) diff --git a/extension/src/xdevice_extension/_core/driver/kunpeng.py b/extension/src/xdevice_extension/_core/driver/kunpeng.py new file mode 100644 index 0000000..b9534b4 --- /dev/null +++ b/extension/src/xdevice_extension/_core/driver/kunpeng.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import platform +import subprocess +import sys +import uuid + +from xdevice import platform_logger +from xdevice import IDriver +from xdevice import Plugin + +from xdevice_extension._core.utils import get_decode + +KUNPENG_TEST = "KunpengTest" +LOG = platform_logger(KUNPENG_TEST) + + +@Plugin(type=Plugin.DRIVER, id=KUNPENG_TEST) +class KunpengTest(IDriver): + """ + KunpengTest is a Test that runs a host-driven test on given kunpeng + servers. + """ + # test driver config + config = None + result = "" + + def __check_environment__(self, device_options): + pass + + def __check_config__(self, config): + pass + + def __execute__(self, request): + self.config = request.config + + # 1.parse config file + mainconfig_file = request.get_config_file() + if not mainconfig_file: + LOG.error("config file not exists") + return + LOG.debug("KunpengTest mainconfig_file FilePath: %s" % mainconfig_file) + + # 2.set params + tmp_id = str(uuid.uuid4()) + tmp_folder = os.path.join(self.config.report_path, "temp") + self.config.tmp_sub_folder = os.path.join(tmp_folder, "task_" + tmp_id) + os.makedirs(self.config.tmp_sub_folder, exist_ok=True) + + # 3.test execution + self._start_kunpengtest_with_cmd(mainconfig_file) + return + + def _start_kunpengtest_with_cmd(self, mainconfig_file): + from xdevice import Variables + # insert& _kunpengtest path for loading kunpengtest module + kunpengtest_module = os.path.join(Variables.modules_dir, + "_kunpengtest") + sys.path.insert(1, kunpengtest_module) + + cmd_parts = [] + if platform.system() == "Windows": + cmd_parts.append("python") + else: + cmd_parts.append("python3") + relative_path = "uniautos/src/Framework/Dev/bin/UniAutosScript.py" + cmd_parts.append(os.path.abspath(os.path.join(kunpengtest_module, + relative_path))) + cmd_parts.append("-c") + cmd_parts.append(mainconfig_file) + cmd_parts.append("-rp") + cmd_parts.append(self.config.tmp_sub_folder) + cmd = " ".join(cmd_parts) + LOG.info("start kunpengtest with cmd: %s" % cmd) + try: + proc = subprocess.Popen(cmd_parts, shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, _) = proc.communicate() + out = get_decode(out).strip() + for line in out.split("\n"): + LOG.info(line) + except (subprocess.CalledProcessError, FileNotFoundError) as error: + LOG.error("kunpeng test error: %s" % error) + + def __result__(self): + return self.result if os.path.exists(self.result) else "" diff --git a/extension/src/xdevice_extension/_core/driver/parser.py b/extension/src/xdevice_extension/_core/driver/parser.py new file mode 100644 index 0000000..c53fb93 --- /dev/null +++ b/extension/src/xdevice_extension/_core/driver/parser.py @@ -0,0 +1,766 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import copy +import re +import time +import datetime +from enum import Enum + +from xdevice import LifeCycle +from xdevice import IParser +from xdevice import platform_logger +from xdevice import Plugin +from xdevice import check_pub_key_exist +from xdevice import StateRecorder +from xdevice import TestDescription +from xdevice import ResultCode + +from xdevice_extension._core.constants import CommonParserType + +__all__ = ["CppTestParser", "CppTestListParser", + "JunitParser", "JSUnitParser"] + +_INFORMATIONAL_MARKER = "[----------]" +_START_TEST_RUN_MARKER = "[==========] Running" +_TEST_RUN_MARKER = "[==========]" +_GTEST_DRYRUN_MARKER = "Running main() " +_START_TEST_MARKER = "[ RUN ]" +_OK_TEST_MARKER = "[ OK ]" +_SKIPPED_TEST_MARKER = "[ SKIPPED ]" +_FAILED_TEST_MARKER = "[ FAILED ]" +_ALT_OK_MARKER = "[ OK ]" +_TIMEOUT_MARKER = "[ TIMEOUT ]" + +_START_JSUNIT_RUN_MARKER = "[start] start run suites" +_START_JSUNIT_SUITE_RUN_MARKER = "[suite start]" +_START_JSUNIT_SUITE_END_MARKER = "[suite end]" +_END_JSUNIT_RUN_MARKER = "[end] run suites end" +_PASS_JSUNIT_MARKER = "[pass]" +_FAIL_JSUNIT_MARKER = "[fail]" +_ACE_LOG_MARKER = "app Log" + +LOG = platform_logger("Parser") + + +@Plugin(type=Plugin.PARSER, id=CommonParserType.cpptest) +class CppTestParser(IParser): + def __init__(self): + self.state_machine = StateRecorder() + self.suite_name = "" + self.listeners = [] + self.product_info = {} + self.is_params = False + + def get_suite_name(self): + return self.suite_name + + def get_listeners(self): + return self.listeners + + def __process__(self, lines): + if not self.state_machine.suites_is_started(): + self.state_machine.trace_logs.extend(lines) + for line in lines: + if not check_pub_key_exist(): + LOG.debug(line) + self.parse(line) + + def __done__(self): + suite_result = self.state_machine.get_suites() + if not suite_result.suites_name: + return + for listener in self.get_listeners(): + suites = copy.copy(suite_result) + listener.__ended__(LifeCycle.TestSuites, test_result=suites, + suites_name=suites.suites_name, + product_info=suites.product_info) + self.state_machine.current_suites = None + + def parse(self, line): + + if self.state_machine.suites_is_started() or line.startswith( + _TEST_RUN_MARKER): + if line.startswith(_START_TEST_RUN_MARKER): + message = line[len(_TEST_RUN_MARKER):].strip() + self.handle_suites_started_tag(message) + elif line.startswith(_INFORMATIONAL_MARKER): + pattern = r"(.*) (\(\d+ ms total\))" + message = line[len(_INFORMATIONAL_MARKER):].strip() + if re.match(pattern, line.strip()): + self.handle_suite_ended_tag(message) + elif re.match(r'(\d+) test[s]? from (.*)', message): + self.handle_suite_started_tag(message) + elif line.startswith(_TEST_RUN_MARKER): + if not self.state_machine.suites_is_running(): + return + message = line[len(_TEST_RUN_MARKER):].strip() + self.handle_suites_ended_tag(message) + elif line.startswith(_START_TEST_MARKER): + # Individual test started + message = line[len(_START_TEST_MARKER):].strip() + self.handle_test_started_tag(message) + else: + self.process_test(line) + + def process_test(self, line): + if _SKIPPED_TEST_MARKER in line: + message = line[line.index(_SKIPPED_TEST_MARKER) + len( + _SKIPPED_TEST_MARKER):].strip() + if not self.state_machine.test_is_running(): + LOG.error( + "Found {} without {} before, wrong GTest log format". + format(line, _START_TEST_MARKER)) + return + self.handle_test_ended_tag(message, ResultCode.SKIPPED) + elif _OK_TEST_MARKER in line: + message = line[line.index(_OK_TEST_MARKER) + len( + _OK_TEST_MARKER):].strip() + if not self.state_machine.test_is_running(): + LOG.error( + "Found {} without {} before, wrong GTest log format". + format(line, _START_TEST_MARKER)) + return + self.handle_test_ended_tag(message, ResultCode.PASSED) + elif _ALT_OK_MARKER in line: + message = line[line.index(_ALT_OK_MARKER) + len( + _ALT_OK_MARKER):].strip() + self.fake_run_marker(message) + self.handle_test_ended_tag(message, ResultCode.PASSED) + elif _FAILED_TEST_MARKER in line: + message = line[line.index(_FAILED_TEST_MARKER) + len( + _FAILED_TEST_MARKER):].strip() + if not self.state_machine.suite_is_running(): + return + if not self.state_machine.test_is_running(): + self.fake_run_marker(message) + self.handle_test_ended_tag(message, ResultCode.FAILED) + elif _TIMEOUT_MARKER in line: + message = line[line.index(_TIMEOUT_MARKER) + len( + _TIMEOUT_MARKER):].strip() + self.fake_run_marker(message) + self.handle_test_ended_tag(message, ResultCode.FAILED) + elif self.state_machine.test_is_running(): + self.append_test_output(line) + + def handle_test_suite_failed(self, error_msg): + error_msg = "Unknown error" if error_msg is None else error_msg + LOG.info("Test run failed: {}".format(error_msg)) + if self.state_machine.test_is_running(): + self.state_machine.test().is_completed = True + for listener in self.get_listeners(): + test_result = copy.copy(self.currentTestResult) + listener.__failed__(LifeCycle.TestCase, test_result) + listener.__ended__(LifeCycle.TestCase, test_result) + self.state_machine.suite().stacktrace = error_msg + self.state_machine.suite().is_completed = True + for listener in self.get_listeners(): + suite_result = copy.copy(self.currentSuiteResult) + listener.__failed__(LifeCycle.TestSuite, suite_result) + listener.__ended__(LifeCycle.TestSuite, suite_result) + + def handle_test_started_tag(self, message): + test_class, test_name, _ = self.parse_test_description( + message) + test_result = self.state_machine.test(reset=True) + test_result.test_class = test_class + test_result.test_name = test_name + for listener in self.get_listeners(): + test_result = copy.copy(test_result) + listener.__started__(LifeCycle.TestCase, test_result) + + @classmethod + def parse_test_description(cls, message): + run_time = 0 + matcher = re.match(r'(.*) \((\d+) ms\)', message) + if matcher: + test_class, test_name = matcher.group(1).rsplit(".", 1) + run_time = int(matcher.group(2)) + else: + test_class, test_name = message.rsplit(".", 1) + return test_class, test_name, run_time + + def handle_test_ended_tag(self, message, test_status): + test_class, test_name, run_time = self.parse_test_description( + message) + test_result = self.state_machine.test() + test_result.run_time = int(run_time) + test_result.code = test_status.value + test_result.current = self.state_machine.running_test_index + 1 + if not test_result.is_running(): + LOG.error( + "Test has no start tag when trying to end test: %s", message) + return + found_unexpected_test = False + if test_result.test_class != test_class: + LOG.error( + "Expected class: {} but got:{} ".format(test_result.test_class, + test_class)) + found_unexpected_test = True + if test_result.test_name != test_name: + LOG.error( + "Expected test: {} but got: {}".format(test_result.test_name, + test_name)) + found_unexpected_test = True + + if found_unexpected_test or ResultCode.FAILED == test_status: + for listener in self.get_listeners(): + result = copy.copy(test_result) + listener.__failed__(LifeCycle.TestCase, result) + elif ResultCode.SKIPPED == test_status: + for listener in self.get_listeners(): + result = copy.copy(test_result) + listener.__skipped__(LifeCycle.TestCase, result) + + self.state_machine.test().is_completed = True + for listener in self.get_listeners(): + result = copy.copy(test_result) + listener.__ended__(LifeCycle.TestCase, result) + self.state_machine.running_test_index += 1 + + def fake_run_marker(self, message): + fake_marker = re.compile(" +").split(message) + self.handle_test_started_tag(fake_marker) + + def handle_suites_started_tag(self, message): + self.state_machine.get_suites(reset=True) + matcher = re.match(r'Running (\d+) test[s]? from .*', message) + expected_test_num = int(matcher.group(1)) if matcher else -1 + if expected_test_num >= 0: + test_suites = self.state_machine.get_suites() + test_suites.suites_name = self.get_suite_name() + test_suites.test_num = expected_test_num + test_suites.product_info = self.product_info + for listener in self.get_listeners(): + suite_report = copy.copy(test_suites) + listener.__started__(LifeCycle.TestSuites, suite_report) + + def handle_suite_started_tag(self, message): + self.state_machine.suite(reset=True) + matcher = re.match(r'(\d+) test[s]? from (.*)', message) + expected_test_num = int(matcher.group(1)) if matcher else -1 + if expected_test_num >= 0: + test_suite = self.state_machine.suite() + test_suite.suite_name = matcher.group(2) + test_suite.test_num = expected_test_num + for listener in self.get_listeners(): + suite_report = copy.copy(test_suite) + listener.__started__(LifeCycle.TestSuite, suite_report) + + def handle_suite_ended_tag(self, message): + self.state_machine.running_test_index = 0 + suite_result = self.state_machine.suite() + matcher = re.match(r'.*\((\d+) ms total\)', message) + if matcher: + suite_result.run_time = int(matcher.group(1)) + suite_result.is_completed = True + for listener in self.get_listeners(): + suite = copy.copy(suite_result) + listener.__ended__(LifeCycle.TestSuite, suite, is_clear=True) + + def handle_suites_ended_tag(self, message): + suites = self.state_machine.get_suites() + matcher = re.match(r'.*\((\d+) ms total\)', message) + if matcher: + suites.run_time = int(matcher.group(1)) + suites.is_completed = True + for listener in self.get_listeners(): + copy_suites = copy.copy(suites) + listener.__ended__(LifeCycle.TestSuites, test_result=copy_suites, + suites_name=suites.suites_name, + product_info=suites.product_info, + suite_report=True) + + def append_test_output(self, message): + if self.state_machine.test().stacktrace: + self.state_machine.test().stacktrace += "\r\n" + self.state_machine.test().stacktrace += message + + @staticmethod + def handle_test_run_failed(error_msg): + if not error_msg: + error_msg = "Unknown error" + if not check_pub_key_exist(): + LOG.debug("error_msg:%s" % error_msg) + + def mark_test_as_blocked(self, test): + if not self.state_machine.current_suite and not test.class_name: + return + suites_result = self.state_machine.get_suites(reset=True) + suites_result.suites_name = self.get_suite_name() + suite_name = self.state_machine.current_suite.suite_name if \ + self.state_machine.current_suite else None + suite_result = self.state_machine.suite(reset=True) + test_result = self.state_machine.test(reset=True) + suite_result.suite_name = suite_name or test.class_name + suite_result.suite_num = 1 + test_result.test_class = test.class_name + test_result.test_name = test.test_name + test_result.stacktrace = "error_msg: run crashed" + test_result.num_tests = 1 + test_result.run_time = 0 + test_result.code = ResultCode.BLOCKED.value + for listener in self.get_listeners(): + suite_report = copy.copy(suites_result) + listener.__started__(LifeCycle.TestSuites, suite_report) + for listener in self.get_listeners(): + suite_report = copy.copy(suite_result) + listener.__started__(LifeCycle.TestSuite, suite_report) + for listener in self.get_listeners(): + test_result = copy.copy(test_result) + listener.__started__(LifeCycle.TestCase, test_result) + for listener in self.get_listeners(): + test_result = copy.copy(test_result) + listener.__ended__(LifeCycle.TestCase, test_result) + for listener in self.get_listeners(): + suite_report = copy.copy(suite_result) + listener.__ended__(LifeCycle.TestSuite, suite_report, + is_clear=True) + self.__done__() + + +@Plugin(type=Plugin.PARSER, id=CommonParserType.cpptest_list) +class CppTestListParser(IParser): + def __init__(self): + self.last_test_class_name = None + self.tests = [] + + def __process__(self, lines): + for line in lines: + if not check_pub_key_exist(): + LOG.debug(line) + self.parse(line) + + def __done__(self): + pass + + def parse(self, line): + class_matcher = re.match('^([a-zA-Z]+.*)\\.$', line) + method_matcher = re.match('\\s+([a-zA-Z_]+[\\S]*)(.*)?(\\s+.*)?$', + line) + if class_matcher: + self.last_test_class_name = class_matcher.group(1) + elif method_matcher: + if not self.last_test_class_name: + LOG.error("parsed new test case name %s but no test class name" + " has been set" % line) + else: + test = TestDescription(self.last_test_class_name, + method_matcher.group(1)) + self.tests.append(test) + else: + if not check_pub_key_exist(): + LOG.debug("line ignored: %s" % line) + + +class StatusCodes(Enum): + FAILURE = -2 + START = 1 + ERROR = -1 + SUCCESS = 0 + IN_PROGRESS = 2 + IGNORE = -3 + BLOCKED = 3 + + +class Prefixes(Enum): + STATUS = "INSTRUMENTATION_STATUS: " + STATUS_CODE = "INSTRUMENTATION_STATUS_CODE: " + STATUS_FAILED = "INSTRUMENTATION_FAILED: " + CODE = "INSTRUMENTATION_CODE: " + RESULT = "INSTRUMENTATION_RESULT: " + TIME_REPORT = "Time: " + + +@Plugin(type=Plugin.PARSER, id=CommonParserType.junit) +class JunitParser(IParser): + def __init__(self): + self.state_machine = StateRecorder() + self.suite_name = "" + self.listeners = [] + self.current_key = None + self.current_value = None + self.start_time = datetime.datetime.now() + self.test_time = 0 + self.test_run_finished = False + + def get_suite_name(self): + return self.suite_name + + def get_listeners(self): + return self.listeners + + def __process__(self, lines): + for line in lines: + if not check_pub_key_exist(): + LOG.debug(line) + self.parse(line) + + def __done__(self): + suite_result = self.state_machine.suite() + suite_result.run_time = self.test_time + suite_result.is_completed = True + for listener in self.get_listeners(): + suite = copy.copy(suite_result) + listener.__ended__(LifeCycle.TestSuite, suite, + suite_report=True) + self.state_machine.current_suite = None + + def parse(self, line): + if line.startswith(Prefixes.STATUS_CODE.value): + self.submit_current_key_value() + self.parse_status_code(line) + elif line.startswith(Prefixes.STATUS.value): + self.submit_current_key_value() + self.parse_key(line, len(Prefixes.STATUS.value)) + elif line.startswith(Prefixes.RESULT.value): + self.test_run_finished = True + elif line.startswith(Prefixes.STATUS_FAILED.value) or \ + line.startswith(Prefixes.CODE.value): + self.submit_current_key_value() + self.test_run_finished = True + elif line.startswith(Prefixes.TIME_REPORT.value): + self.parse_time(line) + else: + if self.current_value: + self.current_value = self.current_value + r"\r\n" + self.current_value = self.current_value + line + elif line: + pass + + def parse_key(self, line, key_start_pos): + key_value = line[key_start_pos:].split("=", 1) + if len(key_value) == 2: + self.current_key = key_value[0] + self.current_value = key_value[1] + + def parse_time(self, line): + message = line[len(Prefixes.TIME_REPORT.value):] + self.test_time = float(message.replace(",", "")) * 1000 + + @staticmethod + def check_legality(name): + if not name or name == "null": + return False + return True + + def parse_status_code(self, line): + value = line[len(Prefixes.STATUS_CODE.value):] + test_info = self.state_machine.test() + test_info.code = int(value) + if test_info.code != StatusCodes.IN_PROGRESS: + if self.check_legality(test_info.test_class) and \ + self.check_legality(test_info.test_name): + self.report_result(test_info) + self.clear_current_test_info() + + def clear_current_test_info(self): + self.state_machine.current_test = None + + def submit_current_key_value(self): + if self.current_key and self.current_value: + status_value = self.current_value + test_info = self.state_machine.test() + if self.current_key == "class": + test_info.test_class = status_value + elif self.current_key == "test": + test_info.test_name = status_value + elif self.current_key == "numtests": + test_info.num_tests = int(status_value) + elif self.current_key == "Error": + self.handle_test_run_failed(status_value) + elif self.current_key == "stack": + test_info.stacktrace = status_value + elif self.current_key == "stream": + pass + self.current_key = None + self.current_value = None + + def report_result(self, test_info): + if not test_info.test_name or not test_info.test_class: + LOG.info("invalid instrumentation status bundle") + return + test_info.is_completed = True + self.report_test_run_started(test_info) + if test_info.code == StatusCodes.START.value: + self.start_time = datetime.datetime.now() + for listener in self.get_listeners(): + result = copy.copy(test_info) + listener.__started__(LifeCycle.TestCase, result) + elif test_info.code == StatusCodes.FAILURE.value: + self.state_machine.running_test_index += 1 + test_info.current = self.state_machine.running_test_index + end_time = datetime.datetime.now() + run_time = (end_time - self.start_time).total_seconds() + test_info.run_time = int(run_time * 1000) + for listener in self.get_listeners(): + result = copy.copy(test_info) + result.code = ResultCode.FAILED.value + listener.__ended__(LifeCycle.TestCase, result) + elif test_info.code == StatusCodes.ERROR.value: + self.state_machine.running_test_index += 1 + test_info.current = self.state_machine.running_test_index + end_time = datetime.datetime.now() + run_time = (end_time - self.start_time).total_seconds() + test_info.run_time = int(run_time * 1000) + for listener in self.get_listeners(): + result = copy.copy(test_info) + result.code = ResultCode.FAILED.value + listener.__ended__(LifeCycle.TestCase, result) + elif test_info.code == StatusCodes.SUCCESS.value: + self.state_machine.running_test_index += 1 + test_info.current = self.state_machine.running_test_index + end_time = datetime.datetime.now() + run_time = (end_time - self.start_time).total_seconds() + test_info.run_time = int(run_time * 1000) + for listener in self.get_listeners(): + result = copy.copy(test_info) + result.code = ResultCode.PASSED.value + listener.__ended__(LifeCycle.TestCase, result) + elif test_info.code == StatusCodes.IGNORE.value: + end_time = datetime.datetime.now() + run_time = (end_time - self.start_time).total_seconds() + test_info.run_time = int(run_time * 1000) + for listener in self.get_listeners(): + result = copy.copy(test_info) + result.code = ResultCode.SKIPPED.value + listener.__skipped__(LifeCycle.TestCase, result) + elif test_info.code == StatusCodes.BLOCKED.value: + test_info.current = self.state_machine.running_test_index + end_time = datetime.datetime.now() + run_time = (end_time - self.start_time).total_seconds() + test_info.run_time = int(run_time * 1000) + for listener in self.get_listeners(): + result = copy.copy(test_info) + result.code = ResultCode.BLOCKED.value + listener.__ended__(LifeCycle.TestCase, result) + + self.output_stack_trace(test_info) + + @classmethod + def output_stack_trace(cls, test_info): + if check_pub_key_exist(): + return + if test_info.stacktrace: + stack_lines = test_info.stacktrace.split(r"\r\n") + LOG.error("stacktrace information is:") + for line in stack_lines: + line.strip() + if line: + LOG.error(line) + + def report_test_run_started(self, test_result): + test_suite = self.state_machine.suite() + if not self.state_machine.suite().is_started: + if not test_suite.test_num or not test_suite.suite_name: + test_suite.suite_name = self.get_suite_name() + test_suite.test_num = test_result.num_tests + for listener in self.get_listeners(): + suite_report = copy.copy(test_suite) + listener.__started__(LifeCycle.TestSuite, suite_report) + + @staticmethod + def handle_test_run_failed(error_msg): + if not error_msg: + error_msg = "Unknown error" + if not check_pub_key_exist(): + LOG.debug("error_msg:%s" % error_msg) + + def mark_test_as_failed(self, test): + test_info = self.state_machine.test() + if test_info: + test_info.test_class = test.class_name + test_info.test_name = test.test_name + test_info.code = StatusCodes.START.value + self.report_result(test_info) + test_info.code = StatusCodes.FAILURE.value + self.report_result(test_info) + self.__done__() + + def mark_test_as_blocked(self, test): + test_info = self.state_machine.test() + if test_info: + test_info.test_class = test.class_name + test_info.test_name = test.test_name + test_info.num_tests = 1 + test_info.run_time = 0 + test_info.code = StatusCodes.START.value + self.report_result(test_info) + test_info.code = StatusCodes.BLOCKED.value + self.report_result(test_info) + self.__done__() + + +@Plugin(type=Plugin.PARSER, id=CommonParserType.jsunit) +class JSUnitParser(IParser): + last_line = "" + pattern = r"(\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}\.\d{3}) " + + def __init__(self): + self.state_machine = StateRecorder() + self.suites_name = "" + self.listeners = [] + + def get_listeners(self): + return self.listeners + + def __process__(self, lines): + if not self.state_machine.suites_is_started(): + self.state_machine.trace_logs.extend(lines) + for line in lines: + self.parse(line) + + def __done__(self): + suite_result = self.state_machine.suite() + suite_result.is_completed = True + for listener in self.get_listeners(): + suite = copy.copy(suite_result) + listener.__ended__(LifeCycle.TestSuite, suite, + suite_report=True) + self.state_machine.current_suite = None + + def parse(self, line): + if (self.state_machine.suites_is_started() or line.find( + _START_JSUNIT_RUN_MARKER) != -1) and line.find( + _ACE_LOG_MARKER) != -1: + if line.find(_START_JSUNIT_RUN_MARKER) != -1: + self.handle_suites_started_tag() + elif line.endswith(_END_JSUNIT_RUN_MARKER): + self.handle_suites_ended_tag() + elif line.find(_START_JSUNIT_SUITE_RUN_MARKER) != -1: + self.handle_suite_started_tag(line.strip()) + elif line.endswith(_START_JSUNIT_SUITE_END_MARKER): + self.handle_suite_ended_tag() + elif _PASS_JSUNIT_MARKER in line or _FAIL_JSUNIT_MARKER \ + in line: + self.handle_one_test_tag(line.strip()) + self.last_line = line + + def parse_test_description(self, message): + pattern = r"\[(pass|fail)\]" + year = time.strftime("%Y") + filter_message = message.split("app Log:")[1].strip() + end_time = "%s-%s" % \ + (year, re.match(self.pattern, message).group().strip()) + start_time = "%s-%s" % \ + (year, re.match(self.pattern, + self.last_line.strip()).group().strip()) + start_timestamp = int(time.mktime( + time.strptime(start_time, "%Y-%m-%d %H:%M:%S.%f"))) * 1000 + int( + start_time.split(".")[-1]) + end_timestamp = int(time.mktime( + time.strptime(end_time, "%Y-%m-%d %H:%M:%S.%f"))) * 1000 + int( + end_time.split(".")[-1]) + run_time = end_timestamp - start_timestamp + _, status_end_index = re.match(pattern, filter_message).span() + status = filter_message[:status_end_index] + test_name = filter_message[status_end_index:] + status_dict = {"pass": ResultCode.PASSED, "fail": ResultCode.FAILED, + "ignore": ResultCode.SKIPPED} + status = status_dict.get(status[1:-1]) + return test_name, status, run_time + + def handle_suites_started_tag(self): + self.state_machine.get_suites(reset=True) + test_suites = self.state_machine.get_suites() + test_suites.suites_name = self.suites_name + test_suites.test_num = 0 + for listener in self.get_listeners(): + suite_report = copy.copy(test_suites) + listener.__started__(LifeCycle.TestSuites, suite_report) + + def handle_suites_ended_tag(self): + suites = self.state_machine.get_suites() + suites.is_completed = True + + for listener in self.get_listeners(): + listener.__ended__(LifeCycle.TestSuites, test_result=suites, + suites_name=suites.suites_name) + + def handle_one_test_tag(self, message): + test_name, status, run_time = \ + self.parse_test_description(message) + test_result = self.state_machine.test(reset=True) + test_suite = self.state_machine.suite() + test_result.test_class = test_suite.suite_name + test_result.test_name = test_name + test_result.run_time = run_time + test_result.code = status.value + test_result.current = self.state_machine.running_test_index + 1 + self.state_machine.suite().run_time += run_time + for listener in self.get_listeners(): + test_result = copy.copy(test_result) + listener.__started__(LifeCycle.TestCase, test_result) + + test_suites = self.state_machine.get_suites() + found_unexpected_test = False + + if found_unexpected_test or ResultCode.FAILED == status: + for listener in self.get_listeners(): + result = copy.copy(test_result) + listener.__failed__(LifeCycle.TestCase, result) + elif ResultCode.SKIPPED == status: + for listener in self.get_listeners(): + result = copy.copy(test_result) + listener.__skipped__(LifeCycle.TestCase, result) + + self.state_machine.test().is_completed = True + test_suite.test_num += 1 + test_suites.test_num += 1 + for listener in self.get_listeners(): + result = copy.copy(test_result) + listener.__ended__(LifeCycle.TestCase, result) + self.state_machine.running_test_index += 1 + + def fake_run_marker(self, message): + fake_marker = re.compile(" +").split(message) + self.processTestStartedTag(fake_marker) + + def handle_suite_started_tag(self, message): + self.state_machine.suite(reset=True) + test_suite = self.state_machine.suite() + if re.match(r".*\[suite start\].*", message): + _, index = re.match(r".*\[suite start\]", message).span() + if message[index:]: + test_suite.suite_name = message[index:] + else: + test_suite.suite_name = self.suite_name + test_suite.test_num = 0 + for listener in self.get_listeners(): + suite_report = copy.copy(test_suite) + listener.__started__(LifeCycle.TestSuite, suite_report) + + def handle_suite_ended_tag(self): + suite_result = self.state_machine.suite() + suites = self.state_machine.get_suites() + suite_result.run_time = suite_result.run_time + suites.run_time += suite_result.run_time + suite_result.is_completed = True + + for listener in self.get_listeners(): + suite = copy.copy(suite_result) + listener.__ended__(LifeCycle.TestSuite, suite, is_clear=True) + + def append_test_output(self, message): + if self.state_machine.test().stacktrace: + self.state_machine.test().stacktrace = \ + "%s\r\n" % self.state_machine.test().stacktrace + self.state_machine.test().stacktrace = \ + ''.join((self.state_machine.test().stacktrace, message)) + + diff --git a/extension/src/xdevice_extension/_core/environment/__init__.py b/extension/src/xdevice_extension/_core/environment/__init__.py new file mode 100644 index 0000000..eb8b49d --- /dev/null +++ b/extension/src/xdevice_extension/_core/environment/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/extension/src/xdevice_extension/_core/environment/device.py b/extension/src/xdevice_extension/_core/environment/device.py new file mode 100644 index 0000000..861f0e4 --- /dev/null +++ b/extension/src/xdevice_extension/_core/environment/device.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import threading + +from xdevice import DeviceOsType +from xdevice import ProductForm +from xdevice import ReportException +from xdevice import IDevice +from xdevice import platform_logger +from xdevice import Plugin +from xdevice import exec_cmd +from xdevice import ConfigConst + +from xdevice_extension._core.environment.dmlib import HdcHelper +from xdevice_extension._core.exception import HdcError +from xdevice_extension._core.environment.dmlib import CollectingOutputReceiver +from xdevice_extension._core.environment.device_state import \ + DeviceAllocationState +from xdevice_extension._core.utils import check_path_legal +from xdevice_extension._core.utils import convert_serial +from xdevice_extension._core.constants import DeviceConnectorType + +__all__ = ["Device"] +TIMEOUT = 300 * 1000 +RETRY_ATTEMPTS = 2 +DEFAULT_UNAVAILABLE_TIMEOUT = 20 * 1000 +BACKGROUND_TIME = 2 * 60 * 1000 +LOG = platform_logger("Device") + + +def perform_device_action(func): + def device_action(self, *args, **kwargs): + if not self.get_recover_state(): + LOG.debug("device %s %s is false" % (self.device_sn, + ConfigConst.recover_state)) + return + # avoid infinite recursion, such as device reboot + abort_on_exception = bool(kwargs.get("abort_on_exception", False)) + if abort_on_exception: + result = func(self, *args, **kwargs) + return result + + tmp = int(kwargs.get("retry", RETRY_ATTEMPTS)) + retry = tmp + 1 if tmp > 0 else 1 + exception = None + for _ in range(retry): + try: + result = func(self, *args, **kwargs) + return result + except ReportException as error: + self.log.exception("Generate report error!", exc_info=False) + exception = error + except (ConnectionResetError, ConnectionRefusedError) as error: + self.log.error("error type: %s, error: %s" % + (error.__class__.__name__, error)) + if self.usb_type == DeviceConnectorType.hdc: + cmd = "hdc reset" + self.log.info("re-execute hdc reset") + exec_cmd(cmd) + if not self.recover_device(): + LOG.debug("set device %s %s false" % ( + self.device_sn, ConfigConst.recover_state)) + self.set_recover_state(False) + raise error + exception = error + except HdcError as error: + self.log.error("error type: %s, error: %s" % + (error.__class__.__name__, error)) + if not self.recover_device(): + LOG.debug("set device %s %s false" % ( + self.device_sn, ConfigConst.recover_state)) + self.set_recover_state(False) + raise error + exception = error + except Exception as error: + self.log.exception("error type: %s, error: %s" % ( + error.__class__.__name__, error), exc_info=False) + exception = error + raise exception + + return device_action + + +@Plugin(type=Plugin.DEVICE, id=DeviceOsType.default) +class Device(IDevice): + """ + Class representing a device. + + Each object of this class represents one device in xDevice, + including handles to hdc, fastboot, and test agent. + + Attributes: + device_sn: A string that's the serial number of the device. + """ + + device_sn = None + host = None + port = None + usb_type = None + is_timeout = False + device_hilog_proc = None + device_os_type = DeviceOsType.default + test_device_state = None + device_allocation_state = DeviceAllocationState.available + label = None + log = platform_logger("Device") + device_state_monitor = None + reboot_timeout = 2 * 60 * 1000 + hilog_file_pipe = None + + model_dict = { + 'default': ProductForm.phone, + 'car': ProductForm.car, + 'tv': ProductForm.television, + 'watch': ProductForm.watch, + 'tablet': ProductForm.tablet + } + + def __init__(self): + self.extend_value = {} + self.device_lock = threading.RLock() + + def __eq__(self, other): + return self.device_sn == other.__get_serial__() and \ + self.device_os_type == other.device_os_type + + def __set_serial__(self, device_sn=""): + self.device_sn = device_sn + return self.device_sn + + def __get_serial__(self): + return self.device_sn + + def get(self, key=None, default=None): + if not key: + return default + value = getattr(self, key, None) + if value: + return value + else: + return self.extend_value.get(key, default) + + def recover_device(self): + if not self.get_recover_state(): + LOG.debug("device %s %s is false, cannot recover device" % ( + self.device_sn, ConfigConst.recover_state)) + return + + LOG.debug("wait device %s to recover" % self.device_sn) + return self.device_state_monitor.wait_for_device_available() + + def get_device_type(self): + model = self.get_property("ro.build.characteristics", + abort_on_exception=True) + self.label = self.model_dict.get(model, None) + + def get_property(self, prop_name, retry=RETRY_ATTEMPTS, + abort_on_exception=False): + """ + Hdc command, ddmlib function. + """ + command = "getprop %s" % prop_name + stdout = self.execute_shell_command( + command, timeout=5 * 1000, output_flag=False, retry=retry, + abort_on_exception=abort_on_exception).strip() + if stdout: + LOG.debug(stdout) + return stdout + + @perform_device_action + def hdc_command(self, command, **kwargs): + timeout = int(kwargs.get("timeout", TIMEOUT)) / 1000 + error_print = bool(kwargs.get("error_print", True)) + join_result = bool(kwargs.get("join_result", False)) + timeout_msg = '' if timeout == 300.0 else \ + " with timeout %ss" % timeout + if self.usb_type == DeviceConnectorType.hdc: + LOG.debug("%s execute command hdc %s%s" % ( + convert_serial(self.device_sn), command, timeout_msg)) + cmd = ["hdc_std", "-t", self.device_sn] + if isinstance(command, list): + cmd.extend(command) + else: + command = command.strip() + cmd.extend(command.split(" ")) + result = exec_cmd(cmd, timeout, error_print, join_result) + if not result: + return result + for line in str(result).split("\n"): + if line.strip(): + LOG.debug(line.strip()) + return result + + @perform_device_action + def execute_shell_command(self, command, timeout=TIMEOUT, + receiver=None, **kwargs): + if not receiver: + collect_receiver = CollectingOutputReceiver() + HdcHelper.execute_shell_command( + self, command, timeout=timeout, + receiver=collect_receiver, **kwargs) + return collect_receiver.output + else: + return HdcHelper.execute_shell_command( + self, command, timeout=timeout, + receiver=receiver, **kwargs) + + def execute_shell_cmd_background(self, command, timeout=TIMEOUT, + receiver=None): + status = HdcHelper.execute_shell_command(self, command, + timeout=timeout, + receiver=receiver) + + self.wait_for_device_not_available(DEFAULT_UNAVAILABLE_TIMEOUT) + self.device_state_monitor.wait_for_device_available(BACKGROUND_TIME) + cmd = "target mount" \ + if self.usb_type == DeviceConnectorType.hdc else "remount" + self.hdc_command(cmd) + self.start_catch_device_log() + return status + + def wait_for_device_not_available(self, wait_time): + return self.device_state_monitor.wait_for_device_not_available( + wait_time) + + def _wait_for_device_online(self, wait_time=None): + return self.device_state_monitor.wait_for_device_online(wait_time) + + def _do_reboot(self): + HdcHelper.reboot(self) + if not self.wait_for_device_not_available( + DEFAULT_UNAVAILABLE_TIMEOUT): + LOG.error("Did not detect device {} becoming unavailable " + "after reboot".format(convert_serial(self.device_sn))) + + def _reboot_until_online(self): + self._do_reboot() + self._wait_for_device_online() + + def reboot(self): + self._reboot_until_online() + self.device_state_monitor.wait_for_device_available( + self.reboot_timeout) + self.enable_hdc_root() + self.start_catch_device_log() + + @perform_device_action + def install_package(self, package_path, command=""): + if package_path is None: + raise HdcError( + "install package: package path cannot be None!") + return HdcHelper.install_package(self, package_path, command) + + @perform_device_action + def uninstall_package(self, package_name): + return HdcHelper.uninstall_package(self, package_name) + + @perform_device_action + def push_file(self, local, remote, **kwargs): + """ + Push a single file. + The top directory won't be created if is_create is False (by default) + and vice versa + """ + if local is None: + raise HdcError("XDevice Local path cannot be None!") + + remote_is_dir = kwargs.get("remote_is_dir", False) + if remote_is_dir: + ret = self.execute_shell_command("test -d %s && echo 0" % remote) + if not (ret != "" and len(str(ret).split()) != 0 and + str(ret).split()[0] == "0"): + self.execute_shell_command("mkdir -p %s" % remote) + + is_create = kwargs.get("is_create", False) + timeout = kwargs.get("timeout", TIMEOUT) + HdcHelper.push_file(self, local, remote, is_create=is_create, + timeout=timeout) + if not self.is_file_exist(remote): + LOG.error("push %s to %s failed" % (local, remote)) + raise HdcError("push %s to %s failed" % (local, remote)) + + @perform_device_action + def pull_file(self, remote, local, **kwargs): + """ + Pull a single file. + The top directory won't be created if is_create is False (by default) + and vice versa + """ + + is_create = kwargs.get("is_create", False) + timeout = kwargs.get("timeout", TIMEOUT) + HdcHelper.pull_file(self, remote, local, is_create=is_create, + timeout=timeout) + + def is_hdc_root(self): + output = self.execute_shell_command("id") + return "uid=0(root)" in output + + def enable_hdc_root(self): + if self.is_hdc_root(): + return True + for index in range(3): + cmd = "smode" \ + if self.usb_type == DeviceConnectorType.hdc else "root" + output = self.hdc_command(cmd) + if self.is_hdc_root(): + return True + LOG.debug( + "hdc root on %s unsuccessful on attempt %d. " + "Output: %s" % ( + convert_serial(self.device_sn), index, output)) + return False + + def is_directory(self, path): + path = check_path_legal(path) + output = self.execute_shell_command("ls -ld {}".format(path)) + if output and output.startswith('d'): + return True + return False + + def is_file_exist(self, file_path): + file_path = check_path_legal(file_path) + command = ["hdc_std", "shell", "ls", file_path] + output = exec_cmd(command) + + if output and "No such file or directory" not in output: + return True + return False + + def start_catch_device_log(self, hilog_file_pipe=None): + """ + Starts hdc log for each device in separate subprocesses and save + the logs in files. + """ + if hilog_file_pipe: + self.hilog_file_pipe = hilog_file_pipe + self._start_catch_device_log() + + def stop_catch_device_log(self): + """ + Stops all hdc log subprocesses. + """ + self._stop_catch_device_log() + + def _start_catch_device_log(self): + pass + + def _stop_catch_device_log(self): + pass + + def get_recover_result(self, retry=RETRY_ATTEMPTS): + command = "getprop ro.product.device" + stdout = self.execute_shell_command(command, timeout=5 * 1000, + output_flag=False, retry=retry, + abort_on_exception=True).strip() + if stdout: + LOG.debug(stdout) + return stdout + + def set_recover_state(self, state): + with self.device_lock: + setattr(self, ConfigConst.recover_state, state) + + def get_recover_state(self, default_state=True): + with self.device_lock: + state = getattr(self, ConfigConst.recover_state, default_state) + return state + + def close(self): + pass diff --git a/extension/src/xdevice_extension/_core/environment/device_monitor.py b/extension/src/xdevice_extension/_core/environment/device_monitor.py new file mode 100644 index 0000000..014d3fc --- /dev/null +++ b/extension/src/xdevice_extension/_core/environment/device_monitor.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +from threading import Condition + +from xdevice_extension._core.environment.device_state import TestDeviceState +from xdevice_extension._core.utils import convert_serial + +CHECK_POLL_TIME = 3 +MAX_CHECK_POLL_TIME = 30 + + +class DeviceStateListener(object): + def __init__(self, expected_state): + self.expected_state = expected_state + self.condition = Condition() + + def state_changed(self, new_state): + if self.expected_state == new_state: + with self.condition: + self.condition.notify_all() + + def get_expected_state(self): + return self.expected_state + + +class DeviceStateMonitor(object): + """ + Provides facilities for monitoring the state of a Device. + """ + + def __init__(self, device): + self.state_listener = [] + self.device_state = device.test_device_state + self.device = device + self.default_online_timeout = 1 * 60 * 1000 + self.default_available_timeout = 6 * 60 * 1000 + + def wait_for_device_state(self, state, wait_time): + listener = DeviceStateListener(state) + if self.device_state == state: + return True + self.device.log.debug( + "wait device %s for %s" % (convert_serial(self.device.device_sn), + state)) + + self.add_device_state_listener(listener) + with listener.condition: + try: + listener.condition.wait(wait_time / 1000) + finally: + self.remove_device_state_listener(listener) + + return self.device_state == state + + def wait_for_device_not_available(self, wait_time): + return self.wait_for_device_state(TestDeviceState.NOT_AVAILABLE, + wait_time) + + def wait_for_device_online(self, wait_time=None): + if not wait_time: + wait_time = self.default_online_timeout + return self.wait_for_device_state(TestDeviceState.ONLINE, wait_time) + + def wait_for_boot_complete(self, wait_time): + counter = 1 + start_time = int(time.time()*1000) + self.device.log.debug("wait for boot complete, and wait time: %s ms" % + wait_time) + while int(time.time()*1000) - start_time < wait_time: + try: + result = self.device.get_recover_result(retry=0) + if result == "1": + return True + except Exception as exception: + self.device.log.error("wait for boot complete exception: %s" + % exception) + time.sleep(min(CHECK_POLL_TIME * counter, MAX_CHECK_POLL_TIME)) + counter = counter + 1 + return False + + def wait_for_device_available(self, wait_time=None): + if not wait_time: + wait_time = self.default_available_timeout + start_time = int(time.time()*1000) + if not self.wait_for_device_online(wait_time): + return False + elapsed_time = int(time.time()*1000) - start_time + if not self.wait_for_boot_complete(wait_time - elapsed_time): + return False + return True + + def remove_device_state_listener(self, listener): + self.state_listener.remove(listener) + + def add_device_state_listener(self, listener): + self.state_listener.append(listener) + + def set_state(self, new_state): + if not new_state: + return + self.device_state = new_state + for listener in self.state_listener: + listener.state_changed(new_state) diff --git a/extension/src/xdevice_extension/_core/environment/device_state.py b/extension/src/xdevice_extension/_core/environment/device_state.py new file mode 100644 index 0000000..79745a6 --- /dev/null +++ b/extension/src/xdevice_extension/_core/environment/device_state.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from dataclasses import dataclass +from enum import Enum +from enum import unique + + +@unique +class TestDeviceState(Enum): + FASTBOOT = "FASTBOOT" + ONLINE = "ONLINE" + RECOVERY = "RECOVERY" + NOT_AVAILABLE = "NOT_AVAILABLE" + + @staticmethod + def get_test_device_state(device_state): + if device_state is None: + return TestDeviceState.NOT_AVAILABLE + elif device_state == DeviceState.ONLINE: + return TestDeviceState.ONLINE + elif device_state == DeviceState.OFFLINE: + return TestDeviceState.NOT_AVAILABLE + elif device_state == DeviceState.RECOVERY: + return TestDeviceState.RECOVERY + elif device_state == DeviceState.BOOTLOADER: + return TestDeviceState.FASTBOOT + else: + return TestDeviceState.NOT_AVAILABLE + + +@unique +class DeviceState(Enum): + BOOTLOADER = "bootloader" + OFFLINE = "offline" + ONLINE = "device" + RECOVERY = "recovery" + + @staticmethod + def get_state(state): + for device_state in DeviceState: + if device_state.value == state: + return device_state + return None + + +@unique +class DeviceEvent(Enum): + """ + Represents a test device event that can change allocation state + """ + CONNECTED_ONLINE = "CONNECTED_ONLINE" + CONNECTED_OFFLINE = "CONNECTED_OFFLINE" + STATE_CHANGE_ONLINE = "STATE_CHANGE_ONLINE" + STATE_CHANGE_OFFLINE = "STATE_CHANGE_OFFLINE" + DISCONNECTED = "DISCONNECTED" + FORCE_AVAILABLE = "FORCE_AVAILABLE" + AVAILABLE_CHECK_PASSED = "AVAILABLE_CHECK_PASSED" + AVAILABLE_CHECK_FAILED = "AVAILABLE_CHECK_FAILED" + AVAILABLE_CHECK_IGNORED = "AVAILABLE_CHECK_IGNORED" + ALLOCATE_REQUEST = "ALLOCATE_REQUEST" + FORCE_ALLOCATE_REQUEST = "FORCE_ALLOCATE_REQUEST" + FREE_AVAILABLE = "FREE_AVAILABLE" + FREE_UNRESPONSIVE = "FREE_UNRESPONSIVE" + FREE_UNAVAILABLE = "FREE_UNAVAILABLE" + FREE_UNKNOWN = "FREE_UNKNOWN" + + +def handle_allocation_event(old_state, event): + new_state = None + if event == DeviceEvent.CONNECTED_ONLINE \ + or event == DeviceEvent.STATE_CHANGE_ONLINE: + if old_state == DeviceAllocationState.allocated: + new_state = old_state + else: + new_state = DeviceAllocationState.checking_availability + elif event == DeviceEvent.CONNECTED_OFFLINE \ + or event == DeviceEvent.STATE_CHANGE_OFFLINE \ + or event == DeviceEvent.DISCONNECTED: + if old_state == DeviceAllocationState.allocated: + new_state = old_state + else: + new_state = DeviceAllocationState.unknown + elif event == DeviceEvent.ALLOCATE_REQUEST: + new_state = DeviceAllocationState.allocated + elif event == DeviceEvent.FREE_AVAILABLE: + new_state = DeviceAllocationState.available + elif event == DeviceEvent.FREE_UNAVAILABLE: + new_state = DeviceAllocationState.unknown + elif event == DeviceEvent.AVAILABLE_CHECK_IGNORED: + new_state = DeviceAllocationState.ignored + elif event == DeviceEvent.AVAILABLE_CHECK_PASSED: + new_state = DeviceAllocationState.available + + return new_state + + +@dataclass +class DeviceAllocationState: + ignored = "Ignored" + available = "Available" + allocated = "Allocated" + checking_availability = "Checking_Availability" + unavailable = "Unavailable" + unknown = "unknown" diff --git a/extension/src/xdevice_extension/_core/environment/dmlib.py b/extension/src/xdevice_extension/_core/environment/dmlib.py new file mode 100644 index 0000000..7b45727 --- /dev/null +++ b/extension/src/xdevice_extension/_core/environment/dmlib.py @@ -0,0 +1,1187 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import platform +import socket +import struct +import threading +import time +import shutil +import stat +from dataclasses import dataclass + +from xdevice import DeviceOsType +from xdevice import ReportException +from xdevice import ExecuteTerminate +from xdevice import platform_logger +from xdevice import Plugin +from xdevice import get_plugin +from xdevice import IShellReceiver +from xdevice import exec_cmd +from xdevice import get_file_absolute_path +from xdevice import ParamError + +from xdevice_extension._core.environment.device_state import DeviceState +from xdevice_extension._core.exception import DeviceError +from xdevice_extension._core.exception import HdcError +from xdevice_extension._core.exception import HdcCommandRejectedException +from xdevice_extension._core.exception import ShellCommandUnresponsiveException +from xdevice_extension._core.utils import is_proc_running +from xdevice_extension._core.utils import convert_ip +from xdevice_extension._core.utils import convert_serial +from xdevice_extension._core.utils import create_dir +from xdevice_extension._core.constants import DeviceConnectorType +from xdevice_extension._core.constants import FilePermission + +ID_OKAY = b'OKAY' +ID_FAIL = b'FAIL' +ID_STAT = b'STAT' +ID_RECV = b'RECV' +ID_DATA = b'DATA' +ID_DONE = b'DONE' +ID_SEND = b'SEND' +ID_LIST = b'LIST' +ID_DENT = b'DENT' + +DEFAULT_ENCODING = "ISO-8859-1" +SYNC_DATA_MAX = 64 * 1024 +REMOTE_PATH_MAX_LENGTH = 1024 +SOCK_DATA_MAX = 256 + +INSTALL_TIMEOUT = 2 * 60 * 1000 +DEFAULT_TIMEOUT = 40 * 1000 + +MAX_CONNECT_ATTEMPT_COUNT = 10 +DATA_UNIT_LENGTH = 4 +HEXADECIMAL_NUMBER = 16 +SPECIAL_FILE_MODE = 41471 +FORMAT_BYTES_LENGTH = 4 +DEFAULT_OFFSET_OF_INT = 4 + +DEFAULT_PORT = 5037 +INVALID_MODE_CODE = -1 +LOG = platform_logger("Hdc") + + +class HdcMonitor: + """ + A Device monitor. + This monitor connects to the Device Connector, gets device and + debuggable process information from it. + """ + MONITOR_MAP = {} + + def __init__(self, host="127.0.0.1", port=None, device_connector=None): + self.channel = dict() + self.channel.setdefault("host", host) + self.channel.setdefault("port", port) + self.main_hdc_connection = None + self.connection_attempt = 0 + self.is_stop = False + self.monitoring = False + self.server = device_connector + self.devices = [] + + @staticmethod + def get_instance(host, port=None, device_connector=None): + if host not in HdcMonitor.MONITOR_MAP: + monitor = HdcMonitor(host, port, device_connector) + HdcMonitor.MONITOR_MAP[host] = monitor + LOG.debug("HdcMonitor map add host %s, map is %s" % + (host, HdcMonitor.MONITOR_MAP)) + + return HdcMonitor.MONITOR_MAP[host] + + def start(self): + """ + Starts the monitoring. + """ + try: + LOG.debug("HdcMonitor usb type is %s" % self.server.usb_type) + bridge_name = 'hdc_std' + if self.server.usb_type == DeviceConnectorType.hdc: + # tell if hdc has already been in the environ path. + env_hdc = shutil.which(bridge_name) + # if not, add xdevice's own hdc path to environ path. + if env_hdc is None: + if os.name == "nt": + hdc = get_file_absolute_path( + "%s.exe" % bridge_name, alt_dir=os.path.join( + "tools", bridge_name, "windows")) + else: + hdc = get_file_absolute_path( + bridge_name, alt_dir=os.path.join + ("tools", bridge_name, "linux")) + xdevice_hdc_path = os.path.dirname(hdc) + os.environ['PATH'] = os.pathsep.join( + (os.environ['PATH'], xdevice_hdc_path) + ) + if not is_proc_running(bridge_name): + self.start_hdc( + connector=bridge_name, + local_port=self.channel.setdefault( + "port", DEFAULT_PORT)) + time.sleep(1) + + server_thread = threading.Thread(target=self.loop_monitor, + name="HdcMonitor", args=()) + server_thread.setDaemon(True) + server_thread.start() + except FileNotFoundError as _: + LOG.error("HdcMonitor can't find connector, init device " + "environment failed!") + + @staticmethod + def stop_hdc(connector="hdc_std"): + """ + Starts the hdc host side server. + """ + if connector == "hdc_std": + if is_proc_running(connector): + try: + LOG.debug("HdcMonitor hdc kill") + exec_cmd([connector, "kill"]) + except ParamError as error: + LOG.debug("HdcMonitor hdc kill error:%s" % error) + except FileNotFoundError as _: + LOG.warning('Cannot kill hdc process, ' + 'please close it manually!') + + def stop(self): + """ + Stops the monitoring. + """ + for host in HdcMonitor.MONITOR_MAP: + LOG.debug("HdcMonitor stop host %s" % host) + monitor = HdcMonitor.MONITOR_MAP[host] + try: + monitor.is_stop = True + if monitor.main_hdc_connection is not None: + monitor.main_hdc_connection.shutdown(2) + monitor.main_hdc_connection.close() + monitor.main_hdc_connection = None + except (socket.error, socket.gaierror, socket.timeout) as _: + LOG.error("HdcMonitor close socket exception") + HdcMonitor.MONITOR_MAP.clear() + LOG.debug("HdcMonitor hdc monitor stop!") + LOG.debug("HdcMonitor map is %s" % HdcMonitor.MONITOR_MAP) + + def loop_monitor(self): + """ + Monitors the devices. This connects to the Debug Bridge + """ + while not self.is_stop: + try: + if self.main_hdc_connection is None: + self.main_hdc_connection = self.open_hdc_connection() + if self.main_hdc_connection is None: + self.connection_attempt += 1 + + if self.connection_attempt > MAX_CONNECT_ATTEMPT_COUNT: + self.is_stop = True + LOG.error( + "HdcMonitor attempt %s, can't connect to hdc " + "for Device List Monitoring" % + str(self.connection_attempt)) + raise HdcError( + "HdcMonitor cannot connect hdc server(%s %s)," + " please check!" % + (self.channel.get("host"), + str(self.channel.get("post")))) + + LOG.debug( + "HdcMonitor Connection attempts: %s" % + str(self.connection_attempt)) + + time.sleep(2) + else: + LOG.debug( + "HdcMonitor Connected to hdc_std for device " + "monitoring, main_hdc_connection is %s" % + self.main_hdc_connection) + + if self.main_hdc_connection and not self.monitoring: + self.monitoring_list_targets() + len_buf = HdcHelper.read(self.main_hdc_connection, + DATA_UNIT_LENGTH) + length = struct.unpack("!I", len_buf)[0] + if length >= 0: + self.monitoring = True + self.process_incoming_target_data(length) + except (HdcError, Exception) as _: + self.handle_exception_monitor_loop() + break + + def handle_exception_monitor_loop(self): + LOG.debug("Handle exception monitor loop: %s" % + self.main_hdc_connection) + if self.main_hdc_connection is None: + return + self.main_hdc_connection.close() + LOG.debug("Handle exception monitor loop, main hdc connection closed, " + "main hdc connection: %s" % self.main_hdc_connection) + + def monitoring_list_targets(self): + HdcHelper.handle_shake(self.main_hdc_connection) + request = HdcHelper.form_hdc_request('list targets') + HdcHelper.write(self.main_hdc_connection, request) + + def device_list_monitoring(self): + request = HdcHelper.form_hdc_request("host:track-devices") + HdcHelper.write(self.main_hdc_connection, request) + resp = HdcHelper.read_hdc_response(self.main_hdc_connection) + if not resp.okay: + LOG.error("HdcMonitor hdc rejected shell command") + raise Exception(resp.message) + else: + LOG.debug( + 'HdcMonitor execute command success:send device_list ' + 'monitoring request') + return True + + def process_incoming_device_data(self, length): + local_array_list = [] + if length > 0: + data_buf = HdcHelper.read(self.main_hdc_connection, length) + data_str = HdcHelper.reply_to_string(data_buf) + lines = data_str.split('\n') + for line in lines: + items = line.strip().split('\t') + if len(items) != 2: + continue + device_instance = self._get_device_instance( + items, DeviceOsType.default) + local_array_list.append(device_instance) + + self.update_devices(local_array_list) + + def process_incoming_target_data(self, length): + local_array_list = [] + if length > 0: + data_buf = HdcHelper.read(self.main_hdc_connection, length) + data_str = HdcHelper.reply_to_string(data_buf) + if 'Empty' not in data_str: + lines = data_str.split('\n') + for line in lines: + items = line.strip().split('\t') + if not items[0] : + continue + items.append(DeviceState.ONLINE.value) + device_instance = self._get_device_instance( + items, DeviceOsType.default) + local_array_list.append(device_instance) + else: + LOG.debug("please check device actually.[%s]" % data_str) + + self.update_devices(local_array_list) + + def _get_device_instance(self, items, os_type): + device = get_plugin(plugin_type=Plugin.DEVICE, plugin_id=os_type)[0] + device_instance = device.__class__() + device_instance.__set_serial__(items[0]) + device_instance.host = self.channel.get("host") + device_instance.port = self.channel.get("port") + LOG.debug("dmlib get_device_instance %s %s %s" % + (device_instance.device_sn, + device_instance.host, device_instance.port)) + device_instance.device_state = DeviceState.get_state(items[1]) + return device_instance + + def update_devices(self, param_array_list): + devices = [item for item in self.devices] + devices.reverse() + for local_device1 in devices: + k = 0 + for local_device2 in param_array_list: + if local_device1.device_sn == local_device2.device_sn and \ + local_device1.device_os_type == \ + local_device2.device_os_type: + k = 1 + if local_device1.device_state != \ + local_device2.device_state: + local_device1.device_state = local_device2.device_state + self.server.device_changed(local_device1) + param_array_list.remove(local_device2) + break + + if k == 0: + self.devices.remove(local_device1) + self.server.device_disconnected(local_device1) + for local_device in param_array_list: + self.devices.append(local_device) + self.server.device_connected(local_device) + + def open_hdc_connection(self): + """ + Attempts to connect to the debug bridge server. Return a connect socket + if success, null otherwise. + """ + sock = None + try: + LOG.debug( + "HdcMonitor connecting to hdc for Device List Monitoring") + LOG.debug("HdcMonitor socket connection host: %s, port: %s" % + (str(convert_ip(self.channel.get("host"))), + str(int(self.channel.get("port"))))) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((self.channel.get("host"), + int(self.channel.get("port")))) + return sock + except (socket.error, socket.gaierror, socket.timeout) as exception: + LOG.error("HdcMonitor hdc socket connection Error: %s, " + "host is %s, port is %s" % (str(exception), + self.channel.get("host"), + self.channel.get("port"))) + return sock + + def start_hdc(self, connector="hdc_std", kill=False, local_port=None): + """Starts the hdc host side server. + + Args: + connector: connector type, like "hdc" + kill: if True, kill exist host side server + local_port: local port to start host side server + + Returns: + None + """ + if connector == "hdc_std": + if kill: + LOG.debug("HdcMonitor hdc kill") + exec_cmd([connector, "kill"]) + LOG.debug("HdcMonitor hdc start") + exec_cmd( + [connector, "-s", "tcp:%s" % local_port, "reset"], + error_print=False) + + +@dataclass +class HdcResponse: + """Response from HDC.""" + okay = ID_OKAY # first 4 bytes in response were "OKAY"? + message = "" # diagnostic string if okay is false + + +class SyncService: + """ + Sync service class to push/pull to/from devices/emulators, + through the debug bridge. + """ + def __init__(self, device, host=None, port=None): + self.device = device + self.host = host + self.port = port + self.sock = None + + def open_sync(self, timeout=DEFAULT_TIMEOUT): + """ + Opens the sync connection. This must be called before any calls to + push[File] / pull[File]. + Return true if the connection opened, false if hdc refuse the + connection. This can happen device is invalid. + """ + LOG.debug("open sync, timeout=%s" % int(timeout/1000)) + self.sock = HdcHelper.socket(host=self.host, port=self.port, + timeout=timeout) + HdcHelper.set_device(self.device, self.sock) + + request = HdcHelper.form_hdc_request("sync:") + HdcHelper.write(self.sock, request) + + resp = HdcHelper.read_hdc_response(self.sock) + if not resp.okay: + self.device.log.error( + "Got unhappy response from HDC sync req: %s" % resp.message) + raise HdcError( + "Got unhappy response from HDC sync req: %s" % resp.message) + + def close(self): + """ + Closes the connection. + """ + if self.sock is not None: + try: + self.sock.close() + except socket.error as error: + LOG.error("socket close error: %s" % error, error_no="00420") + finally: + self.sock = None + + def pull_file(self, remote, local, is_create=False): + """ + Pulls a file. + The top directory won't be created if is_create is False (by default) + and vice versa + """ + mode = self.read_mode(remote) + self.device.log.debug("Remote file %s mode is %d" % (remote, mode)) + if mode == 0: + raise HdcError("Remote object doesn't exist!") + + if str(mode).startswith("168"): + if is_create: + remote_file_split = os.path.split(remote)[-1] \ + if os.path.split(remote)[-1] else os.path.split(remote)[-2] + remote_file_basename = os.path.basename(remote_file_split) + new_local = os.path.join(local, remote_file_basename) + create_dir(new_local) + else: + new_local = local + + collect_receiver = CollectingOutputReceiver() + HdcHelper.execute_shell_command(self.device, "ls %s" % remote, + receiver=collect_receiver) + files = collect_receiver.output.split() + for file_name in files: + self.pull_file("%s/%s" % (remote, file_name), + new_local, is_create=True) + elif mode == SPECIAL_FILE_MODE: + self.device.log.info("skipping special file '%s'" % remote) + else: + if os.path.isdir(local): + local = os.path.join(local, os.path.basename(remote)) + + self.do_pull_file(remote, local) + + def do_pull_file(self, remote, local): + """ + Pulls a remote file + """ + self.device.log.info( + "%s pull %s to %s" % (convert_serial(self.device.device_sn), + remote, local)) + remote_path_content = remote.encode(DEFAULT_ENCODING) + if len(remote_path_content) > REMOTE_PATH_MAX_LENGTH: + raise HdcError("Remote path is too long.") + + msg = self.create_file_req(ID_RECV, remote_path_content) + HdcHelper.write(self.sock, msg) + pull_result = HdcHelper.read(self.sock, DATA_UNIT_LENGTH * 2) + if not self.check_result(pull_result, ID_DATA) and \ + not self.check_result(pull_result, ID_DONE): + raise HdcError(self.read_error_message(pull_result)) + if platform.system() == "Windows": + flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_BINARY + else: + flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND + pulled_file_open = os.open(local, flags, FilePermission.mode_755) + with os.fdopen(pulled_file_open, "wb") as pulled_file: + while True: + if self.check_result(pull_result, ID_DONE): + break + + if not self.check_result(pull_result, ID_DATA): + raise HdcError(self.read_error_message(pull_result)) + + try: + length = self.swap32bit_from_array( + pull_result, DEFAULT_OFFSET_OF_INT) + except IndexError as index_error: + self.device.log.debug("do_pull_file: %s" % + str(pull_result)) + if pull_result == ID_DATA: + pull_result = self.sock.recv(DATA_UNIT_LENGTH) + self.device.log.debug( + "do_pull_file: %s" % str(pull_result)) + length = self.swap32bit_from_array(pull_result, 0) + self.device.log.debug("do_pull_file: %s" % str(length)) + else: + raise IndexError(str(index_error)) + + if length > SYNC_DATA_MAX: + raise HdcError("Receiving too much data.") + + pulled_file.write(HdcHelper.read(self.sock, length)) + pulled_file.flush() + pull_result = self.sock.recv(DATA_UNIT_LENGTH * 2) + + def push_file(self, local, remote, is_create=False): + """ + Push a single file. + The top directory won't be created if is_create is False (by default) + and vice versa + """ + if not os.path.exists(local): + raise HdcError("Local path doesn't exist.") + + if os.path.isdir(local): + if is_create: + local_file_split = os.path.split(local)[-1] \ + if os.path.split(local)[-1] else os.path.split(local)[-2] + local_file_basename = os.path.basename(local_file_split) + remote = "{}/{}".format( + remote, local_file_basename) + HdcHelper.execute_shell_command( + self.device, "mkdir -p %s" % remote) + + for child in os.listdir(local): + file_path = os.path.join(local, child) + if os.path.isdir(file_path): + self.push_file( + file_path, "%s/%s" % (remote, child), + is_create=False) + else: + self.do_push_file(file_path, "%s/%s" % (remote, child)) + else: + self.do_push_file(local, remote) + + def do_push_file(self, local, remote): + """ + Push a single file + + Args: + ------------ + local : string + the local file to push + remote : string + the remote file (length max is 1024) + """ + mode = self.read_mode(remote) + self.device.log.debug("Remote file %s mode is %d" % (remote, mode)) + if self.device.usb_type == DeviceConnectorType.hdc: + self.device.log.debug("%s execute command: hdc push %s %s" % ( + convert_serial(self.device.device_sn), local, remote)) + if str(mode).startswith("168"): + remote = "%s/%s" % (remote, os.path.basename(local)) + + try: + try: + remote_path_content = remote.encode(DEFAULT_ENCODING) + except UnicodeEncodeError: + remote_path_content = remote.encode("UTF-8") + if len(remote_path_content) > REMOTE_PATH_MAX_LENGTH: + raise HdcError("Remote path is too long.") + + # create the header for the action + msg = self.create_send_file_req(ID_SEND, remote_path_content, + FilePermission.mode_644) + + # and send it. We use a custom try/catch block to make the + # difference between file and network IO exceptions. + HdcHelper.write(self.sock, msg) + flags = os.O_RDONLY + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(local, flags, modes), "rb") as test_file: + while True: + if platform.system() == "Linux": + data = test_file.read(1024 * 4) + else: + data = test_file.read(SYNC_DATA_MAX) + + if not data: + break + + buf = struct.pack( + "%ds%ds%ds" % (len(ID_DATA), FORMAT_BYTES_LENGTH, + len(data)), ID_DATA, + self.swap32bits_to_bytes(len(data)), data) + self.sock.send(buf) + except Exception as exception: + self.device.log.error("exception %s" % exception) + raise exception + + msg = self.create_req(ID_DONE, int(time.time())) + HdcHelper.write(self.sock, msg) + result = HdcHelper.read(self.sock, DATA_UNIT_LENGTH * 2) + if not self.check_result(result, ID_OKAY): + self.device.log.error("exception %s" % result) + raise HdcError(self.read_error_message(result)) + + def read_mode(self, path): + """ + Returns the mode of the remote file. + Return an Integer containing the mode if all went well or null + """ + msg = self.create_file_req(ID_STAT, path) + HdcHelper.write(self.sock, msg) + + # read the result, in a byte array containing 4 ints + stat_result = HdcHelper.read(self.sock, DATA_UNIT_LENGTH * 4) + if not self.check_result(stat_result, ID_STAT): + return INVALID_MODE_CODE + + return self.swap32bit_from_array(stat_result, DEFAULT_OFFSET_OF_INT) + + def create_file_req(self, command, path): + """ + Creates the data array for a file request. This creates an array with a + 4 byte command + the remote file name. + + Args: + ------------ + command : + the 4 byte command (ID_STAT, ID_RECV, ...) + path : string + The path, as a byte array, of the remote file on which to execute + the command. + + return: + ------------ + return the byte[] to send to the device through hdc + """ + if isinstance(path, str): + try: + path = path.encode(DEFAULT_ENCODING) + except UnicodeEncodeError as _: + path = path.encode("UTF-8") + + return struct.pack( + "%ds%ds%ds" % (len(command), FORMAT_BYTES_LENGTH, len(path)), + command, self.swap32bits_to_bytes(len(path)), path) + + def create_send_file_req(self, command, path, mode=0o644): + # make the mode into a string + mode_str = ",%s" % str(mode & FilePermission.mode_777) + mode_content = mode_str.encode(DEFAULT_ENCODING) + return struct.pack( + "%ds%ds%ds%ds" % (len(command), FORMAT_BYTES_LENGTH, len(path), + len(mode_content)), + command, self.swap32bits_to_bytes(len(path) + len(mode_content)), + path, mode_content) + + def create_req(self, command, value): + """ + Create a command with a code and an int values + """ + return struct.pack("%ds%ds" % (len(command), FORMAT_BYTES_LENGTH), + command, self.swap32bits_to_bytes(value)) + + @staticmethod + def check_result(result, code): + """ + Checks the result array starts with the provided code + + Args: + ------------ + result : + the result array to check + path : string + the 4 byte code + + return: + ------------ + bool + return true if the code matches + """ + return result[0:4] == code[0:4] + + def read_error_message(self, result): + """ + Reads an error message from the opened Socket. + + Args: + ------------ + result : + the current hdc result. Must contain both FAIL and the length of + the message. + """ + if self.check_result(result, ID_FAIL): + length = self.swap32bit_from_array(result, 4) + if length > 0: + return str(HdcHelper.read(self.sock, length)) + + return None + + @staticmethod + def swap32bits_to_bytes(value): + """ + Swaps an unsigned value around, and puts the result in an bytes that + can be sent to a device. + + Args: + ------------ + value : + the value to swap. + """ + return bytes([value & 0x000000FF, + (value & 0x0000FF00) >> 8, + (value & 0x00FF0000) >> 16, + (value & 0xFF000000) >> 24]) + + @staticmethod + def swap32bit_from_array(value, offset): + """ + Reads a signed 32 bit integer from an array coming from a device. + + Args: + ------------ + value : + the array containing the int + offset: + the offset in the array at which the int starts + + Return: + ------------ + int + the integer read from the array + """ + result = 0 + result |= (int(value[offset])) & 0x000000FF + result |= (int(value[offset + 1]) & 0x000000FF) << 8 + result |= (int(value[offset + 2]) & 0x000000FF) << 16 + result |= (int(value[offset + 3]) & 0x000000FF) << 24 + + return result + + +class HdcHelper: + @staticmethod + def push_file(device, local, remote, is_create=False, + timeout=DEFAULT_TIMEOUT): + if device.usb_type == DeviceConnectorType.hdc: + device.log.info("%s execute command: hdc file send %s %s" % + (convert_serial(device.device_sn), local, remote)) + if not os.path.exists(local): + raise HdcError("Local path doesn't exist.") + sock = HdcHelper.socket(host=device.host, port=device.port, + timeout=DEFAULT_TIMEOUT) + HdcHelper.handle_shake(sock, device.device_sn) + local_end = os.path.split(local)[-1] + remote_end = os.path.split(remote)[-1] + if not local_end == remote_end: + remote = os.path.join(remote, local_end) + request = HdcHelper.form_hdc_request("file send %s %s" % + (local, remote)) + HdcHelper.write(sock, request) + reply = HdcHelper.read(sock, 4) + length = struct.unpack("!I", reply)[0] + data_buf = HdcHelper.read(sock, length) + data_str = HdcHelper.reply_to_string(data_buf) + + @staticmethod + def pull_file(device, remote, local, is_create=False, + timeout=DEFAULT_TIMEOUT): + if device.usb_type == DeviceConnectorType.hdc: + device.log.info("%s execute command: hdc file recv %s to %s" % + (convert_serial(device.device_sn), remote, local)) + + if not os.path.exists(local): + raise HdcError("Local path doesn't exist.") + sock = HdcHelper.socket(host=device.host, port=device.port, + timeout=DEFAULT_TIMEOUT) + HdcHelper.handle_shake(sock, device.device_sn) + local_end = os.path.split(local)[-1] + remote_end = os.path.split(remote)[-1] + if not local_end == remote_end: + remote = os.path.join(remote, local_end) + request = HdcHelper.form_hdc_request("file recv %s %s" % + (local, remote)) + HdcHelper.write(sock, request) + reply = HdcHelper.read(sock, 4) + length = struct.unpack("!I", reply)[0] + data_buf = HdcHelper.read(sock, length) + data_str = HdcHelper.reply_to_string(data_buf) + + @staticmethod + def _install_remote_package(device, remote_file_path, command): + receiver = CollectingOutputReceiver() + cmd = "pm install %s %s" % (command.strip(), remote_file_path) + HdcHelper.execute_shell_command(device, cmd, INSTALL_TIMEOUT, receiver) + return receiver.output + + @staticmethod + def install_package(device, package_file_path, command): + device.log.info("%s install %s" % (convert_serial(device.device_sn), + package_file_path)) + remote_file_path = "/data/local/tmp/%s" % os.path.basename( + package_file_path) + + result = HdcHelper._install_remote_package(device, remote_file_path, + command) + HdcHelper.execute_shell_command(device, "rm %s " % remote_file_path) + return result + + @staticmethod + def uninstall_package(device, package_name): + receiver = CollectingOutputReceiver() + command = "bm uninstall -n %s" % package_name + device.log.info("%s %s" % (convert_serial(device.device_sn), command)) + HdcHelper.execute_shell_command(device, command, INSTALL_TIMEOUT, + receiver) + return receiver.output + + @staticmethod + def reboot(device, into=None): + if device.usb_type == DeviceConnectorType.hdc: + device.log.info("%s execute command: hdc target boot" % + convert_serial(device.device_sn)) + with HdcHelper.socket(host=device.host, port=device.port) as sock: + HdcHelper.set_device(device, sock) + HdcHelper.handle_shake(sock, device.device_sn) + request = HdcHelper.form_hdc_request("reboot") + + HdcHelper.write(sock, request) + + @staticmethod + def execute_shell_command(device, command, timeout=DEFAULT_TIMEOUT, + receiver=None, **kwargs): + """ + Executes a shell command on the device and retrieve the output. + + Args: + ------------ + device : IDevice + on which to execute the command. + command : string + the shell command to execute + timeout : int + max time between command output. If more time passes between + command output, the method will throw + ShellCommandUnresponsiveException (ms). + """ + try: + if not timeout: + timeout = DEFAULT_TIMEOUT + with HdcHelper.socket(host=device.host, port=device.port, + timeout=timeout) as sock: + output_flag = kwargs.get("output_flag", True) + timeout_msg = '' if (timeout/1000) == 300.0 else \ + " with timeout %ss" % str(timeout/1000) + if device.usb_type == DeviceConnectorType.hdc: + message = "%s execute command: hdc shell %s%s" % \ + (convert_serial(device.device_sn), command, + timeout_msg) + if output_flag: + LOG.info(message) + else: + LOG.debug(message) + + HdcHelper.handle_shake(sock, device.device_sn) + request = HdcHelper.form_hdc_request("shell %s" % command) + HdcHelper.write(sock, request) + len_buf = HdcHelper.read(sock, DATA_UNIT_LENGTH) + if len_buf: + length = struct.unpack("!I", len_buf)[0] + resp = HdcResponse() + resp.okay = True + + from xdevice import Scheduler + data = sock.recv(SOCK_DATA_MAX) + while data != b'': + ret = HdcHelper.reply_to_string(data) + if ret: + if receiver: + receiver.__read__(ret) + else: + LOG.debug(ret) + + if not Scheduler.is_execute: + raise ExecuteTerminate() + data = HdcHelper.read(sock, SOCK_DATA_MAX) + + return resp + except socket.timeout as _: + device.log.error("%s shell %s timeout[%sS]" % ( + convert_serial(device.device_sn), command, str(timeout/1000))) + raise ShellCommandUnresponsiveException() + finally: + if receiver: + receiver.__done__() + + @staticmethod + def set_device(device, sock): + """ + tells hdc to talk to a specific device + if the device is not -1, then we first tell hdc we're looking to talk + to a specific device + """ + msg = "host:transport:%s" % device.device_sn + device_query = HdcHelper.form_hdc_request(msg) + HdcHelper.write(sock, device_query) + resp = HdcHelper.read_hdc_response(sock) + if not resp.okay: + raise HdcCommandRejectedException(resp.message) + + @staticmethod + def form_hdc_request(req): + """ + Create an ASCII string preceded by four hex digits. + """ + try: + req = req.encode("utf-8") + fmt = "!I%ss" % len(req) + result = struct.pack(fmt, len(req), req) + except UnicodeEncodeError as ex: + LOG.error(ex) + raise ex + return result + + @staticmethod + def read_hdc_response(sock, read_diag_string=False): + """ + Reads the response from HDC after a command. + + Args: + ------------ + read_diag_string : + If true, we're expecting an OKAY response to be followed by a + diagnostic string. Otherwise, we only expect the diagnostic string + to follow a FAIL. + """ + resp = HdcResponse() + reply = HdcHelper.read(sock, DATA_UNIT_LENGTH) + if HdcHelper.is_okay(reply): + resp.okay = True + else: + read_diag_string = True + resp.okay = False + + while read_diag_string: + len_buf = HdcHelper.read(sock, DATA_UNIT_LENGTH) + len_str = HdcHelper.reply_to_string(len_buf) + msg = HdcHelper.read(sock, int(len_str, HEXADECIMAL_NUMBER)) + resp.message = HdcHelper.reply_to_string(msg) + break + + return resp + + @staticmethod + def write(sock, req, timeout=5): + if isinstance(req, str): + req = req.encode(DEFAULT_ENCODING) + elif isinstance(req, list): + req = bytes(req) + + start_time = time.time() + while req: + if time.time() - start_time > timeout: + LOG.debug("Socket write timeout, timeout:%ss" % timeout) + break + size = sock.send(req) + if size < 0: + raise DeviceError("channel EOF") + + req = req[size:] + time.sleep(5 / 1000) + + @staticmethod + def read(sock, length, timeout=5): + data = b'' + recv_len = 0 + start_time = time.time() + exc_num = 3 + while length - recv_len > 0: + if time.time() - start_time > timeout: + LOG.debug("socket read timeout, timeout:%ss" % timeout) + break + try: + recv = sock.recv(length - recv_len) + if len(recv) > 0: + time.sleep(5 / 1000) + else: + break + except ConnectionResetError as error: + if exc_num <= 0: + raise error + exc_num = exc_num - 1 + recv = b'' + time.sleep(1) + LOG.debug("ConnectionResetError occurs") + + data += recv + recv_len += len(recv) + + return data + + @staticmethod + def is_okay(reply): + """ + Checks to see if the first four bytes in "reply" are OKAY. + """ + return reply[0:4] == ID_OKAY + + @staticmethod + def reply_to_string(reply): + """ + Converts an HDC reply to a string. + """ + try: + return str(reply, encoding=DEFAULT_ENCODING) + except (ValueError, TypeError) as _: + return "" + + @staticmethod + def socket(host=None, port=None, timeout=None): + end = time.time() + 10 * 60 + sock = None + + while host not in HdcMonitor.MONITOR_MAP or \ + HdcMonitor.MONITOR_MAP[host].main_hdc_connection is None: + LOG.debug("Host: %s, port: %s, HdcMonitor map is %s" % ( + host, port, HdcMonitor.MONITOR_MAP)) + if host in HdcMonitor.MONITOR_MAP: + LOG.debug("Monitor main hdc connection is %s" % + HdcMonitor.MONITOR_MAP[host].main_hdc_connection) + if time.time() > end: + raise HdcError("Cannot detect HDC monitor!") + time.sleep(2) + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, int(port))) + except socket.error as exception: + LOG.exception("Connect hdc server error: %s" % str(exception), + exc_info=False) + raise exception + + if sock is None: + raise HdcError("Cannot connect hdc server!") + + if timeout is not None: + sock.setblocking(False) + sock.settimeout(timeout/1000) + + return sock + + @staticmethod + def handle_shake(connection, connect_key=""): + reply = HdcHelper.read(connection, 40) + reply_size = struct.unpack(">I12s24s", reply) + banner_str = b'OHOS HDC' + connect_key = connect_key.encode("utf-8") + size = struct.calcsize('12s24s') + fmt = "!I12s24s" + pack_cmd = struct.pack(fmt, size, banner_str, connect_key) + HdcHelper.write(connection, pack_cmd) + return True + + +class DeviceConnector(object): + __instance = None + __init_flag = False + + def __init__(self, host=None, port=None, usb_type=None): + if DeviceConnector.__init_flag: + return + self.device_listeners = [] + self.device_monitor = None + self.host = host if host else "127.0.0.1" + self.usb_type = usb_type + if port: + self.port = int(port) + elif usb_type == DeviceConnectorType.hdc: + self.port = int(os.getenv("HDC_SERVER_PORT", 8710)) + + def start(self): + self.device_monitor = HdcMonitor.get_instance( + self.host, self.port, device_connector=self) + self.device_monitor.start() + + def terminate(self): + if self.device_monitor: + self.device_monitor.stop() + self.device_monitor = None + + def add_device_change_listener(self, device_change_listener): + self.device_listeners.append(device_change_listener) + + def remove_device_change_listener(self, device_change_listener): + if device_change_listener in self.device_listeners: + self.device_listeners.remove(device_change_listener) + + def device_connected(self, device): + LOG.debug("DeviceConnector device_connected:host %s, port %s, " + "device_sn %s " % (self.host, self.port, device.device_sn)) + if device.host != self.host or device.port != self.port: + LOG.debug("DeviceConnector device error") + for listener in self.device_listeners: + listener.device_connected(device) + + def device_disconnected(self, device): + LOG.debug("DeviceConnector device_disconnected:host %s, port %s, " + "device_sn %s" % (self.host, self.port, device.device_sn)) + if device.host != self.host or device.port != self.port: + LOG.debug("DeviceConnector device error") + for listener in self.device_listeners: + listener.device_disconnected(device) + + def device_changed(self, device): + LOG.debug("DeviceConnector device_changed:host %s, port %s, " + "device_sn %s" % (self.host, self.port, device.device_sn)) + if device.host != self.host or device.port != self.port: + LOG.debug("DeviceConnector device error") + for listener in self.device_listeners: + listener.device_changed(device) + + +class CollectingOutputReceiver(IShellReceiver): + def __init__(self): + self.output = "" + + def __read__(self, output): + self.output = "%s%s" % (self.output, output) + + def __error__(self, message): + pass + + def __done__(self, result_code="", message=""): + pass + + +class DisplayOutputReceiver(IShellReceiver): + def __init__(self): + self.output = "" + self.unfinished_line = "" + + def _process_output(self, output, end_mark="\n"): + content = output + if self.unfinished_line: + content = "".join((self.unfinished_line, content)) + self.unfinished_line = "" + lines = content.split(end_mark) + if content.endswith(end_mark): + # get rid of the tail element of this list contains empty str + return lines[:-1] + else: + self.unfinished_line = lines[-1] + # not return the tail element of this list contains unfinished str, + # so we set position -1 + return lines[:-1] + + def __read__(self, output): + self.output = "%s%s" % (self.output, output) + lines = self._process_output(output) + for line in lines: + line = line.strip() + if line: + LOG.info(line) + + def __error__(self, message): + pass + + def __done__(self, result_code="", message=""): + pass + + +def process_command_ret(ret, receiver): + try: + if ret != "" and receiver: + receiver.__read__(ret) + receiver.__done__() + except Exception as _: + LOG.exception("Error generating log report.", exc_info=False) + raise ReportException() + + if ret != "" and not receiver: + lines = ret.split("\n") + for line in lines: + line = line.strip() + if line: + LOG.debug(line) diff --git a/extension/src/xdevice_extension/_core/environment/emulator.py b/extension/src/xdevice_extension/_core/environment/emulator.py new file mode 100644 index 0000000..501350a --- /dev/null +++ b/extension/src/xdevice_extension/_core/environment/emulator.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from xdevice import IDevice +from xdevice import platform_logger + +from xdevice_extension._core.environment.device_state import \ + DeviceAllocationState +from xdevice_extension._core.environment.device_state import TestDeviceState + +LOG = platform_logger("Emulator") + + +class Emulator(IDevice): + """ + Class representing an emulator. + + Each object of this class represents one emulator in xDevice. + + Attributes: + device_sn: A string that's the serial number of the emulator. + """ + + def __get_serial__(self): + pass + + def __set_serial__(self, device_sn=""): + pass + + def __init__(self, device_sn=""): + self.device_sn = device_sn + self.is_timeout = False + self.device_log_proc = None + self.test_device_state = TestDeviceState.ONLINE + self.device_allocation_state = DeviceAllocationState.available + + def __serial__(self): + return self.device_sn + + def get_device_sn(self): + """ + Returns the serial number of the device. + """ + return self.device_sn diff --git a/extension/src/xdevice_extension/_core/environment/manager_device.py b/extension/src/xdevice_extension/_core/environment/manager_device.py new file mode 100644 index 0000000..ee6e7a2 --- /dev/null +++ b/extension/src/xdevice_extension/_core/environment/manager_device.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import threading + +from xdevice import UserConfigManager +from xdevice import ManagerType +from xdevice import Plugin +from xdevice import get_plugin +from xdevice import IDeviceManager +from xdevice import platform_logger +from xdevice import ParamError +from xdevice import ConfigConst + +from xdevice_extension._core.environment.device_monitor import \ + DeviceStateMonitor +from xdevice_extension._core.environment.device_state import DeviceEvent +from xdevice_extension._core.environment.device_state import TestDeviceState +from xdevice_extension._core.environment.device_state import DeviceState +from xdevice_extension._core.environment.device_state import \ + handle_allocation_event +from xdevice_extension._core.environment.device_state import \ + DeviceAllocationState +from xdevice_extension._core.environment.dmlib import DeviceConnector +from xdevice_extension._core.utils import convert_serial +from xdevice_extension._core.utils import convert_ip +from xdevice_extension._core.utils import convert_port +from xdevice_extension._core.constants import DeviceConnectorType +from xdevice_extension._core.exception import HdcCommandRejectedException + +__all__ = ["ManagerDevice"] + +LOG = platform_logger("ManagerDevice") + + +@Plugin(type=Plugin.MANAGER, id=ManagerType.device) +class ManagerDevice(IDeviceManager): + """ + Class representing device manager + managing the set of available devices for testing + """ + + def __init__(self): + self.devices_list = [] + self.global_device_filter = None + self.lock_con = threading.Condition() + self.list_con = threading.Condition() + self.device_connector = None + self.managed_device_listener = None + self.support_labels = ["phone", "watch", "car", "tv", "tablet", "ivi"] + self.support_types = ["device"] + + def init_environment(self, environment="", user_config_file=""): + self._start_device_monitor(environment, user_config_file) + + def env_stop(self): + self._stop_device_monitor() + + def _start_device_monitor(self, environment="", user_config_file=""): + self.managed_device_listener = ManagedDeviceListener(self) + device = UserConfigManager( + config_file=user_config_file, env=environment).get_device( + "environment/device") + if device: + try: + self.device_connector = DeviceConnector(device.get("ip"), + device.get("port"), + device.get("usb_type")) + self.global_device_filter = UserConfigManager( + config_file=user_config_file, env=environment).get_sn_list( + device.get("sn")) + self.device_connector.add_device_change_listener( + self.managed_device_listener) + self.device_connector.start() + except (ParamError, FileNotFoundError) as error: + self.env_stop() + LOG.debug("start %s error: %s" % ( + device.get("usb_type"), error)) + if device.get("usb_type") == DeviceConnectorType.hdc: + self.device_connector = DeviceConnector( + device.get("ip"), device.get("port"), + "usb-hdc") + self.device_connector.add_device_change_listener( + self.managed_device_listener) + self.device_connector.start() + else: + raise ParamError("Manager device is not supported, please " + "check config user_config.xml", error_no="00108") + + def _stop_device_monitor(self): + self.device_connector.remove_device_change_listener( + self.managed_device_listener) + self.device_connector.terminate() + + def find(self, idevice): + LOG.debug("find: apply list con lock") + self.list_con.acquire() + try: + for device in self.devices_list: + if device.device_sn == idevice.device_sn and \ + device.device_os_type == idevice.device_os_type: + return device + finally: + LOG.debug("find: release list con lock") + self.list_con.release() + + def apply_device(self, device_option, timeout=10): + + LOG.debug("apply_device: apply lock con lock") + self.lock_con.acquire() + try: + device = self.allocate_device_option(device_option) + if device: + return device + LOG.debug("wait for available device founded") + self.lock_con.wait(timeout) + return self.allocate_device_option(device_option) + finally: + LOG.debug("apply_device: release lock con lock") + self.lock_con.release() + + def allocate_device_option(self, device_option): + """ + Request a device for testing that meets certain criteria. + """ + + LOG.debug("allocate_device_option: apply list con lock") + if not self.list_con.acquire(timeout=5): + LOG.debug("allocate_device_option: list con wait timeout") + return None + try: + allocated_device = None + + for device in self.devices_list: + if device_option.matches(device): + self.handle_device_event(device, + DeviceEvent.ALLOCATE_REQUEST) + LOG.debug("allocate device sn: %s, type: %s" % ( + device.__get_serial__(), device.__class__)) + return device + return allocated_device + + finally: + LOG.debug("allocate_device_option: release list con lock") + self.list_con.release() + + def release_device(self, device): + LOG.debug("release_device: apply list con lock") + self.list_con.acquire() + try: + if device.test_device_state == TestDeviceState.ONLINE: + self.handle_device_event(device, DeviceEvent.FREE_AVAILABLE) + else: + self.handle_device_event(device, DeviceEvent.FREE_UNAVAILABLE) + + device.device_id = None + + LOG.debug("free device sn: %s, type: %s" % ( + device.__get_serial__(), device.__class__.__name__)) + + finally: + LOG.debug("release_device: release list con lock") + self.list_con.release() + + def find_device(self, device_sn, device_os_type): + for device in self.devices_list: + if device.device_sn == device_sn and \ + device.device_os_type == device_os_type: + return device + + def append_device_by_sort(self, device_instance): + if (not self.global_device_filter or + not self.devices_list or + device_instance.device_sn not in self.global_device_filter): + self.devices_list.append(device_instance) + else: + device_dict = dict(zip( + self.global_device_filter, + list(range(1, len(self.global_device_filter)+1)))) + for index in range(len(self.devices_list)): + if self.devices_list[index].device_sn not in \ + self.global_device_filter: + self.devices_list.insert(index, device_instance) + break + if device_dict[device_instance.device_sn] < \ + device_dict[self.devices_list[index].device_sn]: + self.devices_list.insert(index, device_instance) + break + else: + self.devices_list.append(device_instance) + + def find_or_create(self, idevice): + LOG.debug("find_or_create: apply list con lock") + self.list_con.acquire() + try: + device = self.find_device(idevice.device_sn, + idevice.device_os_type) + if device is None: + device = get_plugin( + plugin_type=Plugin.DEVICE, + plugin_id=idevice.device_os_type)[0] + device_instance = device.__class__() + device_instance.__set_serial__(idevice.device_sn) + device_instance.host = idevice.host + device_instance.port = idevice.port + device_instance.usb_type = self.device_connector.usb_type + LOG.debug("create device(%s) host is %s, " + "port is %s, device sn is %s, usb type is %s" % + (device_instance, device_instance.host, + device_instance.port, device_instance.device_sn, + device_instance.usb_type)) + device_instance.device_state = DeviceState.get_state( + idevice.device_state) + device_instance.test_device_state = \ + TestDeviceState.get_test_device_state( + device_instance.device_state) + device_instance.device_state_monitor = \ + DeviceStateMonitor(device_instance) + if idevice.device_state == DeviceState.ONLINE: + device_instance.get_device_type() + self.append_device_by_sort(device_instance) + device = device_instance + else: + LOG.debug("find device(%s), host is %s, " + "port is %s, device sn is %s, usb type is %s" % + (device, device.host, device.port, device.device_sn, + device.usb_type)) + return device + except HdcCommandRejectedException as hcr_error: + LOG.debug("%s occurs error. Reason:%s" % + (idevice.device_sn, hcr_error)) + finally: + LOG.debug("find_or_create: release list con lock") + self.list_con.release() + + def remove(self, idevice): + LOG.debug("remove: apply list con lock") + self.list_con.acquire() + try: + self.devices_list.remove(idevice) + finally: + LOG.debug("remove: release list con lock") + self.list_con.release() + + def handle_device_event(self, device, event): + state_changed = None + old_state = device.device_allocation_state + new_state = handle_allocation_event(old_state, event) + + if new_state == DeviceAllocationState.checking_availability: + if self.global_device_filter and \ + device.device_sn not in self.global_device_filter: + event = DeviceEvent.AVAILABLE_CHECK_IGNORED + else: + event = DeviceEvent.AVAILABLE_CHECK_PASSED + new_state = handle_allocation_event(new_state, event) + + if old_state != new_state: + state_changed = True + device.device_allocation_state = new_state + + if state_changed is True and \ + new_state == DeviceAllocationState.available: + # notify_device_state_change + LOG.debug("handle_device_event apply lock_con") + self.lock_con.acquire() + LOG.debug("find available device") + self.lock_con.notify_all() + LOG.debug("handle_device_event release lock_con") + self.lock_con.release() + + if device.device_allocation_state == \ + DeviceAllocationState.unknown: + self.remove(device) + return + + def launch_emulator(self): + pass + + def kill_emulator(self): + pass + + def list_devices(self): + print("devices:") + print("{0:<20}{1:<16}{2:<16}{3:<16}{4:<16}{5:<16}{6:<16}".format( + "Serial", "OsType", "State", "Allocation", "Product", "host", + "port")) + for device in self.devices_list: + print("{0:<20}{1:<16}{2:<16}{3:<16}{4:<16}{5:<16}{6:<16}".format( + convert_serial(device.device_sn), device.device_os_type, + device.test_device_state.value, + device.device_allocation_state, + device.label if device.label else 'None', + convert_ip(device.host), convert_port(device.port))) + + +class ManagedDeviceListener(object): + """ + A class to listen for and act on device presence updates from ddmlib + """ + + def __init__(self, manager): + self.manager = manager + + def device_changed(self, idevice): + test_device = self.manager.find_or_create(idevice) + if test_device is None: + return + new_state = TestDeviceState.get_test_device_state(idevice.device_state) + test_device.test_device_state = new_state + if new_state == TestDeviceState.ONLINE: + self.manager.handle_device_event(test_device, + DeviceEvent.STATE_CHANGE_ONLINE) + elif new_state == TestDeviceState.NOT_AVAILABLE: + self.manager.handle_device_event(test_device, + DeviceEvent.STATE_CHANGE_OFFLINE) + test_device.device_state_monitor.set_state( + test_device.test_device_state) + LOG.debug("device changed to %s: %s %s %s %s" % ( + new_state, convert_serial(idevice.device_sn), + idevice.device_os_type, idevice.host, idevice.port)) + + def device_connected(self, idevice): + test_device = self.manager.find_or_create(idevice) + if test_device is None: + return + new_state = TestDeviceState.get_test_device_state(idevice.device_state) + test_device.test_device_state = new_state + if test_device.test_device_state == TestDeviceState.ONLINE: + self.manager.handle_device_event(test_device, + DeviceEvent.CONNECTED_ONLINE) + elif new_state == TestDeviceState.NOT_AVAILABLE: + self.manager.handle_device_event(test_device, + DeviceEvent.CONNECTED_OFFLINE) + test_device.device_state_monitor.set_state( + test_device.test_device_state) + LOG.debug("device connected: %s %s %s %s" % ( + convert_serial(idevice.device_sn), idevice.device_os_type, + idevice.host, idevice.port)) + LOG.debug("set device %s %s to true" % ( + idevice.device_sn, ConfigConst.recover_state)) + test_device.set_recover_state(True) + + def device_disconnected(self, disconnected_device): + test_device = self.manager.find(disconnected_device) + if test_device is not None: + test_device.test_device_state = TestDeviceState.NOT_AVAILABLE + self.manager.handle_device_event(test_device, + DeviceEvent.DISCONNECTED) + test_device.device_state_monitor.set_state( + TestDeviceState.NOT_AVAILABLE) + LOG.debug("device disconnected: %s %s %s %s" % ( + convert_serial(disconnected_device.device_sn), + disconnected_device.device_os_type, + disconnected_device.host, disconnected_device.port)) diff --git a/extension/src/xdevice_extension/_core/exception.py b/extension/src/xdevice_extension/_core/exception.py new file mode 100644 index 0000000..25a9e6b --- /dev/null +++ b/extension/src/xdevice_extension/_core/exception.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from xdevice import DeviceError + + +class HdcError(DeviceError): + """ + Raised when there is an error in hdc operations. + """ + + def __init__(self, error_msg, error_no=""): + super(HdcError, self).__init__(error_msg, error_no) + self.error_msg = error_msg + self.error_no = error_no + + def __str__(self): + return str(self.error_msg) + + +class HdcCommandRejectedException(HdcError): + """ + Exception thrown when hdc refuses a command. + """ + + def __init__(self, error_msg, error_no=""): + super(HdcCommandRejectedException, self).__init__(error_msg, error_no) + self.error_msg = error_msg + self.error_no = error_no + + def __str__(self): + return str(self.error_msg) + + +class ShellCommandUnresponsiveException(HdcError): + """ + Exception thrown when a shell command executed on a device takes too long + to send its output. + """ + def __init__(self, error_msg="ShellCommandUnresponsiveException", + error_no=""): + super(ShellCommandUnresponsiveException, self).\ + __init__(error_msg, error_no) + self.error_msg = error_msg + self.error_no = error_no + + def __str__(self): + return str(self.error_msg) + + +class DeviceUnresponsiveException(HdcError): + """ + Exception thrown when a shell command executed on a device takes too long + to send its output. + """ + def __init__(self, error_msg="DeviceUnresponsiveException", error_no=""): + super(DeviceUnresponsiveException, self).__init__(error_msg, error_no) + self.error_msg = error_msg + self.error_no = error_no + + def __str__(self): + return str(self.error_msg) + + +class AppInstallError(DeviceError): + def __init__(self, error_msg, error_no=""): + super(AppInstallError, self).__init__(error_msg, error_no) + self.error_msg = error_msg + self.error_no = error_no + + def __str__(self): + return str(self.error_msg) + + +class HapNotSupportTest(DeviceError): + def __init__(self, error_msg, error_no=""): + super(HapNotSupportTest, self).__init__(error_msg, error_no) + self.error_msg = error_msg + self.error_no = error_no + + def __str__(self): + return str(self.error_msg) diff --git a/extension/src/xdevice_extension/_core/executor/__init__.py b/extension/src/xdevice_extension/_core/executor/__init__.py new file mode 100644 index 0000000..eb8b49d --- /dev/null +++ b/extension/src/xdevice_extension/_core/executor/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/extension/src/xdevice_extension/_core/executor/listener.py b/extension/src/xdevice_extension/_core/executor/listener.py new file mode 100644 index 0000000..1e8dd58 --- /dev/null +++ b/extension/src/xdevice_extension/_core/executor/listener.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from xdevice import Plugin +from xdevice import LifeCycle +from xdevice import IListener +from xdevice import platform_logger +from xdevice import TestDescription + +from xdevice_extension._core.constants import ListenerType + +__all__ = ["CollectingTestListener", "CollectingLiteGTestListener"] + +LOG = platform_logger("Listener") + + +@Plugin(type=Plugin.LISTENER, id=ListenerType.collect) +class CollectingTestListener(IListener): + """ + listener test status information to the console + """ + + def __init__(self): + self.tests = [] + + def __started__(self, lifecycle, test_result): + if lifecycle == LifeCycle.TestCase: + if not test_result.test_class or not test_result.test_name: + return + test = TestDescription(test_result.test_class, + test_result.test_name) + if test not in self.tests: + self.tests.append(test) + + def __ended__(self, lifecycle, test_result=None, **kwargs): + pass + + def __skipped__(self, lifecycle, test_result): + pass + + def __failed__(self, lifecycle, test_result): + pass + + def get_current_run_results(self): + return self.tests + + +@Plugin(type=Plugin.LISTENER, id=ListenerType.collect_lite) +class CollectingLiteGTestListener(IListener): + """ + listener test status information to the console + """ + + def __init__(self): + self.tests = [] + + def __started__(self, lifecycle, test_result): + pass + + def __ended__(self, lifecycle, test_result=None, **kwargs): + del kwargs + if lifecycle == LifeCycle.TestCase: + if not test_result.test_class or not test_result.test_name: + return + test = TestDescription(test_result.test_class, + test_result.test_name) + if test not in self.tests: + self.tests.append(test) + + def __skipped__(self, lifecycle, test_result): + pass + + def __failed__(self, lifecycle, test_result): + if lifecycle == LifeCycle.TestCase: + if not test_result.test_class or not test_result.test_name: + return + test = TestDescription(test_result.test_class, + test_result.test_name) + if test not in self.tests: + self.tests.append(test) + + def get_current_run_results(self): + return self.tests diff --git a/extension/src/xdevice_extension/_core/testkit/__init__.py b/extension/src/xdevice_extension/_core/testkit/__init__.py new file mode 100644 index 0000000..eb8b49d --- /dev/null +++ b/extension/src/xdevice_extension/_core/testkit/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/extension/src/xdevice_extension/_core/testkit/kit.py b/extension/src/xdevice_extension/_core/testkit/kit.py new file mode 100644 index 0000000..0546717 --- /dev/null +++ b/extension/src/xdevice_extension/_core/testkit/kit.py @@ -0,0 +1,960 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import platform +import re +import signal +import subprocess +import zipfile +import stat +import time +import json +from dataclasses import dataclass +from tempfile import TemporaryDirectory +from tempfile import NamedTemporaryFile + +from xdevice import ITestKit +from xdevice import platform_logger +from xdevice import Plugin +from xdevice import ParamError +from xdevice import get_file_absolute_path +from xdevice import get_config_value +from xdevice import exec_cmd + +from xdevice_extension._core.constants import CKit +from xdevice_extension._core.environment.dmlib import CollectingOutputReceiver +from xdevice_extension._core.utils import check_path_legal +from xdevice_extension._core.utils import modify_props +from xdevice_extension._core.exception import AppInstallError +from xdevice_extension._core.constants import DeviceConnectorType +from xdevice_extension._core.utils import convert_serial + + +__all__ = ["STSKit", "PushKit", "PropertyCheckKit", "ShellKit", "WifiKit", + "ConfigKit", "AppInstallKit", "junit_para_parse", + "gtest_para_parse", "junit_dex_para_parse", "reset_junit_para"] + +LOG = platform_logger("Kit") + + +@Plugin(type=Plugin.TEST_KIT, id=CKit.sts) +class STSKit(ITestKit): + def __init__(self): + self.sts_version = "" + self.throw_error = "" + + def __check_config__(self, config): + self.sts_version = get_config_value('sts-version', config) + self.throw_error = get_config_value('throw-error', config) + if len(self.sts_version) < 1: + raise TypeError( + "The sts_version: {} is invalid".format(self.sts_version)) + + def __setup__(self, device, **kwargs): + del kwargs + LOG.debug("STSKit setup, device:{}, params:{}". + format(device, self.get_plugin_config().__dict__)) + device_spl = device.get_property(Props.security_patch) + if device_spl is None or device_spl == "": + LOG.error("The device security {} is invalid".format(device_spl)) + raise ParamError( + "The device security patch version {} is invalid".format( + device_spl)) + rex = '^[a-zA-Z\\d\\.]+_([\\d]+-[\\d]+)$' + match = re.match(rex, self.sts_version) + if match is None: + LOG.error("The sts version {} does match the rule".format( + self.sts_version)) + raise ParamError("The sts version {} does match the rule".format( + self.sts_version)) + sts_version_date_user = match.group(1).join("-01") + sts_version_date_kernel = match.group(1).join("-05") + if device_spl in [sts_version_date_user, sts_version_date_kernel]: + LOG.info( + "The device SPL version {} match the sts version {}".format( + device_spl, self.sts_version)) + else: + err_msg = "The device SPL version {} does not match the sts " \ + "version {}".format(device_spl, self.sts_version) + LOG.error(err_msg) + raise ParamError(err_msg) + + def __teardown__(self, device): + LOG.debug("STSKit teardown: device:{}, params:{}".format + (device, self.get_plugin_config().__dict__)) + + +@Plugin(type=Plugin.TEST_KIT, id=CKit.push) +class PushKit(ITestKit): + def __init__(self): + self.pre_push = "" + self.push_list = "" + self.post_push = "" + self.is_uninstall = "" + self.paths = "" + self.pushed_file = [] + self.abort_on_push_failure = True + + def __check_config__(self, config): + self.pre_push = get_config_value('pre-push', config) + self.push_list = get_config_value('push', config) + self.post_push = get_config_value('post-push', config) + self.is_uninstall = get_config_value('uninstall', config, + is_list=False, default=True) + self.abort_on_push_failure = get_config_value( + 'abort-on-push-failure', config, is_list=False, default=True) + if isinstance(self.abort_on_push_failure, str): + self.abort_on_push_failure = False if \ + self.abort_on_push_failure.lower() == "false" else True + + self.paths = get_config_value('paths', config) + self.pushed_file = [] + + def __setup__(self, device, **kwargs): + del kwargs + LOG.debug("PushKit setup, device:{}".format(device.device_sn)) + for command in self.pre_push: + run_command(device, command) + for push_info in self.push_list: + files = re.split('->|=>', push_info) + if len(files) != 2: + LOG.error("The push spec is invalid: {}".format(push_info)) + continue + src = files[0].strip() + dst = files[1].strip() if files[1].strip().startswith("/") else \ + files[1].strip() + Props.dest_root + LOG.debug( + "Trying to push the file local {} to remote {}".format(src, + dst)) + + try: + real_src_path = get_file_absolute_path(src, self.paths) + except ParamError as error: + if self.abort_on_push_failure: + raise + else: + LOG.warning(error, error_no=error.error_no) + continue + remount(device) + device.push_file(real_src_path, dst) + if os.path.isdir(real_src_path): + self.add_pushed_dir(real_src_path, dst) + else: + if device.is_directory(dst): + self.pushed_file.append( + os.path.join(dst, os.path.basename(real_src_path))) + else: + self.pushed_file.append(dst) + LOG.debug("Push file finished from {} to {}".format(src, dst)) + for command in self.post_push: + run_command(device, command) + + def add_pushed_dir(self, src, dst): + for root, _, files in os.walk(src): + for file_path in files: + self.pushed_file.append( + os.path.join(root, file_path).replace(src, dst)) + + def __teardown__(self, device): + LOG.debug("PushKit teardown: device:{}".format(device.device_sn)) + if self.is_uninstall: + remount(device) + for file_name in self.pushed_file: + LOG.debug("Trying to remove file {}".format(file_name)) + file_name = file_name.replace("\\", "/") + + for _ in range( + Props.trying_remove_maximum_times): + collect_receiver = CollectingOutputReceiver() + file_name = check_path_legal(file_name) + device.execute_shell_command("rm -rf {}".format( + file_name), receiver=collect_receiver, + output_flag=False) + if not collect_receiver.output: + LOG.debug( + "Removed file {} successfully".format(file_name)) + break + else: + LOG.error("Removed file {} successfully". + format(collect_receiver.output)) + else: + LOG.error("Failed to remove file {}".format(file_name)) + + def __add_pushed_file__(self, device, src, dst): + if device.is_directory(dst): + dst = dst + os.path.basename(src) if dst.endswith( + "/") else dst + "/" + os.path.basename(src) + self.pushed_file.append(dst) + + def __add_dir_pushed_files__(self, device, src, dst): + if device.file_exist(device, dst): + for _, dirs, files in os.walk(src): + for file_path in files: + if dst.endswith("/"): + dst = "%s%s" % (dst, os.path.basename(file_path)) + else: + dst = "%s/%s" % (dst, os.path.basename(file_path)) + self.pushed_file.append(dst) + for dir_name in dirs: + self.__add_dir_pushed_files__(device, dir_name, dst) + else: + self.pushed_file.append(dst) + + +@Plugin(type=Plugin.TEST_KIT, id=CKit.propertycheck) +class PropertyCheckKit(ITestKit): + def __init__(self): + self.prop_name = "" + self.expected_value = "" + self.throw_error = "" + + def __check_config__(self, config): + self.prop_name = get_config_value('property-name', config, + is_list=False) + self.expected_value = get_config_value('expected-value', config, + is_list=False) + self.throw_error = get_config_value('throw-error', config, + is_list=False) + + def __setup__(self, device, **kwargs): + del kwargs + LOG.debug("PropertyCheckKit setup, device:{}".format(device.device_sn)) + if not self.prop_name: + LOG.warning("The option of property-name not setting") + return + prop_value = device.get_property(self.prop_name) + if not prop_value: + LOG.warning( + "The property {} not found on device, cannot check the value". + format(self.prop_name)) + return + + if prop_value != self.expected_value: + msg = "The value found for property {} is {}, not same with the " \ + "expected {}".format(self.prop_name, prop_value, + self.expected_value) + LOG.warning(msg) + if self.throw_error and self.throw_error.lower() == 'true': + raise Exception(msg) + + @classmethod + def __teardown__(cls, device): + LOG.debug("PropertyCheckKit teardown: device:{}".format( + device.device_sn)) + + +@Plugin(type=Plugin.TEST_KIT, id=CKit.shell) +class ShellKit(ITestKit): + def __init__(self): + self.command_list = [] + self.tear_down_command = [] + self.paths = None + + def __check_config__(self, config): + self.command_list = get_config_value('run-command', config) + self.tear_down_command = get_config_value('teardown-command', config) + self.paths = get_config_value('paths', config) + + def __setup__(self, device, **kwargs): + del kwargs + LOG.debug("ShellKit setup, device:{}".format(device.device_sn)) + if len(self.command_list) == 0: + LOG.info("No setup_command to run, skipping!") + return + for command in self.command_list: + run_command(device, command) + + def __teardown__(self, device): + LOG.debug("ShellKit teardown: device:{}".format(device.device_sn)) + if len(self.tear_down_command) == 0: + LOG.info("No teardown_command to run, skipping!") + return + for command in self.tear_down_command: + run_command(device, command) + + +@Plugin(type=Plugin.TEST_KIT, id=CKit.wifi) +class WifiKit(ITestKit): + def __init__(self): + pass + + def __check_config__(self, config): + self.certfilename = get_config_value( + 'certfilename', config, False, + default=None) + self.certpassword = get_config_value( + 'certpassword', config, False, + default=None) + self.wifiname = get_config_value( + 'wifiname', config, False, + default=None) + + def __setup__(self, device, **kwargs): + request = kwargs.get("request", None) + if not request: + LOG.error("WifiKit need input request") + return + testargs = request.get("testargs", {}) + self.certfilename = \ + testargs.pop("certfilename", [self.certfilename])[0] + self.wifiname = \ + testargs.pop("wifiname", [self.wifiname])[0] + self.certpassword = \ + testargs.pop("certpassword", [self.certpassword])[0] + del kwargs + LOG.debug("WifiKit setup, device:{}".format(device.device_sn)) + + try: + wifi_app_path = get_file_absolute_path( + Props.Paths.service_wifi_app_path) + except ParamError: + wifi_app_path = None + + if wifi_app_path is None: + LOG.error("The resource wifi app file does not exist!") + return + + try: + pfx_path = get_file_absolute_path( + "tools/wifi/%s" % self.certfilename + ) if self.certfilename else None + except ParamError: + pfx_path = None + + if pfx_path is None: + LOG.error("The resource wifi pfx file does not exist!") + return + pfx_dest_path = \ + "/storage/emulated/0/%s" % self.certfilename + if self.wifiname is None: + LOG.error("The wifi name is not given!") + return + if self.certpassword is None: + LOG.error("The wifi password is not given!") + return + + device.install_package(wifi_app_path, command="-r") + device.push_file(pfx_path, pfx_dest_path) + device.execute_shell_command("svc wifi enable") + for _ in range(Props.maximum_connect_wifi_times): + connect_wifi_cmd = Props.connect_wifi_cmd % ( + pfx_dest_path, + self.certpassword, + self.wifiname + ) + if device.execute_shell_command(connect_wifi_cmd): + LOG.info("Connect wifi successfully") + break + else: + LOG.error("Connect wifi failed") + + @classmethod + def __teardown__(cls, device): + LOG.debug("WifiKit teardown: device:{}".format(device.device_sn)) + LOG.info("Disconnect wifi") + device.execute_shell_command("svc wifi disable") + + +@dataclass +class Props: + @dataclass + class Paths: + system_build_prop_path = "/%s/%s" % ("system", "build.prop") + service_wifi_app_path = "tools/wifi/%s" % "Service-wifi.app" + + dest_root = "/%s/%s/" % ("data", "data") + mnt_external_storage = "EXTERNAL_STORAGE" + trying_remove_maximum_times = 3 + maximum_connect_wifi_times = 3 + connect_wifi_cmd = "am instrument -e request \"{module:Wifi, " \ + "method:connectWifiByCertificate, params:{'certPath':" \ + "'%s'," \ + "'certPassword':'%s'," \ + "'wifiName':'%s'}}\" " \ + "-w com.xdeviceservice.service/.MainInstrumentation" + security_patch = "ro.build.version.security_patch" + + +@Plugin(type=Plugin.TEST_KIT, id=CKit.config) +class ConfigKit(ITestKit): + def __init__(self): + self.is_connect_wifi = "" + self.is_disconnect_wifi = "" + self.wifi_kit = WifiKit() + self.min_external_store_space = "" + self.is_disable_dialing = "" + self.is_test_harness = "" + self.is_audio_silent = "" + self.is_disable_dalvik_verifier = "" + self.build_prop_list = "" + self.is_enable_hook = "" + self.cust_prop_file = "" + self.is_prop_changed = False + self.local_system_prop_file = "" + self.cust_props = "" + self.is_reboot_delay = "" + self.is_remount = "" + self.local_cust_prop_file = {} + + def __check_config__(self, config): + self.is_connect_wifi = get_config_value('connect-wifi', config, + is_list=False, default=False) + self.is_disconnect_wifi = get_config_value( + 'disconnect-wifi-after-test', config, is_list=False, default=True) + self.wifi_kit = WifiKit() + self.min_external_store_space = get_config_value( + 'min-external-store-space', config) + self.is_disable_dialing = get_config_value('disable-dialing', config) + self.is_test_harness = get_config_value('set-test-harness', config) + self.is_audio_silent = get_config_value('audio-silent', config) + self.is_disable_dalvik_verifier = get_config_value( + 'disable-dalvik-verifier', config) + self.build_prop_list = get_config_value('build-prop', config) + self.cust_prop_file = get_config_value('cust-prop-file', config) + self.cust_props = get_config_value('cust-prop', config) + self.is_enable_hook = get_config_value('enable-hook', config) + self.is_reboot_delay = get_config_value('reboot-delay', config) + self.is_remount = get_config_value('remount', config, default=True) + self.local_system_prop_file = NamedTemporaryFile(prefix='build', + suffix='.prop', + delete=False).name + + def __setup__(self, device, **kwargs): + del kwargs + LOG.debug("ConfigKit setup, device:{}".format(device.device_sn)) + if self.is_remount: + remount(device) + self.is_prop_changed = self.modify_system_prop(device) + self.is_prop_changed = self.modify_cust_prop( + device) or self.is_prop_changed + + keep_screen_on(device) + if self.is_enable_hook: + pass + if self.is_prop_changed: + device.reboot() + + def __teardown__(self, device): + LOG.debug("ConfigKit teardown: device:{}".format(device.device_sn)) + if self.is_remount: + remount(device) + if self.is_connect_wifi and self.is_disconnect_wifi: + self.wifi_kit.__teardown__(device) + if self.is_prop_changed: + device.push_file(self.local_system_prop_file, + Props.Paths.system_build_prop_path) + device.execute_shell_command( + " ".join(["chmod 644", Props.Paths.system_build_prop_path])) + os.remove(self.local_system_prop_file) + + for target_file, temp_file in self.local_cust_prop_file.items(): + device.push_file(temp_file, target_file) + device.execute_shell_command( + " ".join(["chmod 644", target_file])) + os.remove(temp_file) + + def modify_system_prop(self, device): + prop_changed = False + new_props = {} + if self.is_disable_dialing: + new_props['ro.telephony.disable-call'] = 'true' + if self.is_test_harness: + new_props['ro.monkey'] = '1' + new_props['ro.test_harness'] = '1' + if self.is_audio_silent: + new_props['ro.audio.silent'] = '1' + if self.is_disable_dalvik_verifier: + new_props['dalvik.vm.dexopt-flags'] = 'v=n' + for prop in self.build_prop_list: + if prop is None or prop.find("=") < 0 or len(prop.split("=")) != 2: + LOG.warning("The build prop:{} not match the format " + "'key=value'".format(prop)) + continue + new_props[prop.split("=")[0]] = prop.split("=")[1] + if new_props: + prop_changed = modify_props(device, self.local_system_prop_file, + Props.Paths.system_build_prop_path, + new_props) + return prop_changed + + def modify_cust_prop(self, device): + prop_changed = False + cust_files = {} + new_props = {} + for cust_prop_file in self.cust_prop_file: + # the correct format should be "CustName:/cust/prop/absolutepath" + if len(cust_prop_file.split(":")) != 2: + LOG.error( + "The value %s of option cust-prop-file is incorrect" % + cust_prop_file) + continue + cust_files[cust_prop_file.split(":")[0]] = \ + cust_prop_file.split(":")[1] + for prop in self.cust_props: + # the correct format should be "CustName:key=value" + prop_infos = re.split(r'[:|=]', prop) + if len(prop_infos) != 3: + LOG.error( + "The value {} of option cust-prop is incorrect".format( + prop)) + continue + file_name, key, value = prop_infos + if file_name not in cust_files: + LOG.error( + "The custName {} must be in cust-prop-file option".format( + file_name)) + continue + props = new_props.setdefault(file_name, {}) + props[key] = value + + for name in new_props.keys(): + cust_file = cust_files.get(name) + temp_cust_file = NamedTemporaryFile(prefix='cust', suffix='.prop', + delete=False).name + self.local_cust_prop_file[cust_file] = temp_cust_file + prop_changed = modify_props(device, temp_cust_file, cust_file, + new_props[name]) or prop_changed + return prop_changed + + +@Plugin(type=Plugin.TEST_KIT, id=CKit.appinstall) +class AppInstallKit(ITestKit): + def __init__(self): + self.app_list = "" + self.is_clean = "" + self.alt_dir = "" + self.ex_args = "" + self.installed_app = [] + self.paths = "" + self.is_pri_app = "" + self.pushed_hap_file = [] + + def __check_config__(self, options): + self.app_list = get_config_value('test-file-name', options) + self.is_clean = get_config_value('cleanup-apps', options, False) + self.alt_dir = get_config_value('alt-dir', options, False) + if self.alt_dir and self.alt_dir.startswith("resource/"): + self.alt_dir = self.alt_dir[len("resource/"):] + self.ex_args = get_config_value('install-arg', options) + self.installed_app = [] + self.paths = get_config_value('paths', options) + self.is_pri_app = get_config_value('install-as-privapp', options, + False, default=False) + + def __setup__(self, device, **kwargs): + del kwargs + LOG.debug("AppInstallKit setup, device:{}".format(device.device_sn)) + if len(self.app_list) == 0: + LOG.info("No app to install, skipping!") + return + for app in self.app_list: + if self.alt_dir: + app_file = get_file_absolute_path(app, self.paths, + self.alt_dir) + else: + app_file = get_file_absolute_path(app, self.paths) + if app_file is None: + LOG.error("The app file {} does not exist".format(app)) + continue + if app_file.endswith(".hap"): + self.install_hap(device, app_file) + else: + result = device.install_package( + app_file, get_install_args( + device, app_file, self.ex_args)) + if not result.startswith("Success"): + raise AppInstallError( + "Failed to install %s on %s. Reason:%s" % + (app_file, device.__get_serial__(), result)) + self.installed_app.append(app_file) + + def __teardown__(self, device): + LOG.debug("AppInstallKit teardown: device:{}".format(device.device_sn)) + if self.is_clean and str(self.is_clean).lower() == "true": + for app in self.installed_app: + app_name = get_app_name(app) + + if app_name: + result = device.uninstall_package(app_name) + if not result.startswith("Success"): + LOG.warning("error uninstalling package %s %s" % + (device.__get_serial__(), result)) + else: + LOG.warning("Can't find app_name for %s" % app) + if self.is_pri_app: + remount(device) + for pushed_file in self.pushed_hap_file: + device.execute_shell_command("rm -r %s" % pushed_file) + + def install_hap(self, device, hap_file): + if self.is_pri_app: + LOG.info("Install hap as privileged app {}".format(hap_file)) + hap_name = os.path.basename(hap_file).replace(".hap", "") + try: + with TemporaryDirectory(prefix=hap_name) as temp_dir: + zif_file = zipfile.ZipFile(hap_file) + zif_file.extractall(path=temp_dir) + entry_app = os.path.join(temp_dir, "Entry.app") + push_dest_dir = os.path.join("/system/priv-app/", hap_name) + device.execute_shell_command("rm -rf " + push_dest_dir, + output_flag=False) + device.push_file(entry_app, os.path.join( + push_dest_dir + os.path.basename(entry_app))) + device.push_file(hap_file, os.path.join( + push_dest_dir + os.path.basename(hap_file))) + self.pushed_hap_file.append(os.path.join( + push_dest_dir + os.path.basename(hap_file))) + device.reboot() + except RuntimeError as exception: + msg = "Install hap app failed with error {}".format(exception) + LOG.error(msg) + raise Exception(msg) + except Exception as exception: + msg = "Install hap app failed with exception {}".format( + exception) + LOG.error(msg) + raise Exception(msg) + finally: + zif_file.close() + else: + # push_dest = "/%s" % "sdcard" + # push_dest = "%s/%s" % (push_dest, os.path.basename(hap_file)) + # device.push_file(hap_file, push_dest) + # self.pushed_hap_file.append(push_dest) + # output = device.execute_shell_command("bm install -p " + push_dest) + output = device.hdc_command("install %s" % hap_file) + # if not output.startswith("Success"): + # output = output.strip() + # if "[ERROR_GET_BUNDLE_INSTALLER_FAILED]" not in output.upper(): + # raise AppInstallError( + # "Failed to install %s on %s. Reason:%s" % + # (push_dest, device.__get_serial__(), output)) + # else: + # LOG.info("'[ERROR_GET_BUNDLE_INSTALLER_FAILED]' occurs, " + # "retry install hap") + # exec_out = self.retry_install_hap( + # device, "bm install -p " + push_dest) + # if not exec_out.startswith("Success"): + # raise AppInstallError( + # "Retry failed,Can't install %s on %s. Reason:%s" % + # (push_dest, device.__get_serial__(), exec_out)) + # else: + LOG.debug("Install %s, and output = %s" % (hap_file, output)) + + @classmethod + def retry_install_hap(cls, device, command): + if device.usb_type == DeviceConnectorType.hdc: + # hdc -t UID -s tcp:IP:PORT + real_command = ["hdc", "-t", str(device.device_sn), "-s", + "tcp:%s:%s" % (str(device.host), str(device.port)), + "shell", command] + message = "%s execute command: %s" % \ + (convert_serial(device.device_sn), " ".join(real_command)) + LOG.info(message) + exec_out = "" + for wait_count in range(1, 4): + LOG.debug("retry times:%s, wait %ss" % + (wait_count, (wait_count * 10))) + time.sleep(wait_count * 10) + exec_out = exec_cmd(real_command) + if exec_out and exec_out.startswith("Success"): + break + if not exec_out: + exec_out = "System is not in %s" % ["Windows", "Linux", "Darwin"] + LOG.info("retry install hap result is: [%s]" % exec_out.strip()) + return exec_out + + +def remount(device): + device.enable_hdc_root() + cmd = "target mount" \ + if device.usb_type == DeviceConnectorType.hdc else "remount" + device.hdc_command(cmd) + device.execute_shell_command("remount") + device.execute_shell_command("mount -o rw,remount /cust") + device.execute_shell_command("mount -o rw,remount /product") + device.execute_shell_command("mount -o rw,remount /hw_product") + device.execute_shell_command("mount -o rw,remount /version") + device.execute_shell_command("mount -o rw,remount /%s" % "system") + + +def keep_screen_on(device): + device.execute_shell_command("svc power stayon true") + + +def get_install_args(device, app_name, original_args=None): + """To obtain all the args of app install + Args: + original_args: the argus configure in .config file + device : the device will be installed app + app_name : the name of the app which will be installed + Returns: + All the args + """ + if original_args is None: + original_args = [] + new_args = original_args[:] + try: + sdk_version = device.get_property("ro.build.version.sdk") + if int(sdk_version) > 22: + new_args.append("-g") + except TypeError as type_error: + LOG.error("Obtain the sdk version failed with exception {}".format( + type_error)) + except ValueError as value_error: + LOG.error("Obtain the sdk version failed with exception {}".format( + value_error)) + if app_name.endswith(".apex"): + new_args.append("--apex") + return " ".join(new_args) + + +def run_command(device, command): + LOG.debug("The command:{} is running".format(command)) + stdout = None + if command.strip() == "remount": + remount(device) + elif command.strip() == "reboot": + device.reboot() + elif command.strip() == "reboot-delay": + pass + else: + stdout = device.execute_shell_command(command) + LOG.debug("Run command result: %s" % (stdout if stdout else "")) + return stdout + + +def junit_para_parse(device, junit_paras, prefix_char="-e"): + """To parse the para of junit + Args: + device: the device running + junit_paras: the para dict of junit + prefix_char: the prefix char of parsed cmd + Returns: + the new para using in a command like -e testFile xxx + -e coverage true... + """ + ret_str = [] + path = "/%s/%s/%s/%s" % ("data", "local", "tmp", "ajur") + include_file = "%s/%s" % (path, "includes.txt") + exclude_file = "%s/%s" % (path, "excludes.txt") + + if not isinstance(junit_paras, dict): + LOG.warning("The para of junit is not the dict format as required") + return "" + # Disable screen keyguard + disable_key_guard = junit_paras.get('disable-keyguard') + if not disable_key_guard or disable_key_guard[0].lower() != 'false': + from xdevice_extension._core.driver.drivers import disable_keyguard + disable_keyguard(device) + + for para_name in junit_paras.keys(): + path = "/%s/%s/%s/%s/" % ("data", "local", "tmp", "ajur") + if para_name.strip() == 'test-file-include-filter': + for file_name in junit_paras[para_name]: + device.push_file(file_name, include_file) + device.execute_shell_command( + 'chown -R shell:shell %s' % path) + ret_str.append(" ".join([prefix_char, 'testFile', include_file])) + elif para_name.strip() == "test-file-exclude-filter": + for file_name in junit_paras[para_name]: + device.push_file(file_name, include_file) + device.execute_shell_command( + 'chown -R shell:shell %s' % path) + ret_str.append(" ".join([prefix_char, 'notTestFile', + exclude_file])) + elif para_name.strip() == "test" or para_name.strip() == "class": + result = _get_class(junit_paras, prefix_char, para_name.strip()) + ret_str.append(result) + elif para_name.strip() == "include-annotation": + ret_str.append(" ".join([prefix_char, "annotation", + ",".join(junit_paras[para_name])])) + elif para_name.strip() == "exclude-annotation": + ret_str.append(" ".join([prefix_char, "notAnnotation", + ",".join(junit_paras[para_name])])) + else: + ret_str.append(" ".join([prefix_char, para_name, + ",".join(junit_paras[para_name])])) + + return " ".join(ret_str) + + +def junit_dex_para_parse(device, junit_paras, prefix_char="--"): + """To parse the para of junit + Args: + device: the device running + junit_paras: the para dict of junit + prefix_char: the prefix char of parsed cmd + Returns: + the new para using in a command like -e testFile xxx + -e coverage true... + """ + ret_str = [] + path = "/%s/%s/%s/%s" % ("data", "local", "tmp", "ajur") + include_file = "%s/%s" % (path, "includes.txt") + exclude_file = "%s/%s" % (path, "excludes.txt") + + if not isinstance(junit_paras, dict): + LOG.warning("The para of junit is not the dict format as required") + return "" + # Disable screen keyguard + disable_key_guard = junit_paras.get('disable-keyguard') + if not disable_key_guard or disable_key_guard[0].lower() != 'false': + from xdevice_extension._core.driver.drivers import disable_keyguard + disable_keyguard(device) + + for para_name in junit_paras.keys(): + path = "/%s/%s/%s/%s/" % ("data", "local", "tmp", "ajur") + if para_name.strip() == 'test-file-include-filter': + for file_name in junit_paras[para_name]: + device.push_file(file_name, include_file) + device.execute_shell_command( + 'chown -R shell:shell %s' % path) + ret_str.append(prefix_char + " ".join(['testFile', include_file])) + elif para_name.strip() == "test-file-exclude-filter": + for file_name in junit_paras[para_name]: + device.push_file(file_name, include_file) + device.execute_shell_command( + 'chown -R shell:shell %s' % path) + ret_str.append(prefix_char + " ".join(['notTestFile', + exclude_file])) + elif para_name.strip() == "test" or para_name.strip() == "class": + result = _get_class(junit_paras, prefix_char, para_name.strip()) + ret_str.append(result) + elif para_name.strip() == "include-annotation": + ret_str.append(prefix_char + " ".join( + ['annotation', ",".join(junit_paras[para_name])])) + elif para_name.strip() == "exclude-annotation": + ret_str.append(prefix_char + " ".join( + ['notAnnotation', ",".join(junit_paras[para_name])])) + else: + ret_str.append(prefix_char + " ".join( + [para_name, ",".join(junit_paras[para_name])])) + + return " ".join(ret_str) + + +def timeout_callback(proc): + try: + LOG.error("Error: execute command timeout.") + LOG.error(proc.pid) + if platform.system() != "Windows": + os.killpg(proc.pid, signal.SIGKILL) + else: + subprocess.call( + ["C:\\Windows\\System32\\taskkill", "/F", "/T", "/PID", + str(proc.pid)], shell=False) + except (FileNotFoundError, KeyboardInterrupt, AttributeError) as error: + LOG.exception("timeout callback exception: %s" % error, exc_info=False) + + +def _get_class(junit_paras, prefix_char, para_name): + if not junit_paras.get(para_name): + return "" + result = "" + if prefix_char == "-e": + result = " %s class " % prefix_char + elif prefix_char == "--": + result = " %sclass " % prefix_char + elif prefix_char == "-s": + result = " %s class " % prefix_char + test_items = [] + for test in junit_paras.get(para_name): + test_item = test.split("#") + if len(test_item) == 1 or len(test_item) == 2: + test_item = "%s" % test + test_items.append(test_item) + elif len(test_item) == 3: + test_item = "%s#%s" % (test_item[1], test_item[2]) + test_items.append(test_item) + else: + raise ParamError("The parameter %s %s is error" % ( + prefix_char, para_name)) + if not result: + LOG.debug("there is unsolved prefix char: %s ." % prefix_char) + return result + ",".join(test_items) + + +def gtest_para_parse(gtest_paras, runner): + """To parse the para of gtest + Args: + gtest_paras: the para dict of gtest + Returns: + the new para using in gtest + """ + ret_str = [] + if not isinstance(gtest_paras, dict): + LOG.warning("The para of gtest is not the dict format as required") + return "" + for para in gtest_paras.keys(): + if para.strip() == 'test-file-include-filter': + case_list = [] + files = gtest_paras.get(para) + for case_file in files: + flags = os.O_RDONLY + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(case_file, flags, modes), + "r") as file_desc: + case_list.extend(file_desc.read().splitlines()) + + runner.add_instrumentation_arg("gtest_filter", ":".join(case_list)) + + return " ".join(ret_str) + + +def reset_junit_para(junit_para_str, prefix_char="-e", ignore_keys=None): + if not ignore_keys and not isinstance(ignore_keys, list): + ignore_keys = ["class", "test"] + lines = junit_para_str.split(prefix_char) + normal_lines = [] + for line in lines: + line = line.strip() + if line: + items = line.split() + if items[0].strip() in ignore_keys: + continue + normal_lines.append("{} {}".format(prefix_char, line)) + return " ".join(normal_lines) + + +def get_app_name(hap_app): + hap_name = os.path.basename(hap_app).replace(".hap", "") + app_name = "" + with TemporaryDirectory(prefix=hap_name) as temp_dir: + zif_file = zipfile.ZipFile(hap_app) + zif_file.extractall(path=temp_dir) + config_json_file = os.path.join(temp_dir, "config.json") + if not os.path.exists(config_json_file): + LOG.debug("config.json not in %s.hap" % hap_name) + else: + flags = os.O_RDONLY + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(config_json_file, flags, modes), + "r") as file_desc: + attrs = json.loads(file_desc.read()) + if "app" in attrs.keys() and \ + "bundleName" in attrs.get("app", dict()).keys(): + app_name = attrs["app"]["bundleName"] + LOG.info("obtain the app name {} from json " + "successfully".format(app_name)) + else: + LOG.debug("'app' or 'bundleName' not " + "in %s.hap/config.json" % hap_name) + zif_file.close() + return app_name diff --git a/extension/src/xdevice_extension/_core/utils.py b/extension/src/xdevice_extension/_core/utils.py new file mode 100644 index 0000000..42b3050 --- /dev/null +++ b/extension/src/xdevice_extension/_core/utils.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import platform +import subprocess +import signal +import stat +from tempfile import NamedTemporaryFile + +from xdevice import ShellHandler +from xdevice import platform_logger +from xdevice import get_plugin +from xdevice import Plugin + +LOG = platform_logger("Utils") + + +def get_filename_extension(file_path): + _, fullname = os.path.split(file_path) + filename, ext = os.path.splitext(fullname) + return filename, ext + + +def start_standing_subprocess(cmd, pipe=subprocess.PIPE, return_result=False): + """Starts a non-blocking subprocess that is going to continue running after + this function returns. + + A subprocess group is actually started by setting sid, so we can kill all + the processes spun out from the subprocess when stopping it. This is + necessary in case users pass in pipe commands. + + Args: + cmd: Command to start the subprocess with. + pipe: pipe to get execution result + return_result: return execution result or not + + Returns: + The subprocess that got started. + """ + sys_type = platform.system() + process = subprocess.Popen(cmd, stdout=pipe, shell=False, + preexec_fn=None if sys_type == "Windows" + else os.setsid) + if not return_result: + return process + else: + rev = process.stdout.read() + return rev.decode("utf-8").strip() + + +def stop_standing_subprocess(process): + """Stops a subprocess started by start_standing_subprocess. + + Catches and ignores the PermissionError which only happens on Macs. + + Args: + process: Subprocess to terminate. + """ + try: + sys_type = platform.system() + signal_value = signal.SIGINT if sys_type == "Windows" \ + else signal.SIGTERM + os.kill(process.pid, signal_value) + except (PermissionError, AttributeError, FileNotFoundError, + SystemError) as error: + LOG.error("stop standing subprocess error '%s'" % error) + + +def get_decode(stream): + if not isinstance(stream, str) and not isinstance(stream, bytes): + ret = str(stream) + else: + try: + ret = stream.decode("utf-8", errors="ignore") + except (ValueError, AttributeError, TypeError): + ret = str(stream) + return ret + + +def is_proc_running(pid, name=None): + if platform.system() == "Windows": + proc_sub = subprocess.Popen(["C:\\Windows\\System32\\tasklist"], + stdout=subprocess.PIPE, + shell=False) + proc = subprocess.Popen(["C:\\Windows\\System32\\findstr", "%s" % pid], + stdin=proc_sub.stdout, + stdout=subprocess.PIPE, shell=False) + else: + # /bin/ps -ef | /bin/grep -v grep | /bin/grep -w pid + proc_sub = subprocess.Popen(["/bin/ps", "-ef"], + stdout=subprocess.PIPE, + shell=False) + proc_v_sub = subprocess.Popen(["/bin/grep", "-v", "grep"], + stdin=proc_sub.stdout, + stdout=subprocess.PIPE, + shell=False) + proc = subprocess.Popen(["/bin/grep", "-w", "%s" % pid], + stdin=proc_v_sub.stdout, + stdout=subprocess.PIPE, shell=False) + (out, _) = proc.communicate() + out = get_decode(out).strip() + LOG.debug("check %s proc running output: %s", pid, out) + if out == "": + return False + else: + return True if name is None else out.find(name) != -1 + + +def create_dir(path): + """Creates a directory if it does not exist already. + + Args: + path: The path of the directory to create. + """ + full_path = os.path.abspath(os.path.expanduser(path)) + if not os.path.exists(full_path): + os.makedirs(full_path, exist_ok=True) + + +def modify_props(device, local_prop_file, target_prop_file, new_props): + """To change the props if need + Args: + device: the device to modify props + local_prop_file : the local file to save the old props + target_prop_file : the target prop file to change + new_props : the new props + Returns: + True : prop file changed + False : prop file no need to change + """ + is_changed = False + device.pull_file(target_prop_file, local_prop_file) + old_props = {} + changed_prop_key = [] + lines = [] + flags = os.O_RDONLY + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(local_prop_file, flags, modes), "r") as old_file: + lines = old_file.readlines() + if lines: + lines[-1] = lines[-1] + '\n' + for line in lines: + line = line.strip() + if not line.startswith("#") and line.find("=") > 0: + key_value = line.split("=") + if len(key_value) == 2: + old_props[line.split("=")[0]] = line.split("=")[1] + + for key, value in new_props.items(): + if key not in old_props.keys(): + lines.append("".join([key, "=", value, '\n'])) + is_changed = True + elif old_props.get(key) != value: + changed_prop_key.append(key) + is_changed = True + + if is_changed: + local_temp_prop_file = NamedTemporaryFile(mode='w', prefix='build', + suffix='.tmp', delete=False) + for index, line in enumerate(lines): + if not line.startswith("#") and line.find("=") > 0: + key = line.split("=")[0] + if key in changed_prop_key: + lines[index] = "".join([key, "=", new_props[key], '\n']) + local_temp_prop_file.writelines(lines) + local_temp_prop_file.close() + device.push_file(local_temp_prop_file.name, target_prop_file) + device.execute_shell_command(" ".join(["chmod 644", target_prop_file])) + LOG.info("Changed the system property as required successfully") + os.remove(local_temp_prop_file.name) + + return is_changed + + +def convert_ip(origin_ip): + addr = origin_ip.strip().split(".") + if len(addr) == 4: + return addr[0] + "." + '*'*len(addr[1]) + "." + '*'*len(addr[2]) + \ + "." + addr[-1] + else: + return origin_ip + + +def convert_port(port): + _port = str(port) + if len(_port) >= 2: + return "{}{}{}".format(_port[0], "*" * (len(_port) - 2), _port[-1]) + else: + return "*{}".format(_port[-1]) + + +def convert_serial(serial): + if serial.startswith("local_"): + return "local_" + '*'*(len(serial)-6) + elif serial.startswith("remote_"): + return "remote_" + convert_ip(serial.split("_")[1]) + else: + length = len(serial)//3 + return serial[0:length] + "*"*(len(serial)-length*2) + serial[-length:] + + +def get_shell_handler(request, parser_type): + suite_name = request.root.source.test_name + parsers = get_plugin(Plugin.PARSER, parser_type) + if parsers: + parsers = parsers[:1] + parser_instances = [] + for listener in request.listeners: + listener.device_sn = request.config.environment.devices[0].device_sn + for parser in parsers: + parser_instance = parser.__class__() + parser_instance.suite_name = suite_name + parser_instance.listeners = request.listeners + parser_instances.append(parser_instance) + handler = ShellHandler(parser_instances) + return handler + + +def check_path_legal(path): + if path and " " in path: + return "\"%s\"" % path + return path diff --git a/figures/icon-caution.gif b/figures/icon-caution.gif old mode 100755 new mode 100644 diff --git a/figures/icon-danger.gif b/figures/icon-danger.gif old mode 100755 new mode 100644 diff --git a/figures/icon-note.gif b/figures/icon-note.gif old mode 100755 new mode 100644 diff --git a/figures/icon-notice.gif b/figures/icon-notice.gif old mode 100755 new mode 100644 diff --git a/figures/icon-tip.gif b/figures/icon-tip.gif old mode 100755 new mode 100644 diff --git a/figures/icon-warning.gif b/figures/icon-warning.gif old mode 100755 new mode 100644 diff --git a/run.bat b/run.bat old mode 100755 new mode 100644 diff --git a/run.sh b/run.sh old mode 100755 new mode 100644 index ddcf955..5621043 --- a/run.sh +++ b/run.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env sh +#!/bin/bash # -# Copyright (c) 2020-2021 Huawei Device Co., Ltd. +# Copyright (C) 2020-2021 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +14,8 @@ # limitations under the License. # +set -e + error() { echo "$1" @@ -29,10 +31,10 @@ fi $PYTHON -c 'import sys; exit(1) if sys.version_info.major < 3 or sys.version_info.minor < 7 else exit(0)' || \ error "Python3.7 or higher version required!" -cd $(dirname "$0") || error "Failure to change direcory!" +cd $(dirname "$0") || error "Failure to change directory!" $PYTHON -c "import easy_install" || error "Please install setuptools first!" -if [ ! -d $TOOLS ]; then +if [ ! -d "$TOOLS" ]; then error "$TOOLS directory not exists" fi @@ -53,3 +55,4 @@ do done $PYTHON -m xdevice "$@" +exit 0 diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index c06bb1d..3b283c6 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def main(): setup(name='xdevice', description='xdevice test framework', url='', - package_dir={'': 'src', 'adapter': 'adapter'}, + package_dir={'': 'src'}, packages=['xdevice', 'xdevice._core', 'xdevice._core.build', diff --git a/src/xdevice/__main__.py b/src/xdevice/__main__.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/command/console.py b/src/xdevice/_core/command/console.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/config/config_manager.py b/src/xdevice/_core/config/config_manager.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/config/resource_manager.py b/src/xdevice/_core/config/resource_manager.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/constants.py b/src/xdevice/_core/constants.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/driver/device_test.py b/src/xdevice/_core/driver/device_test.py old mode 100755 new mode 100644 index f19475b..4a87450 --- a/src/xdevice/_core/driver/device_test.py +++ b/src/xdevice/_core/driver/device_test.py @@ -243,6 +243,10 @@ class DeviceTestDriver(IDriver): if configs["testcases_path"]: sys.path.insert(1, configs["testcases_path"]) + # apply data to devicetest module about resource path + request = configs.get('request', None) + if request: + sys.ecotest_resource_path = request.config.resource_path # run devicetest from _devicetest.devicetest.main import DeviceTest device_test = DeviceTest(test_list=test_list, configs=configs, diff --git a/src/xdevice/_core/driver/drivers_lite.py b/src/xdevice/_core/driver/drivers_lite.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/driver/parser_lite.py b/src/xdevice/_core/driver/parser_lite.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/environment/device_lite.py b/src/xdevice/_core/environment/device_lite.py old mode 100755 new mode 100644 index 1c48f34..b35189d --- a/src/xdevice/_core/environment/device_lite.py +++ b/src/xdevice/_core/environment/device_lite.py @@ -43,7 +43,8 @@ from _core.utils import check_mode LOG = platform_logger("DeviceLite") TIMEOUT = 90 RETRY_ATTEMPTS = 0 -HDC = "litehdc.exe" +HDC = "hdc_std.exe" +DEFAULT_BAUD_RATE = 115200 def get_hdc_path(): @@ -510,7 +511,7 @@ class ComController: self.com = None self.serial_port = device.get("com") if device.get("com") else None self.baud_rate = int(device.get("baud_rate")) if device.get( - "baud_rate") else 115200 + "baud_rate") else DEFAULT_BAUD_RATE self.timeout = int(device.get("timeout")) if device.get( "timeout") else TIMEOUT diff --git a/src/xdevice/_core/environment/dmlib_lite.py b/src/xdevice/_core/environment/dmlib_lite.py old mode 100755 new mode 100644 index 123c41d..db9fce8 --- a/src/xdevice/_core/environment/dmlib_lite.py +++ b/src/xdevice/_core/environment/dmlib_lite.py @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + import json import time import re @@ -25,6 +26,7 @@ from _core.exception import LiteDeviceTimeout from _core.exception import LiteDeviceConnectError from _core.exception import LiteDeviceExecuteCommandError from _core.logger import platform_logger +from _core.utils import convert_port __all__ = ["generate_report", "LiteHelper"] @@ -279,7 +281,7 @@ class LiteHelper: error_no="00402") LOG.info("local_%s execute command shell %s with timeout %ss" % - (com.port, command, str(timeout))) + (convert_port(com.port), command, str(timeout))) if isinstance(command, str): command = command.encode("utf-8") @@ -302,7 +304,8 @@ class LiteHelper: error_no="00402") LOG.info( - "local_%s execute command shell %s" % (com.port, command)) + "local_%s execute command shell %s" % (convert_port(com.port), + command)) command = command.encode("utf-8") if command[-2:] != b"\r\n": command = command.rstrip() + b'\r\n' @@ -340,8 +343,9 @@ class LiteHelper: elif "put" == command_type: url = base_url + target_save_path + command - data = open(local_source_dir + command, "rb") - response = requests.put(url=url, headers=headers, data=data) + with open(local_source_dir + command, "rb") as data: + response = requests.put(url=url, headers=headers, + data=data) if response.status_code == 200: LOG.info("{} upload file to {} " "success".format(local_source_dir, @@ -355,10 +359,10 @@ class LiteHelper: url = base_url + target_save_path + command response = requests.get(url=url, headers=headers, stream=True) if response.status_code == 200: - file_name = open(local_source_dir + command + "bak", "wb") - for file_data in response.iter_content(chunk_size=512): - file_name.write(file_data) - file_name.close() + with open(local_source_dir + command + "bak", "wb") \ + as file_name: + for file_data in response.iter_content(chunk_size=512): + file_name.write(file_data) LOG.info("from {} download file to {} success".format( target_save_path + command, local_source_dir)) diff --git a/src/xdevice/_core/environment/manager_env.py b/src/xdevice/_core/environment/manager_env.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/environment/manager_lite.py b/src/xdevice/_core/environment/manager_lite.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/exception.py b/src/xdevice/_core/exception.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/executor/concurrent.py b/src/xdevice/_core/executor/concurrent.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/executor/listener.py b/src/xdevice/_core/executor/listener.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/executor/request.py b/src/xdevice/_core/executor/request.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/executor/scheduler.py b/src/xdevice/_core/executor/scheduler.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/executor/source.py b/src/xdevice/_core/executor/source.py old mode 100755 new mode 100644 index 9c050de..3c0ad8b --- a/src/xdevice/_core/executor/source.py +++ b/src/xdevice/_core/executor/source.py @@ -383,6 +383,8 @@ def _get_test_type(config_file, test_driver, ext): if ext in [".py", ".js", ".dex", ".hap", ".bin"] \ and ext in EXT_TYPE_DICT.keys(): test_type = EXT_TYPE_DICT[ext] + elif ext in EXT_TYPE_DICT.keys(): + test_type = DeviceTestType.hap_test else: test_type = DeviceTestType.cpp_test return test_type diff --git a/src/xdevice/_core/interface.py b/src/xdevice/_core/interface.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/logger.py b/src/xdevice/_core/logger.py old mode 100755 new mode 100644 index 12c3c59..fa3988a --- a/src/xdevice/_core/logger.py +++ b/src/xdevice/_core/logger.py @@ -378,7 +378,8 @@ class EncryptFileHandler(RotatingFileHandler): # baseFilename is the attribute in FileHandler base_file_name = getattr(self, "baseFilename", None) - return open(base_file_name, self.mode) + with open(base_file_name, self.mode) as encrypt_file: + return encrypt_file def emit(self, record): try: diff --git a/src/xdevice/_core/plugin.py b/src/xdevice/_core/plugin.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/report/__main__.py b/src/xdevice/_core/report/__main__.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/report/encrypt.py b/src/xdevice/_core/report/encrypt.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/report/reporter_helper.py b/src/xdevice/_core/report/reporter_helper.py index 1aaf751..020acca 100755 --- a/src/xdevice/_core/report/reporter_helper.py +++ b/src/xdevice/_core/report/reporter_helper.py @@ -551,8 +551,8 @@ class VisionHelper: if not os.path.exists(self.template_name): LOG.error("template file not exists") return "" - - file_context = open(self.template_name).read() + with open(self.template_name) as file_temp: + file_context = file_temp.read() file_context = self._render_key("", ReportConstant.title_name, title_name, file_context) file_context = self._render_exec_info(file_context, exec_info) diff --git a/src/xdevice/_core/report/suite_reporter.py b/src/xdevice/_core/report/suite_reporter.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/resource/config/user_config.xml b/src/xdevice/_core/resource/config/user_config.xml old mode 100755 new mode 100644 index 5439585..61b89cb --- a/src/xdevice/_core/resource/config/user_config.xml +++ b/src/xdevice/_core/resource/config/user_config.xml @@ -1,66 +1,63 @@ - - - - - - - - cmd - 115200 - 8 - 1 - 20 - - - - deploy - 115200 - - - - - - cmd - 115200 - 8 - 1 - 1 - - - - - - - - - - - - - - - - - - - - - - INFO - + + + + + + + + cmd + 115200 + 8 + 1 + 20 + + + + deploy + 115200 + + + + + + cmd + 115200 + 8 + 1 + 1 + + + + + + + + + + + + + + + + + + + + + + INFO + \ No newline at end of file diff --git a/src/xdevice/_core/resource/template/report.html b/src/xdevice/_core/resource/template/report.html index 8f72eb4..fc59b37 100644 --- a/src/xdevice/_core/resource/template/report.html +++ b/src/xdevice/_core/resource/template/report.html @@ -26,7 +26,6 @@ } div.logo { - background-image: url('data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAeAB4AAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAEAAAAAAAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAAdAFADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9+JjkfXFfI/j7/gop8TPhZ4M1bxJ4i/Zh+IGj6DodtJe319c+J9EWO3hQFmc4uCeg6AEk8AEkCvriVhs/zxXyd+0BpTft2ftMWvwniLS/DD4d3FvrPjyRG/da1eg+ZZ6RkcMikCWYcj5Qp2sor0stp051H7aKcUrtu+iXazWr2Xm0cmMnOMV7N2k9Ftq/n0W78j0T4fftT+I/ifovwb1jR/hb4kbQfijZS6hq17c3kELeDoxbrNAbiM/NL5xO1dmMdTyQK9qjO4dDg45r5t/aqa7h/bk/Zhjt7q6t7RtQ8QCeCKVo4ZgNOXaHQHawHbIOO1ef/C/9ni2/bl+M3xi8WePPEXjaOTwn4xuPCfhS10bxFeaVH4Yis44v9LgWB1H2iR33s8gfJUDG3CjT6hCdNV2+WNr6Xb1m4pb+Xlou++X1qUZeztd3t2Xwpt/1fU9t/Yj/AGl9U/ak8CeLtW1bSbHR5vDvjbWvDEMVrK8izw2N00McrbuQ7qMsBwD0r2gPsHQ9K+B/2XtN8PeBf2B/iLY/EDxl4q0jTP8AhZWs29/qmh3Mljq+tTC/AMUZtgZQ906kMkG1sSMFZRyHfADT9N+D37evw70/wD4Y+K/gDwv430fV7fWtH8XXdzJa6nJbRxzwXUEdxcTssqElXY7CFkUY+d89OKymPtqqpOyjzW00fKrvW+9ttH52uZUcfJQhz7ytfXXV227fM+9hJz07Zo87FfC/wZ/ZS0/9q747/H+38d+IvGl/4Z0nxzcR6doNprt1YWkM8ltDuumMLq8rKixpHG5MUeHIQs5I5e1+OXxC8Of8E2fD2gx+IvFF/wCIdS+Ij/Dhtfs5d+vPp4vpkLwSSN/x9tDF5KSM4IZ1berAOIWSubUKc05XinpZLmV1r1tbXT0uX/aPKuaUdLSt/wBuu3yv0P0SEnqCKcDmvg/4UfC7VPhZ+0j8PdQ+FXw5+N3g3Rbq9ex8a23ifVGvNL1OyeFtt04lvZyLmKUKwdApIZhyCVP3ZG22uDG4RUHG0rpq/S61as0m7d99rHVh8R7VO6tb+tNEYMlp4mKH/TtDU9iLKXI/OXH518kfBT9mL4z/AA88N31h4R+JE2l282oz3d/HNplhJcz3kjZllmeWJ5Hkc87ix4AA4AA+2GTcKrQ6RDBfSXCrtmlUCRh/Hjpn3HrV4PMJ4eMoxjF81t0ntfv6k4jCRqtNtq3Z23Pnq9+CvjTU/iD8GdV8T+I7TVPEHhGbVHkun09V+0tNAU/eCJkThdoGxV6c5OTXNax+zp48tPi9421T4d+MLzwzpvia6WXxJb2dnAVurzYFeSzM5YwzFfvP3bngBMfVU+mQ3F1DNIu6S3JMbf3cjBos9NhsvM8tdvnyGR/dj1P6VdPNKkNUls1aytrJy220b07GcsDGW7e97312tv8An3PkGw/ZM+w/syz+G7PULi3+y+NG1zQbjyGN9p9+Jg0EjNJIytt5DeYGJycnODXReGfgF8QP+GkvBXijxp4tGu+IdFh1CK1c2CRabHDLb7CsUUbKVcnJdmJLER4O1Qo+mBoFqLOSAwo0Mzl3RhuDMTkmm2/h61t7uOZIyskYIQli23IwcZJrWWcVWpJ297m1a195Wdn0v17kxy+Caavpb00d0eP/ALPvwo1rwB49+J19b6jpjv4l8StqMwmsXwrGGNMLiXp8vfnn8/O7D9mCOT9l3WvCuqXgmt7zxTd6rZS2ls0WoW1+940sMkD+ZtVlY8Eg/LuznOK+rLbS4bOWZ412tcP5jn1PrVU+GrOS2WHy8RrL54APR+uf1rGOYTjLmT1vF/8AgKsjSWDi429fx3PC/hp8MfjppXjHR7jxR8RIdW0mz3F7WHSbSBZmMbKv2llAdwCQf3TJ8wUkEZFeyfZPExP/AB/aD/4BTf8Ax2taLQoIZVdfO3KcjMzn9CauYrmxWJdafM4peisvuRrRo+zVrt+ruf/Z'); width: 80px; height: 30px; } @@ -468,4 +467,4 @@ - \ No newline at end of file + diff --git a/src/xdevice/_core/testkit/json_parser.py b/src/xdevice/_core/testkit/json_parser.py old mode 100755 new mode 100644 diff --git a/src/xdevice/_core/testkit/kit_lite.py b/src/xdevice/_core/testkit/kit_lite.py old mode 100755 new mode 100644 index c43ee8d..25ad4c8 --- a/src/xdevice/_core/testkit/kit_lite.py +++ b/src/xdevice/_core/testkit/kit_lite.py @@ -284,7 +284,7 @@ class MountKit(ITestKit): for mount_file in self.mount_list: source = mount_file.get("source") if not source: - raise TypeError("The source of MountKit cant be empty " + raise TypeError("The source of MountKit can not be empty " "in Test.json!") source = source.replace("$testcases/", "").\ replace("$resources/", "") @@ -314,10 +314,10 @@ class MountKit(ITestKit): if (str(get_local_ip()) == linux_host) and ( linux_directory == ("/data%s" % testcases_dir)): return - ip = remote_info.get("ip", "") + remote_ip = remote_info.get("ip", "") port = remote_info.get("port", "") remote_dir = remote_info.get("dir", "") - if not ip or not port or not remote_dir: + if not remote_ip or not port or not remote_dir: LOG.warning("nfs server's ip or port or dir is empty") return for _file in file_local_paths: @@ -327,7 +327,7 @@ class MountKit(ITestKit): if not is_remote.lower() == "false": try: import paramiko - client = paramiko.Transport(ip, int(port)) + client = paramiko.Transport(remote_ip, int(port)) client.connect(username=remote_info.get("username"), password=remote_info.get("password")) sftp = paramiko.SFTPClient.from_transport(client) @@ -383,9 +383,8 @@ class MountKit(ITestKit): command="umount {}".format(mounted_dir), timeout=2) if result.find("Resource busy") == -1: - device.execute_command_with_timeout(command="rm -r {}". - format(mounted_dir) - , timeout=1) + device.execute_command_with_timeout( + command="rm -r {}".format(mounted_dir), timeout=1) if status: break LOG.info("umount failed,try " diff --git a/src/xdevice/_core/utils.py b/src/xdevice/_core/utils.py old mode 100755 new mode 100644 diff --git a/src/xdevice/variables.py b/src/xdevice/variables.py old mode 100755 new mode 100644 index 6712ad8..594ad67 --- a/src/xdevice/variables.py +++ b/src/xdevice/variables.py @@ -23,12 +23,16 @@ from dataclasses import dataclass __all__ = ["Variables"] SRC_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +SRC_ADAPTER_DIR = os.path.abspath(os.path.join(SRC_DIR, "adapter")) MODULES_DIR = os.path.abspath(os.path.dirname(__file__)) TOP_DIR = os.path.abspath( os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +TOP_ADAPTER_DIR = os.path.abspath(os.path.join(TOP_DIR, "adapter")) sys.path.insert(0, SRC_DIR) sys.path.insert(1, MODULES_DIR) sys.path.insert(2, TOP_DIR) +sys.path.insert(3, SRC_ADAPTER_DIR) +sys.path.insert(4, TOP_ADAPTER_DIR) @dataclass