From 1705c7946ac86dc4e9b63eeff2282593f5b4ff0c Mon Sep 17 00:00:00 2001 From: Geoff Brown Date: Wed, 18 Apr 2018 14:43:33 -0600 Subject: [PATCH] Bug 1445716 - Add runjunit.py, a test harness for junit tests on Android devices; r=jmaher --- testing/mochitest/moz.build | 1 + testing/mochitest/rungeckoview.py | 6 +- testing/mochitest/runjunit.py | 405 ++++++++++++++++++++++++++++++ 3 files changed, 409 insertions(+), 3 deletions(-) create mode 100644 testing/mochitest/runjunit.py diff --git a/testing/mochitest/moz.build b/testing/mochitest/moz.build index d293985ea7b8..71fa79181cf2 100644 --- a/testing/mochitest/moz.build +++ b/testing/mochitest/moz.build @@ -56,6 +56,7 @@ TEST_HARNESS_FILES.testing.mochitest += [ 'pywebsocket_wrapper.py', 'redirect.html', 'rungeckoview.py', + 'runjunit.py', 'runrobocop.py', 'runtests.py', 'runtestsremote.py', diff --git a/testing/mochitest/rungeckoview.py b/testing/mochitest/rungeckoview.py index c9778f11579b..1d9885510f6b 100644 --- a/testing/mochitest/rungeckoview.py +++ b/testing/mochitest/rungeckoview.py @@ -147,9 +147,9 @@ class GeckoviewTestRunner: expected = 'PASS' (passed, message) = test() if passed: - pass_count = pass_count + 1 + pass_count += 1 else: - fail_count = fail_count + 1 + fail_count += 1 status = 'PASS' if passed else 'FAIL' self.log.test_end(self.test_name, status, expected, message) @@ -195,7 +195,7 @@ class GeckoviewTestRunner: try: shutil.rmtree(dump_dir) except Exception: - self.log.warn("unable to remove directory: %s" % dump_dir) + self.log.warning("unable to remove directory: %s" % dump_dir) return crashed def cleanup(self): diff --git a/testing/mochitest/runjunit.py b/testing/mochitest/runjunit.py new file mode 100644 index 000000000000..d1a1616c5a4b --- /dev/null +++ b/testing/mochitest/runjunit.py @@ -0,0 +1,405 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import json +import os +import posixpath +import re +import shutil +import sys +import tempfile +import traceback + +import mozcrash +import mozinfo +import mozlog +import moznetwork +from mozdevice import ADBAndroid +from mozprofile import Profile, Preferences, DEFAULT_PORTS +from runtests import MochitestDesktop, update_mozinfo + +here = os.path.abspath(os.path.dirname(__file__)) + +try: + from mozbuild.base import ( + MozbuildObject, + MachCommandConditions as conditions, + ) + build_obj = MozbuildObject.from_environment(cwd=here) +except ImportError: + build_obj = None + conditions = None + + +class JUnitTestRunner(MochitestDesktop): + """ + A test harness to run geckoview junit tests on a remote device. + """ + + def __init__(self, log, options): + self.log = log + verbose = False + if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': + verbose = True + self.device = ADBAndroid(adb=options.adbPath or 'adb', + device=options.deviceSerial, + test_root=options.remoteTestRoot, + verbose=verbose) + self.options = options + self.log.debug("options=%s" % vars(options)) + update_mozinfo() + self.remote_profile = posixpath.join(self.device.test_root, 'junit-profile') + self.server_init() + + self.cleanup() + self.device.clear_logcat() + self.build_profile() + self.startServers( + self.options, + debuggerInfo=None, + ignoreSSLTunnelExts=True) + self.log.debug("Servers started") + + def server_init(self): + """ + Additional initialization required to satisfy MochitestDesktop.startServers + """ + self._locations = None + self.server = None + self.wsserver = None + self.websocketProcessBridge = None + self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90 + if self.options.remoteWebServer is None: + if os.name != "nt": + self.options.remoteWebServer = moznetwork.get_ip() + else: + raise Exception("--remote-webserver must be specified") + self.options.webServer = self.options.remoteWebServer + self.options.webSocketPort = '9988' + self.options.httpdPath = None + self.options.keep_open = False + self.options.pidFile = "" + self.options.subsuite = None + self.options.xrePath = None + if build_obj and 'MOZ_HOST_BIN' in os.environ: + self.options.xrePath = os.environ['MOZ_HOST_BIN'] + if not self.options.utilityPath: + self.options.utilityPath = self.options.xrePath + if not self.options.xrePath: + self.options.xrePath = self.options.utilityPath + if build_obj: + self.options.certPath = os.path.join(build_obj.topsrcdir, + 'build', 'pgo', 'certs') + + def build_profile(self): + """ + Create a local profile with test prefs and proxy definitions and + push it to the remote device. + """ + preferences = [ + os.path.join( + here, + 'profile_data', + 'prefs_general.js')] + + prefs = {} + for path in preferences: + prefs.update(Preferences.read_prefs(path)) + + interpolation = { + "server": "%s:%s" % + (self.options.webServer, self.options.httpPort)} + + prefs = json.loads(json.dumps(prefs) % interpolation) + for pref in prefs: + prefs[pref] = Preferences.cast(prefs[pref]) + + proxy = {'remote': self.options.webServer, + 'http': self.options.httpPort, + 'https': self.options.sslPort, + 'ws': self.options.sslPort + } + + self.profile = Profile(preferences=prefs, proxy=proxy) + self.options.profilePath = self.profile.profile + + if self.fillCertificateDB(self.options): + self.log.error("Certificate integration failed") + + self.device.mkdir(self.remote_profile, parents=True) + self.device.push(self.profile.profile, self.remote_profile) + self.log.debug("profile %s -> %s" % + (str(self.profile.profile), str(self.remote_profile))) + + def cleanup(self): + try: + self.stopServers() + self.log.debug("Servers stopped") + self.device.stop_application(self.options.app) + self.device.rm(self.remote_profile, force=True, recursive=True) + if hasattr(self, 'profile'): + del self.profile + except Exception: + traceback.print_exc() + self.log.info("Caught and ignored an exception during cleanup") + + def build_command_line(self, test_filters): + """ + Construct and return the 'am instrument' command line. + """ + cmd = "am instrument -w -r" + # profile location + cmd = cmd + " -e args '-profile %s'" % self.remote_profile + # multi-process + e10s = 'true' if self.options.e10s else 'false' + cmd = cmd + " -e use_multiprocess %s" % e10s + # test filters: limit run to specific test(s) + for f in test_filters: + # filter can be class-name or 'class-name#method-name' (single test) + cmd = cmd + " -e class %s" % f + # environment + env = {} + env["MOZ_CRASHREPORTER"] = "1" + env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" + env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" + env["XPCOM_DEBUG_BREAK"] = "stack" + env["DISABLE_UNSAFE_CPOW_WARNINGS"] = "1" + env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" + env["MOZ_IN_AUTOMATION"] = "1" + env["R_LOG_VERBOSE"] = "1" + env["R_LOG_LEVEL"] = "6" + env["R_LOG_DESTINATION"] = "stderr" + for (env_count, (env_key, env_val)) in enumerate(env.iteritems()): + cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val) + # runner + cmd = cmd + " %s/%s" % (self.options.app, self.options.runner) + return cmd + + def run_tests(self, test_filters=None): + """ + Run the tests. + """ + if not self.device.is_app_installed(self.options.app): + raise Exception("%s is not installed" % + self.options.app) + if self.device.process_exist(self.options.app): + raise Exception("%s already running before starting tests" % + self.options.app) + + self.test_started = False + self.pass_count = 0 + self.fail_count = 0 + self.class_name = "" + self.test_name = "" + self.current_full_name = "" + + def callback(line): + # Output callback: Parse the raw junit log messages, translating into + # treeherder-friendly test start/pass/fail messages. + + self.log.process_output(self.options.app, str(line)) + expected = 'PASS' + # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something" + match = re.match(r'INSTRUMENTATION_STATUS:\s*class=(.*)', line) + if match: + self.class_name = match.group(1) + # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something" + match = re.match(r'INSTRUMENTATION_STATUS:\s*test=(.*)', line) + if match: + self.test_name = match.group(1) + # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..." + match = re.match(r'INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)', line) + if match: + status = match.group(1) + full_name = "%s.%s" % (self.class_name, self.test_name) + if full_name == self.current_full_name: + if status == '0': + message = '' + status = 'PASS' + self.pass_count += 1 + else: + message = 'status %s' % status + status = 'FAIL' + self.fail_count += 1 + self.log.test_end(full_name, status, expected, message) + self.test_started = False + else: + if self.test_started: + # next test started without reporting previous status + self.fail_count += 1 + self.log.test_end(self.current_full_name, 'FAIL', expected, + "missing test completion status") + self.log.test_start(full_name) + self.test_started = True + self.current_full_name = full_name + + # Ideally all test names should be reported to suite_start, but these test + # names are not known in advance. + self.log.suite_start(["geckoview-junit"]) + try: + cmd = self.build_command_line(test_filters) + self.log.info("launching %s" % cmd) + self.device.shell(cmd, timeout=self.options.max_time, stdout_callback=callback) + self.log.info("Passed: %d" % self.pass_count) + self.log.info("Failed: %d" % self.fail_count) + finally: + self.log.suite_end() + + if self.check_for_crashes(): + self.fail_count = 1 + + return 1 if self.fail_count else 0 + + def check_for_crashes(self): + logcat = self.device.get_logcat() + if logcat: + if mozcrash.check_for_java_exception(logcat, self.current_full_name): + return True + symbols_path = self.options.symbolsPath + try: + dump_dir = tempfile.mkdtemp() + remote_dir = posixpath.join(self.remote_profile, 'minidumps') + if not self.device.is_dir(remote_dir): + # If crash reporting is enabled (MOZ_CRASHREPORTER=1), the + # minidumps directory is automatically created when the app + # (first) starts, so its lack of presence is a hint that + # something went wrong. + print "Automation Error: No crash directory (%s) found on remote device" % \ + remote_dir + return True + self.device.pull(remote_dir, dump_dir) + crashed = mozcrash.log_crashes(self.log, dump_dir, symbols_path, + test=self.current_full_name) + finally: + try: + shutil.rmtree(dump_dir) + except Exception: + self.log.warning("unable to remove directory: %s" % dump_dir) + return crashed + + +class JunitArgumentParser(argparse.ArgumentParser): + """ + An argument parser for geckoview-junit. + """ + def __init__(self, **kwargs): + super(JunitArgumentParser, self).__init__(**kwargs) + + self.add_argument("--appname", + action="store", + type=str, + dest="app", + default="org.mozilla.geckoview.test", + help="Test package name.") + self.add_argument("--adbpath", + action="store", + type=str, + dest="adbPath", + default=None, + help="Path to adb executable.") + self.add_argument("--deviceSerial", + action="store", + type=str, + dest="deviceSerial", + help="adb serial number of remote device.") + self.add_argument("--remoteTestRoot", + action="store", + type=str, + dest="remoteTestRoot", + help="Remote directory to use as test root " + "(eg. /mnt/sdcard/tests or /data/local/tests).") + self.add_argument("--disable-e10s", + action="store_false", + dest="e10s", + default=True, + help="Disable multiprocess mode in test app.") + self.add_argument("--max-time", + action="store", + type=str, + dest="max_time", + default="2400", + help="Max time in seconds to wait for tests (default 2400s).") + self.add_argument("--runner", + action="store", + type=str, + dest="runner", + default="android.support.test.runner.AndroidJUnitRunner", + help="Test runner name.") + self.add_argument("--symbols-path", + action="store", + type=str, + dest="symbolsPath", + default=None, + help="Path to directory containing breakpad symbols, " + "or the URL of a zip file containing symbols.") + self.add_argument("--utility-path", + action="store", + type=str, + dest="utilityPath", + default=None, + help="Path to directory containing host utility programs.") + # Additional options for server. + self.add_argument("--certificate-path", + action="store", + type=str, + dest="certPath", + default=None, + help="Path to directory containing certificate store."), + self.add_argument("--http-port", + action="store", + type=str, + dest="httpPort", + default=DEFAULT_PORTS['http'], + help="Port of the web server for http traffic.") + self.add_argument("--remote-webserver", + action="store", + type=str, + dest="remoteWebServer", + help="IP address of the webserver.") + self.add_argument("--ssl-port", + action="store", + type=str, + dest="sslPort", + default=DEFAULT_PORTS['https'], + help="Port of the web server for https traffic.") + # Remaining arguments are test filters. + self.add_argument("test_filters", + nargs="*", + help="Test filter(s): class and/or method names of test(s) to run.") + + mozlog.commandline.add_logging_group(self) + + +def run_test_harness(parser, options): + if hasattr(options, 'log'): + log = options.log + else: + log = mozlog.commandline.setup_logging("runjunit", options, + {"tbpl": sys.stdout}) + runner = JUnitTestRunner(log, options) + result = -1 + try: + result = runner.run_tests(options.test_filters) + except KeyboardInterrupt: + log.info("runjunit.py | Received keyboard interrupt") + result = -1 + except Exception: + traceback.print_exc() + log.error( + "runjunit.py | Received unexpected exception while running tests") + result = 1 + finally: + runner.cleanup() + return result + + +def main(args=sys.argv[1:]): + parser = JunitArgumentParser() + options = parser.parse_args() + return run_test_harness(parser, options) + + +if __name__ == "__main__": + sys.exit(main())