Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-07 20:17:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
47b3d8c977
gecko-dev/testing/mozharness/scripts/android_panda.py
James Graham d3ac719a31 Bug 1193215 - Support for passing test directories through mach try, r=chmanchester 
This adds support for web-platform-tests to mach try. It changes the implementation
so that instead of passing paths to manifests, the user passes arbitary paths in the
source tree, and tests under that path are run, with test discovery mainly left to
the harness.
2015-09-25 16:32:57 +01:00

541 lines
23 KiB
Python
Raw Blame History

#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
import copy
import getpass
import os
import re
import signal
import subprocess
import sys
import time
import socket
# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))
from mozharness.mozilla.buildbot import TBPL_SUCCESS, BuildbotMixin
from mozharness.base.errors import BaseErrorList
from mozharness.base.log import INFO, ERROR, FATAL
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.mozbase import MozbaseMixin
from mozharness.mozilla.testing.mozpool import MozpoolMixin
from mozharness.mozilla.testing.device import SUTDeviceMozdeviceMixin
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
SUITE_CATEGORIES = ['mochitest', 'reftest', 'crashtest', 'jsreftest', 'robocop', 'instrumentation', 'xpcshell', 'jittest', 'cppunittest']
class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, BuildbotMixin, SUTDeviceMozdeviceMixin, MozbaseMixin):
test_suites = SUITE_CATEGORIES
config_options = [
[["--mozpool-api-url"], {
"dest": "mozpool_api_url",
"help": "Override mozpool api url",
}],
[["--mozpool-device"], {
"dest": "mozpool_device",
"help": "Set Panda device to run tests on",
}],
[["--mozpool-assignee"], {
"dest": "mozpool_assignee",
"help": "Set mozpool assignee (requestor name, free-form)",
}],
[["--total-chunks"], {
"action": "store",
"dest": "total_chunks",
"help": "Number of total chunks",
}],
[["--this-chunk"], {
"action": "store",
"dest": "this_chunk",
"help": "Number of this chunk",
}],
[["--extra-args"], {
"action": "store",
"dest": "extra_args",
"help": "Extra arguments",
}],
[['--mochitest-suite', ], {
"action": "extend",
"dest": "specified_mochitest_suites",
"type": "string",
"help": "Specify which mochi suite to run. "
"Suites are defined in the config file.\n"
"Examples: 'all', 'plain1', 'plain5', 'chrome', or 'a11y'"}
],
[['--reftest-suite', ], {
"action": "extend",
"dest": "specified_reftest_suites",
"type": "string",
"help": "Specify which reftest suite to run. "
"Suites are defined in the config file.\n"
"Examples: 'all', 'crashplan', or 'jsreftest'"}
],
[['--crashtest-suite', ], {
"action": "extend",
"dest": "specified_crashtest_suites",
"type": "string",
"help": "Specify which crashtest suite to run. "
"Suites are defined in the config file\n."
"Examples: 'crashtest'"}
],
[['--jsreftest-suite', ], {
"action": "extend",
"dest": "specified_jsreftest_suites",
"type": "string",
"help": "Specify which jsreftest suite to run. "
"Suites are defined in the config file\n."
"Examples: 'jsreftest'"}
],
[['--robocop-suite', ], {
"action": "extend",
"dest": "specified_robocop_suites",
"type": "string",
"help": "Specify which robocop suite to run. "
"Suites are defined in the config file\n."
"Examples: 'robocop'"}
],
[['--instrumentation-suite', ], {
"action": "extend",
"dest": "specified_instrumentation_suites",
"type": "string",
"help": "Specify which instrumentation suite to run. "
"Suites are defined in the config file\n."
"Examples: 'browser', 'background'"}
],
[['--xpcshell-suite', ], {
"action": "extend",
"dest": "specified_xpcshell_suites",
"type": "string",
"help": "Specify which xpcshell suite to run. "
"Suites are defined in the config file\n."
"Examples: 'xpcshell'"}
],
[['--jittest-suite', ], {
"action": "extend",
"dest": "specified_jittest_suites",
"type": "string",
"help": "Specify which jittest suite to run. "
"Suites are defined in the config file\n."
"Examples: 'jittest'"}
],
[['--cppunittest-suite', ], {
"action": "extend",
"dest": "specified_cppunittest_suites",
"type": "string",
"help": "Specify which cpp unittest suite to run. "
"Suites are defined in the config file\n."
"Examples: 'cppunittest'"}
],
[['--run-all-suites', ], {
"action": "store_true",
"dest": "run_all_suites",
"default": False,
"help": "This will run all suites that are specified "
"in the config file. You do not need to specify "
"any other suites. Beware, this may take a while ;)"}
],
] + copy.deepcopy(testing_config_options) + \
copy.deepcopy(blobupload_config_options)
error_list = []
mozpool_handler = None
virtualenv_modules = [
'mozpoolclient',
]
def __init__(self, require_config_file=False):
super(PandaTest, self).__init__(
config_options=self.config_options,
all_actions=['clobber',
'read-buildbot-config',
'download-and-extract',
'create-virtualenv',
'request-device',
'run-test',
'close-request'],
default_actions=['clobber',
'read-buildbot-config',
'download-and-extract',
'create-virtualenv',
'request-device',
'run-test',
'close-request'],
require_config_file=require_config_file,
config={'virtualenv_modules': self.virtualenv_modules})
self.mozpool_assignee = self.config.get('mozpool_assignee', getpass.getuser())
self.request_url = None
self.installer_url = self.config.get("installer_url")
self.test_url = self.config.get("test_url")
self.mozpool_device = self.config.get("mozpool_device")
self.symbols_url = self.config.get('symbols_url')
def postflight_read_buildbot_config(self):
super(PandaTest, self).postflight_read_buildbot_config()
self.mozpool_device = self.config.get('mozpool_device', self.buildbot_config.get('properties')["slavename"])
dirs = self.query_abs_dirs()
#touch the shutdown file
shutdown_file = os.path.join(dirs['shutdown_dir'], 'shutdown.stamp')
try:
self.info("*** Touching the shutdown file **")
open(shutdown_file, 'w').close()
except Exception, e:
self.warning("We failed to create the shutdown file: str(%s)" % str(e))
def request_device(self):
self.retrieve_android_device(b2gbase="")
env = self.query_env()
cmd = [self.query_exe('python'), self.config.get("verify_path")]
if self.run_command(cmd, env=env):
self.critical("Preparing to abort run due to failed verify check.")
self.close_request()
self.fatal("Dying due to failing verification")
else:
self.info("Successfully verified the device")
def _sut_prep_steps(self):
device_time = self.set_device_epoch_time()
self.info("Current time on device: %s - %s" %
(device_time, time.strftime("%x %H:%M:%S", time.gmtime(float(device_time)))))
def download_and_extract(self):
"""
Provides the target suite categories to TestingMixin.download_
"""
if self.config.get('run_all_suites'):
target_categories = SUITE_CATEGORIES
else:
target_categories = [cat for cat in SUITE_CATEGORIES
if self._query_specified_suites(cat) is not None]
super(PandaTest, self).download_and_extract(suite_categories=target_categories)
def _query_try_flavor(self, category, suite):
flavors = {
"mochitest": [("plain.*", "mochitest"),
("browser-chrome.*", "browser-chrome"),
("mochitest-devtools-chrome.*", "devtools-chrome"),
("chrome", "chrome")],
"xpcshell": [("xpcshell", "xpcshell")],
"reftest": [("reftest", "reftest"),
("crashtest", "crashtest")]
}
for suite_pattern, flavor in flavors.get(category, []):
if re.compile(suite_pattern).match(suite):
return flavor
def _run_category_suites(self, suite_category, preflight_run_method=None):
"""run suite(s) to a specific category"""
env = self.query_env(partial_env={'DM_TRANS': "sut", 'TEST_DEVICE': self.mozpool_device})
self.info("Running tests...")
suites = self._query_specified_suites(suite_category)
level = INFO
if preflight_run_method:
preflight_run_method(suites)
if suites:
self.info('#### Running %s suites' % suite_category)
for suite in suites:
dirs = self.query_abs_dirs()
self._download_unzip_hostutils()
abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
should_install_app = True
if 'cppunittest' in suite:
should_install_app = False
if 'robocop' in suite:
self._download_robocop_apk()
if 'jittest' in suite:
should_install_app = False
if should_install_app:
self._install_app()
cmd = abs_base_cmd[:]
flavor = self._query_try_flavor(suite_category, suite)
try_options, try_tests = self.try_args(flavor)
cmd.extend(self.query_options(suites[suite],
try_options))
cmd.extend(self.query_tests_args(try_tests))
tests = self.config["suite_definitions"][suite_category].get("tests", [])
cmd += tests
tbpl_status, log_level = None, None
error_list = BaseErrorList + [{
'regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"),
'level': ERROR,
}]
c = self.config
if c.get('minidump_stackwalk_path'):
env['MINIDUMP_STACKWALK'] = c['minidump_stackwalk_path']
env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
env = self.query_env(partial_env=env, log_level=INFO)
if env.has_key('PYTHONPATH'):
del env['PYTHONPATH']
parser = self.get_test_output_parser(suite_category,
config=self.config,
error_list=error_list,
log_obj=self.log_obj)
return_code = self.run_command(cmd,
cwd=dirs['abs_test_install_dir'],
env=env,
output_parser=parser)
tbpl_status, log_level = parser.evaluate_parser(return_code)
if tbpl_status != TBPL_SUCCESS:
self.info("Output logcat...")
try:
lines = self.get_logcat()
self.info("*** STARTING LOGCAT ***")
for l in lines:
self.info(l)
self.info("*** END LOGCAT ***")
except Exception, e:
self.warning("We failed to run logcat: str(%s)" % str(e))
parser.append_tinderboxprint_line(suite)
self.buildbot_status(tbpl_status, level=level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_category, suite, tbpl_status), level=log_level)
def _query_specified_suites(self, category):
# logic goes: if at least one '--{category}-suite' was given,
# then run only that(those) given suite(s). Elif no suites were
# specified and the --run-all-suites flag was given,
# run all {category} suites. Anything else, run no suites.
c = self.config
all_suites = c.get('all_%s_suites' % (category))
specified_suites = c.get('specified_%s_suites' % (category)) # list
suites = None
if specified_suites:
if 'all' in specified_suites:
# useful if you want a quick way of saying run all suites
# of a specific category.
suites = all_suites
else:
# suites gets a dict of everything from all_suites where a key
# is also in specified_suites
suites = dict((key, all_suites.get(key)) for key in
specified_suites if key in all_suites.keys())
else:
if c.get('run_all_suites'): # needed if you dont specify any suites
suites = all_suites
return suites
def run_test(self):
# do we need to set the device time? command doesn't work anyways
# self._sut_prep_steps()
env = self.query_env()
env["DM_TRANS"] = "sut"
env["TEST_DEVICE"] = self.mozpool_device
self.mkdir_p(self.abs_dirs['abs_blob_upload_dir'])
self._start_logcat()
self.info("Running tests...")
for category in SUITE_CATEGORIES:
self._run_category_suites(category)
self._stop_logcat()
def _start_logcat(self):
# Start logcat.py as a separate process continuously pulling logcat from
# the device and writing to a file. Output is written directly to
# the blobber upload directory so that it is uploaded automatically
# at the end of the job.
device_ip = socket.gethostbyname(self.mozpool_device)
logcat_path = os.path.join(self.abs_dirs['abs_blob_upload_dir'], 'logcat.log')
logcat_cmd = ['python', '-u', self.config.get("logcat_path"), \
device_ip, logcat_path, '-v time']
self.info('Starting logcat: %s' % str(logcat_cmd))
self.logcat_proc = subprocess.Popen(logcat_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def _stop_logcat(self):
# Signal logcat.py so that it can cleanup (kill the device logcat process)
self.logcat_proc.send_signal(signal.SIGINT)
self.logcat_proc.kill()
out, err = self.logcat_proc.communicate()
self.info("logcat.py output:\n%s\n%s\n" % (out, err))
def _download_unzip_hostutils(self):
c = self.config
dirs = self.query_abs_dirs()
self.host_utils_url = c['hostutils_url']
#create the hostutils dir, get the zip and extract it
self.mkdir_p(dirs['abs_hostutils_dir'])
self._download_unzip(self.host_utils_url, dirs['abs_hostutils_dir'])
def _install_app(self):
c = self.config
base_work_dir = c['base_work_dir']
cmd = ['python', self.config.get("install_app_path"), self.device_ip, 'build/' + str(self.filename_apk), self.app_name]
self.run_command(cmd, cwd=base_work_dir, halt_on_failure=True, fatal_exit_code=3)
def _download_robocop_apk(self):
dirs = self.query_abs_dirs()
self.apk_url = self.installer_url[:self.installer_url.rfind('/')]
robocop_url = self.apk_url + '/robocop.apk'
self.info("Downloading robocop...")
self.download_file(robocop_url, 'robocop.apk', dirs['abs_work_dir'], error_level=FATAL)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(PandaTest, self).query_abs_dirs()
dirs = {}
dirs['abs_test_install_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'tests')
dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
dirs['abs_mochitest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'mochitest')
dirs['abs_reftest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'reftest')
dirs['abs_crashtest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'reftest')
dirs['abs_jsreftest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'reftest')
dirs['abs_xpcshell_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'xpcshell')
dirs['abs_xre_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'xre')
dirs['abs_utility_path'] = os.path.join(
abs_dirs['abs_work_dir'], 'bin')
dirs['abs_certificate_path'] = os.path.join(
abs_dirs['abs_work_dir'], 'certs')
dirs['abs_hostutils_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'hostutils')
dirs['abs_robocop_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'mochitest')
dirs['abs_instrumentation_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'instrumentation')
dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
dirs['abs_jittest_dir'] = os.path.join(dirs['abs_test_install_dir'], "jit-test", "jit-test")
dirs['shutdown_dir'] = abs_dirs['abs_work_dir'].rsplit("/", 2)[0]
dirs['abs_cppunittest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'cppunittest')
for key in dirs.keys():
if key not in abs_dirs:
abs_dirs[key] = dirs[key]
self.abs_dirs = abs_dirs
return self.abs_dirs
def _query_symbols_url(self):
"""query the full symbols URL based upon binary URL"""
# may break with name convention changes but is one less 'input' for script
if self.symbols_url:
return self.symbols_url
def _query_abs_base_cmd(self, suite_category, suite):
#check for apk first with if ?
c = self.config
dirs = self.query_abs_dirs()
options = []
run_file = c["suite_definitions"][suite_category]["run_filename"]
base_cmd = ['python', '-u']
base_cmd.append(os.path.join((dirs["abs_%s_dir" % suite_category]), run_file))
self.device_ip = socket.gethostbyname(self.mozpool_device)
#applies to mochitest, reftest, jsreftest
# TestingMixin._download_and_extract_symbols() will set
# self.symbols_path when downloading/extracting.
hostnumber = 0
mozpool_device_list = self.mozpool_device.split('-')
if len(mozpool_device_list) == 2:
hostnumber = int(mozpool_device_list[1])
http_port = '30%03i' % hostnumber
ssl_port = '31%03i' % hostnumber
#get filename from installer_url
self.filename_apk = self.installer_url.split('/')[-1]
#find appname from package-name.txt - assumes download-and-extract has completed successfully
apk_dir = self.abs_dirs['abs_work_dir']
self.apk_path = os.path.join(apk_dir, self.filename_apk)
unzip = self.query_exe("unzip")
package_path = os.path.join(apk_dir, 'package-name.txt')
unzip_cmd = [unzip, '-q', '-o', self.apk_path]
self.run_command(unzip_cmd, cwd=apk_dir, halt_on_failure=True, fatal_exit_code=3)
self.app_name = str(self.read_from_file(package_path, verbose=True)).rstrip()
raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
'%s_raw.log' % suite)
error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
'%s_errorsummary.log' % suite)
str_format_values = {
'device_ip': self.device_ip,
'hostname': self.mozpool_device,
'symbols_path': self._query_symbols_url(),
'http_port': http_port,
'ssl_port': ssl_port,
'app_name': self.app_name,
'apk_name': self.filename_apk,
'apk_path': self.apk_path,
'raw_log_file': raw_log_file,
'error_summary_file': error_summary_file,
}
if "suite_definitions" in c and \
suite_category in c["suite_definitions"]: # new in-tree format
for option in c["suite_definitions"][suite_category]["options"]:
options.append(option % str_format_values)
abs_base_cmd = base_cmd + options
return abs_base_cmd
else:
self.warning("Suite options for %s could not be determined."
"\nIf you meant to have options for this suite, "
"please make sure they are specified in your "
"config." % suite_category)
###### helper methods
def _pre_config_lock(self, rw_config):
super(PandaTest, self)._pre_config_lock(rw_config)
c = self.config
if not c.get('run_all_suites'):
return # configs are valid
for category in SUITE_CATEGORIES:
specific_suites = c.get('specified_%s_suites' % (category))
if specific_suites:
if specific_suites != 'all':
self.fatal("Config options are not valid. Please ensure"
" that if the '--run-all-suites' flag was enabled,"
" then do not specify to run only specific suites "
"like:\n '--mochitest-suite browser-chrome'")
def close_request(self):
if self.request_url:
mph = self.query_mozpool_handler(self.mozpool_device)
mph.close_request(self.request_url)
self.info("Request '%s' deleted on cleanup" % self.request_url)
self.request_url = None
else:
self.info("request_url doesn't exist. Already closed?")
def _build_arg(self, option, value):
"""
Build a command line argument
"""
if not value:
return []
return [str(option), str(value)]
if __name__ == '__main__':
pandaTest = PandaTest()
pandaTest.run_and_exit()
Reference in New Issue View Git Blame Copy Permalink