Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-12-03 02:25:34 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1191324 - Extend Marionette to allow automation of telemetry tests - changes; r=automatedtester

Browse Source 
MozReview-Commit-ID: oDGtQ2Vnq4

Marionette harness has undergone lots of changes, so I decided to take current sources from M-C and do the copy + changes patches again with my own changes to get a Session harness.

--HG--
extra : transplant_source : D5%AA%A1E%96%8E%FC%06%BC%AF%C1T%DA%83%AF%1C%AB9%B5
This commit is contained in:
André Reinald 2016-05-31 19:15:37 +02:00
parent a424592bc2
commit b73e384ace
9 changed files with 160 additions and 693 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
testing/marionette/harness/marionette/runtests.py
View File

@ -31,9 +31,11 @@ class MarionetteHarness(object):
def __init__(self,
runner_class=MarionetteTestRunner,
parser_class=MarionetteArguments,
testcase_class=MarionetteTestCase,
args=None):
self._runner_class = runner_class
self._parser_class = parser_class
self._testcase_class = testcase_class
self.args = args or self.parse_args()
def parse_args(self, logger_defaults=None):
@ -57,7 +59,7 @@ class MarionetteHarness(object):
def process_args(self):
if self.args.get('pydebugger'):
MarionetteTestCase.pydebugger = __import__(self.args['pydebugger'])
self._testcase_class.pydebugger = __import__(self.args['pydebugger'])
def run(self):
try:
@ -75,7 +77,7 @@ class MarionetteHarness(object):
def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
harness_class=MarionetteHarness, args=None):
harness_class=MarionetteHarness, testcase_class=MarionetteTestCase, args=None):
"""
Call the harness to parse args and run tests.
@ -86,7 +88,7 @@ def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
"""
logger = mozlog.commandline.setup_logging('Marionette test runner', {})
try:
failed = harness_class(runner_class, parser_class, args=args).run()
failed = harness_class(runner_class, parser_class, testcase_class, args=args).run()
if failed > 0:
sys.exit(10)
except Exception:

36
testing/marionette/harness/session/__init__.py
View File

@ -2,33 +2,29 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
__version__ = '3.0.0'
from .marionette_test import (
CommonTestCase,
from marionette.marionette_test import (
expectedFailure,
MarionetteJSTestCase,
MarionetteTestCase,
skip,
skip_if_desktop,
SkipTest,
skip_unless_protocol,
)
from .runner import (
BaseMarionetteArguments,
BaseMarionetteTestRunner,
BrowserMobProxyTestCaseMixin,
EnduranceArguments,
EnduranceTestCaseMixin,
HTMLReportingArguments,
HTMLReportingTestResultMixin,
HTMLReportingTestRunnerMixin,
Marionette,
MarionetteTest,
MarionetteTestResult,
MarionetteTextTestRunner,
MemoryEnduranceTestCaseMixin,
from marionette.runner import (
TestManifest,
TestResult,
TestResultCollection,
)
from .session_test import (
SessionJSTestCase,
SessionTestCase,
)
from .runner import (
BaseSessionArguments,
BaseSessionTestRunner,
SessionTest,
SessionTestResult,
SessionTextTestRunner,
)

26
testing/marionette/harness/session/runner/__init__.py
View File

@ -3,25 +3,9 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from .base import (
BaseMarionetteArguments,
BaseMarionetteTestRunner,
Marionette,
MarionetteTest,
MarionetteTestResult,
MarionetteTextTestRunner,
TestManifest,
TestResult,
TestResultCollection,
)
from .mixins import (
EnduranceArguments,
EnduranceTestCaseMixin,
HTMLReportingArguments,
HTMLReportingTestResultMixin,
HTMLReportingTestRunnerMixin,
MemoryEnduranceTestCaseMixin,
BrowserMobProxyTestCaseMixin,
BrowserMobProxyArguments,
BrowserMobTestCase,
BaseSessionArguments,
BaseSessionTestRunner,
SessionTest,
SessionTestResult,
SessionTextTestRunner,
)

284
testing/marionette/harness/session/runner/base.py
View File

@ -26,26 +26,13 @@ from moztest.adapters.unit import StructuredTestRunner, StructuredTestResult
from moztest.results import TestResultCollection, TestResult, relevant_line
import mozversion
import httpd
from marionette.runner import httpd
here = os.path.abspath(os.path.dirname(__file__))
def update_mozinfo(path=None):
"""walk up directories to find mozinfo.json and update the info"""
path = path or here
dirs = set()
while path != os.path.expanduser('~'):
if path in dirs:
break
dirs.add(path)
path = os.path.split(path)[0]
return mozinfo.find_and_update_from_json(*dirs)
class MarionetteTest(TestResult):
class SessionTest(TestResult):
@property
def test_name(self):
@ -56,13 +43,12 @@ class MarionetteTest(TestResult):
else:
return self.name
class MarionetteTestResult(StructuredTestResult, TestResultCollection):
class SessionTestResult(StructuredTestResult, TestResultCollection):
resultClass = MarionetteTest
resultClass = SessionTest
def __init__(self, *args, **kwargs):
self.marionette = kwargs.pop('marionette')
TestResultCollection.__init__(self, 'MarionetteTest')
TestResultCollection.__init__(self, 'SessionTest')
self.passed = 0
self.testsRun = 0
self.result_modifiers = [] # used by mixins to modify the result
@ -144,31 +130,31 @@ class MarionetteTestResult(StructuredTestResult, TestResultCollection):
def addError(self, test, err):
self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='ERROR')
super(MarionetteTestResult, self).addError(test, err)
super(SessionTestResult, self).addError(test, err)
def addFailure(self, test, err):
self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='UNEXPECTED-FAIL')
super(MarionetteTestResult, self).addFailure(test, err)
super(SessionTestResult, self).addFailure(test, err)
def addSuccess(self, test):
self.passed += 1
self.add_test_result(test, result_actual='PASS')
super(MarionetteTestResult, self).addSuccess(test)
super(SessionTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
"""Called when an expected failure/error occured."""
self.add_test_result(test, output=self._exc_info_to_string(err, test),
result_actual='KNOWN-FAIL')
super(MarionetteTestResult, self).addExpectedFailure(test, err)
super(SessionTestResult, self).addExpectedFailure(test, err)
def addUnexpectedSuccess(self, test):
"""Called when a test was expected to fail, but succeed."""
self.add_test_result(test, result_actual='UNEXPECTED-PASS')
super(MarionetteTestResult, self).addUnexpectedSuccess(test)
super(SessionTestResult, self).addUnexpectedSuccess(test)
def addSkip(self, test, reason):
self.add_test_result(test, output=reason, result_actual='SKIPPED')
super(MarionetteTestResult, self).addSkip(test, reason)
super(SessionTestResult, self).addSkip(test, reason)
def getInfo(self, test):
return test.test_name
@ -203,40 +189,30 @@ class MarionetteTestResult(StructuredTestResult, TestResultCollection):
def stopTest(self, *args, **kwargs):
unittest._TextTestResult.stopTest(self, *args, **kwargs)
if self.marionette.check_for_crash():
# this tells unittest.TestSuite not to continue running tests
self.shouldStop = True
test = next((a for a in args if isinstance(a, unittest.TestCase)),
None)
if test:
self.addError(test, sys.exc_info())
class MarionetteTextTestRunner(StructuredTestRunner):
class SessionTextTestRunner(StructuredTestRunner):
resultclass = MarionetteTestResult
resultclass = SessionTestResult
def __init__(self, **kwargs):
self.marionette = kwargs.pop('marionette')
self.capabilities = kwargs.pop('capabilities')
self.binary = kwargs.pop('binary')
StructuredTestRunner.__init__(self, **kwargs)
def _makeResult(self):
return self.resultclass(self.stream,
self.descriptions,
self.verbosity,
marionette=self.marionette,
logger=self.logger,
result_callbacks=self.result_callbacks)
def run(self, test):
result = super(MarionetteTextTestRunner, self).run(test)
result = super(SessionTextTestRunner, self).run(test)
result.printLogs(test)
return result
class BaseMarionetteArguments(ArgumentParser):
class BaseSessionArguments(ArgumentParser):
socket_timeout_default = 360.0
def __init__(self, **kwargs):
@ -257,11 +233,6 @@ class BaseMarionetteArguments(ArgumentParser):
action='count',
help='Increase verbosity to include debug messages with -v, '
'and trace messages with -vv.')
self.add_argument('--address',
help='host:port of running Gecko instance to connect to')
self.add_argument('--device',
dest='device_serial',
help='serial ID of a device to use for adb / fastboot')
self.add_argument('--app',
help='application to use')
self.add_argument('--app-arg',
@ -319,8 +290,6 @@ class BaseMarionetteArguments(ArgumentParser):
self.add_argument('--this-chunk',
type=int,
help='which chunk to run')
self.add_argument('--sources',
help='path to sources.xml (Firefox OS only)')
self.add_argument('--server-root',
help='url to a webserver or path to a document root from which content '
'resources are served (default: {}).'.format(os.path.join(
@ -412,8 +381,8 @@ class BaseMarionetteArguments(ArgumentParser):
print '{0} does not exist'.format(path)
sys.exit(1)
if not args.address and not args.binary:
print 'must specify --binary, or --address'
if not args.binary:
print 'must specify --binary'
sys.exit(1)
if args.total_chunks is not None and args.this_chunk is None:
@ -447,9 +416,9 @@ class BaseMarionetteArguments(ArgumentParser):
return args
class BaseMarionetteTestRunner(object):
class BaseSessionTestRunner(object):
textrunnerclass = MarionetteTextTestRunner
textrunnerclass = SessionTextTestRunner
driverclass = Marionette
def __init__(self, address=None,
@ -461,7 +430,7 @@ class BaseMarionetteTestRunner(object):
sdcard=None, this_chunk=1, total_chunks=1, sources=None,
server_root=None, gecko_log=None, result_callbacks=None,
prefs=None, test_tags=None,
socket_timeout=BaseMarionetteArguments.socket_timeout_default,
socket_timeout=BaseSessionArguments.socket_timeout_default,
startup_timeout=None, addons=None, workspace=None,
verbose=0, e10s=True, **kwargs):
self.address = address
@ -472,15 +441,12 @@ class BaseMarionetteTestRunner(object):
self.addons = addons
self.logger = logger
self.httpd = None
self.marionette = None
self.logdir = logdir
self.repeat = repeat
self.test_kwargs = kwargs
self.symbols_path = symbols_path
self.timeout = timeout
self.socket_timeout = socket_timeout
self._device = None
self._capabilities = None
self._appinfo = None
self._appName = None
self.shuffle = shuffle
@ -506,7 +472,7 @@ class BaseMarionetteTestRunner(object):
def gather_debug(test, status):
rv = {}
marionette = test._marionette_weakref()
marionette = test.marionette
# In the event we're gathering debug without starting a session, skip marionette commands
if marionette.session is not None:
@ -578,49 +544,6 @@ class BaseMarionetteTestRunner(object):
e.message))
return data
@property
def capabilities(self):
if self._capabilities:
return self._capabilities
self.marionette.start_session()
self._capabilities = self.marionette.session_capabilities
self.marionette.delete_session()
return self._capabilities
@property
def appinfo(self):
if self._appinfo:
return self._appinfo
self.marionette.start_session()
with self.marionette.using_context('chrome'):
self._appinfo = self.marionette.execute_script("""
try {
return Services.appinfo;
} catch (e) {
return null;
}""")
self.marionette.delete_session()
self._appinfo = self._appinfo or {}
return self._appinfo
@property
def device(self):
if self._device:
return self._device
self._device = self.capabilities.get('device')
return self._device
@property
def appName(self):
if self._appName:
return self._appName
self._appName = self.capabilities.get('browserName')
return self._appName
@property
def bin(self):
return self._bin
@ -634,11 +557,6 @@ class BaseMarionetteTestRunner(object):
"""
self._bin = path
self.tests = []
if hasattr(self, 'marionette') and self.marionette:
self.marionette.cleanup()
if self.marionette.instance:
self.marionette.instance = None
self.marionette = None
def reset_test_stats(self):
self.passed = 0
@ -649,133 +567,22 @@ class BaseMarionetteTestRunner(object):
self.skipped = 0
self.failures = []
def _build_kwargs(self):
if self.logdir and not os.access(self.logdir, os.F_OK):
os.mkdir(self.logdir)
kwargs = {
'timeout': self.timeout,
'socket_timeout': self.socket_timeout,
'prefs': self.prefs,
'startup_timeout': self.startup_timeout,
'verbose': self.verbose,
}
if self.bin:
kwargs.update({
'host': 'localhost',
'port': 2828,
'app': self.app,
'app_args': self.app_args,
'bin': self.bin,
'profile': self.profile,
'addons': self.addons,
'gecko_log': self.gecko_log,
})
if self.address:
host, port = self.address.split(':')
kwargs.update({
'host': host,
'port': int(port),
})
if not self.bin:
try:
#establish a socket connection so we can vertify the data come back
connection = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
connection.connect((host,int(port)))
connection.close()
except Exception, e:
raise Exception("Connection attempt to %s:%s failed with error: %s" %(host,port,e))
if self.workspace:
kwargs['workspace'] = self.workspace_path
return kwargs
def start_marionette(self):
self.marionette = self.driverclass(**self._build_kwargs())
def launch_test_container(self):
if self.marionette.session is None:
self.marionette.start_session()
self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
result = self.marionette.execute_async_script("""
if((navigator.mozSettings == undefined) || (navigator.mozSettings == null) || (navigator.mozApps == undefined) || (navigator.mozApps == null)) {
marionetteScriptFinished(false);
return;
}
let setReq = navigator.mozSettings.createLock().set({'lockscreen.enabled': false});
setReq.onsuccess = function() {
let appName = 'Test Container';
let activeApp = window.wrappedJSObject.Service.currentApp;
// if the Test Container is already open then do nothing
if(activeApp.name === appName){
marionetteScriptFinished(true);
}
let appsReq = navigator.mozApps.mgmt.getAll();
appsReq.onsuccess = function() {
let apps = appsReq.result;
for (let i = 0; i < apps.length; i++) {
let app = apps[i];
if (app.manifest.name === appName) {
app.launch();
window.addEventListener('appopen', function apploadtime(){
window.removeEventListener('appopen', apploadtime);
marionetteScriptFinished(true);
});
return;
}
}
marionetteScriptFinished(false);
}
appsReq.onerror = function() {
marionetteScriptFinished(false);
}
}
setReq.onerror = function() {
marionetteScriptFinished(false);
}""", script_timeout=60000)
if not result:
raise Exception("Could not launch test container app")
def record_crash(self):
crash = True
try:
crash = self.marionette.check_for_crash()
self.crashed += int(crash)
except Exception:
traceback.print_exc()
return crash
def run_tests(self, tests):
assert len(tests) > 0
assert len(self.test_handlers) > 0
self.reset_test_stats()
self.start_time = time.time()
need_external_ip = True
if not self.marionette:
self.start_marionette()
# if we're working against a desktop version, we usually don't need
# an external ip
if self.capabilities['device'] == "desktop":
need_external_ip = False
self.logger.info('Initial Profile Destination is '
'"{}"'.format(self.marionette.profile_path))
# Gaia sets server_root and that means we shouldn't spin up our own httpd
if not self.httpd:
if self.server_root is None or os.path.isdir(self.server_root):
self.logger.info("starting httpd")
self.start_httpd(need_external_ip)
self.marionette.baseurl = self.httpd.get_url()
self.logger.info("running httpd on %s" % self.marionette.baseurl)
self.httpd = self.create_httpd(False)
self.base_url = self.httpd.get_url()
self.logger.info("running httpd on %s" % self.base_url)
else:
self.marionette.baseurl = self.server_root
self.logger.info("using remote content from %s" % self.marionette.baseurl)
self.base_url = self.server_root
self.logger.info("using remote content from %s" % self.base_url)
device_info = None
@ -792,13 +599,8 @@ setReq.onerror = function() {
% '\n '.join(invalid_tests))
self.logger.info("running with e10s: {}".format(self.e10s))
version_info = mozversion.get_version(binary=self.bin,
sources=self.sources,
dm_type=os.environ.get('DM_TRANS', 'adb') )
self.logger.suite_start(self.tests,
version_info=version_info,
device_info=device_info)
self.logger.suite_start(self.tests)
for test in self.manifest_skipped_tests:
name = os.path.basename(test['path'])
@ -850,16 +652,9 @@ setReq.onerror = function() {
for failed_test in self.failures:
self.logger.info('%s' % failed_test[0])
self.record_crash()
self.end_time = time.time()
self.elapsedtime = self.end_time - self.start_time
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
self.marionette.cleanup()
for run_tests in self.mixin_run_tests:
run_tests(tests)
if self.shuffle:
@ -868,11 +663,6 @@ setReq.onerror = function() {
self.logger.info('mode: {}'.format('e10s' if self.e10s else 'non-e10s'))
self.logger.suite_end()
def start_httpd(self, need_external_ip):
warnings.warn("start_httpd has been deprecated in favour of create_httpd",
DeprecationWarning)
self.httpd = self.create_httpd(need_external_ip)
def create_httpd(self, need_external_ip):
host = "127.0.0.1"
if need_external_ip:
@ -894,7 +684,6 @@ setReq.onerror = function() {
self.add_test(filepath)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
@ -904,13 +693,9 @@ setReq.onerror = function() {
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
json_path = update_mozinfo(filepath)
self.logger.info("mozinfo updated with the following: {}".format(None))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
device=self.device,
app=self.appName,
e10s=self.e10s,
**mozinfo.info)
if len(manifest_tests) == 0:
@ -941,7 +726,9 @@ setReq.onerror = function() {
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
self.test_kwargs['binary'] = self.bin
self.test_kwargs['expected'] = expected
self.test_kwargs['base_url'] = self.base_url
self.test_kwargs['test_container'] = test_container
mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
for handler in self.test_handlers:
@ -950,16 +737,14 @@ setReq.onerror = function() {
filepath,
suite,
testloader,
self.marionette,
self.testvars,
**self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(logger=self.logger,
marionette=self.marionette,
capabilities=self.capabilities,
result_callbacks=self.result_callbacks)
result_callbacks=self.result_callbacks,
binary=self.bin)
if test_container:
self.launch_test_container()
@ -993,8 +778,6 @@ setReq.onerror = function() {
for test in tests:
self.run_test(test['filepath'], test['expected'], test['test_container'])
if self.record_crash():
break
def run_test_sets(self):
if len(self.tests) < 1:
@ -1019,7 +802,4 @@ setReq.onerror = function() {
if self.httpd:
self.httpd.stop()
if self.marionette:
self.marionette.cleanup()
__del__ = cleanup

93
testing/marionette/harness/session/runtests.py
View File

@ -4,95 +4,22 @@
import sys
from marionette import __version__
from marionette_driver import __version__ as driver_version
from marionette.marionette_test import MarionetteTestCase, MarionetteJSTestCase
from marionette.runner import (
BaseMarionetteTestRunner,
BaseMarionetteArguments,
BrowserMobProxyArguments,
)
import mozlog
from session.session_test import SessionTestCase, SessionJSTestCase
from session.runner import BaseSessionTestRunner, BaseSessionArguments
from marionette.runtests import MarionetteHarness, cli
class MarionetteTestRunner(BaseMarionetteTestRunner):
class SessionTestRunner(BaseSessionTestRunner):
def __init__(self, **kwargs):
BaseMarionetteTestRunner.__init__(self, **kwargs)
self.test_handlers = [MarionetteTestCase, MarionetteJSTestCase]
BaseSessionTestRunner.__init__(self, **kwargs)
self.test_handlers = [SessionTestCase, SessionJSTestCase]
class MarionetteArguments(BaseMarionetteArguments):
class SessionArguments(BaseSessionArguments):
def __init__(self, **kwargs):
BaseMarionetteArguments.__init__(self, **kwargs)
self.register_argument_container(BrowserMobProxyArguments())
BaseSessionArguments.__init__(self, **kwargs)
class MarionetteHarness(object):
def __init__(self,
runner_class=MarionetteTestRunner,
parser_class=MarionetteArguments,
args=None):
self._runner_class = runner_class
self._parser_class = parser_class
self.args = args or self.parse_args()
def parse_args(self, logger_defaults=None):
parser = self._parser_class(usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...')
parser.add_argument('--version', action='version',
help="Show version information.",
version="%(prog)s {version}"
" (using marionette-driver: {driver_version}, ".format(
version=__version__,
driver_version=driver_version
))
mozlog.commandline.add_logging_group(parser)
args = parser.parse_args()
parser.verify_usage(args)
logger = mozlog.commandline.setup_logging(
args.logger_name, args, logger_defaults or {"tbpl": sys.stdout})
args.logger = logger
return vars(args)
def process_args(self):
if self.args.get('pydebugger'):
MarionetteTestCase.pydebugger = __import__(self.args['pydebugger'])
def run(self):
try:
self.process_args()
tests = self.args.pop('tests')
runner = self._runner_class(**self.args)
runner.run_tests(tests)
return runner.failed + runner.crashed
except Exception:
logger = self.args.get('logger')
if logger:
logger.error('Failure during test execution.',
exc_info=True)
raise
def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
harness_class=MarionetteHarness, args=None):
"""
Call the harness to parse args and run tests.
The following exit codes are expected:
- Test failures: 10
- Harness/other failures: 1
- Success: 0
"""
logger = mozlog.commandline.setup_logging('Marionette test runner', {})
try:
failed = harness_class(runner_class, parser_class, args=args).run()
if failed > 0:
sys.exit(10)
except Exception:
logger.error('Failure during harness setup', exc_info=True)
sys.exit(1)
sys.exit(0)
if __name__ == "__main__":
cli()
cli(runner_class=SessionTestRunner, parser_class=SessionArguments,
harness_class=MarionetteHarness, testcase_class=SessionTestCase, args=None)

284
testing/marionette/harness/session/session_test.py
View File

@ -15,6 +15,8 @@ import weakref
import warnings
from mozprofile import FirefoxProfile
from mozrunner import FirefoxRunner
from marionette_driver.errors import (
MarionetteException, TimeoutException,
JavascriptException, NoSuchElementException, NoSuchWindowException,
@ -28,221 +30,19 @@ from marionette_driver.wait import Wait
from marionette_driver.expected import element_present, element_not_present
from mozlog import get_default_logger
from marionette.marionette_test import (
SkipTest,
_ExpectedFailure,
_UnexpectedSuccess,
skip,
expectedFailure,
parameterized,
with_parameters,
wraps_parameterized,
MetaParameterized,
JSTest
)
class SkipTest(Exception):
"""
Raise this exception in a test to skip it.
Usually you can use TestResult.skip() or one of the skipping decorators
instead of raising this directly.
"""
pass
class _ExpectedFailure(Exception):
"""
Raise this when a test is expected to fail.
This is an implementation detail.
"""
def __init__(self, exc_info):
super(_ExpectedFailure, self).__init__()
self.exc_info = exc_info
class _UnexpectedSuccess(Exception):
"""
The test was supposed to fail, but it didn't!
"""
pass
def skip(reason):
"""Unconditionally skip a test."""
def decorator(test_item):
if not isinstance(test_item, (type, types.ClassType)):
@functools.wraps(test_item)
def skip_wrapper(*args, **kwargs):
raise SkipTest(reason)
test_item = skip_wrapper
test_item.__unittest_skip__ = True
test_item.__unittest_skip_why__ = reason
return test_item
return decorator
def expectedFailure(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except Exception:
raise _ExpectedFailure(sys.exc_info())
raise _UnexpectedSuccess
return wrapper
def skip_if_chrome(target):
def wrapper(self, *args, **kwargs):
if self.marionette._send_message("getContext", key="value") == "chrome":
raise SkipTest("skipping test in chrome context")
return target(self, *args, **kwargs)
return wrapper
def skip_if_desktop(target):
def wrapper(self, *args, **kwargs):
if self.marionette.session_capabilities.get('b2g') is None:
raise SkipTest('skipping due to desktop')
return target(self, *args, **kwargs)
return wrapper
def skip_if_e10s(target):
def wrapper(self, *args, **kwargs):
with self.marionette.using_context('chrome'):
multi_process_browser = self.marionette.execute_script("""
try {
return Services.appinfo.browserTabsRemoteAutostart;
} catch (e) {
return false;
}""")
if multi_process_browser:
raise SkipTest('skipping due to e10s')
return target(self, *args, **kwargs)
return wrapper
def skip_unless_protocol(predicate):
"""Given a predicate passed the current protocol level, skip the
test if the predicate does not match."""
def decorator(test_item):
@functools.wraps(test_item)
def skip_wrapper(self):
level = self.marionette.client.protocol
if not predicate(level):
raise SkipTest('skipping because protocol level is %s' % level)
return self
return skip_wrapper
return decorator
def skip_unless_browser_pref(pref, predicate=bool):
"""
Skip a test based on the value of a browser preference.
:param pref: the preference name
:param predicate: a function that should return false to skip the test.
The function takes one parameter, the preference value.
Defaults to the python built-in bool function.
Note that the preference must exist, else a failure is raised.
Example: ::
class TestSomething(MarionetteTestCase):
@skip_unless_browser_pref("accessibility.tabfocus",
lambda value: value >= 7)
def test_foo(self):
pass # test implementation here
"""
def wrapper(target):
@functools.wraps(target)
def wrapped(self, *args, **kwargs):
value = self.marionette.get_pref(pref)
if value is None:
self.fail("No such browser preference: %r" % pref)
if not predicate(value):
raise SkipTest("browser preference %r: %r" % (pref, value))
return target(self, *args, **kwargs)
return wrapped
return wrapper
def parameterized(func_suffix, *args, **kwargs):
"""
A decorator that can generate methods given a base method and some data.
**func_suffix** is used as a suffix for the new created method and must be
unique given a base method. if **func_suffix** countains characters that
are not allowed in normal python function name, these characters will be
replaced with "_".
This decorator can be used more than once on a single base method. The class
must have a metaclass of :class:`MetaParameterized`.
Example::
# This example will generate two methods:
#
# - MyTestCase.test_it_1
# - MyTestCase.test_it_2
#
class MyTestCase(MarionetteTestCase):
@parameterized("1", 5, named='name')
@parameterized("2", 6, named='name2')
def test_it(self, value, named=None):
print value, named
:param func_suffix: will be used as a suffix for the new method
:param \*args: arguments to pass to the new method
:param \*\*kwargs: named arguments to pass to the new method
"""
def wrapped(func):
if not hasattr(func, 'metaparameters'):
func.metaparameters = []
func.metaparameters.append((func_suffix, args, kwargs))
return func
return wrapped
def with_parameters(parameters):
"""
A decorator that can generate methods given a base method and some data.
Acts like :func:`parameterized`, but define all methods in one call.
Example::
# This example will generate two methods:
#
# - MyTestCase.test_it_1
# - MyTestCase.test_it_2
#
DATA = [("1", [5], {'named':'name'}), ("2", [6], {'named':'name2'})]
class MyTestCase(MarionetteTestCase):
@with_parameters(DATA)
def test_it(self, value, named=None):
print value, named
:param parameters: list of tuples (**func_suffix**, **args**, **kwargs**)
defining parameters like in :func:`todo`.
"""
def wrapped(func):
func.metaparameters = parameters
return func
return wrapped
def wraps_parameterized(func, func_suffix, args, kwargs):
"""Internal: for MetaParameterized"""
def wrapper(self):
return func(self, *args, **kwargs)
wrapper.__name__ = func.__name__ + '_' + str(func_suffix)
wrapper.__doc__ = '[%s] %s' % (func_suffix, func.__doc__)
return wrapper
class MetaParameterized(type):
"""
A metaclass that allow a class to use decorators like :func:`parameterized`
or :func:`with_parameters` to generate new methods.
"""
RE_ESCAPE_BAD_CHARS = re.compile(r'[\.\(\) -/]')
def __new__(cls, name, bases, attrs):
for k, v in attrs.items():
if callable(v) and hasattr(v, 'metaparameters'):
for func_suffix, args, kwargs in v.metaparameters:
func_suffix = cls.RE_ESCAPE_BAD_CHARS.sub('_', func_suffix)
wrapper = wraps_parameterized(v, func_suffix, args, kwargs)
if wrapper.__name__ in attrs:
raise KeyError("%s is already a defined method on %s" %
(wrapper.__name__, name))
attrs[wrapper.__name__] = wrapper
del attrs[k]
return type.__new__(cls, name, bases, attrs)
class JSTest:
head_js_re = re.compile(r"MARIONETTE_HEAD_JS(\s*)=(\s*)['|\"](.*?)['|\"];")
@ -261,9 +61,10 @@ class CommonTestCase(unittest.TestCase):
unittest.TestCase.__init__(self, methodName)
self.loglines = []
self.duration = 0
self.start_time = 0
self.expected = kwargs.pop('expected', 'pass')
self.logger = get_default_logger()
self.profile = FirefoxProfile()
self.binary = kwargs.pop('binary', None)
def _enter_pm(self):
if self.pydebugger:
@ -406,7 +207,7 @@ class CommonTestCase(unittest.TestCase):
return m is not None
@classmethod
def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars):
def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars):
"""
Adds all the tests in the specified file to the specified suite.
"""
@ -434,7 +235,7 @@ class CommonTestCase(unittest.TestCase):
# a persistent circular reference which in turn would prevent
# proper garbage collection.
self.start_time = time.time()
self.marionette = self._marionette_weakref()
self.marionette = Marionette(bin=self.binary, profile=self.profile)
if self.marionette.session is None:
self.marionette.start_session()
if self.marionette.timeout is not None:
@ -445,7 +246,7 @@ class CommonTestCase(unittest.TestCase):
self.marionette.timeouts(self.marionette.TIMEOUT_PAGE, 30000)
def tearDown(self):
pass
self.marionette.cleanup()
def cleanTest(self):
self._deleteSession()
@ -607,13 +408,12 @@ class CommonTestCase(unittest.TestCase):
class MarionetteTestCase(CommonTestCase):
class SessionTestCase(CommonTestCase):
match_re = re.compile(r"test_(.*)\.py$")
def __init__(self, marionette_weakref, methodName='runTest',
def __init__(self, methodName='runTest',
filepath='', **kwargs):
self._marionette_weakref = marionette_weakref
self.marionette = None
self.methodName = methodName
self.filepath = filepath
@ -622,7 +422,7 @@ class MarionetteTestCase(CommonTestCase):
CommonTestCase.__init__(self, methodName, **kwargs)
@classmethod
def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars, **kwargs):
def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
# since we use imp.load_source to load test modules, if a module
# is loaded with the same name as another one the module would just be
# reloaded.
@ -645,33 +445,15 @@ class MarionetteTestCase(CommonTestCase):
issubclass(obj, unittest.TestCase)):
testnames = testloader.getTestCaseNames(obj)
for testname in testnames:
suite.addTest(obj(weakref.ref(marionette),
methodName=testname,
suite.addTest(obj(methodName=testname,
filepath=filepath,
testvars=testvars,
**kwargs))
def setUp(self):
CommonTestCase.setUp(self)
self.marionette.test_name = self.test_name
self.marionette.execute_script("log('TEST-START: %s:%s')" %
(self.filepath.replace('\\', '\\\\'), self.methodName),
sandbox="simpletest")
def tearDown(self):
if not self.marionette.check_for_crash():
try:
self.marionette.clear_imported_scripts()
self.marionette.execute_script("log('TEST-END: %s:%s')" %
(self.filepath.replace('\\', '\\\\'),
self.methodName),
sandbox="simpletest")
self.marionette.test_name = None
except (MarionetteException, IOError):
# We have tried to log the test end when there is no listener
# object that we can access
pass
CommonTestCase.tearDown(self)
def wait_for_condition(self, method, timeout=30):
@ -684,36 +466,24 @@ class MarionetteTestCase(CommonTestCase):
else:
raise TimeoutException("wait_for_condition timed out")
class MarionetteJSTestCase(CommonTestCase):
class SessionJSTestCase(CommonTestCase):
match_re = re.compile(r"test_(.*)\.js$")
def __init__(self, marionette_weakref, methodName='runTest', jsFile=None, **kwargs):
def __init__(self, methodName='runTest', jsFile=None, **kwargs):
assert(jsFile)
self.jsFile = jsFile
self._marionette_weakref = marionette_weakref
self.marionette = None
self.test_container = kwargs.pop('test_container', None)
CommonTestCase.__init__(self, methodName)
@classmethod
def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars, **kwargs):
suite.addTest(cls(weakref.ref(marionette), jsFile=filepath, **kwargs))
def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
suite.addTest(cls(jsFile=filepath, **kwargs))
def runTest(self):
if self.marionette.session is None:
self.marionette.start_session()
self.marionette.execute_script(
"log('TEST-START: %s');" % self.jsFile.replace('\\', '\\\\'),
sandbox="simpletest")
self.run_js_test(self.jsFile)
self.marionette.execute_script(
"log('TEST-END: %s');" % self.jsFile.replace('\\', '\\\\'),
sandbox="simpletest")
self.marionette.test_name = None
def get_test_class_name(self):
# returns a dot separated folders as class name
dirname = os.path.dirname(self.jsFile).replace('\\', '/')

41
testing/marionette/harness/session/tests/test_session.py
View File

@ -5,51 +5,14 @@
import itertools
from marionette_driver import errors
from marionette.marionette_test import MarionetteTestCase as TC
from session.session_test import SessionTestCase as TC
class TestProtocol1Errors(TC):
class TestHelloWorld(TC):
def setUp(self):
TC.setUp(self)
self.op = self.marionette.protocol
self.marionette.protocol = 1
def tearDown(self):
self.marionette.protocol = self.op
TC.tearDown(self)
def test_malformed_packet(self):
for t in [{}, {"error": None}]:
with self.assertRaisesRegexp(errors.MarionetteException, "Malformed packet"):
self.marionette._handle_error(t)
def test_known_error_code(self):
with self.assertRaises(errors.NoSuchElementException):
self.marionette._handle_error(
{"error": {"status": errors.NoSuchElementException.code[0]}})
def test_known_error_status(self):
with self.assertRaises(errors.NoSuchElementException):
self.marionette._handle_error(
{"error": {"status": errors.NoSuchElementException.status}})
def test_unknown_error_code(self):
with self.assertRaises(errors.MarionetteException):
self.marionette._handle_error({"error": {"status": 123456}})
def test_unknown_error_status(self):
with self.assertRaises(errors.MarionetteException):
self.marionette._handle_error({"error": {"status": "barbera"}})
class TestProtocol2Errors(TC):
def setUp(self):
TC.setUp(self)
self.op = self.marionette.protocol
self.marionette.protocol = 2
def tearDown(self):
self.marionette.protocol = self.op
TC.tearDown(self)
def test_malformed_packet(self):

17
testing/marionette/harness/session/tests/unit-tests.ini
View File

@ -1,15 +1,2 @@
; marionette unit tests
[include:unit/unit-tests.ini]
test_container = true
; layout tests
[include:../../../../../layout/base/tests/marionette/manifest.ini]
; loop tests
[include:../../../../../browser/extensions/loop/manifest.ini]
; microformats tests
[include:../../../../../toolkit/components/microformats/manifest.ini]
; migration tests
[include:../../../../../browser/components/migration/tests/marionette/manifest.ini]
; session unit tests
[test_session.py]

64
testing/marionette/mach_commands.py
View File

@ -20,7 +20,7 @@ from mach.decorators import (
)
def setup_argument_parser():
def setup_marionette_argument_parser():
from marionette.runner.base import BaseMarionetteArguments
return BaseMarionetteArguments()
@ -43,7 +43,6 @@ def run_marionette(tests, testtype=None, address=None, binary=None, topsrcdir=No
args = parser.parse_args(args=tests)
args.binary = binary
path, exe = os.path.split(args.binary)
for k, v in kwargs.iteritems():
setattr(args, k, v)
@ -59,6 +58,50 @@ def run_marionette(tests, testtype=None, address=None, binary=None, topsrcdir=No
else:
return 0
def setup_session_argument_parser():
from session.runner.base import BaseSessionArguments
return BaseSessionArguments()
def run_session(tests, testtype=None, address=None, binary=None, topsrcdir=None, **kwargs):
from mozlog.structured import commandline
from marionette.runtests import (
MarionetteHarness
)
from session.runtests import (
SessionTestRunner,
BaseSessionArguments,
SessionArguments,
SessionTestCase,
)
parser = BaseSessionArguments()
commandline.add_logging_group(parser)
if not tests:
tests = [os.path.join(topsrcdir,
'testing/marionette/harness/session/tests/unit-tests.ini')]
args = parser.parse_args(args=tests)
args.binary = binary
for k, v in kwargs.iteritems():
setattr(args, k, v)
parser.verify_usage(args)
args.logger = commandline.setup_logging("Session Unit Tests",
args,
{"mach": sys.stdout})
failed = MarionetteHarness(runner_class=SessionTestRunner, parser_class=SessionArguments,
testcase_class=SessionTestCase, args=vars(args)).run()
if failed > 0:
return 1
else:
return 0
@CommandProvider
class B2GCommands(MachCommandBase):
def __init__(self, context):
@ -95,7 +138,7 @@ class MachCommands(MachCommandBase):
@Command('marionette-test', category='testing',
description='Run a Marionette test (Check UI or the internal JavaScript using marionette).',
conditions=[conditions.is_firefox],
parser=setup_argument_parser,
parser=setup_marionette_argument_parser,
)
def run_marionette_test(self, tests, **kwargs):
if 'test_objects' in kwargs:
@ -106,3 +149,18 @@ class MachCommands(MachCommandBase):
kwargs['binary'] = self.get_binary_path('app')
return run_marionette(tests, topsrcdir=self.topsrcdir, **kwargs)
@Command('session-test', category='testing',
description='Run a Session test (Check Telemetry using marionette).',
conditions=[conditions.is_firefox],
parser=setup_session_argument_parser,
)
def run_session_test(self, tests, **kwargs):
if 'test_objects' in kwargs:
tests = []
for obj in kwargs['test_objects']:
tests.append(obj['file_relpath'])
del kwargs['test_objects']
kwargs['binary'] = self.get_binary_path('app')
return run_session(tests, topsrcdir=self.topsrcdir, **kwargs)