Bug 1191324 - Extend Marionette to allow automation of telemetry tests - changes; r=automatedtester

MozReview-Commit-ID: oDGtQ2Vnq4

Marionette harness has undergone lots of changes, so I decided to take current sources from M-C and do the copy + changes patches again with my own changes to get a Session harness.

--HG--
extra : transplant_source : D5%AA%A1E%96%8E%FC%06%BC%AF%C1T%DA%83%AF%1C%AB9%B5

testing/marionette/harness/marionette/runtests.py

@@ -31,9 +31,11 @@ class MarionetteHarness(object):
     def __init__(self,
                  runner_class=MarionetteTestRunner,
                  parser_class=MarionetteArguments,
+                 testcase_class=MarionetteTestCase,
                  args=None):
         self._runner_class = runner_class
         self._parser_class = parser_class
+        self._testcase_class = testcase_class
         self.args = args or self.parse_args()
 
     def parse_args(self, logger_defaults=None):
@@ -57,7 +59,7 @@ class MarionetteHarness(object):
 
     def process_args(self):
         if self.args.get('pydebugger'):
-            MarionetteTestCase.pydebugger = __import__(self.args['pydebugger'])
+            self._testcase_class.pydebugger = __import__(self.args['pydebugger'])
 
     def run(self):
         try:
@@ -75,7 +77,7 @@ class MarionetteHarness(object):
 
 
 def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
-        harness_class=MarionetteHarness, args=None):
+        harness_class=MarionetteHarness, testcase_class=MarionetteTestCase, args=None):
     """
     Call the harness to parse args and run tests.
 
@@ -86,7 +88,7 @@ def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
     """
     logger = mozlog.commandline.setup_logging('Marionette test runner', {})
     try:
-        failed = harness_class(runner_class, parser_class, args=args).run()
+        failed = harness_class(runner_class, parser_class, testcase_class, args=args).run()
         if failed > 0:
             sys.exit(10)
     except Exception:

testing/marionette/harness/session/__init__.py

@@ -2,33 +2,29 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = '3.0.0'
+from marionette.marionette_test import (
-
-from .marionette_test import (
-    CommonTestCase,
     expectedFailure,
-    MarionetteJSTestCase,
-    MarionetteTestCase,
     skip,
     skip_if_desktop,
     SkipTest,
     skip_unless_protocol,
 )
-from .runner import (
+
-    BaseMarionetteArguments,
+from marionette.runner import (
-    BaseMarionetteTestRunner,
-    BrowserMobProxyTestCaseMixin,
-    EnduranceArguments,
-    EnduranceTestCaseMixin,
-    HTMLReportingArguments,
-    HTMLReportingTestResultMixin,
-    HTMLReportingTestRunnerMixin,
-    Marionette,
-    MarionetteTest,
-    MarionetteTestResult,
-    MarionetteTextTestRunner,
-    MemoryEnduranceTestCaseMixin,
     TestManifest,
     TestResult,
     TestResultCollection,
 )
+
+from .session_test import (
+    SessionJSTestCase,
+    SessionTestCase,
+)
+
+from .runner import (
+    BaseSessionArguments,
+    BaseSessionTestRunner,
+    SessionTest,
+    SessionTestResult,
+    SessionTextTestRunner,
+)

testing/marionette/harness/session/runner/__init__.py

@@ -3,25 +3,9 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from .base import (
-    BaseMarionetteArguments,
+    BaseSessionArguments,
-    BaseMarionetteTestRunner,
+    BaseSessionTestRunner,
-    Marionette,
+    SessionTest,
-    MarionetteTest,
+    SessionTestResult,
-    MarionetteTestResult,
+    SessionTextTestRunner,
-    MarionetteTextTestRunner,
-    TestManifest,
-    TestResult,
-    TestResultCollection,
-)
-
-from .mixins import (
-    EnduranceArguments,
-    EnduranceTestCaseMixin,
-    HTMLReportingArguments,
-    HTMLReportingTestResultMixin,
-    HTMLReportingTestRunnerMixin,
-    MemoryEnduranceTestCaseMixin,
-    BrowserMobProxyTestCaseMixin,
-    BrowserMobProxyArguments,
-    BrowserMobTestCase,
 )

testing/marionette/harness/session/runner/base.py

@@ -26,26 +26,13 @@ from moztest.adapters.unit import StructuredTestRunner, StructuredTestResult
 from moztest.results import TestResultCollection, TestResult, relevant_line
 import mozversion
 
-import httpd
+from marionette.runner import httpd
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-def update_mozinfo(path=None):
-    """walk up directories to find mozinfo.json and update the info"""
 
-    path = path or here
+class SessionTest(TestResult):
-    dirs = set()
-    while path != os.path.expanduser('~'):
-        if path in dirs:
-            break
-        dirs.add(path)
-        path = os.path.split(path)[0]
-
-    return mozinfo.find_and_update_from_json(*dirs)
-
-
-class MarionetteTest(TestResult):
 
     @property
     def test_name(self):
@@ -56,13 +43,12 @@ class MarionetteTest(TestResult):
         else:
             return self.name
 
-class MarionetteTestResult(StructuredTestResult, TestResultCollection):
+class SessionTestResult(StructuredTestResult, TestResultCollection):
 
-    resultClass = MarionetteTest
+    resultClass = SessionTest
 
     def __init__(self, *args, **kwargs):
-        self.marionette = kwargs.pop('marionette')
+        TestResultCollection.__init__(self, 'SessionTest')
-        TestResultCollection.__init__(self, 'MarionetteTest')
         self.passed = 0
         self.testsRun = 0
         self.result_modifiers = [] # used by mixins to modify the result
@@ -144,31 +130,31 @@ class MarionetteTestResult(StructuredTestResult, TestResultCollection):
 
     def addError(self, test, err):
         self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='ERROR')
-        super(MarionetteTestResult, self).addError(test, err)
+        super(SessionTestResult, self).addError(test, err)
 
     def addFailure(self, test, err):
         self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='UNEXPECTED-FAIL')
-        super(MarionetteTestResult, self).addFailure(test, err)
+        super(SessionTestResult, self).addFailure(test, err)
 
     def addSuccess(self, test):
         self.passed += 1
         self.add_test_result(test, result_actual='PASS')
-        super(MarionetteTestResult, self).addSuccess(test)
+        super(SessionTestResult, self).addSuccess(test)
 
     def addExpectedFailure(self, test, err):
         """Called when an expected failure/error occured."""
         self.add_test_result(test, output=self._exc_info_to_string(err, test),
                              result_actual='KNOWN-FAIL')
-        super(MarionetteTestResult, self).addExpectedFailure(test, err)
+        super(SessionTestResult, self).addExpectedFailure(test, err)
 
     def addUnexpectedSuccess(self, test):
         """Called when a test was expected to fail, but succeed."""
         self.add_test_result(test, result_actual='UNEXPECTED-PASS')
-        super(MarionetteTestResult, self).addUnexpectedSuccess(test)
+        super(SessionTestResult, self).addUnexpectedSuccess(test)
 
     def addSkip(self, test, reason):
         self.add_test_result(test, output=reason, result_actual='SKIPPED')
-        super(MarionetteTestResult, self).addSkip(test, reason)
+        super(SessionTestResult, self).addSkip(test, reason)
 
     def getInfo(self, test):
         return test.test_name
@@ -203,40 +189,30 @@ class MarionetteTestResult(StructuredTestResult, TestResultCollection):
 
     def stopTest(self, *args, **kwargs):
         unittest._TextTestResult.stopTest(self, *args, **kwargs)
-        if self.marionette.check_for_crash():
-            # this tells unittest.TestSuite not to continue running tests
-            self.shouldStop = True
-            test = next((a for a in args if isinstance(a, unittest.TestCase)),
-                        None)
-            if test:
-                self.addError(test, sys.exc_info())
 
 
-class MarionetteTextTestRunner(StructuredTestRunner):
+class SessionTextTestRunner(StructuredTestRunner):
 
-    resultclass = MarionetteTestResult
+    resultclass = SessionTestResult
 
     def __init__(self, **kwargs):
-        self.marionette = kwargs.pop('marionette')
+        self.binary = kwargs.pop('binary')
-        self.capabilities = kwargs.pop('capabilities')
-
         StructuredTestRunner.__init__(self, **kwargs)
 
     def _makeResult(self):
         return self.resultclass(self.stream,
                                 self.descriptions,
                                 self.verbosity,
-                                marionette=self.marionette,
                                 logger=self.logger,
                                 result_callbacks=self.result_callbacks)
 
     def run(self, test):
-        result = super(MarionetteTextTestRunner, self).run(test)
+        result = super(SessionTextTestRunner, self).run(test)
         result.printLogs(test)
         return result
 
 
-class BaseMarionetteArguments(ArgumentParser):
+class BaseSessionArguments(ArgumentParser):
     socket_timeout_default = 360.0
 
     def __init__(self, **kwargs):
@@ -257,11 +233,6 @@ class BaseMarionetteArguments(ArgumentParser):
                         action='count',
                         help='Increase verbosity to include debug messages with -v, '
                             'and trace messages with -vv.')
-        self.add_argument('--address',
-                        help='host:port of running Gecko instance to connect to')
-        self.add_argument('--device',
-                        dest='device_serial',
-                        help='serial ID of a device to use for adb / fastboot')
         self.add_argument('--app',
                         help='application to use')
         self.add_argument('--app-arg',
@@ -319,8 +290,6 @@ class BaseMarionetteArguments(ArgumentParser):
         self.add_argument('--this-chunk',
                         type=int,
                         help='which chunk to run')
-        self.add_argument('--sources',
-                        help='path to sources.xml (Firefox OS only)')
         self.add_argument('--server-root',
                         help='url to a webserver or path to a document root from which content '
                         'resources are served (default: {}).'.format(os.path.join(
@@ -412,8 +381,8 @@ class BaseMarionetteArguments(ArgumentParser):
                 print '{0} does not exist'.format(path)
                 sys.exit(1)
 
-        if not args.address and not args.binary:
+        if not args.binary:
-            print 'must specify --binary, or --address'
+            print 'must specify --binary'
             sys.exit(1)
 
         if args.total_chunks is not None and args.this_chunk is None:
@@ -447,9 +416,9 @@ class BaseMarionetteArguments(ArgumentParser):
         return args
 
 
-class BaseMarionetteTestRunner(object):
+class BaseSessionTestRunner(object):
 
-    textrunnerclass = MarionetteTextTestRunner
+    textrunnerclass = SessionTextTestRunner
     driverclass = Marionette
 
     def __init__(self, address=None,
@@ -461,7 +430,7 @@ class BaseMarionetteTestRunner(object):
                  sdcard=None, this_chunk=1, total_chunks=1, sources=None,
                  server_root=None, gecko_log=None, result_callbacks=None,
                  prefs=None, test_tags=None,
-                 socket_timeout=BaseMarionetteArguments.socket_timeout_default,
+                 socket_timeout=BaseSessionArguments.socket_timeout_default,
                  startup_timeout=None, addons=None, workspace=None,
                  verbose=0, e10s=True, **kwargs):
         self.address = address
@@ -472,15 +441,12 @@ class BaseMarionetteTestRunner(object):
         self.addons = addons
         self.logger = logger
         self.httpd = None
-        self.marionette = None
         self.logdir = logdir
         self.repeat = repeat
         self.test_kwargs = kwargs
         self.symbols_path = symbols_path
         self.timeout = timeout
         self.socket_timeout = socket_timeout
-        self._device = None
-        self._capabilities = None
         self._appinfo = None
         self._appName = None
         self.shuffle = shuffle
@@ -506,7 +472,7 @@ class BaseMarionetteTestRunner(object):
 
         def gather_debug(test, status):
             rv = {}
-            marionette = test._marionette_weakref()
+            marionette = test.marionette
 
             # In the event we're gathering debug without starting a session, skip marionette commands
             if marionette.session is not None:
@@ -578,49 +544,6 @@ class BaseMarionetteTestRunner(object):
                                                        e.message))
         return data
 
-    @property
-    def capabilities(self):
-        if self._capabilities:
-            return self._capabilities
-
-        self.marionette.start_session()
-        self._capabilities = self.marionette.session_capabilities
-        self.marionette.delete_session()
-        return self._capabilities
-
-    @property
-    def appinfo(self):
-        if self._appinfo:
-            return self._appinfo
-
-        self.marionette.start_session()
-        with self.marionette.using_context('chrome'):
-            self._appinfo = self.marionette.execute_script("""
-            try {
-              return Services.appinfo;
-            } catch (e) {
-              return null;
-            }""")
-        self.marionette.delete_session()
-        self._appinfo = self._appinfo or {}
-        return self._appinfo
-
-    @property
-    def device(self):
-        if self._device:
-            return self._device
-
-        self._device = self.capabilities.get('device')
-        return self._device
-
-    @property
-    def appName(self):
-        if self._appName:
-            return self._appName
-
-        self._appName = self.capabilities.get('browserName')
-        return self._appName
-
     @property
     def bin(self):
         return self._bin
@@ -634,11 +557,6 @@ class BaseMarionetteTestRunner(object):
         """
         self._bin = path
         self.tests = []
-        if hasattr(self, 'marionette') and self.marionette:
-            self.marionette.cleanup()
-            if self.marionette.instance:
-                self.marionette.instance = None
-        self.marionette = None
 
     def reset_test_stats(self):
         self.passed = 0
@@ -649,133 +567,22 @@ class BaseMarionetteTestRunner(object):
         self.skipped = 0
         self.failures = []
 
-    def _build_kwargs(self):
-        if self.logdir and not os.access(self.logdir, os.F_OK):
-            os.mkdir(self.logdir)
-
-        kwargs = {
-            'timeout': self.timeout,
-            'socket_timeout': self.socket_timeout,
-            'prefs': self.prefs,
-            'startup_timeout': self.startup_timeout,
-            'verbose': self.verbose,
-        }
-        if self.bin:
-            kwargs.update({
-                'host': 'localhost',
-                'port': 2828,
-                'app': self.app,
-                'app_args': self.app_args,
-                'bin': self.bin,
-                'profile': self.profile,
-                'addons': self.addons,
-                'gecko_log': self.gecko_log,
-            })
-
-        if self.address:
-            host, port = self.address.split(':')
-            kwargs.update({
-                'host': host,
-                'port': int(port),
-            })
-
-            if not self.bin:
-                try:
-                    #establish a socket connection so we can vertify the data come back
-                    connection = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
-                    connection.connect((host,int(port)))
-                    connection.close()
-                except Exception, e:
-                    raise Exception("Connection attempt to %s:%s failed with error: %s" %(host,port,e))
-        if self.workspace:
-            kwargs['workspace'] = self.workspace_path
-        return kwargs
-
-    def start_marionette(self):
-        self.marionette = self.driverclass(**self._build_kwargs())
-
-    def launch_test_container(self):
-        if self.marionette.session is None:
-            self.marionette.start_session()
-        self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
-
-        result = self.marionette.execute_async_script("""
-if((navigator.mozSettings == undefined) || (navigator.mozSettings == null) || (navigator.mozApps == undefined) || (navigator.mozApps == null)) {
-    marionetteScriptFinished(false);
-    return;
-}
-let setReq = navigator.mozSettings.createLock().set({'lockscreen.enabled': false});
-setReq.onsuccess = function() {
-    let appName = 'Test Container';
-    let activeApp = window.wrappedJSObject.Service.currentApp;
-
-    // if the Test Container is already open then do nothing
-    if(activeApp.name === appName){
-        marionetteScriptFinished(true);
-    }
-
-    let appsReq = navigator.mozApps.mgmt.getAll();
-    appsReq.onsuccess = function() {
-        let apps = appsReq.result;
-        for (let i = 0; i < apps.length; i++) {
-            let app = apps[i];
-            if (app.manifest.name === appName) {
-                app.launch();
-                window.addEventListener('appopen', function apploadtime(){
-                    window.removeEventListener('appopen', apploadtime);
-                    marionetteScriptFinished(true);
-                });
-                return;
-            }
-        }
-        marionetteScriptFinished(false);
-    }
-    appsReq.onerror = function() {
-        marionetteScriptFinished(false);
-    }
-}
-setReq.onerror = function() {
-    marionetteScriptFinished(false);
-}""", script_timeout=60000)
-
-        if not result:
-            raise Exception("Could not launch test container app")
-
-    def record_crash(self):
-        crash = True
-        try:
-            crash = self.marionette.check_for_crash()
-            self.crashed += int(crash)
-        except Exception:
-            traceback.print_exc()
-        return crash
-
     def run_tests(self, tests):
         assert len(tests) > 0
         assert len(self.test_handlers) > 0
         self.reset_test_stats()
         self.start_time = time.time()
 
-        need_external_ip = True
-        if not self.marionette:
-            self.start_marionette()
-            # if we're working against a desktop version, we usually don't need
-            # an external ip
-            if self.capabilities['device'] == "desktop":
-                need_external_ip = False
-        self.logger.info('Initial Profile Destination is '
-                         '"{}"'.format(self.marionette.profile_path))
-
         # Gaia sets server_root and that means we shouldn't spin up our own httpd
         if not self.httpd:
             if self.server_root is None or os.path.isdir(self.server_root):
                 self.logger.info("starting httpd")
-                self.start_httpd(need_external_ip)
+                self.httpd = self.create_httpd(False)
-                self.marionette.baseurl = self.httpd.get_url()
+                self.base_url = self.httpd.get_url()
-                self.logger.info("running httpd on %s" % self.marionette.baseurl)
+                self.logger.info("running httpd on %s" % self.base_url)
             else:
-                self.marionette.baseurl = self.server_root
+                self.base_url = self.server_root
-                self.logger.info("using remote content from %s" % self.marionette.baseurl)
+                self.logger.info("using remote content from %s" % self.base_url)
 
         device_info = None
 
@@ -792,13 +599,8 @@ setReq.onerror = function() {
                             % '\n  '.join(invalid_tests))
 
         self.logger.info("running with e10s: {}".format(self.e10s))
-        version_info = mozversion.get_version(binary=self.bin,
-                                              sources=self.sources,
-                                              dm_type=os.environ.get('DM_TRANS', 'adb') )
 
-        self.logger.suite_start(self.tests,
+        self.logger.suite_start(self.tests)
-                                version_info=version_info,
-                                device_info=device_info)
 
         for test in self.manifest_skipped_tests:
             name = os.path.basename(test['path'])
@@ -850,16 +652,9 @@ setReq.onerror = function() {
             for failed_test in self.failures:
                 self.logger.info('%s' % failed_test[0])
 
-        self.record_crash()
         self.end_time = time.time()
         self.elapsedtime = self.end_time - self.start_time
 
-        if self.marionette.instance:
-            self.marionette.instance.close()
-            self.marionette.instance = None
-
-        self.marionette.cleanup()
-
         for run_tests in self.mixin_run_tests:
             run_tests(tests)
         if self.shuffle:
@@ -868,11 +663,6 @@ setReq.onerror = function() {
         self.logger.info('mode: {}'.format('e10s' if self.e10s else 'non-e10s'))
         self.logger.suite_end()
 
-    def start_httpd(self, need_external_ip):
-        warnings.warn("start_httpd has been deprecated in favour of create_httpd",
-            DeprecationWarning)
-        self.httpd = self.create_httpd(need_external_ip)
-
     def create_httpd(self, need_external_ip):
         host = "127.0.0.1"
         if need_external_ip:
@@ -894,7 +684,6 @@ setReq.onerror = function() {
                         self.add_test(filepath)
             return
 
-
         file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
 
         if file_ext == '.ini':
@@ -904,13 +693,9 @@ setReq.onerror = function() {
             filters = []
             if self.test_tags:
                 filters.append(tags(self.test_tags))
-            json_path = update_mozinfo(filepath)
-            self.logger.info("mozinfo updated with the following: {}".format(None))
             manifest_tests = manifest.active_tests(exists=False,
                                                    disabled=True,
                                                    filters=filters,
-                                                   device=self.device,
-                                                   app=self.appName,
                                                    e10s=self.e10s,
                                                    **mozinfo.info)
             if len(manifest_tests) == 0:
@@ -941,7 +726,9 @@ setReq.onerror = function() {
 
         testloader = unittest.TestLoader()
         suite = unittest.TestSuite()
+        self.test_kwargs['binary'] = self.bin
         self.test_kwargs['expected'] = expected
+        self.test_kwargs['base_url'] = self.base_url
         self.test_kwargs['test_container'] = test_container
         mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
         for handler in self.test_handlers:
@@ -950,16 +737,14 @@ setReq.onerror = function() {
                                            filepath,
                                            suite,
                                            testloader,
-                                           self.marionette,
                                            self.testvars,
                                            **self.test_kwargs)
                 break
 
         if suite.countTestCases():
             runner = self.textrunnerclass(logger=self.logger,
-                                          marionette=self.marionette,
+                                          result_callbacks=self.result_callbacks,
-                                          capabilities=self.capabilities,
+                                          binary=self.bin)
-                                          result_callbacks=self.result_callbacks)
 
             if test_container:
                 self.launch_test_container()
@@ -993,8 +778,6 @@ setReq.onerror = function() {
 
         for test in tests:
             self.run_test(test['filepath'], test['expected'], test['test_container'])
-            if self.record_crash():
-                break
 
     def run_test_sets(self):
         if len(self.tests) < 1:
@@ -1019,7 +802,4 @@ setReq.onerror = function() {
         if self.httpd:
             self.httpd.stop()
 
-        if self.marionette:
-            self.marionette.cleanup()
-
     __del__ = cleanup

testing/marionette/harness/session/runtests.py

@@ -4,95 +4,22 @@
 
 import sys
 
-from marionette import __version__
+from session.session_test import SessionTestCase, SessionJSTestCase
-from marionette_driver import __version__ as driver_version
+from session.runner import BaseSessionTestRunner, BaseSessionArguments
-from marionette.marionette_test import MarionetteTestCase, MarionetteJSTestCase
+from marionette.runtests import MarionetteHarness, cli
-from marionette.runner import (
-    BaseMarionetteTestRunner,
-    BaseMarionetteArguments,
-    BrowserMobProxyArguments,
-)
-import mozlog
 
 
-class MarionetteTestRunner(BaseMarionetteTestRunner):
+class SessionTestRunner(BaseSessionTestRunner):
     def __init__(self, **kwargs):
-        BaseMarionetteTestRunner.__init__(self, **kwargs)
+        BaseSessionTestRunner.__init__(self, **kwargs)
-        self.test_handlers = [MarionetteTestCase, MarionetteJSTestCase]
+        self.test_handlers = [SessionTestCase, SessionJSTestCase]
 
 
-class MarionetteArguments(BaseMarionetteArguments):
+class SessionArguments(BaseSessionArguments):
     def __init__(self, **kwargs):
-        BaseMarionetteArguments.__init__(self, **kwargs)
+        BaseSessionArguments.__init__(self, **kwargs)
-        self.register_argument_container(BrowserMobProxyArguments())
 
 
-class MarionetteHarness(object):
-    def __init__(self,
-                 runner_class=MarionetteTestRunner,
-                 parser_class=MarionetteArguments,
-                 args=None):
-        self._runner_class = runner_class
-        self._parser_class = parser_class
-        self.args = args or self.parse_args()
-
-    def parse_args(self, logger_defaults=None):
-        parser = self._parser_class(usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...')
-        parser.add_argument('--version', action='version',
-            help="Show version information.",
-            version="%(prog)s {version}"
-                    " (using marionette-driver: {driver_version}, ".format(
-                        version=__version__,
-                        driver_version=driver_version
-                    ))
-        mozlog.commandline.add_logging_group(parser)
-        args = parser.parse_args()
-        parser.verify_usage(args)
-
-        logger = mozlog.commandline.setup_logging(
-            args.logger_name, args, logger_defaults or {"tbpl": sys.stdout})
-
-        args.logger = logger
-        return vars(args)
-
-    def process_args(self):
-        if self.args.get('pydebugger'):
-            MarionetteTestCase.pydebugger = __import__(self.args['pydebugger'])
-
-    def run(self):
-        try:
-            self.process_args()
-            tests = self.args.pop('tests')
-            runner = self._runner_class(**self.args)
-            runner.run_tests(tests)
-            return runner.failed + runner.crashed
-        except Exception:
-            logger = self.args.get('logger')
-            if logger:
-                logger.error('Failure during test execution.',
-                                       exc_info=True)
-            raise
-
-
-def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
-        harness_class=MarionetteHarness, args=None):
-    """
-    Call the harness to parse args and run tests.
-
-    The following exit codes are expected:
-    - Test failures: 10
-    - Harness/other failures: 1
-    - Success: 0
-    """
-    logger = mozlog.commandline.setup_logging('Marionette test runner', {})
-    try:
-        failed = harness_class(runner_class, parser_class, args=args).run()
-        if failed > 0:
-            sys.exit(10)
-    except Exception:
-        logger.error('Failure during harness setup', exc_info=True)
-        sys.exit(1)
-    sys.exit(0)
-
 if __name__ == "__main__":
-    cli()
+    cli(runner_class=SessionTestRunner, parser_class=SessionArguments,
+        harness_class=MarionetteHarness, testcase_class=SessionTestCase, args=None)

testing/marionette/harness/session/session_test.py

@@ -15,6 +15,8 @@ import weakref
 import warnings
 
 
+from mozprofile import FirefoxProfile
+from mozrunner import FirefoxRunner
 from marionette_driver.errors import (
         MarionetteException, TimeoutException,
         JavascriptException, NoSuchElementException, NoSuchWindowException,
@@ -28,221 +30,19 @@ from marionette_driver.wait import Wait
 from marionette_driver.expected import element_present, element_not_present
 from mozlog import get_default_logger
 
+from marionette.marionette_test import (
+        SkipTest,
+        _ExpectedFailure,
+        _UnexpectedSuccess,
+        skip,
+        expectedFailure,
+        parameterized,
+        with_parameters,
+        wraps_parameterized,
+        MetaParameterized,
+        JSTest
+        )
 
-class SkipTest(Exception):
-    """
-    Raise this exception in a test to skip it.
-
-    Usually you can use TestResult.skip() or one of the skipping decorators
-    instead of raising this directly.
-    """
-    pass
-
-class _ExpectedFailure(Exception):
-    """
-    Raise this when a test is expected to fail.
-
-    This is an implementation detail.
-    """
-
-    def __init__(self, exc_info):
-        super(_ExpectedFailure, self).__init__()
-        self.exc_info = exc_info
-
-class _UnexpectedSuccess(Exception):
-    """
-    The test was supposed to fail, but it didn't!
-    """
-    pass
-
-def skip(reason):
-    """Unconditionally skip a test."""
-    def decorator(test_item):
-        if not isinstance(test_item, (type, types.ClassType)):
-            @functools.wraps(test_item)
-            def skip_wrapper(*args, **kwargs):
-                raise SkipTest(reason)
-            test_item = skip_wrapper
-
-        test_item.__unittest_skip__ = True
-        test_item.__unittest_skip_why__ = reason
-        return test_item
-    return decorator
-
-def expectedFailure(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            func(*args, **kwargs)
-        except Exception:
-            raise _ExpectedFailure(sys.exc_info())
-        raise _UnexpectedSuccess
-    return wrapper
-
-def skip_if_chrome(target):
-    def wrapper(self, *args, **kwargs):
-        if self.marionette._send_message("getContext", key="value") == "chrome":
-            raise SkipTest("skipping test in chrome context")
-        return target(self, *args, **kwargs)
-    return wrapper
-
-def skip_if_desktop(target):
-    def wrapper(self, *args, **kwargs):
-        if self.marionette.session_capabilities.get('b2g') is None:
-            raise SkipTest('skipping due to desktop')
-        return target(self, *args, **kwargs)
-    return wrapper
-
-def skip_if_e10s(target):
-    def wrapper(self, *args, **kwargs):
-        with self.marionette.using_context('chrome'):
-            multi_process_browser = self.marionette.execute_script("""
-            try {
-              return Services.appinfo.browserTabsRemoteAutostart;
-            } catch (e) {
-              return false;
-            }""")
-
-        if multi_process_browser:
-            raise SkipTest('skipping due to e10s')
-        return target(self, *args, **kwargs)
-    return wrapper
-
-def skip_unless_protocol(predicate):
-    """Given a predicate passed the current protocol level, skip the
-    test if the predicate does not match."""
-    def decorator(test_item):
-        @functools.wraps(test_item)
-        def skip_wrapper(self):
-            level = self.marionette.client.protocol
-            if not predicate(level):
-                raise SkipTest('skipping because protocol level is %s' % level)
-            return self
-        return skip_wrapper
-    return decorator
-
-def skip_unless_browser_pref(pref, predicate=bool):
-    """
-    Skip a test based on the value of a browser preference.
-
-    :param pref: the preference name
-    :param predicate: a function that should return false to skip the test.
-                      The function takes one parameter, the preference value.
-                      Defaults to the python built-in bool function.
-
-    Note that the preference must exist, else a failure is raised.
-
-    Example: ::
-
-      class TestSomething(MarionetteTestCase):
-          @skip_unless_browser_pref("accessibility.tabfocus",
-                                    lambda value: value >= 7)
-          def test_foo(self):
-              pass  # test implementation here
-    """
-    def wrapper(target):
-        @functools.wraps(target)
-        def wrapped(self, *args, **kwargs):
-            value = self.marionette.get_pref(pref)
-            if value is None:
-                self.fail("No such browser preference: %r" % pref)
-            if not predicate(value):
-                raise SkipTest("browser preference %r: %r" % (pref, value))
-            return target(self, *args, **kwargs)
-        return wrapped
-    return wrapper
-
-def parameterized(func_suffix, *args, **kwargs):
-    """
-    A decorator that can generate methods given a base method and some data.
-
-    **func_suffix** is used as a suffix for the new created method and must be
-    unique given a base method. if **func_suffix** countains characters that
-    are not allowed in normal python function name, these characters will be
-    replaced with "_".
-
-    This decorator can be used more than once on a single base method. The class
-    must have a metaclass of :class:`MetaParameterized`.
-
-    Example::
-
-      # This example will generate two methods:
-      #
-      # - MyTestCase.test_it_1
-      # - MyTestCase.test_it_2
-      #
-      class MyTestCase(MarionetteTestCase):
-          @parameterized("1", 5, named='name')
-          @parameterized("2", 6, named='name2')
-          def test_it(self, value, named=None):
-              print value, named
-
-    :param func_suffix: will be used as a suffix for the new method
-    :param \*args: arguments to pass to the new method
-    :param \*\*kwargs: named arguments to pass to the new method
-    """
-    def wrapped(func):
-        if not hasattr(func, 'metaparameters'):
-            func.metaparameters = []
-        func.metaparameters.append((func_suffix, args, kwargs))
-        return func
-    return wrapped
-
-def with_parameters(parameters):
-    """
-    A decorator that can generate methods given a base method and some data.
-    Acts like :func:`parameterized`, but define all methods in one call.
-
-    Example::
-
-      # This example will generate two methods:
-      #
-      # - MyTestCase.test_it_1
-      # - MyTestCase.test_it_2
-      #
-
-      DATA = [("1", [5], {'named':'name'}), ("2", [6], {'named':'name2'})]
-
-      class MyTestCase(MarionetteTestCase):
-          @with_parameters(DATA)
-          def test_it(self, value, named=None):
-              print value, named
-
-    :param parameters: list of tuples (**func_suffix**, **args**, **kwargs**)
-                       defining parameters like in :func:`todo`.
-    """
-    def wrapped(func):
-        func.metaparameters = parameters
-        return func
-    return wrapped
-
-def wraps_parameterized(func, func_suffix, args, kwargs):
-    """Internal: for MetaParameterized"""
-    def wrapper(self):
-        return func(self, *args, **kwargs)
-    wrapper.__name__ = func.__name__ + '_' + str(func_suffix)
-    wrapper.__doc__ = '[%s] %s' % (func_suffix, func.__doc__)
-    return wrapper
-
-class MetaParameterized(type):
-    """
-    A metaclass that allow a class to use decorators like :func:`parameterized`
-    or :func:`with_parameters` to generate new methods.
-    """
-    RE_ESCAPE_BAD_CHARS = re.compile(r'[\.\(\) -/]')
-    def __new__(cls, name, bases, attrs):
-        for k, v in attrs.items():
-            if callable(v) and hasattr(v, 'metaparameters'):
-                for func_suffix, args, kwargs in v.metaparameters:
-                    func_suffix = cls.RE_ESCAPE_BAD_CHARS.sub('_', func_suffix)
-                    wrapper = wraps_parameterized(v, func_suffix, args, kwargs)
-                    if wrapper.__name__ in attrs:
-                        raise KeyError("%s is already a defined method on %s" %
-                                        (wrapper.__name__, name))
-                    attrs[wrapper.__name__] = wrapper
-                del attrs[k]
-
-        return type.__new__(cls, name, bases, attrs)
 
 class JSTest:
     head_js_re = re.compile(r"MARIONETTE_HEAD_JS(\s*)=(\s*)['|\"](.*?)['|\"];")
@@ -261,9 +61,10 @@ class CommonTestCase(unittest.TestCase):
         unittest.TestCase.__init__(self, methodName)
         self.loglines = []
         self.duration = 0
-        self.start_time = 0
         self.expected = kwargs.pop('expected', 'pass')
         self.logger = get_default_logger()
+        self.profile = FirefoxProfile()
+        self.binary = kwargs.pop('binary', None)
 
     def _enter_pm(self):
         if self.pydebugger:
@@ -406,7 +207,7 @@ class CommonTestCase(unittest.TestCase):
         return m is not None
 
     @classmethod
-    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars):
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars):
         """
         Adds all the tests in the specified file to the specified suite.
         """
@@ -434,7 +235,7 @@ class CommonTestCase(unittest.TestCase):
         # a persistent circular reference which in turn would prevent
         # proper garbage collection.
         self.start_time = time.time()
-        self.marionette = self._marionette_weakref()
+        self.marionette = Marionette(bin=self.binary, profile=self.profile)
         if self.marionette.session is None:
             self.marionette.start_session()
         if self.marionette.timeout is not None:
@@ -445,7 +246,7 @@ class CommonTestCase(unittest.TestCase):
             self.marionette.timeouts(self.marionette.TIMEOUT_PAGE, 30000)
 
     def tearDown(self):
-        pass
+        self.marionette.cleanup()
 
     def cleanTest(self):
         self._deleteSession()
@@ -607,13 +408,12 @@ class CommonTestCase(unittest.TestCase):
 
 
 
-class MarionetteTestCase(CommonTestCase):
+class SessionTestCase(CommonTestCase):
 
     match_re = re.compile(r"test_(.*)\.py$")
 
-    def __init__(self, marionette_weakref, methodName='runTest',
+    def __init__(self, methodName='runTest',
                  filepath='', **kwargs):
-        self._marionette_weakref = marionette_weakref
         self.marionette = None
         self.methodName = methodName
         self.filepath = filepath
@@ -622,7 +422,7 @@ class MarionetteTestCase(CommonTestCase):
         CommonTestCase.__init__(self, methodName, **kwargs)
 
     @classmethod
-    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars, **kwargs):
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
         # since we use imp.load_source to load test modules, if a module
         # is loaded with the same name as another one the module would just be
         # reloaded.
@@ -645,33 +445,15 @@ class MarionetteTestCase(CommonTestCase):
                 issubclass(obj, unittest.TestCase)):
                 testnames = testloader.getTestCaseNames(obj)
                 for testname in testnames:
-                    suite.addTest(obj(weakref.ref(marionette),
+                    suite.addTest(obj(methodName=testname,
-                                  methodName=testname,
                                   filepath=filepath,
                                   testvars=testvars,
                                   **kwargs))
 
     def setUp(self):
         CommonTestCase.setUp(self)
-        self.marionette.test_name = self.test_name
-        self.marionette.execute_script("log('TEST-START: %s:%s')" %
-                                       (self.filepath.replace('\\', '\\\\'), self.methodName),
-                                       sandbox="simpletest")
 
     def tearDown(self):
-        if not self.marionette.check_for_crash():
-            try:
-                self.marionette.clear_imported_scripts()
-                self.marionette.execute_script("log('TEST-END: %s:%s')" %
-                                               (self.filepath.replace('\\', '\\\\'),
-                                                self.methodName),
-                                               sandbox="simpletest")
-                self.marionette.test_name = None
-            except (MarionetteException, IOError):
-                # We have tried to log the test end when there is no listener
-                # object that we can access
-                pass
-
         CommonTestCase.tearDown(self)
 
     def wait_for_condition(self, method, timeout=30):
@@ -684,36 +466,24 @@ class MarionetteTestCase(CommonTestCase):
         else:
             raise TimeoutException("wait_for_condition timed out")
 
-class MarionetteJSTestCase(CommonTestCase):
+class SessionJSTestCase(CommonTestCase):
 
     match_re = re.compile(r"test_(.*)\.js$")
 
-    def __init__(self, marionette_weakref, methodName='runTest', jsFile=None, **kwargs):
+    def __init__(self, methodName='runTest', jsFile=None, **kwargs):
         assert(jsFile)
         self.jsFile = jsFile
-        self._marionette_weakref = marionette_weakref
         self.marionette = None
         self.test_container = kwargs.pop('test_container', None)
         CommonTestCase.__init__(self, methodName)
 
     @classmethod
-    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars, **kwargs):
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
-        suite.addTest(cls(weakref.ref(marionette), jsFile=filepath, **kwargs))
+        suite.addTest(cls(jsFile=filepath, **kwargs))
 
     def runTest(self):
-        if self.marionette.session is None:
-            self.marionette.start_session()
-        self.marionette.execute_script(
-            "log('TEST-START: %s');" % self.jsFile.replace('\\', '\\\\'),
-            sandbox="simpletest")
-
         self.run_js_test(self.jsFile)
 
-        self.marionette.execute_script(
-            "log('TEST-END: %s');" % self.jsFile.replace('\\', '\\\\'),
-            sandbox="simpletest")
-        self.marionette.test_name = None
-
     def get_test_class_name(self):
         # returns a dot separated folders as class name
         dirname = os.path.dirname(self.jsFile).replace('\\', '/')

testing/marionette/harness/session/tests/test_session.py

@@ -5,51 +5,14 @@
 import itertools
 
 from marionette_driver import errors
-from marionette.marionette_test import MarionetteTestCase as TC
+from session.session_test import SessionTestCase as TC
 
 
-class TestProtocol1Errors(TC):
+class TestHelloWorld(TC):
     def setUp(self):
         TC.setUp(self)
-        self.op = self.marionette.protocol
-        self.marionette.protocol = 1
 
     def tearDown(self):
-        self.marionette.protocol = self.op
-        TC.tearDown(self)
-
-    def test_malformed_packet(self):
-        for t in [{}, {"error": None}]:
-            with self.assertRaisesRegexp(errors.MarionetteException, "Malformed packet"):
-                self.marionette._handle_error(t)
-
-    def test_known_error_code(self):
-        with self.assertRaises(errors.NoSuchElementException):
-            self.marionette._handle_error(
-                {"error": {"status": errors.NoSuchElementException.code[0]}})
-
-    def test_known_error_status(self):
-        with self.assertRaises(errors.NoSuchElementException):
-            self.marionette._handle_error(
-                {"error": {"status": errors.NoSuchElementException.status}})
-
-    def test_unknown_error_code(self):
-        with self.assertRaises(errors.MarionetteException):
-            self.marionette._handle_error({"error": {"status": 123456}})
-
-    def test_unknown_error_status(self):
-        with self.assertRaises(errors.MarionetteException):
-            self.marionette._handle_error({"error": {"status": "barbera"}})
-
-
-class TestProtocol2Errors(TC):
-    def setUp(self):
-        TC.setUp(self)
-        self.op = self.marionette.protocol
-        self.marionette.protocol = 2
-
-    def tearDown(self):
-        self.marionette.protocol = self.op
         TC.tearDown(self)
 
     def test_malformed_packet(self):

testing/marionette/harness/session/tests/unit-tests.ini

@@ -1,15 +1,2 @@
-; marionette unit tests
+; session unit tests
-[include:unit/unit-tests.ini]
+[test_session.py]
-test_container = true
-
-; layout tests
-[include:../../../../../layout/base/tests/marionette/manifest.ini]
-
-; loop tests
-[include:../../../../../browser/extensions/loop/manifest.ini]
-
-; microformats tests
-[include:../../../../../toolkit/components/microformats/manifest.ini]
-
-; migration tests
-[include:../../../../../browser/components/migration/tests/marionette/manifest.ini]

testing/marionette/mach_commands.py

@@ -20,7 +20,7 @@ from mach.decorators import (
 )
 
 
-def setup_argument_parser():
+def setup_marionette_argument_parser():
     from marionette.runner.base import BaseMarionetteArguments
     return BaseMarionetteArguments()
 
@@ -43,7 +43,6 @@ def run_marionette(tests, testtype=None, address=None, binary=None, topsrcdir=No
     args = parser.parse_args(args=tests)
 
     args.binary = binary
-    path, exe = os.path.split(args.binary)
 
     for k, v in kwargs.iteritems():
         setattr(args, k, v)
@@ -59,6 +58,50 @@ def run_marionette(tests, testtype=None, address=None, binary=None, topsrcdir=No
     else:
         return 0
 
+def setup_session_argument_parser():
+    from session.runner.base import BaseSessionArguments
+    return BaseSessionArguments()
+
+def run_session(tests, testtype=None, address=None, binary=None, topsrcdir=None, **kwargs):
+    from mozlog.structured import commandline
+
+    from marionette.runtests import (
+        MarionetteHarness
+    )
+
+    from session.runtests import (
+        SessionTestRunner,
+        BaseSessionArguments,
+        SessionArguments,
+        SessionTestCase,
+    )
+
+    parser = BaseSessionArguments()
+    commandline.add_logging_group(parser)
+
+    if not tests:
+        tests = [os.path.join(topsrcdir,
+                 'testing/marionette/harness/session/tests/unit-tests.ini')]
+
+    args = parser.parse_args(args=tests)
+
+    args.binary = binary
+
+    for k, v in kwargs.iteritems():
+        setattr(args, k, v)
+
+    parser.verify_usage(args)
+
+    args.logger = commandline.setup_logging("Session Unit Tests",
+                                            args,
+                                            {"mach": sys.stdout})
+    failed = MarionetteHarness(runner_class=SessionTestRunner, parser_class=SessionArguments,
+                               testcase_class=SessionTestCase, args=vars(args)).run()
+    if failed > 0:
+        return 1
+    else:
+        return 0
+
 @CommandProvider
 class B2GCommands(MachCommandBase):
     def __init__(self, context):
@@ -95,7 +138,7 @@ class MachCommands(MachCommandBase):
     @Command('marionette-test', category='testing',
         description='Run a Marionette test (Check UI or the internal JavaScript using marionette).',
         conditions=[conditions.is_firefox],
-        parser=setup_argument_parser,
+        parser=setup_marionette_argument_parser,
     )
     def run_marionette_test(self, tests, **kwargs):
         if 'test_objects' in kwargs:
@@ -106,3 +149,18 @@ class MachCommands(MachCommandBase):
 
         kwargs['binary'] = self.get_binary_path('app')
         return run_marionette(tests, topsrcdir=self.topsrcdir, **kwargs)
+
+    @Command('session-test', category='testing',
+        description='Run a Session test (Check Telemetry using marionette).',
+        conditions=[conditions.is_firefox],
+        parser=setup_session_argument_parser,
+    )
+    def run_session_test(self, tests, **kwargs):
+        if 'test_objects' in kwargs:
+            tests = []
+            for obj in kwargs['test_objects']:
+                tests.append(obj['file_relpath'])
+            del kwargs['test_objects']
+
+        kwargs['binary'] = self.get_binary_path('app')
+        return run_session(tests, topsrcdir=self.topsrcdir, **kwargs)