Backed out 7 changesets (bug 1181516, bug 1198257, bug 1193257, bug 1194166, bug 1193223, bug 1193224, bug 1181520) for test bustages on Android on a CLOSED TREE

Backed out changeset efe681ac063f (bug 1194166)
Backed out changeset bfcbce193ccf (bug 1193257)
Backed out changeset bb035419fd9c (bug 1193224)
Backed out changeset eb2d48b71c37 (bug 1198257)
Backed out changeset 0d6532c1e99e (bug 1193223)
Backed out changeset da3dd2f650d0 (bug 1181516)
Backed out changeset 29c33bbaac01 (bug 1181520)

layout/tools/reftest/Makefile.in

@@ -7,7 +7,6 @@ _DEST_DIR = $(DEPTH)/_tests/reftest
 
 _HARNESS_FILES = \
   $(srcdir)/runreftest.py \
-  $(srcdir)/reftestcommandline.py \
   $(srcdir)/remotereftest.py \
   $(srcdir)/runreftestb2g.py \
   $(srcdir)/b2g_desktop.py \

layout/tools/reftest/b2g_desktop.py

@@ -9,7 +9,7 @@ import sys
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-from runreftest import RefTest
+from runreftest import RefTest, ReftestOptions
 
 from marionette_driver import expected
 from marionette_driver.by import By
@@ -51,10 +51,12 @@ class B2GDesktopReftest(RefTest):
             f.close()
         self.marionette.execute_script(self.test_script)
 
-    def run_tests(self, tests, options):
+    def run_tests(self, test_path, options):
-        manifests = self.resolver.resolveManifests(options, tests)
+        reftestlist = self.getManifestPath(test_path)
+        if not reftestlist.startswith('file://'):
+            reftestlist = 'file://%s' % reftestlist
 
-        self.profile = self.create_profile(options, manifests,
+        self.profile = self.create_profile(options, reftestlist,
                                            profile_to_clone=options.profile)
         env = self.buildBrowserEnv(options, self.profile.profile)
         kp_kwargs = { 'processOutputLine': [self._on_output],
@@ -105,8 +107,8 @@ class B2GDesktopReftest(RefTest):
         log.info("%s | Running tests: end.", os.path.basename(__file__))
         return status
 
-    def create_profile(self, options, manifests, profile_to_clone=None):
+    def create_profile(self, options, reftestlist, profile_to_clone=None):
-        profile = RefTest.createReftestProfile(self, options, manifests,
+        profile = RefTest.createReftestProfile(self, options, reftestlist,
                                                profile_to_clone=profile_to_clone)
 
         prefs = {}
@@ -134,6 +136,7 @@ class B2GDesktopReftest(RefTest):
         prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org"
         prefs["reftest.browser.iframe.enabled"] = False
         prefs["reftest.remote"] = False
+        prefs["reftest.uri"] = "%s" % reftestlist
         # Set a future policy version to avoid the telemetry prompt.
         prefs["toolkit.telemetry.prompted"] = 999
         prefs["toolkit.telemetry.notifiedOptOut"] = 999
@@ -189,7 +192,7 @@ class MuletReftest(B2GDesktopReftest):
         Wait(self.marionette, timeout).until(expected.element_present(
             By.CSS_SELECTOR, '#homescreen[loading-state=false]'))
 
-def run_desktop_reftests(parser, options):
+def run_desktop_reftests(parser, options, args):
     marionette_args = {}
     if options.marionette:
         host, port = options.marionette.split(':')
@@ -201,7 +204,9 @@ def run_desktop_reftests(parser, options):
     else:
         reftest = B2GDesktopReftest(marionette_args)
 
-    parser.validate(options, reftest)
+    options = ReftestOptions.verifyCommonOptions(parser, options, reftest)
+    if options == None:
+        sys.exit(1)
 
     # add a -bin suffix if b2g-bin exists, but just b2g was specified
     if options.app[-4:] != '-bin':
@@ -214,4 +219,4 @@ def run_desktop_reftests(parser, options):
     if options.desktop and not options.profile:
         raise Exception("must specify --profile when specifying --desktop")
 
-    sys.exit(reftest.run_tests(options.tests, options))
+    sys.exit(reftest.run_tests(args[0], options))

layout/tools/reftest/mach_commands.py

@@ -23,7 +23,8 @@ from mach.decorators import (
     Command,
 )
 
-import reftestcommandline
+
+DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
 
 ADB_NOT_FOUND = '''
 The %s command requires the adb binary to be on your path.
@@ -80,10 +81,43 @@ class ReftestRunner(MozbuildObject):
         self.tests_dir = os.path.join(self.topobjdir, '_tests')
         self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
 
+    def _manifest_file(self, suite):
+        """Returns the manifest file used for a given test suite."""
+        files = {
+            'reftest': 'reftest.list',
+            'reftest-ipc': 'reftest.list',
+            'crashtest': 'crashtests.list',
+            'crashtest-ipc': 'crashtests.list',
+            'jstestbrowser': 'jstests.list'
+        }
+        assert suite in files
+        return files[suite]
+
+    def _find_manifest(self, suite, test_file):
+        """Return a tuple of (manifest-path, filter-string) for running test_file.
+
+        test_file can be a relative path to a single test file or manifest from
+        the top source directory, an absolute path to the same, or a directory
+        containing a manifest.
+        """
+        assert test_file
+        path_arg = self._wrap_path_argument(test_file)
+        relpath = path_arg.relpath()
+
+        if os.path.isdir(path_arg.srcdir_path()):
+            return (mozpath.join(relpath, self._manifest_file(suite)), None)
+
+        if relpath.endswith('.list'):
+            return (relpath, None)
+
+        return (self._find_manifest(suite, mozpath.dirname(test_file))[0],
+                mozpath.basename(test_file))
+
     def _make_shell_string(self, s):
         return "'%s'" % re.sub("'", r"'\''", s)
 
-    def run_b2g_test(self, b2g_home=None, xre_path=None, **kwargs):
+    def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None,
+                     suite=None, filter=None, **kwargs):
         """Runs a b2g reftest.
 
         filter is a regular expression (in JS syntax, as could be passed to the
@@ -96,189 +130,324 @@ class ReftestRunner(MozbuildObject):
         suite is the type of reftest to run. It can be one of ('reftest',
         'crashtest').
         """
-        if kwargs["suite"] not in ('reftest', 'crashtest'):
+        if suite not in ('reftest', 'crashtest'):
             raise Exception('None or unrecognized reftest suite type.')
 
-        sys.path.insert(0, self.reftest_dir)
-
-        test_subdir = {"reftest": os.path.join('layout', 'reftests'),
-                       "crashtest": os.path.join('layout', 'crashtest')}[kwargs["suite"]]
-
         # Find the manifest file
-        if not kwargs["tests"]:
+        if not test_file:
-            if not os.path.exists(os.path.join(self.topsrcdir, test_subdir)):
+            if suite == 'reftest':
-                test_file = mozpath.relpath(os.path.abspath(test_subdir),
+                test_file = mozpath.join('layout', 'reftests')
-                                            self.topsrcdir)
+            elif suite == 'crashtest':
-            kwargs["tests"] = [test_subdir]
+                test_file = mozpath.join('testing', 'crashtest')
 
+        if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
+            test_file = mozpath.relpath(os.path.abspath(test_file),
+                                             self.topsrcdir)
+
+        (manifest, single_file_filter) = self._find_manifest(suite, test_file)
+        if not os.path.exists(mozpath.join(self.topsrcdir, manifest)):
+            raise Exception('No manifest file was found at %s.' % manifest)
+        if single_file_filter:
+            if filter:
+                raise Exception('Cannot run single files in conjunction with --filter')
+            filter = single_file_filter
+
+        # Need to chdir to reftest_dir otherwise imports fail below.
+        os.chdir(self.reftest_dir)
+
+        # The imp module can spew warnings if the modules below have
+        # already been imported, ignore them.
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+
+            import imp
+            path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
+            with open(path, 'r') as fh:
+                imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
+            import reftest
+
+        # Set up the reftest options.
+        parser = reftest.B2GOptions()
+        options, args = parser.parse_args([])
+
+        # Tests need to be served from a subdirectory of the server. Symlink
+        # topsrcdir here to get around this.
         tests = os.path.join(self.reftest_dir, 'tests')
         if not os.path.isdir(tests):
             os.symlink(self.topsrcdir, tests)
+        args.insert(0, os.path.join('tests', manifest))
 
-        for i, path in enumerate(kwargs["tests"]):
+        for k, v in kwargs.iteritems():
-            # Non-absolute paths are relative to the packaged directory, which
+            setattr(options, k, v)
-            # has an extra tests/ at the start
-            if os.path.exists(os.path.abspath(path)):
-                path = os.path.relpath(path, os.path.join(self.topsrcdir))
-            kwargs["tests"][i] = os.path.join('tests', path)
 
         if conditions.is_b2g_desktop(self):
-            return self.run_b2g_desktop(**kwargs)
+            if self.substs.get('ENABLE_MARIONETTE') != '1':
+                print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
+                                             self.mozconfig['path']))
+                return 1
 
-        return self.run_b2g_remote(b2g_home, xre_path, **kwargs)
+            options.profile = options.profile or os.environ.get('GAIA_PROFILE')
-
+            if not options.profile:
-    def run_b2g_desktop(self, **kwargs):
-        if self.substs.get('ENABLE_MARIONETTE') != '1':
-            print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
-                                         self.mozconfig['path']))
-            return 1
-
-        if not kwargs["profile"]:
-            gaia_profile = os.environ.get('GAIA_PROFILE')
-            if not gaia_profile:
                 print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop')
                 return 1
-            kwargs["profile"] = gaia_profile
 
+            if os.path.isfile(os.path.join(options.profile, 'extensions', \
+                    'httpd@gaiamobile.org')):
+                print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
+                                               options.profile))
+                return 1
 
-        if os.path.isfile(os.path.join(kwargs["profile"], 'extensions',
+            options.desktop = True
-                                       'httpd@gaiamobile.org')):
+            options.app = self.get_binary_path()
-            print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
+            if options.oop:
-                                           kwargs["profile"]))
+                options.browser_arg = '-oop'
-            return 1
+            if not options.app.endswith('-bin'):
+                options.app = '%s-bin' % options.app
+            if not os.path.isfile(options.app):
+                options.app = options.app[:-len('-bin')]
 
-        kwargs["desktop"] = True
+            return reftest.run_desktop_reftests(parser, options, args)
-        kwargs["app"] = self.get_binary_path()
-        if kwargs["oop"]:
-            options.browser_arg = '-oop'
-        if not kwargs["app"].endswith('-bin'):
-            kwargs["app"] = '%s-bin' % options.app
-        if not os.path.isfile(kwargs["app"]):
-            options.app = kwargs["app"][:-len('-bin')]
 
-        return runreftestb2g.run(**kwargs)
-
-    def run_b2g_remote(self, b2g_home, xre_path, **kwargs):
-        import runreftestb2g
 
         try:
             which.which('adb')
         except which.WhichError:
             # TODO Find adb automatically if it isn't on the path
-            raise Exception(ADB_NOT_FOUND % ('%s-remote' % kwargs["suite"], b2g_home))
+            raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
 
-        kwargs["b2gPath"] = b2g_home
+        options.b2gPath = b2g_home
-        kwargs["logdir"] = self.reftest_dir
+        options.logdir = self.reftest_dir
-        kwargs["httpdPath"] = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
+        options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
-        kwargs["xrePath"] = xre_path
+        options.xrePath = xre_path
-        kwargs["ignoreWindowSize"] = True
+        options.ignoreWindowSize = True
+        options.filter = filter
 
         # Don't enable oop for crashtest until they run oop in automation
-        if kwargs["suite"] == 'reftest':
+        if suite == 'reftest':
-            kwargs["oop"] = True
+            options.oop = True
 
-        return runreftestb2g.run_remote(**kwargs)
+        return reftest.run_remote_reftests(parser, options, args)
 
-    def run_desktop_test(self, **kwargs):
+    def run_desktop_test(self, test_file=None, filter=None, suite=None,
-        """Runs a reftest."""
+            debugger=None, debugger_args=None, parallel=False, shuffle=False,
-        import runreftest
+            e10s=False, extraPrefs=None, this_chunk=None, total_chunks=None):
+        """Runs a reftest.
 
-        if kwargs["suite"] not in ('reftest', 'crashtest', 'jstestbrowser'):
+        test_file is a path to a test file. It can be a relative path from the
+        top source directory, an absolute filename, or a directory containing
+        test files.
+
+        filter is a regular expression (in JS syntax, as could be passed to the
+        RegExp constructor) to select which reftests to run from the manifest.
+
+        suite is the type of reftest to run. It can be one of ('reftest',
+        'crashtest', 'jstestbrowser').
+
+        debugger is the program name (in $PATH) or the full path of the
+        debugger to run.
+
+        debugger_args are the arguments passed to the debugger.
+
+        parallel indicates whether tests should be run in parallel or not.
+
+        shuffle indicates whether to run tests in random order.
+        """
+
+        if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc', 'jstestbrowser'):
             raise Exception('None or unrecognized reftest suite type.')
 
-        default_manifest = {
+        env = {}
-            "reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"),
+        extra_args = []
-            "crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"),
-            "jstestbrowser": (self.topobjdir, "dist", "test-stage", "jsreftest", "tests",
-                              "jstests.list")
-        }
 
-        kwargs["extraProfileFiles"] = [os.path.join(self.topobjdir, "dist", "plugins")]
+        if test_file:
-        kwargs["symbolsPath"] = os.path.join(self.topobjdir, "crashreporter-symbols")
+            (path, single_file_filter) = self._find_manifest(suite, test_file)
+            if not os.path.exists(mozpath.join(self.topsrcdir, path)):
+                raise Exception('No manifest file was found at %s.' % path)
+            if single_file_filter:
+                if filter:
+                    raise Exception('Cannot run single files in conjunction with --filter')
+                filter = single_file_filter
+            env[b'TEST_PATH'] = path
+        if filter:
+            extra_args.extend(['--filter', self._make_shell_string(filter)])
 
-        if not kwargs["tests"]:
+        pass_thru = False
-            kwargs["tests"] = [os.path.join(*default_manifest[kwargs["suite"]])]
 
-        if kwargs["suite"] == "jstestbrowser":
+        if debugger:
-            kwargs["extraProfileFiles"].append(os.path.join(self.topobjdir, "dist",
+            extra_args.append('--debugger=\'%s\'' % debugger)
-                                                            "test-stage", "jsreftest",
+            pass_thru = True
-                                                            "tests", "user.js"))
+            if debugger_args:
+                # Use _make_shell_string (which quotes) so that we
+                # handle multiple args being passed to the debugger.
+                extra_args.extend(['--debugger-args', self._make_shell_string(debugger_args)])
+        else:
+            if debugger_args:
+                print("--debugger-args passed, but no debugger specified.")
+                return 1
 
-        if not kwargs["runTestsInParallel"]:
+        if parallel:
-            kwargs["logFile"] = "%s.log" % kwargs["suite"]
+            extra_args.append('--run-tests-in-parallel')
 
-        #Remove the stdout handler from the internal logger and let mach deal with it
+        if shuffle:
-        runreftest.log.removeHandler(runreftest.log.handlers[0])
+            extra_args.append('--shuffle')
-        self.log_manager.enable_unstructured()
-        rv = runreftest.run(**kwargs)
-        self.log_manager.disable_unstructured()
 
-        return rv
+        if e10s:
+            extra_args.append('--e10s')
+
+        if extraPrefs:
+            for pref in extraPrefs:
+                extra_args.extend(['--setpref', pref])
+
+        if this_chunk:
+            extra_args.append('--this-chunk=%s' % this_chunk)
+
+        if total_chunks:
+            extra_args.append('--total-chunks=%s' % total_chunks)
+
+        if extra_args:
+            args = [os.environ.get(b'EXTRA_TEST_ARGS', '')]
+            args.extend(extra_args)
+            env[b'EXTRA_TEST_ARGS'] = ' '.join(args)
+
+        # TODO hook up harness via native Python
+        return self._run_make(directory='.', target=suite, append_env=env,
+            pass_thru=pass_thru, ensure_exit_code=False)
 
 
-def process_test_objects(kwargs):
+def ReftestCommand(func):
-    """|mach test| works by providing a test_objects argument, from
+    """Decorator that adds shared command arguments to reftest commands."""
-    which the test path must be extracted and converted into a normal
-    reftest tests argument."""
 
-    if "test_objects" in kwargs:
+    debugger = CommandArgument('--debugger', metavar='DEBUGGER',
-        if kwargs["tests"] is None:
+        help=DEBUGGER_HELP)
-            kwargs["tests"] = []
+    func = debugger(func)
-        kwargs["tests"].extend(item["path"] for item in kwargs["test_objects"])
+
-        del kwargs["test_objects"]
+    debugger_args = CommandArgument('--debugger-args', metavar='DEBUGGER_ARGS',
+        help='Arguments to pass to the debugger.')
+    func = debugger_args(func)
+
+    flter = CommandArgument('--filter', metavar='REGEX',
+        help='A JS regular expression to match test URLs against, to select '
+             'a subset of tests to run.')
+    func = flter(func)
+
+    path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',
+        help='Reftest manifest file, or a directory in which to select '
+             'reftest.list. If omitted, the entire test suite is executed.')
+    func = path(func)
+
+    parallel = CommandArgument('--parallel', action='store_true',
+        help='Run tests in parallel.')
+    func = parallel(func)
+
+    shuffle = CommandArgument('--shuffle', action='store_true',
+        help='Run tests in random order.')
+    func = shuffle(func)
+
+    e10s = CommandArgument('--e10s', action='store_true',
+        help='Use content processes.')
+    func = e10s(func)
+
+    extraPrefs = CommandArgument('--setpref', action='append',
+        default=[], dest='extraPrefs', metavar='PREF=VALUE',
+        help='Set prefs in the reftest profile.')
+    func = extraPrefs(func)
+
+    totalChunks = CommandArgument('--total-chunks',
+        help = 'How many chunks to split the tests up into.')
+    func = totalChunks(func)
+
+    thisChunk = CommandArgument('--this-chunk',
+        help = 'Which chunk to run between 1 and --total-chunks.')
+    func = thisChunk(func)
+
+    return func
+
+def B2GCommand(func):
+    """Decorator that adds shared command arguments to b2g reftest commands."""
+
+    busybox = CommandArgument('--busybox', default=None,
+        help='Path to busybox binary to install on device')
+    func = busybox(func)
+
+    logdir = CommandArgument('--logdir', default=None,
+        help='directory to store log files')
+    func = logdir(func)
+
+    sdcard = CommandArgument('--sdcard', default="10MB",
+        help='Define size of sdcard: 1MB, 50MB...etc')
+    func = sdcard(func)
+
+    emulator_res = CommandArgument('--emulator-res', default='800x1000',
+        help='Emulator resolution of the format \'<width>x<height>\'')
+    func = emulator_res(func)
+
+    marionette = CommandArgument('--marionette', default=None,
+        help='host:port to use when connecting to Marionette')
+    func = marionette(func)
+
+    totalChunks = CommandArgument('--total-chunks', dest='totalChunks',
+        type = int,
+        help = 'How many chunks to split the tests up into.')
+    func = totalChunks(func)
+
+    thisChunk = CommandArgument('--this-chunk', dest='thisChunk',
+        type = int,
+        help = 'Which chunk to run between 1 and --total-chunks.')
+    func = thisChunk(func)
+
+    flter = CommandArgument('--filter', metavar='REGEX',
+        help='A JS regular expression to match test URLs against, to select '
+             'a subset of tests to run.')
+    func = flter(func)
+
+    oop = CommandArgument('--enable-oop', action='store_true', dest='oop',
+        help = 'Run tests in out-of-process mode.')
+    func = oop(func)
+
+    path = CommandArgument('test_file', default=None, nargs='?',
+        metavar='TEST',
+        help='Test to run. Can be specified as a single file, a ' \
+            'directory, or omitted. If omitted, the entire test suite is ' \
+            'executed.')
+    func = path(func)
+
+    return func
 
 
 @CommandProvider
 class MachCommands(MachCommandBase):
-    @Command('reftest',
+    @Command('reftest', category='testing', description='Run reftests (layout and graphics correctness).')
-             category='testing',
+    @ReftestCommand
-             description='Run reftests (layout and graphics correctness).',
+    def run_reftest(self, test_file, **kwargs):
-             parser=reftestcommandline.DesktopArgumentsParser)
+        return self._run_reftest(test_file, suite='reftest', **kwargs)
-    def run_reftest(self, **kwargs):
-        kwargs["suite"] = "reftest"
-        return self._run_reftest(**kwargs)
 
-    @Command('jstestbrowser',
+    @Command('jstestbrowser', category='testing',
-             category='testing',
+        description='Run js/src/tests in the browser.')
-             description='Run js/src/tests in the browser.',
+    @ReftestCommand
-             parser=reftestcommandline.DesktopArgumentsParser)
+    def run_jstestbrowser(self, test_file, **kwargs):
-    def run_jstestbrowser(self, **kwargs):
+        return self._run_reftest(test_file, suite='jstestbrowser', **kwargs)
-        self._mach_context.commands.dispatch("build",
-                                             self._mach_context,
-                                             what=["stage-jstests"])
-        kwargs["suite"] = "jstestbrowser"
-        return self._run_reftest(**kwargs)
 
-    @Command('reftest-ipc',
+    @Command('reftest-ipc', category='testing',
-             category='testing',
+        description='Run IPC reftests (layout and graphics correctness, separate process).')
-             description='Run IPC reftests (layout and graphics correctness, separate process).',
+    @ReftestCommand
-             parser=reftestcommandline.DesktopArgumentsParser)
+    def run_ipc(self, test_file, **kwargs):
-    def run_ipc(self, **kwargs):
+        return self._run_reftest(test_file, suite='reftest-ipc', **kwargs)
-        kwargs["ipc"] = True
-        kwargs["suite"] = "reftest"
-        return self._run_reftest(**kwargs)
 
-    @Command('crashtest',
+    @Command('crashtest', category='testing',
-             category='testing',
+        description='Run crashtests (Check if crashes on a page).')
-             description='Run crashtests (Check if crashes on a page).',
+    @ReftestCommand
-             parser=reftestcommandline.DesktopArgumentsParser)
+    def run_crashtest(self, test_file, **kwargs):
-    def run_crashtest(self, **kwargs):
+        return self._run_reftest(test_file, suite='crashtest', **kwargs)
-        kwargs["suite"] = "crashtest"
-        return self._run_reftest(**kwargs)
 
-    @Command('crashtest-ipc',
+    @Command('crashtest-ipc', category='testing',
-             category='testing',
+        description='Run IPC crashtests (Check if crashes on a page, separate process).')
-             description='Run IPC crashtests (Check if crashes on a page, separate process).',
+    @ReftestCommand
-             parser=reftestcommandline.DesktopArgumentsParser)
+    def run_crashtest_ipc(self, test_file, **kwargs):
-    def run_crashtest_ipc(self, **kwargs):
+        return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
-        kwargs["ipc"] = True
-        kwargs["suite"] = "crashtest"
-        return self._run_reftest(**kwargs)
 
-    def _run_reftest(self, **kwargs):
+    def _run_reftest(self, test_file=None, suite=None, **kwargs):
-        process_test_objects(kwargs)
         reftest = self._spawn(ReftestRunner)
-        return reftest.run_desktop_test(**kwargs)
+        return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
 
 
 # TODO For now b2g commands will only work with the emulator,
@@ -297,31 +466,27 @@ class B2GCommands(MachCommandBase):
             setattr(self, attr, getattr(context, attr, None))
 
     @Command('reftest-remote', category='testing',
-             description='Run a remote reftest (b2g layout and graphics correctness, remote device).',
+        description='Run a remote reftest (b2g layout and graphics correctness, remote device).',
-             conditions=[conditions.is_b2g, is_emulator],
+        conditions=[conditions.is_b2g, is_emulator])
-             parser=reftestcommandline.B2GArgumentParser)
+    @B2GCommand
-    def run_reftest_remote(self, **kwargs):
+    def run_reftest_remote(self, test_file, **kwargs):
-        kwargs["suite"] = "reftest"
+        return self._run_reftest(test_file, suite='reftest', **kwargs)
-        return self._run_reftest(**kwargs)
 
     @Command('reftest-b2g-desktop', category='testing',
-             description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).',
+        description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).',
-             conditions=[conditions.is_b2g_desktop],
+        conditions=[conditions.is_b2g_desktop])
-             parser=reftestcommandline.B2GArgumentParser)
+    @B2GCommand
-    def run_reftest_b2g_desktop(self, **kwargs):
+    def run_reftest_b2g_desktop(self, test_file, **kwargs):
-        kwargs["suite"] = "reftest"
+        return self._run_reftest(test_file, suite='reftest', **kwargs)
-        return self._run_reftest(**kwargs)
 
     @Command('crashtest-remote', category='testing',
-             description='Run a remote crashtest (Check if b2g crashes on a page, remote device).',
+        description='Run a remote crashtest (Check if b2g crashes on a page, remote device).',
-             conditions=[conditions.is_b2g, is_emulator],
+        conditions=[conditions.is_b2g, is_emulator])
-             parser=reftestcommandline.B2GArgumentParser)
+    @B2GCommand
     def run_crashtest_remote(self, test_file, **kwargs):
-        kwargs["suite"] = "crashtest"
+        return self._run_reftest(test_file, suite='crashtest', **kwargs)
-        return self._run_reftest(**kwargs)
 
-    def _run_reftest(self, **kwargs):
+    def _run_reftest(self, test_file=None, suite=None, **kwargs):
-        process_test_objects(kwargs)
         if self.device_name:
             if self.device_name.startswith('emulator'):
                 emulator = 'arm'
@@ -330,4 +495,5 @@ class B2GCommands(MachCommandBase):
                 kwargs['emulator'] = emulator
 
         reftest = self._spawn(ReftestRunner)
-        return reftest.run_b2g_test(self.b2g_home, self.xre_path, **kwargs)
+        return reftest.run_b2g_test(self.b2g_home, self.xre_path,
+            test_file, suite=suite, **kwargs)

layout/tools/reftest/reftest-cmdline.js

@@ -21,6 +21,37 @@ RefTestCmdLineHandler.prototype =
 
   /* nsICommandLineHandler */
   handle : function handler_handle(cmdLine) {
+    var args = { };
+    args.wrappedJSObject = args;
+    try {
+      var uristr = cmdLine.handleFlagWithParam("reftest", false);
+      if (uristr == null)
+        return;
+      try {
+        args.uri = cmdLine.resolveURI(uristr).spec;
+      }
+      catch (e) {
+        return;
+      }
+    }
+    catch (e) {
+      cmdLine.handleFlag("reftest", true);
+    }
+
+    try {
+      var nocache = cmdLine.handleFlag("reftestnocache", false);
+      args.nocache = nocache;
+    }
+    catch (e) {
+    }
+
+    try {
+      var skipslowtests = cmdLine.handleFlag("reftestskipslowtests", false);
+      args.skipslowtests = skipslowtests;
+    }
+    catch (e) {
+    }
+
     /* Ignore the platform's online/offline status while running reftests. */
     var ios = Components.classes["@mozilla.org/network/io-service;1"]
               .getService(Components.interfaces.nsIIOService2);
@@ -47,7 +78,7 @@ RefTestCmdLineHandler.prototype =
 
     function loadReftests() {
       wwatch.openWindow(null, "chrome://reftest/content/reftest.xul", "_blank",
-                        "chrome,dialog=no,all", {});
+                        "chrome,dialog=no,all", args);
     }
 
     var remote = false;

layout/tools/reftest/reftest.js

@@ -48,7 +48,7 @@ var gShuffle = false;
 var gTotalChunks = 0;
 var gThisChunk = 0;
 var gContainingWindow = null;
-var gURLFilterRegex = {};
+var gURLFilterRegex = null;
 const FOCUS_FILTER_ALL_TESTS = "all";
 const FOCUS_FILTER_NEEDS_FOCUS_TESTS = "needs-focus";
 const FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS = "non-needs-focus";
@@ -69,7 +69,6 @@ var gCanvas1, gCanvas2;
 // RecordResult.
 var gCurrentCanvas = null;
 var gURLs;
-var gManifestsLoaded = {};
 // Map from URI spec to the number of times it remains to be used
 var gURIUseCounts;
 // Map from URI spec to the canvas rendered for that URI
@@ -390,6 +389,10 @@ function InitAndStartRefTests()
         gThisChunk = 0;
     }
 
+    try {
+        gURLFilterRegex = new RegExp(prefs.getCharPref("reftest.filter"));
+    } catch(e) {}
+
     try {
         gFocusFilterMode = prefs.getCharPref("reftest.focusFilterMode");
     } catch(e) {}
@@ -446,7 +449,8 @@ function Shuffle(array)
 
 function StartTests()
 {
-    var manifests;
+    var uri;
+#if BOOTSTRAP
     /* These prefs are optional, so we don't need to spit an error to the log */
     try {
         var prefs = Components.classes["@mozilla.org/preferences-service;1"].
@@ -473,32 +477,41 @@ function StartTests()
         gRunSlowTests = false;
     }
 
+    try {
+        uri = prefs.getCharPref("reftest.uri");
+    } catch(e) {
+        uri = "";
+    }
+
+    if (uri == "") {
+        gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.uri pref.  Please ensure your profile is setup properly\n");
+        DoneTests();
+    }
+#else
+    try {
+        // Need to read the manifest once we have gHttpServerPort..
+        var args = window.arguments[0].wrappedJSObject;
+
+        if ("nocache" in args && args["nocache"])
+            gNoCanvasCache = true;
+
+        if ("skipslowtests" in args && args.skipslowtests)
+            gRunSlowTests = false;
+
+        uri = args.uri;
+    } catch (e) {
+        ++gTestResults.Exception;
+        gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | EXCEPTION: " + ex + "\n");
+        DoneTests();
+    }
+#endif
+
     if (gShuffle) {
         gNoCanvasCache = true;
     }
 
-    gURLs = [];
-
     try {
-        var manifests = JSON.parse(prefs.getCharPref("reftest.manifests"));
+        ReadTopManifest(uri);
-        gURLFilterRegex = manifests[null];
-    } catch(e) {
-        gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.manifests pref.  Please ensure your profile is setup properly\n");
-        DoneTests();
-    }
-
-    try {
-        var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null;
-        var manifestURLs = Object.keys(manifests);
-
-        // Ensure we read manifests from higher up the directory tree first so that we
-        // process includes before reading the included manifest again
-        manifestURLs.sort(function(a,b) {return a.length - b.length})
-        manifestURLs.forEach(function(manifestURL) {
-            gDumpLog("Readings manifest" + manifestURL + "\n");
-            var filter = manifests[manifestURL] ? new RegExp(manifests[manifestURL]) : null;
-            ReadTopManifest(manifestURL, [globalFilter, filter, false]);
-        });
         BuildUseCounts();
 
         // Filter tests which will be skipped to get a more even distribution when chunking
@@ -751,25 +764,18 @@ function AddPrefSettings(aWhere, aPrefName, aPrefValExpression, aSandbox, aTestP
     return true;
 }
 
-function ReadTopManifest(aFileURL, aFilter)
+function ReadTopManifest(aFileURL)
 {
+    gURLs = new Array();
     var url = gIOService.newURI(aFileURL, null, null);
     if (!url)
         throw "Expected a file or http URL for the manifest.";
-    ReadManifest(url, EXPECTED_PASS, aFilter);
+    ReadManifest(url, EXPECTED_PASS);
 }
 
-function AddTestItem(aTest, aFilter)
+function AddTestItem(aTest)
 {
-    if (!aFilter)
+    if (gURLFilterRegex && !gURLFilterRegex.test(aTest.url1.spec))
-        aFilter = [null, [], false];
-
-    globalFilter = aFilter[0];
-    manifestFilter = aFilter[1];
-    invertManifest = aFilter[2];
-    if ((globalFilter && !globalFilter.test(aTest.url1.spec)) ||
-        (manifestFilter &&
-         !(invertManifest ^ manifestFilter.test(aTest.url1.spec))))
         return;
     if (gFocusFilterMode == FOCUS_FILTER_NEEDS_FOCUS_TESTS &&
         !aTest.needsFocus)
@@ -782,19 +788,8 @@ function AddTestItem(aTest, aFilter)
 
 // Note: If you materially change the reftest manifest parsing,
 // please keep the parser in print-manifest-dirs.py in sync.
-function ReadManifest(aURL, inherited_status, aFilter)
+function ReadManifest(aURL, inherited_status)
 {
-    // Ensure each manifest is only read once. This assumes that manifests that are
-    // included with an unusual inherited_status or filters will be read via their
-    // include before they are read directly in the case of a duplicate
-    if (gManifestsLoaded.hasOwnProperty(aURL.spec)) {
-        if (gManifestsLoaded[aURL.spec] === null)
-            return;
-        else
-            aFilter = [aFilter[0], aFilter[1], true];
-    }
-    gManifestsLoaded[aURL.spec] = aFilter[1];
-
     var secMan = CC[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
                      .getService(CI.nsIScriptSecurityManager);
 
@@ -1010,7 +1005,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
             var incURI = gIOService.newURI(items[1], null, listURL);
             secMan.checkLoadURIWithPrincipal(principal, incURI,
                                              CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
-            ReadManifest(incURI, expected_status, aFilter);
+            ReadManifest(incURI, expected_status);
         } else if (items[0] == TYPE_LOAD) {
             if (items.length != 2)
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to load";
@@ -1040,7 +1035,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
                           fuzzyMaxPixels: fuzzy_max_pixels,
                           url1: testURI,
                           url2: null,
-                          chaosMode: chaosMode }, aFilter);
+                          chaosMode: chaosMode });
         } else if (items[0] == TYPE_SCRIPT) {
             if (items.length != 2)
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to script";
@@ -1067,7 +1062,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
                           fuzzyMaxPixels: fuzzy_max_pixels,
                           url1: testURI,
                           url2: null,
-                          chaosMode: chaosMode }, aFilter);
+                          chaosMode: chaosMode });
         } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) {
             if (items.length != 3)
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0];
@@ -1097,7 +1092,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
                           fuzzyMaxPixels: fuzzy_max_pixels,
                           url1: testURI,
                           url2: refURI,
-                          chaosMode: chaosMode }, aFilter);
+                          chaosMode: chaosMode });
         } else {
             throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unknown test type " + items[0];
         }

layout/tools/reftest/reftestcommandline.py

@@ -1,740 +0,0 @@
-import argparse
-import os
-from collections import OrderedDict
-from urlparse import urlparse
-
-here = os.path.abspath(os.path.dirname(__file__))
-
-
-class ReftestArgumentsParser(argparse.ArgumentParser):
-    def __init__(self, **kwargs):
-        super(ReftestArgumentsParser, self).__init__(**kwargs)
-
-        # Try to import a MozbuildObject. Success indicates that we are
-        # running from a source tree. This allows some defaults to be set
-        # from the source tree.
-        try:
-            from mozbuild.base import MozbuildObject
-            self.build_obj = MozbuildObject.from_environment(cwd=here)
-        except ImportError:
-            self.build_obj = None
-
-        self.add_argument("--xre-path",
-                          action="store",
-                          type=str,
-                          dest="xrePath",
-                          # individual scripts will set a sane default
-                          default=None,
-                          help="absolute path to directory containing XRE (probably xulrunner)")
-
-        self.add_argument("--symbols-path",
-                          action="store",
-                          type=str,
-                          dest="symbolsPath",
-                          default=None,
-                          help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
-
-        self.add_argument("--debugger",
-                          action="store",
-                          dest="debugger",
-                          help="use the given debugger to launch the application")
-
-        self.add_argument("--debugger-args",
-                          action="store",
-                          dest="debuggerArgs",
-                          help="pass the given args to the debugger _before_ "
-                          "the application on the command line")
-
-        self.add_argument("--debugger-interactive",
-                          action="store_true",
-                          dest="debuggerInteractive",
-                          help="prevents the test harness from redirecting "
-                          "stdout and stderr for interactive debuggers")
-
-        self.add_argument("--appname",
-                          action="store",
-                          type=str,
-                          dest="app",
-                          default=None,
-                          help="absolute path to application, overriding default")
-
-        self.add_argument("--extra-profile-file",
-                          action="append",
-                          dest="extraProfileFiles",
-                          default=[],
-                          help="copy specified files/dirs to testing profile")
-
-        self.add_argument("--timeout",
-                          action="store",
-                          dest="timeout",
-                          type=int,
-                          default=5 * 60,  # 5 minutes per bug 479518
-                          help="reftest will timeout in specified number of seconds. [default %(default)s].")
-
-        self.add_argument("--leak-threshold",
-                          action="store",
-                          type=int,
-                          dest="defaultLeakThreshold",
-                          default=0,
-                          help="fail if the number of bytes leaked in default "
-                          "processes through refcounted objects (or bytes "
-                          "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
-                          "is greater than the given number")
-
-        self.add_argument("--utility-path",
-                          action="store",
-                          type=str,
-                          dest="utilityPath",
-                          default="bindir",
-                          help="absolute path to directory containing utility "
-                          "programs (xpcshell, ssltunnel, certutil)")
-
-        self.add_argument("--total-chunks",
-                          type=int,
-                          dest="totalChunks",
-                          help="how many chunks to split the tests up into")
-
-        self.add_argument("--this-chunk",
-                          type=int,
-                          dest="thisChunk",
-                          help="which chunk to run between 1 and --total-chunks")
-
-        self.add_argument("--log-file",
-                          action="store",
-                          type=str,
-                          dest="logFile",
-                          default=None,
-                          help="file to log output to in addition to stdout")
-
-        self.add_argument("--skip-slow-tests",
-                          dest="skipSlowTests",
-                          action="store_true",
-                          default=False,
-                          help="skip tests marked as slow when running")
-
-        self.add_argument("--ignore-window-size",
-                          dest="ignoreWindowSize",
-                          action="store_true",
-                          default=False,
-                          help="ignore the window size, which may cause spurious failures and passes")
-
-        self.add_argument("--install-extension",
-                          action="append",
-                          dest="extensionsToInstall",
-                          default=[],
-                          help="install the specified extension in the testing profile. "
-                          "The extension file's name should be <id>.xpi where <id> is "
-                          "the extension's id as indicated in its install.rdf. "
-                          "An optional path can be specified too.")
-
-        self.add_argument("--setenv",
-                          action="append",
-                          type=str,
-                          default=[],
-                          dest="environment",
-                          metavar="NAME=VALUE",
-                          help="sets the given variable in the application's "
-                          "environment")
-
-        self.add_argument("--filter",
-                          action="store",
-                          type=str,
-                          dest="filter",
-                          help="specifies a regular expression (as could be passed to the JS "
-                          "RegExp constructor) to test against URLs in the reftest manifest; "
-                          "only test items that have a matching test URL will be run.")
-
-        self.add_argument("--shuffle",
-                          action="store_true",
-                          default=False,
-                          dest="shuffle",
-                          help="run reftests in random order")
-
-        self.add_argument("--focus-filter-mode",
-                          action="store",
-                          type=str,
-                          dest="focusFilterMode",
-                          default="all",
-                          help="filters tests to run by whether they require focus. "
-                          "Valid values are `all', `needs-focus', or `non-needs-focus'. "
-                          "Defaults to `all'.")
-
-        self.add_argument("--e10s",
-                          action="store_true",
-                          default=False,
-                          dest="e10s",
-                          help="enables content processes")
-
-        self.add_argument("--setpref",
-                          action="append",
-                          type=str,
-                          default=[],
-                          dest="extraPrefs",
-                          metavar="PREF=VALUE",
-                          help="defines an extra user preference")
-
-        self.add_argument("--reftest-extension-path",
-                          action="store",
-                          dest="reftestExtensionPath",
-                          help="Path to the reftest extension")
-
-        self.add_argument("--special-powers-extension-path",
-                          action="store",
-                          dest="specialPowersExtensionPath",
-                          help="Path to the special powers extension")
-
-        self.add_argument("--suite",
-                          choices=["reftest", "crashtest", "jstestbrowser"],
-                          default=None,
-                          help=argparse.SUPPRESS)
-
-        self.add_argument("tests",
-                          metavar="TEST_PATH",
-                          nargs="*",
-                          help="Path to test file, manifest file, or directory containing tests")
-
-    def get_ip(self):
-        import moznetwork
-        if os.name != "nt":
-            return moznetwork.get_ip()
-        else:
-            self.error(
-                "ERROR: you must specify a --remote-webserver=<ip address>\n")
-
-    def set_default_suite(self, options):
-        manifests = OrderedDict([("reftest.list", "reftest"),
-                                 ("crashtests.list", "crashtest"),
-                                 ("jstests.list", "jstestbrowser")])
-
-        for test_path in options.tests:
-            file_name = os.path.basename(test_path)
-            if file_name in manifests:
-                options.suite = manifests[file_name]
-                return
-
-        for test_path in options.tests:
-            for manifest_file, suite in manifests.iteritems():
-                if os.path.exists(os.path.join(test_path, manifest_file)):
-                    options.suite = suite
-                    return
-
-        self.error("Failed to determine test suite; supply --suite to set this explicitly")
-
-    def validate(self, options, reftest):
-        import sys
-
-        if not options.tests:
-            # Can't just set this in the argument parser because mach will set a default
-            self.error("Must supply at least one path to a manifest file, test directory, or test file to run.")
-
-        if options.suite is None:
-            self.set_default_suite(options)
-
-        if options.totalChunks is not None and options.thisChunk is None:
-            self.error(
-                "thisChunk must be specified when totalChunks is specified")
-
-        if options.totalChunks:
-            if not 1 <= options.thisChunk <= options.totalChunks:
-                self.error("thisChunk must be between 1 and totalChunks")
-
-        if options.logFile:
-            options.logFile = reftest.getFullPath(options.logFile)
-
-        if options.xrePath is not None:
-            if not os.access(options.xrePath, os.F_OK):
-                self.error("--xre-path '%s' not found" % options.xrePath)
-            if not os.path.isdir(options.xrePath):
-                self.error("--xre-path '%s' is not a directory" %
-                           options.xrePath)
-            options.xrePath = reftest.getFullPath(options.xrePath)
-
-        if options.reftestExtensionPath is None:
-            if self.build_obj is not None:
-                options.reftestExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
-                                                            "reftest", "reftest")
-            else:
-                options.reftestExtensionPath = os.path.join(here, "reftest")
-
-        if (options.specialPowersExtensionPath is None and
-            options.suite in ["crashtest", "jstestbrowser"]):
-            if self.build_obj is not None:
-                options.specialPowersExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
-                                                                  "reftest", "specialpowers")
-            else:
-                options.specialPowersExtensionPath = os.path.join(
-                    here, "specialpowers")
-
-        options.leakThresholds = {
-            "default": options.defaultLeakThreshold,
-            "tab": 5000,  # See dependencies of bug 1051230.
-        }
-
-
-class DesktopArgumentsParser(ReftestArgumentsParser):
-    def __init__(self, **kwargs):
-        super(DesktopArgumentsParser, self).__init__(**kwargs)
-
-        self.add_argument("--run-tests-in-parallel",
-                          action="store_true",
-                          default=False,
-                          dest="runTestsInParallel",
-                          help="run tests in parallel if possible")
-
-        self.add_argument("--ipc",
-                          action="store_true",
-                          default=False,
-                          help="Run in out-of-processes mode")
-
-    def _prefs_oop(self):
-        import mozinfo
-        prefs = ["layers.async-pan-zoom.enabled=true",
-                 "browser.tabs.remote.autostart=true"]
-        if mozinfo.os == "win":
-            prefs.append("layers.acceleration.disabled=true")
-
-        return prefs
-
-    def _prefs_gpu(self):
-        if mozinfo.os != "win":
-            return ["layers.acceleration.force-enabled=true"]
-        return []
-
-    def validate(self, options, reftest):
-        super(DesktopArgumentsParser, self).validate(options, reftest)
-
-        if options.ipc:
-            for item in self._prefs_oop():
-                if item not in options.extraPrefs:
-                    options.extraPrefs.append(item)
-
-        if options.runTestsInParallel:
-            if options.logFile is not None:
-                self.error("cannot specify logfile with parallel tests")
-            if options.totalChunks is not None or options.thisChunk is not None:
-                self.error(
-                    "cannot specify thisChunk or totalChunks with parallel tests")
-            if options.focusFilterMode != "all":
-                self.error("cannot specify focusFilterMode with parallel tests")
-            if options.debugger is not None:
-                self.error("cannot specify a debugger with parallel tests")
-
-        if not options.tests:
-            self.error("No test files specified.")
-
-        if options.app is None:
-            bin_dir = (self.build_obj.get_binary_path() if
-                       self.build_obj and self.build_obj.substs[
-                           'MOZ_BUILD_APP'] != 'mobile/android'
-                       else None)
-
-            if bin_dir:
-                options.app = bin_dir
-            else:
-                self.error(
-                    "could not find the application path, --appname must be specified")
-
-        options.app = reftest.getFullPath(options.app)
-        if not os.path.exists(options.app):
-            self.error("""Error: Path %(app)s doesn't exist.
-            Are you executing $objdir/_tests/reftest/runreftest.py?"""
-                       % {"app": options.app})
-
-        if options.xrePath is None:
-            options.xrePath = os.path.dirname(options.app)
-
-        if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
-            options.symbolsPath = reftest.getFullPath(options.symbolsPath)
-
-        options.utilityPath = reftest.getFullPath(options.utilityPath)
-
-
-class B2GArgumentParser(ReftestArgumentsParser):
-    def __init__(self, **kwargs):
-        super(B2GArgumentParser, self).__init__(**kwargs)
-
-        self.add_argument("--browser-arg",
-                          action="store",
-                          type=str,
-                          dest="browser_arg",
-                          help="Optional command-line arg to pass to the browser")
-
-        self.add_argument("--b2gpath",
-                          action="store",
-                          type=str,
-                          dest="b2gPath",
-                          help="path to B2G repo or qemu dir")
-
-        self.add_argument("--marionette",
-                          action="store",
-                          type=str,
-                          dest="marionette",
-                          help="host:port to use when connecting to Marionette")
-
-        self.add_argument("--emulator",
-                          action="store",
-                          type=str,
-                          dest="emulator",
-                          help="Architecture of emulator to use: x86 or arm")
-
-        self.add_argument("--emulator-res",
-                          action="store",
-                          type=str,
-                          dest="emulator_res",
-                          help="Emulator resolution of the format '<width>x<height>'")
-
-        self.add_argument("--no-window",
-                          action="store_true",
-                          dest="noWindow",
-                          default=False,
-                          help="Pass --no-window to the emulator")
-
-        self.add_argument("--adbpath",
-                          action="store",
-                          type=str,
-                          dest="adb_path",
-                          default="adb",
-                          help="path to adb")
-
-        self.add_argument("--deviceIP",
-                          action="store",
-                          type=str,
-                          dest="deviceIP",
-                          help="ip address of remote device to test")
-
-        self.add_argument("--devicePort",
-                          action="store",
-                          type=str,
-                          dest="devicePort",
-                          default="20701",
-                          help="port of remote device to test")
-
-        self.add_argument("--remote-logfile",
-                          action="store",
-                          type=str,
-                          dest="remoteLogFile",
-                          help="Name of log file on the device relative to the device root.  PLEASE ONLY USE A FILENAME.")
-
-        self.add_argument("--remote-webserver",
-                          action="store",
-                          type=str,
-                          dest="remoteWebServer",
-                          help="ip address where the remote web server is hosted at")
-
-        self.add_argument("--http-port",
-                          action="store",
-                          type=str,
-                          dest="httpPort",
-                          help="ip address where the remote web server is hosted at")
-
-        self.add_argument("--ssl-port",
-                          action="store",
-                          type=str,
-                          dest="sslPort",
-                          help="ip address where the remote web server is hosted at")
-
-        self.add_argument("--pidfile",
-                          action="store",
-                          type=str,
-                          dest="pidFile",
-                          default="",
-                          help="name of the pidfile to generate")
-
-        self.add_argument("--gecko-path",
-                          action="store",
-                          type=str,
-                          dest="geckoPath",
-                          help="the path to a gecko distribution that should "
-                          "be installed on the emulator prior to test")
-
-        self.add_argument("--logdir",
-                          action="store",
-                          type=str,
-                          dest="logdir",
-                          help="directory to store log files")
-
-        self.add_argument('--busybox',
-                          action='store',
-                          type=str,
-                          dest='busybox',
-                          help="Path to busybox binary to install on device")
-
-        self.add_argument("--httpd-path",
-                          action="store",
-                          type=str,
-                          dest="httpdPath",
-                          help="path to the httpd.js file")
-
-        self.add_argument("--profile",
-                          action="store",
-                          type=str,
-                          dest="profile",
-                          help="for desktop testing, the path to the "
-                          "gaia profile to use")
-
-        self.add_argument("--desktop",
-                          action="store_true",
-                          dest="desktop",
-                          default=False,
-                          help="Run the tests on a B2G desktop build")
-
-        self.add_argument("--mulet",
-                          action="store_true",
-                          dest="mulet",
-                          default=False,
-                          help="Run the tests on a B2G desktop build")
-
-        self.add_argument("--enable-oop",
-                          action="store_true",
-                          dest="oop",
-                          default=False,
-                          help="Run the tests out of process")
-
-        self.set_defaults(remoteTestRoot=None,
-                          logFile="reftest.log",
-                          autorun=True,
-                          closeWhenDone=True,
-                          testPath="")
-
-    def validate_remote(self, options, automation):
-        if not options.app:
-            options.app = automation.DEFAULT_APP
-
-        if not options.remoteTestRoot:
-            options.remoteTestRoot = automation._devicemanager.deviceRoot + \
-                "/reftest"
-
-        options.remoteProfile = options.remoteTestRoot + "/profile"
-
-        productRoot = options.remoteTestRoot + "/" + automation._product
-        if options.utilityPath is None:
-            options.utilityPath = productRoot + "/bin"
-
-        if not options.httpPort:
-            options.httpPort = automation.DEFAULT_HTTP_PORT
-
-        if not options.sslPort:
-            options.sslPort = automation.DEFAULT_SSL_PORT
-
-        if options.remoteWebServer is None:
-            options.remoteWebServer = self.get_ip()
-
-        options.webServer = options.remoteWebServer
-
-        if options.geckoPath and not options.emulator:
-            self.error(
-                "You must specify --emulator if you specify --gecko-path")
-
-        if options.logdir and not options.emulator:
-            self.error("You must specify --emulator if you specify --logdir")
-
-        if options.remoteLogFile is None:
-            options.remoteLogFile = "reftest.log"
-
-        options.localLogName = options.remoteLogFile
-        options.remoteLogFile = options.remoteTestRoot + \
-            '/' + options.remoteLogFile
-
-        # Ensure that the options.logfile (which the base class uses) is set to
-        # the remote setting when running remote. Also, if the user set the
-        # log file name there, use that instead of reusing the remotelogfile as
-        # above.
-        if (options.logFile):
-            # If the user specified a local logfile name use that
-            options.localLogName = options.logFile
-        options.logFile = options.remoteLogFile
-
-        # Only reset the xrePath if it wasn't provided
-        if options.xrePath is None:
-            options.xrePath = options.utilityPath
-        options.xrePath = os.path.abspath(options.xrePath)
-
-        if options.pidFile != "":
-            f = open(options.pidFile, 'w')
-            f.write("%s" % os.getpid())
-            f.close()
-
-        # httpd-path is specified by standard makefile targets and may be specified
-        # on the command line to select a particular version of httpd.js. If not
-        # specified, try to select the one from from the xre bundle, as
-        # required in bug 882932.
-        if not options.httpdPath:
-            options.httpdPath = os.path.join(options.xrePath, "components")
-
-        return options
-
-
-class RemoteArgumentsParser(ReftestArgumentsParser):
-    def __init__(self, **kwargs):
-        super(RemoteArgumentsParser, self).__init__()
-
-        # app, xrePath and utilityPath variables are set in main function
-        self.set_defaults(logFile="reftest.log",
-                          app="",
-                          xrePath="",
-                          utilityPath="",
-                          localLogName=None)
-
-        self.add_argument("--remote-app-path",
-                          action="store",
-                          type=str,
-                          dest="remoteAppPath",
-                          help="Path to remote executable relative to device root using only forward slashes.  Either this or app must be specified, but not both.")
-
-        self.add_argument("--deviceIP",
-                          action="store",
-                          type=str,
-                          dest="deviceIP",
-                          help="ip address of remote device to test")
-
-        self.add_argument("--deviceSerial",
-                          action="store",
-                          type=str,
-                          dest="deviceSerial",
-                          help="adb serial number of remote device to test")
-
-        self.add_argument("--devicePort",
-                          action="store",
-                          type=str,
-                          default="20701",
-                          dest="devicePort",
-                          help="port of remote device to test")
-
-        self.add_argument("--remote-product-name",
-                          action="store",
-                          type=str,
-                          dest="remoteProductName",
-                          default="fennec",
-                          help="Name of product to test - either fennec or firefox, defaults to fennec")
-
-        self.add_argument("--remote-webserver",
-                          action="store",
-                          type=str,
-                          dest="remoteWebServer",
-                          help="IP Address of the webserver hosting the reftest content")
-
-        self.add_argument("--http-port",
-                          action="store",
-                          type=str,
-                          dest="httpPort",
-                          help="port of the web server for http traffic")
-
-        self.add_argument("--ssl-port",
-                          action="store",
-                          type=str,
-                          dest="sslPort",
-                          help="Port for https traffic to the web server")
-
-        self.add_argument("--remote-logfile",
-                          action="store",
-                          type=str,
-                          dest="remoteLogFile",
-                          default="reftest.log",
-                          help="Name of log file on the device relative to device root.  PLEASE USE ONLY A FILENAME.")
-
-        self.add_argument("--pidfile",
-                          action="store",
-                          type=str,
-                          dest="pidFile",
-                          default="",
-                          help="name of the pidfile to generate")
-
-        self.add_argument("--bootstrap",
-                          action="store_true",
-                          dest="bootstrap",
-                          default=False,
-                          help="test with a bootstrap addon required for native Fennec")
-
-        self.add_argument("--dm_trans",
-                          action="store",
-                          type=str,
-                          dest="dm_trans",
-                          default="sut",
-                          help="the transport to use to communicate with device: [adb|sut]; default=sut")
-
-        self.add_argument("--remoteTestRoot",
-                          action="store",
-                          type=str,
-                          dest="remoteTestRoot",
-                          help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
-
-        self.add_argument("--httpd-path",
-                          action="store",
-                          type=str,
-                          dest="httpdPath",
-                          help="path to the httpd.js file")
-
-    def validate_remote(self, options, automation):
-        # Ensure our defaults are set properly for everything we can infer
-        if not options.remoteTestRoot:
-            options.remoteTestRoot = automation._devicemanager.deviceRoot + \
-                '/reftest'
-        options.remoteProfile = options.remoteTestRoot + "/profile"
-
-        if options.remoteWebServer is None:
-            options.remoteWebServer = self.get_ip()
-
-        # Verify that our remotewebserver is set properly
-        if options.remoteWebServer == '127.0.0.1':
-            self.error("ERROR: Either you specified the loopback for the remote webserver or ",
-                       "your local IP cannot be detected.  Please provide the local ip in --remote-webserver")
-
-        if not options.httpPort:
-            options.httpPort = automation.DEFAULT_HTTP_PORT
-
-        if not options.sslPort:
-            options.sslPort = automation.DEFAULT_SSL_PORT
-
-        # One of remoteAppPath (relative path to application) or the app (executable) must be
-        # set, but not both.  If both are set, we destroy the user's selection for app
-        # so instead of silently destroying a user specificied setting, we
-        # error.
-        if options.remoteAppPath and options.app:
-            self.error(
-                "ERROR: You cannot specify both the remoteAppPath and the app")
-        elif options.remoteAppPath:
-            options.app = options.remoteTestRoot + "/" + options.remoteAppPath
-        elif options.app is None:
-            # Neither remoteAppPath nor app are set -- error
-            self.error("ERROR: You must specify either appPath or app")
-
-        if options.xrePath is None:
-            self.error(
-                "ERROR: You must specify the path to the controller xre directory")
-        else:
-            # Ensure xrepath is a full path
-            options.xrePath = os.path.abspath(options.xrePath)
-
-        options.localLogName = options.remoteLogFile
-        options.remoteLogFile = options.remoteTestRoot + \
-            '/' + options.remoteLogFile
-
-        # Ensure that the options.logfile (which the base class uses) is set to
-        # the remote setting when running remote. Also, if the user set the
-        # log file name there, use that instead of reusing the remotelogfile as
-        # above.
-        if options.logFile:
-            # If the user specified a local logfile name use that
-            options.localLogName = options.logFile
-
-        options.logFile = options.remoteLogFile
-
-        if options.pidFile != "":
-            with open(options.pidFile, 'w') as f:
-                f.write(str(os.getpid()))
-
-        # httpd-path is specified by standard makefile targets and may be specified
-        # on the command line to select a particular version of httpd.js. If not
-        # specified, try to select the one from hostutils.zip, as required in
-        # bug 882932.
-        if not options.httpdPath:
-            options.httpdPath = os.path.join(options.utilityPath, "components")
-
-        if not options.ignoreWindowSize:
-            parts = automation._devicemanager.getInfo(
-                'screen')['screen'][0].split()
-            width = int(parts[0].split(':')[1])
-            height = int(parts[1].split(':')[1])
-            if (width < 1050 or height < 1050):
-                self.error("ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (
-                    width, height))

layout/tools/reftest/remotereftest.py

@@ -9,9 +9,10 @@ import tempfile
 import traceback
 
 # We need to know our current directory so that we can serve our test files from it.
-SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 
-from runreftest import RefTest, ReftestResolver
+from runreftest import RefTest
+from runreftest import ReftestOptions
 from automation import Automation
 import devicemanager
 import droid
@@ -19,26 +20,159 @@ import mozinfo
 import moznetwork
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
 
-import reftestcommandline
+class RemoteOptions(ReftestOptions):
+    def __init__(self, automation):
+        ReftestOptions.__init__(self)
+        self.automation = automation
 
-class RemoteReftestResolver(ReftestResolver):
+        defaults = {}
-    def absManifestPath(self, path):
+        defaults["logFile"] = "reftest.log"
-        script_abs_path = os.path.join(SCRIPT_DIRECTORY, path)
+        # app, xrePath and utilityPath variables are set in main function
-        if os.path.exists(script_abs_path):
+        defaults["app"] = ""
-            rv = script_abs_path
+        defaults["xrePath"] = ""
-        elif os.path.exists(os.path.abspath(path)):
+        defaults["utilityPath"] = ""
-            rv = os.path.abspath(path)
+        defaults["runTestsInParallel"] = False
+
+        self.add_option("--remote-app-path", action="store",
+                    type = "string", dest = "remoteAppPath",
+                    help = "Path to remote executable relative to device root using only forward slashes.  Either this or app must be specified, but not both.")
+        defaults["remoteAppPath"] = None
+
+        self.add_option("--deviceIP", action="store",
+                    type = "string", dest = "deviceIP",
+                    help = "ip address of remote device to test")
+        defaults["deviceIP"] = None
+
+        self.add_option("--deviceSerial", action="store",
+                    type = "string", dest = "deviceSerial",
+                    help = "adb serial number of remote device to test")
+        defaults["deviceSerial"] = None
+
+        self.add_option("--devicePort", action="store",
+                    type = "string", dest = "devicePort",
+                    help = "port of remote device to test")
+        defaults["devicePort"] = 20701
+
+        self.add_option("--remote-product-name", action="store",
+                    type = "string", dest = "remoteProductName",
+                    help = "Name of product to test - either fennec or firefox, defaults to fennec")
+        defaults["remoteProductName"] = "fennec"
+
+        self.add_option("--remote-webserver", action="store",
+                    type = "string", dest = "remoteWebServer",
+                    help = "IP Address of the webserver hosting the reftest content")
+        defaults["remoteWebServer"] = moznetwork.get_ip()
+
+        self.add_option("--http-port", action = "store",
+                    type = "string", dest = "httpPort",
+                    help = "port of the web server for http traffic")
+        defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
+
+        self.add_option("--ssl-port", action = "store",
+                    type = "string", dest = "sslPort",
+                    help = "Port for https traffic to the web server")
+        defaults["sslPort"] = automation.DEFAULT_SSL_PORT
+
+        self.add_option("--remote-logfile", action="store",
+                    type = "string", dest = "remoteLogFile",
+                    help = "Name of log file on the device relative to device root.  PLEASE USE ONLY A FILENAME.")
+        defaults["remoteLogFile"] = None
+
+        self.add_option("--pidfile", action = "store",
+                    type = "string", dest = "pidFile",
+                    help = "name of the pidfile to generate")
+        defaults["pidFile"] = ""
+
+        self.add_option("--bootstrap", action="store_true", dest = "bootstrap",
+                    help = "test with a bootstrap addon required for native Fennec")
+        defaults["bootstrap"] = False
+
+        self.add_option("--dm_trans", action="store",
+                    type = "string", dest = "dm_trans",
+                    help = "the transport to use to communicate with device: [adb|sut]; default=sut")
+        defaults["dm_trans"] = "sut"
+
+        self.add_option("--remoteTestRoot", action = "store",
+                    type = "string", dest = "remoteTestRoot",
+                    help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
+        defaults["remoteTestRoot"] = None
+
+        self.add_option("--httpd-path", action = "store",
+                    type = "string", dest = "httpdPath",
+                    help = "path to the httpd.js file")
+        defaults["httpdPath"] = None
+
+        defaults["localLogName"] = None
+
+        self.set_defaults(**defaults)
+
+    def verifyRemoteOptions(self, options):
+        if options.runTestsInParallel:
+            self.error("Cannot run parallel tests here")
+
+        # Ensure our defaults are set properly for everything we can infer
+        if not options.remoteTestRoot:
+            options.remoteTestRoot = self.automation._devicemanager.deviceRoot + '/reftest'
+        options.remoteProfile = options.remoteTestRoot + "/profile"
+
+        # Verify that our remotewebserver is set properly
+        if (options.remoteWebServer == None or
+            options.remoteWebServer == '127.0.0.1'):
+            print "ERROR: Either you specified the loopback for the remote webserver or ",
+            print "your local IP cannot be detected.  Please provide the local ip in --remote-webserver"
+            return None
+
+        # One of remoteAppPath (relative path to application) or the app (executable) must be
+        # set, but not both.  If both are set, we destroy the user's selection for app
+        # so instead of silently destroying a user specificied setting, we error.
+        if (options.remoteAppPath and options.app):
+            print "ERROR: You cannot specify both the remoteAppPath and the app"
+            return None
+        elif (options.remoteAppPath):
+            options.app = options.remoteTestRoot + "/" + options.remoteAppPath
+        elif (options.app == None):
+            # Neither remoteAppPath nor app are set -- error
+            print "ERROR: You must specify either appPath or app"
+            return None
+
+        if (options.xrePath == None):
+            print "ERROR: You must specify the path to the controller xre directory"
+            return None
         else:
-            print >> sys.stderr, "Could not find manifest %s" % script_abs_path
+            # Ensure xrepath is a full path
-            sys.exit(1)
+            options.xrePath = os.path.abspath(options.xrePath)
-        return os.path.normpath(rv)
 
-    def manifestURL(self, options, path):
+        # Default to <deviceroot>/reftest/reftest.log
-        # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
+        if (options.remoteLogFile == None):
-        # It's possible for this url to have a leading "..", but reftest.js will fix that up
+            options.remoteLogFile = 'reftest.log'
-        relPath = os.path.relpath(path, SCRIPT_DIRECTORY)
-        return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath)
 
+        options.localLogName = options.remoteLogFile
+        options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
+
+        # Ensure that the options.logfile (which the base class uses) is set to
+        # the remote setting when running remote. Also, if the user set the
+        # log file name there, use that instead of reusing the remotelogfile as above.
+        if (options.logFile):
+            # If the user specified a local logfile name use that
+            options.localLogName = options.logFile
+
+        options.logFile = options.remoteLogFile
+
+        if (options.pidFile != ""):
+            f = open(options.pidFile, 'w')
+            f.write("%s" % os.getpid())
+            f.close()
+
+        # httpd-path is specified by standard makefile targets and may be specified
+        # on the command line to select a particular version of httpd.js. If not
+        # specified, try to select the one from hostutils.zip, as required in bug 882932.
+        if not options.httpdPath:
+            options.httpdPath = os.path.join(options.utilityPath, "components")
+
+        # TODO: Copied from main, but I think these are no longer used in a post xulrunner world
+        #options.xrePath = options.remoteTestRoot + self.automation._product + '/xulrunner'
+        #options.utilityPath = options.testRoot + self.automation._product + '/bin'
+        return options
 
 class ReftestServer:
     """ Web server used to serve Reftests, for closer fidelity to the real web.
@@ -126,7 +260,6 @@ class ReftestServer:
 
 class RemoteReftest(RefTest):
     remoteApp = ''
-    resolver_cls = RemoteReftestResolver
 
     def __init__(self, automation, devicemanager, options, scriptDir):
         RefTest.__init__(self)
@@ -209,12 +342,8 @@ class RemoteReftest(RefTest):
     def stopWebServer(self, options):
         self.server.stop()
 
-    def createReftestProfile(self, options, manifest):
+    def createReftestProfile(self, options, reftestlist):
-        profile = RefTest.createReftestProfile(self,
+        profile = RefTest.createReftestProfile(self, options, reftestlist, server=options.remoteWebServer, port=options.httpPort)
-                                               options,
-                                               manifest,
-                                               server=options.remoteWebServer,
-                                               port=options.httpPort)
         profileDir = profile.profile
 
         prefs = {}
@@ -226,6 +355,7 @@ class RemoteReftest(RefTest):
         # Set a future policy version to avoid the telemetry prompt.
         prefs["toolkit.telemetry.prompted"] = 999
         prefs["toolkit.telemetry.notifiedOptOut"] = 999
+        prefs["reftest.uri"] = "%s" % reftestlist
         prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
 
         # Point the url-classifier to the local testing server for fast failures
@@ -280,6 +410,9 @@ class RemoteReftest(RefTest):
             print "Automation Error: Failed to copy extra files to device"
             raise
 
+    def getManifestPath(self, path):
+        return path
+
     def printDeviceInfo(self, printLogcat=False):
         try:
             if printLogcat:
@@ -340,10 +473,10 @@ class RemoteReftest(RefTest):
             except:
                 print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
 
-def main():
+def main(args):
     automation = RemoteAutomation(None)
-    parser = reftestcommandline.RemoteArgumentsParser()
+    parser = RemoteOptions(automation)
-    options = parser.parse_args()
+    options, args = parser.parse_args()
 
     if (options.dm_trans == 'sut' and options.deviceIP == None):
         print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option"
@@ -369,7 +502,18 @@ def main():
         automation.setProduct(options.remoteProductName)
 
     # Set up the defaults and ensure options are set
-    parser.validate_remote(options, automation)
+    options = parser.verifyRemoteOptions(options)
+    if (options == None):
+        print "ERROR: Invalid options specified, use --help for a list of valid options"
+        return 1
+
+    if not options.ignoreWindowSize:
+        parts = dm.getInfo('screen')['screen'][0].split()
+        width = int(parts[0].split(':')[1])
+        height = int(parts[1].split(':')[1])
+        if (width < 1050 or height < 1050):
+            print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height)
+            return 1
 
     # Check that Firefox is installed
     expected = options.app.split('/')[-1]
@@ -382,7 +526,7 @@ def main():
     automation.setRemoteProfile(options.remoteProfile)
     automation.setRemoteLog(options.remoteLogFile)
     reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY)
-    parser.validate(options, reftest)
+    options = parser.verifyCommonOptions(options, reftest)
 
     if mozinfo.info['debug']:
         print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout
@@ -391,6 +535,17 @@ def main():
     # Hack in a symbolic link for jsreftest
     os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")))
 
+    # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
+    manifest = args[0]
+    if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])):
+        manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0]
+    elif os.path.exists(args[0]):
+        manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/')
+        manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath
+    else:
+        print "ERROR: Could not find test manifest '%s'" % manifest
+        return 1
+
     # Start the webserver
     retVal = reftest.startWebServer(options)
     if retVal:
@@ -406,8 +561,11 @@ def main():
 #    manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list"
     retVal = 0
     try:
+        cmdlineArgs = ["-reftest", manifest]
+        if options.bootstrap:
+            cmdlineArgs = []
         dm.recordLogcat()
-        retVal = reftest.runTests(options.tests, options)
+        retVal = reftest.runTests(manifest, options, cmdlineArgs)
     except:
         print "Automation Error: Exception caught while running tests"
         traceback.print_exc()
@@ -420,5 +578,5 @@ def main():
     return retVal
 
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main(sys.argv[1:]))
 

layout/tools/reftest/runreftest.py

@@ -6,8 +6,9 @@
 Runs the reftest test harness.
 """
 
+from optparse import OptionParser
+from urlparse import urlparse
 import collections
-import json
 import multiprocessing
 import os
 import re
@@ -18,9 +19,8 @@ import sys
 import threading
 
 SCRIPT_DIRECTORY = os.path.abspath(
-    os.path.realpath(os.path.dirname(__file__)))
+    os.path.realpath(os.path.dirname(sys.argv[0])))
-if SCRIPT_DIRECTORY not in sys.path:
+sys.path.insert(0, SCRIPT_DIRECTORY)
-    sys.path.insert(0, SCRIPT_DIRECTORY)
 
 import mozcrash
 import mozdebug
@@ -32,7 +32,13 @@ import mozrunner
 from mozrunner.utils import get_stack_fixer_function, test_environment
 from mozscreenshot import printstatus, dump_screen
 
-import reftestcommandline
+here = os.path.abspath(os.path.dirname(__file__))
+
+try:
+    from mozbuild.base import MozbuildObject
+    build_obj = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+    build_obj = None
 
 # set up logging handler a la automation.py.in for compatability
 import logging
@@ -126,86 +132,14 @@ class ReftestThread(threading.Thread):
             if summaryHeadRegex.search(line) is None:
                 yield line
 
-class ReftestResolver(object):
-    def defaultManifest(self, suite):
-        return {"reftest": "reftest.list",
-                "crashtest": "crashtests.list",
-                "jstestbrowser": "jstests.list"}[suite]
-
-    def directoryManifest(self, suite, path):
-        return os.path.join(path, self.defaultManifest(suite))
-
-    def findManifest(self, suite, test_file, subdirs=True):
-        """Return a tuple of (manifest-path, filter-string) for running test_file.
-
-        test_file is a path to a test or a manifest file
-        """
-        rv = []
-        default_manifest = self.defaultManifest(suite)
-        if not os.path.isabs(test_file):
-            test_file = self.absManifestPath(test_file)
-
-        if os.path.isdir(test_file):
-            for dirpath, dirnames, filenames in os.walk(test_file):
-                if default_manifest in filenames:
-                    rv.append((os.path.join(dirpath, default_manifest), None))
-                    # We keep recursing into subdirectories which means that in the case
-                    # of include directives we get the same manifest multiple times.
-                    # However reftest.js will only read each manifest once
-
-        elif test_file.endswith('.list'):
-            if os.path.exists(test_file):
-                rv = [(test_file, None)]
-        else:
-            dirname, pathname = os.path.split(test_file)
-            found = True
-            while not os.path.exists(os.path.join(dirname, default_manifest)):
-                dirname, suffix = os.path.split(dirname)
-                pathname = os.path.join(suffix, pathname)
-                if os.path.dirname(dirname) == dirname:
-                    found = False
-                    break
-            if found:
-                rv = [(os.path.join(dirname, default_manifest),
-                       r".*(?:/|\\)%s$" % pathname)]
-
-        return rv
-
-    def absManifestPath(self, path):
-        return os.path.normpath(os.path.abspath(path))
-
-    def manifestURL(self, options, path):
-        return "file://%s" % path
-
-    def resolveManifests(self, options, tests):
-        suite = options.suite
-        manifests = {}
-        for testPath in tests:
-            for manifest, filter_str in self.findManifest(suite, testPath):
-                manifest = self.manifestURL(options, manifest)
-                if manifest not in manifests:
-                    manifests[manifest] = set()
-                manifests[manifest].add(filter_str)
-
-        for key in manifests.iterkeys():
-            if os.path.split(key)[1] != self.defaultManifest(suite):
-                print >> sys.stderr, "Invalid manifest for suite %s, %s" %(options.suite, key)
-                sys.exit(1)
-            if None in manifests[key]:
-                manifests[key] = None
-            else:
-                manifests[key] = "|".join(list(manifests[key]))
-        return manifests
 
 class RefTest(object):
     oldcwd = os.getcwd()
-    resolver_cls = ReftestResolver
 
     def __init__(self):
         self.update_mozinfo()
         self.lastTestSeen = 'reftest'
         self.haveDumpedScreen = False
-        self.resolver = self.resolver_cls()
 
     def update_mozinfo(self):
         """walk up directories to find mozinfo.json update the info"""
@@ -224,15 +158,30 @@ class RefTest(object):
         "Get an absolute path relative to self.oldcwd."
         return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
 
-    def createReftestProfile(self, options, manifests, server='localhost', port=0,
+    def getManifestPath(self, path):
-                             profile_to_clone=None):
+        "Get the path of the manifest, and for remote testing this function is subclassed to point to remote manifest"
-        """Sets up a profile for reftest.
+        path = self.getFullPath(path)
+        if os.path.isdir(path):
+            defaultManifestPath = os.path.join(path, 'reftest.list')
+            if os.path.exists(defaultManifestPath):
+                path = defaultManifestPath
+            else:
+                defaultManifestPath = os.path.join(path, 'crashtests.list')
+                if os.path.exists(defaultManifestPath):
+                    path = defaultManifestPath
+        return path
 
-        :param options: Object containing command line options
+    def makeJSString(self, s):
-        :param manifests: Dictionary of the form {manifest_path: [filters]}
+        return '"%s"' % re.sub(r'([\\"])', r'\\\1', s)
-        :param server: Server name to use for http tests
+
-        :param profile_to_clone: Path to a profile to use as the basis for the
+    def createReftestProfile(self, options, manifest, server='localhost', port=0,
-                                 test profile"""
+                             special_powers=True, profile_to_clone=None):
+        """
+          Sets up a profile for reftest.
+          'manifest' is the path to the reftest.list file we want to test with.  This is used in
+          the remote subclass in remotereftest.py so we can write it to a preference for the
+          bootstrap extension.
+        """
 
         locations = mozprofile.permissions.ServerLocations()
         locations.add_host(server, scheme='http', port=port)
@@ -251,10 +200,11 @@ class RefTest(object):
             prefs['reftest.logFile'] = options.logFile
         if options.ignoreWindowSize:
             prefs['reftest.ignoreWindowSize'] = True
+        if options.filter:
+            prefs['reftest.filter'] = options.filter
         if options.shuffle:
             prefs['reftest.shuffle'] = True
         prefs['reftest.focusFilterMode'] = options.focusFilterMode
-        prefs['reftest.manifests'] = json.dumps(manifests)
 
         # Ensure that telemetry is disabled, so we don't connect to the telemetry
         # server in the middle of the tests.
@@ -299,10 +249,13 @@ class RefTest(object):
                 thispref[1].strip())
 
         # install the reftest extension bits into the profile
-        addons = [options.reftestExtensionPath]
+        addons = []
+        addons.append(os.path.join(SCRIPT_DIRECTORY, "reftest"))
 
-        if options.specialPowersExtensionPath is not None:
+        # I would prefer to use "--install-extension reftest/specialpowers", but that requires tight coordination with
-            addons.append(options.specialPowersExtensionPath)
+        # release engineering and landing on multiple branches at once.
+        if special_powers and (manifest.endswith('crashtests.list') or manifest.endswith('jstests.list')):
+            addons.append(os.path.join(SCRIPT_DIRECTORY, 'specialpowers'))
             # SpecialPowers requires insecure automation-only features that we
             # put behind a pref.
             prefs['security.turn_off_all_security_so_that_viruses_can_take_over_this_computer'] = True
@@ -383,7 +336,7 @@ class RefTest(object):
         if profileDir:
             shutil.rmtree(profileDir, True)
 
-    def runTests(self, tests, options, cmdlineArgs=None):
+    def runTests(self, testPath, options, cmdlineArgs=None):
         # Despite our efforts to clean up servers started by this script, in practice
         # we still see infrequent cases where a process is orphaned and interferes
         # with future tests, typically because the old server is keeping the port in use.
@@ -392,12 +345,8 @@ class RefTest(object):
         self.killNamedOrphans('ssltunnel')
         self.killNamedOrphans('xpcshell')
 
-        manifests = self.resolver.resolveManifests(options, tests)
+        if not options.runTestsInParallel:
-        if options.filter:
+            return self.runSerialTests(testPath, options, cmdlineArgs)
-            manifests[""] = options.filter
-
-        if not hasattr(options, "runTestsInParallel") or not options.runTestsInParallel:
-            return self.runSerialTests(manifests, options, cmdlineArgs)
 
         cpuCount = multiprocessing.cpu_count()
 
@@ -428,6 +377,7 @@ class RefTest(object):
                 jobArgs.remove("--run-tests-in-parallel")
             except:
                 pass
+            jobArgs.insert(-1, "--no-run-tests-in-parallel")
             jobArgs[0:0] = [sys.executable, "-u"]
 
         threads = [ReftestThread(args) for args in perProcessArgs[1:]]
@@ -650,7 +600,7 @@ class RefTest(object):
             status = 1
         return status
 
-    def runSerialTests(self, manifests, options, cmdlineArgs=None):
+    def runSerialTests(self, testPath, options, cmdlineArgs=None):
         debuggerInfo = None
         if options.debugger:
             debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
@@ -658,9 +608,10 @@ class RefTest(object):
 
         profileDir = None
         try:
+            reftestlist = self.getManifestPath(testPath)
             if cmdlineArgs == None:
-                cmdlineArgs = []
+                cmdlineArgs = ['-reftest', reftestlist]
-            profile = self.createReftestProfile(options, manifests)
+            profile = self.createReftestProfile(options, reftestlist)
             profileDir = profile.profile  # name makes more sense
 
             # browser environment
@@ -708,26 +659,209 @@ class RefTest(object):
                 continue
 
 
-def run(**kwargs):
+class ReftestOptions(OptionParser):
-    # Mach gives us kwargs; this is a way to turn them back into an
+
-    # options object
+    def __init__(self):
-    parser = reftestcommandline.DesktopArgumentsParser()
+        OptionParser.__init__(self)
-    reftest = RefTest()
+        defaults = {}
-    parser.set_defaults(**kwargs)
+        self.add_option("--xre-path",
-    options = parser.parse_args(kwargs["tests"])
+                        action="store", type="string", dest="xrePath",
-    parser.validate(options, reftest)
+                        # individual scripts will set a sane default
-    return reftest.runTests(options.tests, options)
+                        default=None,
+                        help="absolute path to directory containing XRE (probably xulrunner)")
+        self.add_option("--symbols-path",
+                        action="store", type="string", dest="symbolsPath",
+                        default=None,
+                        help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+        self.add_option("--debugger",
+                        action="store", dest="debugger",
+                        help="use the given debugger to launch the application")
+        self.add_option("--debugger-args",
+                        action="store", dest="debuggerArgs",
+                        help="pass the given args to the debugger _before_ "
+                        "the application on the command line")
+        self.add_option("--debugger-interactive",
+                        action="store_true", dest="debuggerInteractive",
+                        help="prevents the test harness from redirecting "
+                        "stdout and stderr for interactive debuggers")
+        self.add_option("--appname",
+                        action="store", type="string", dest="app",
+                        help="absolute path to application, overriding default")
+        # Certain paths do not make sense when we're cross compiling Fennec.  This
+        # logic is cribbed from the example in
+        # python/mozbuild/mozbuild/mach_commands.py.
+        defaults['app'] = build_obj.get_binary_path() if \
+            build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None
+
+        self.add_option("--extra-profile-file",
+                        action="append", dest="extraProfileFiles",
+                        default=[],
+                        help="copy specified files/dirs to testing profile")
+        self.add_option("--timeout",
+                        action="store", dest="timeout", type="int",
+                        default=5 * 60,  # 5 minutes per bug 479518
+                        help="reftest will timeout in specified number of seconds. [default %default s].")
+        self.add_option("--leak-threshold",
+                        action="store", type="int", dest="defaultLeakThreshold",
+                        default=0,
+                        help="fail if the number of bytes leaked in default "
+                        "processes through refcounted objects (or bytes "
+                               "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
+                               "is greater than the given number")
+        self.add_option("--utility-path",
+                        action="store", type="string", dest="utilityPath",
+                        help="absolute path to directory containing utility "
+                        "programs (xpcshell, ssltunnel, certutil)")
+        defaults["utilityPath"] = build_obj.bindir if \
+            build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None
+
+        self.add_option("--total-chunks",
+                        type="int", dest="totalChunks",
+                        help="how many chunks to split the tests up into")
+        defaults["totalChunks"] = None
+
+        self.add_option("--this-chunk",
+                        type="int", dest="thisChunk",
+                        help="which chunk to run between 1 and --total-chunks")
+        defaults["thisChunk"] = None
+
+        self.add_option("--log-file",
+                        action="store", type="string", dest="logFile",
+                        default=None,
+                        help="file to log output to in addition to stdout")
+        defaults["logFile"] = None
+
+        self.add_option("--skip-slow-tests",
+                        dest="skipSlowTests", action="store_true",
+                        help="skip tests marked as slow when running")
+        defaults["skipSlowTests"] = False
+
+        self.add_option("--ignore-window-size",
+                        dest="ignoreWindowSize", action="store_true",
+                        help="ignore the window size, which may cause spurious failures and passes")
+        defaults["ignoreWindowSize"] = False
+
+        self.add_option("--install-extension",
+                        action="append", dest="extensionsToInstall",
+                        help="install the specified extension in the testing profile. "
+                        "The extension file's name should be <id>.xpi where <id> is "
+                               "the extension's id as indicated in its install.rdf. "
+                               "An optional path can be specified too.")
+        defaults["extensionsToInstall"] = []
+
+        self.add_option("--run-tests-in-parallel",
+                        action="store_true", dest="runTestsInParallel",
+                        help="run tests in parallel if possible")
+        self.add_option("--no-run-tests-in-parallel",
+                        action="store_false", dest="runTestsInParallel",
+                        help="do not run tests in parallel")
+        defaults["runTestsInParallel"] = False
+
+        self.add_option("--setenv",
+                        action="append", type="string",
+                        dest="environment", metavar="NAME=VALUE",
+                        help="sets the given variable in the application's "
+                        "environment")
+        defaults["environment"] = []
+
+        self.add_option("--filter",
+                        action="store", type="string", dest="filter",
+                        help="specifies a regular expression (as could be passed to the JS "
+                        "RegExp constructor) to test against URLs in the reftest manifest; "
+                               "only test items that have a matching test URL will be run.")
+        defaults["filter"] = None
+
+        self.add_option("--shuffle",
+                        action="store_true", dest="shuffle",
+                        help="run reftests in random order")
+        defaults["shuffle"] = False
+
+        self.add_option("--focus-filter-mode",
+                        action="store", type="string", dest="focusFilterMode",
+                        help="filters tests to run by whether they require focus. "
+                        "Valid values are `all', `needs-focus', or `non-needs-focus'. "
+                               "Defaults to `all'.")
+        defaults["focusFilterMode"] = "all"
+
+        self.add_option("--e10s",
+                        action="store_true",
+                        dest="e10s",
+                        help="enables content processes")
+        defaults["e10s"] = False
+
+        self.add_option("--setpref",
+                        action="append", type="string",
+                        default=[],
+                        dest="extraPrefs", metavar="PREF=VALUE",
+                        help="defines an extra user preference")
+
+        self.set_defaults(**defaults)
+
+    def verifyCommonOptions(self, options, reftest):
+        if options.totalChunks is not None and options.thisChunk is None:
+            self.error("thisChunk must be specified when totalChunks is specified")
+
+        if options.totalChunks:
+            if not 1 <= options.thisChunk <= options.totalChunks:
+                self.error("thisChunk must be between 1 and totalChunks")
+
+        if options.logFile:
+            options.logFile = reftest.getFullPath(options.logFile)
+
+        if options.xrePath is not None:
+            if not os.access(options.xrePath, os.F_OK):
+                self.error("--xre-path '%s' not found" % options.xrePath)
+            if not os.path.isdir(options.xrePath):
+                self.error("--xre-path '%s' is not a directory" %
+                           options.xrePath)
+            options.xrePath = reftest.getFullPath(options.xrePath)
+
+        if options.runTestsInParallel:
+            if options.logFile is not None:
+                self.error("cannot specify logfile with parallel tests")
+            if options.totalChunks is not None and options.thisChunk is None:
+                self.error("cannot specify thisChunk or totalChunks with parallel tests")
+            if options.focusFilterMode != "all":
+                self.error("cannot specify focusFilterMode with parallel tests")
+            if options.debugger is not None:
+                self.error("cannot specify a debugger with parallel tests")
+
+        options.leakThresholds = {
+            "default": options.defaultLeakThreshold,
+            "tab": 5000,  # See dependencies of bug 1051230.
+        }
+
+        return options
 
 
 def main():
-    parser = reftestcommandline.DesktopArgumentsParser()
+    parser = ReftestOptions()
     reftest = RefTest()
 
-    options = parser.parse_args()
+    options, args = parser.parse_args()
-    parser.validate(options, reftest)
+    if len(args) != 1:
+        print >>sys.stderr, "No reftest.list specified."
+        sys.exit(1)
 
-    sys.exit(reftest.runTests(options.tests, options))
+    options = parser.verifyCommonOptions(options, reftest)
+    if options.app is None:
+        parser.error("could not find the application path, --appname must be specified")
 
+    options.app = reftest.getFullPath(options.app)
+    if not os.path.exists(options.app):
+        print """Error: Path %(app)s doesn't exist.
+Are you executing $objdir/_tests/reftest/runreftest.py?""" \
+                % {"app": options.app}
+        sys.exit(1)
+
+    if options.xrePath is None:
+        options.xrePath = os.path.dirname(options.app)
+
+    if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
+        options.symbolsPath = reftest.getFullPath(options.symbolsPath)
+    options.utilityPath = reftest.getFullPath(options.utilityPath)
+
+    sys.exit(reftest.runTests(args[0], options))
 
 if __name__ == "__main__":
     main()

layout/tools/reftest/runreftestb2g.py

@@ -10,18 +10,207 @@ import traceback
 
 # We need to know our current directory so that we can serve our test files from it.
 here = os.path.abspath(os.path.dirname(__file__))
-if here not in sys.path:
-    sys.path.insert(0, here)
 
 from automation import Automation
 from b2gautomation import B2GRemoteAutomation
 from b2g_desktop import run_desktop_reftests
-from remotereftest import RemoteReftestResolver, ReftestServer
 from runreftest import RefTest
-import reftestcommandline
+from runreftest import ReftestOptions
+from remotereftest import ReftestServer
 
 from mozdevice import DeviceManagerADB, DMError
 from marionette import Marionette
+import moznetwork
+
+class B2GOptions(ReftestOptions):
+
+    def __init__(self, **kwargs):
+        defaults = {}
+        ReftestOptions.__init__(self)
+        # This is only used for procName in run_remote_reftests.
+        defaults["app"] = Automation.DEFAULT_APP
+
+        self.add_option("--browser-arg", action="store",
+                    type = "string", dest = "browser_arg",
+                    help = "Optional command-line arg to pass to the browser")
+        defaults["browser_arg"] = None
+
+        self.add_option("--b2gpath", action="store",
+                    type = "string", dest = "b2gPath",
+                    help = "path to B2G repo or qemu dir")
+        defaults["b2gPath"] = None
+
+        self.add_option("--marionette", action="store",
+                    type = "string", dest = "marionette",
+                    help = "host:port to use when connecting to Marionette")
+        defaults["marionette"] = None
+
+        self.add_option("--emulator", action="store",
+                    type="string", dest = "emulator",
+                    help = "Architecture of emulator to use: x86 or arm")
+        defaults["emulator"] = None
+        self.add_option("--emulator-res", action="store",
+                    type="string", dest = "emulator_res",
+                    help = "Emulator resolution of the format '<width>x<height>'")
+        defaults["emulator_res"] = None
+
+        self.add_option("--no-window", action="store_true",
+                    dest = "noWindow",
+                    help = "Pass --no-window to the emulator")
+        defaults["noWindow"] = False
+
+        self.add_option("--adbpath", action="store",
+                    type = "string", dest = "adb_path",
+                    help = "path to adb")
+        defaults["adb_path"] = "adb"
+
+        self.add_option("--deviceIP", action="store",
+                    type = "string", dest = "deviceIP",
+                    help = "ip address of remote device to test")
+        defaults["deviceIP"] = None
+
+        self.add_option("--devicePort", action="store",
+                    type = "string", dest = "devicePort",
+                    help = "port of remote device to test")
+        defaults["devicePort"] = 20701
+
+        self.add_option("--remote-logfile", action="store",
+                    type = "string", dest = "remoteLogFile",
+                    help = "Name of log file on the device relative to the device root.  PLEASE ONLY USE A FILENAME.")
+        defaults["remoteLogFile"] = None
+
+        self.add_option("--remote-webserver", action = "store",
+                    type = "string", dest = "remoteWebServer",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["remoteWebServer"] = None
+
+        self.add_option("--http-port", action = "store",
+                    type = "string", dest = "httpPort",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["httpPort"] = None
+
+        self.add_option("--ssl-port", action = "store",
+                    type = "string", dest = "sslPort",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["sslPort"] = None
+
+        self.add_option("--pidfile", action = "store",
+                    type = "string", dest = "pidFile",
+                    help = "name of the pidfile to generate")
+        defaults["pidFile"] = ""
+        self.add_option("--gecko-path", action="store",
+                        type="string", dest="geckoPath",
+                        help="the path to a gecko distribution that should "
+                        "be installed on the emulator prior to test")
+        defaults["geckoPath"] = None
+        self.add_option("--logdir", action="store",
+                        type="string", dest="logdir",
+                        help="directory to store log files")
+        defaults["logdir"] = None
+        self.add_option('--busybox', action='store',
+                        type='string', dest='busybox',
+                        help="Path to busybox binary to install on device")
+        defaults['busybox'] = None
+        self.add_option("--httpd-path", action = "store",
+                    type = "string", dest = "httpdPath",
+                    help = "path to the httpd.js file")
+        defaults["httpdPath"] = None
+        self.add_option("--profile", action="store",
+                    type="string", dest="profile",
+                    help="for desktop testing, the path to the "
+                         "gaia profile to use")
+        defaults["profile"] = None
+        self.add_option("--desktop", action="store_true",
+                        dest="desktop",
+                        help="Run the tests on a B2G desktop build")
+        defaults["desktop"] = False
+        self.add_option("--mulet", action="store_true",
+                        dest="mulet",
+                        help="Run the tests on a B2G desktop build")
+        defaults["mulet"] = False
+        self.add_option("--enable-oop", action="store_true",
+                        dest="oop",
+                        help="Run the tests out of process")
+        defaults["oop"] = False
+        defaults["remoteTestRoot"] = None
+        defaults["logFile"] = "reftest.log"
+        defaults["autorun"] = True
+        defaults["closeWhenDone"] = True
+        defaults["testPath"] = ""
+        defaults["runTestsInParallel"] = False
+
+        self.set_defaults(**defaults)
+
+    def verifyRemoteOptions(self, options, auto):
+        if options.runTestsInParallel:
+            self.error("Cannot run parallel tests here")
+
+        if not options.remoteTestRoot:
+            options.remoteTestRoot = auto._devicemanager.deviceRoot + "/reftest"
+
+        options.remoteProfile = options.remoteTestRoot + "/profile"
+
+        productRoot = options.remoteTestRoot + "/" + auto._product
+        if options.utilityPath is None:
+            options.utilityPath = productRoot + "/bin"
+
+        if options.remoteWebServer == None:
+            if os.name != "nt":
+                options.remoteWebServer = moznetwork.get_ip()
+            else:
+                print "ERROR: you must specify a --remote-webserver=<ip address>\n"
+                return None
+
+        options.webServer = options.remoteWebServer
+
+        if not options.httpPort:
+            options.httpPort = auto.DEFAULT_HTTP_PORT
+
+        if not options.sslPort:
+            options.sslPort = auto.DEFAULT_SSL_PORT
+
+        if options.geckoPath and not options.emulator:
+            self.error("You must specify --emulator if you specify --gecko-path")
+
+        if options.logdir and not options.emulator:
+            self.error("You must specify --emulator if you specify --logdir")
+
+        #if not options.emulator and not options.deviceIP:
+        #    print "ERROR: you must provide a device IP"
+        #    return None
+
+        if options.remoteLogFile == None:
+            options.remoteLogFile = "reftest.log"
+
+        options.localLogName = options.remoteLogFile
+        options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
+
+        # Ensure that the options.logfile (which the base class uses) is set to
+        # the remote setting when running remote. Also, if the user set the
+        # log file name there, use that instead of reusing the remotelogfile as above.
+        if (options.logFile):
+            # If the user specified a local logfile name use that
+            options.localLogName = options.logFile
+        options.logFile = options.remoteLogFile
+
+        # Only reset the xrePath if it wasn't provided
+        if options.xrePath == None:
+            options.xrePath = options.utilityPath
+        options.xrePath = os.path.abspath(options.xrePath)
+
+        if options.pidFile != "":
+            f = open(options.pidFile, 'w')
+            f.write("%s" % os.getpid())
+            f.close()
+
+        # httpd-path is specified by standard makefile targets and may be specified
+        # on the command line to select a particular version of httpd.js. If not
+        # specified, try to select the one from from the xre bundle, as required in bug 882932.
+        if not options.httpdPath:
+            options.httpdPath = os.path.join(options.xrePath, "components")
+
+        return options
+
 
 class ProfileConfigParser(ConfigParser.RawConfigParser):
     """Subclass of RawConfigParser that outputs .ini files in the exact
@@ -54,7 +243,6 @@ class B2GRemoteReftest(RefTest):
     localProfile = None
     remoteApp = ''
     profile = None
-    resolver_cls = RemoteReftestResolver
 
     def __init__(self, automation, devicemanager, options, scriptDir):
         RefTest.__init__(self)
@@ -230,9 +418,10 @@ class B2GRemoteReftest(RefTest):
             pass
 
 
-    def createReftestProfile(self, options, manifests):
+    def createReftestProfile(self, options, reftestlist):
-        profile = RefTest.createReftestProfile(self, options, manifests,
+        profile = RefTest.createReftestProfile(self, options, reftestlist,
-                                               server=options.remoteWebServer)
+                                               server=options.remoteWebServer,
+                                               special_powers=False)
         profileDir = profile.profile
 
         prefs = {}
@@ -248,7 +437,7 @@ class B2GRemoteReftest(RefTest):
         prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org"
         prefs["reftest.browser.iframe.enabled"] = False
         prefs["reftest.remote"] = True
-
+        prefs["reftest.uri"] = "%s" % reftestlist
         # Set a future policy version to avoid the telemetry prompt.
         prefs["toolkit.telemetry.prompted"] = 999
         prefs["toolkit.telemetry.notifiedOptOut"] = 999
@@ -306,6 +495,9 @@ class B2GRemoteReftest(RefTest):
             print "Automation Error: Failed to copy extra files to device"
             raise
 
+    def getManifestPath(self, path):
+        return path
+
     def environment(self, **kwargs):
      return self.automation.environment(**kwargs)
 
@@ -324,7 +516,7 @@ class B2GRemoteReftest(RefTest):
         return status
 
 
-def run_remote_reftests(parser, options):
+def run_remote_reftests(parser, options, args):
     auto = B2GRemoteAutomation(None, "fennec", context_chrome=True)
 
     # create our Marionette instance
@@ -367,7 +559,11 @@ def run_remote_reftests(parser, options):
         dm = DeviceManagerADB(**kwargs)
     auto.setDeviceManager(dm)
 
-    parser.validate_remote(options, auto)
+    options = parser.verifyRemoteOptions(options, auto)
+
+    if (options == None):
+        print "ERROR: Invalid options specified, use --help for a list of valid options"
+        sys.exit(1)
 
     # TODO fix exception
     if not options.ignoreWindowSize:
@@ -384,7 +580,7 @@ def run_remote_reftests(parser, options):
     auto.logFinish = "REFTEST TEST-START | Shutdown"
 
     reftest = B2GRemoteReftest(auto, dm, options, here)
-    parser.validate(options, reftest)
+    options = parser.verifyCommonOptions(options, reftest)
 
     logParent = os.path.dirname(options.remoteLogFile)
     dm.mkDir(logParent);
@@ -394,6 +590,16 @@ def run_remote_reftests(parser, options):
     # Hack in a symbolic link for jsreftest
     os.system("ln -s %s %s" % (os.path.join('..', 'jsreftest'), os.path.join(here, 'jsreftest')))
 
+    # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
+    manifest = args[0]
+    if os.path.exists(os.path.join(here, args[0])):
+        manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, args[0])
+    elif os.path.exists(args[0]):
+        manifestPath = os.path.abspath(args[0]).split(here)[1].strip('/')
+        manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, manifestPath)
+    else:
+        print "ERROR: Could not find test manifest '%s'" % manifest
+        return 1
 
     # Start the webserver
     retVal = 1
@@ -405,7 +611,11 @@ def run_remote_reftests(parser, options):
         if (dm.processExist(procName)):
             dm.killProcess(procName)
 
-        retVal = reftest.runTests(options.tests, options)
+        cmdlineArgs = ["-reftest", manifest]
+        if getattr(options, 'bootstrap', False):
+            cmdlineArgs = []
+
+        retVal = reftest.runTests(manifest, options, cmdlineArgs)
     except:
         print "Automation Error: Exception caught while running tests"
         traceback.print_exc()
@@ -419,21 +629,13 @@ def run_remote_reftests(parser, options):
     reftest.stopWebServer(options)
     return retVal
 
-def run_remote(**kwargs):
+def main(args=sys.argv[1:]):
-    # Tests need to be served from a subdirectory of the server. Symlink
+    parser = B2GOptions()
-    # topsrcdir here to get around this.
+    options, args = parser.parse_args(args)
-    parser = reftestcommandline.B2GArgumentParser()
-    parser.set_defaults(**kwargs)
-    options = parser.parse_args(kwargs["tests"])
-    return run_remote_reftests(parser, options)
-
-def main():
-    parser = reftestcommandline.B2GArgumentParser()
-    options = parser.parse_args()
 
     if options.desktop or options.mulet:
-        return run_desktop_reftests(parser, options)
+        return run_desktop_reftests(parser, options, args)
-    return run_remote_reftests(parser, options)
+    return run_remote_reftests(parser, options, args)
 
 
 if __name__ == "__main__":

testing/mach_commands.py

@@ -96,7 +96,7 @@ TEST_SUITES = {
     'reftest': {
         'aliases': ('RR', 'rr', 'Rr'),
         'mach_command': 'reftest',
-        'kwargs': {'tests': None},
+        'kwargs': {'test_file': None},
     },
     'reftest-ipc': {
         'aliases': ('Ripc',),
@@ -139,10 +139,7 @@ TEST_FLAVORS = {
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'mochitest', 'test_paths': []},
     },
-    'reftest': {
+    'reftest': { },
-        'mach_command': 'reftest',
-        'kwargs': {'tests': []}
-    },
     'steeplechase': { },
     'web-platform-tests': {
         'mach_command': 'web-platform-tests',

testing/mozharness/configs/android/android_panda_releng.py

@@ -97,9 +97,8 @@ config = {
                 "--http-port=%(http_port)s",
                 "--ssl-port=%(ssl_port)s",
                 "--symbols-path=%(symbols_path)s",
-                "--suite=crashtest",
+                "reftest/tests/testing/crashtest/crashtests.list"
             ],
-            "tests": ["reftest/tests/testing/crashtest/crashtests.list",],
             "run_filename": "remotereftest.py",
             "testsdir": "reftest"
         },
@@ -129,12 +128,11 @@ config = {
                 "--ignore-window-size",
                 "--bootstrap",
                 "--extra-profile-file=jsreftest/tests/user.js",
+                "jsreftest/tests/jstests.list",
                 "--http-port=%(http_port)s",
                 "--ssl-port=%(ssl_port)s",
-                "--symbols-path=%(symbols_path)s",
+                "--symbols-path=%(symbols_path)s"
-                "--suite=jstestbrowser",
             ],
-            "tests": ["jsreftest/tests/jstests.list",],
             "run_filename": "remotereftest.py",
             "testsdir": "reftest"
         },
@@ -168,9 +166,8 @@ config = {
                 "--http-port=%(http_port)s",
                 "--ssl-port=%(ssl_port)s",
                 "--symbols-path=%(symbols_path)s",
-                "--suite=reftest",
+                "reftest/tests/layout/reftests/reftest.list"
             ],
-            "tests": ["reftest/tests/layout/reftests/reftest.list"],
             "run_filename": "remotereftest.py",
             "testsdir": "reftest"
         },

testing/mozharness/configs/android/androidarm.py

@@ -140,9 +140,8 @@ config = {
                 "%(modules_dir)s",
                 "--symbols-path=%(symbols_path)s",
                 "--total-chunks=16",
-                "--suite=reftest",
+                "tests/layout/reftests/reftest.list",
             ],
-            "tests": ["tests/layout/reftests/reftest.list"],
         },
         "crashtest": {
             "run_filename": "remotereftest.py",
@@ -162,9 +161,8 @@ config = {
                 "%(modules_dir)s",
                 "--symbols-path=%(symbols_path)s",
                 "--total-chunks=2",
-                "--suite=crashtest",
+                "tests/testing/crashtest/crashtests.list",
             ],
-            "tests": ["tests/testing/crashtest/crashtests.list"],
         },
         "jsreftest": {
             "run_filename": "remotereftest.py",
@@ -183,11 +181,10 @@ config = {
                 "--httpd-path",
                 "%(modules_dir)s",
                 "--symbols-path=%(symbols_path)s",
+                "../jsreftest/tests/jstests.list",
                 "--total-chunks=6",
                 "--extra-profile-file=jsreftest/tests/user.js",
-                "--suite=jstestbrowser",
             ],
-            "tests": ["../jsreftest/tests/jstests.list"],
         },
         "xpcshell": {
             "run_filename": "remotexpcshelltests.py",
@@ -318,67 +315,83 @@ config = {
         },
         "reftest-1": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=1"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=1",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-2": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=2"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=2",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-3": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=3"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=3",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-4": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=4"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=4",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-5": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=5"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=5",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-6": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=6"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=6",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-7": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=7"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=7",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-8": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=8"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=8",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-9": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=9"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=9",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-10": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=10"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=10",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-11": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=11"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=11",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-12": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=12"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=12",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-13": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=13"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=13",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-14": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=14"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=14",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-15": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=15"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=15",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-16": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=16", "--this-chunk=16"],
+            "extra_args": ["--total-chunks=16", "--this-chunk=16",
+                "tests/layout/reftests/reftest.list"]
         },
         "crashtest-1": {
             "category": "crashtest",

testing/mozharness/configs/android/androidarm_4_3.py

@@ -138,9 +138,8 @@ config = {
                 "--httpd-path", "%(modules_dir)s",
                 "--symbols-path=%(symbols_path)s",
                 "--total-chunks=16",
-                "--suite=reftest",
+                "tests/layout/reftests/reftest.list",
             ],
-            "tests": ["tests/layout/reftests/reftest.list",],
         },
         "crashtest": {
             "run_filename": "remotereftest.py",
@@ -159,9 +158,8 @@ config = {
                 "%(modules_dir)s",
                 "--symbols-path=%(symbols_path)s",
                 "--total-chunks=2",
-                "--suite=crashtest",
+                "tests/testing/crashtest/crashtests.list",
             ],
-            "tests": ["tests/testing/crashtest/crashtests.list",],
         },
         "jsreftest": {
             "run_filename": "remotereftest.py",
@@ -175,11 +173,10 @@ config = {
                 "--utility-path=%(utility_path)s", "--http-port=%(http_port)s",
                 "--ssl-port=%(ssl_port)s", "--httpd-path", "%(modules_dir)s",
                 "--symbols-path=%(symbols_path)s",
+                "../jsreftest/tests/jstests.list",
                 "--total-chunks=6",
                 "--extra-profile-file=jsreftest/tests/user.js",
-                "--suite=jstestbrowser",
             ],
-            "tests": ["../jsreftest/tests/jstests.list",],
         },
         "xpcshell": {
             "run_filename": "remotexpcshelltests.py",
@@ -378,195 +375,243 @@ config = {
         },
         "reftest-1": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=1"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=1",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-2": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=2"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=2",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-3": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=3"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=3",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-4": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=4"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=4",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-5": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=5"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=5",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-6": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=6"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=6",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-7": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=7"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=7",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-8": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=8"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=8",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-9": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=9"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=9",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-10": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=10"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=10",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-11": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=11"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=11",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-12": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=12"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=12",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-13": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=13"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=13",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-14": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=14"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=14",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-15": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=15"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=15",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-16": {
             "category": "reftest",
-            "extra_args": ["--total-chunks=48", "--this-chunk=16"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=16",
+                "tests/layout/reftests/reftest.list"]
         },
         "reftest-17": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=17"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=17",
+            "tests/layout/reftests/reftest.list"]
         },
        "reftest-18": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=18"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=18",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-19": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=19"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=19",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-20": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=20"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=20",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-21": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=21"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=21",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-22": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=22"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=22",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-23": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=23"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=23",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-24": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=24"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=24",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-25": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=25"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=25",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-26": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=26"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=26",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-27": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=27"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=27",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-28": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=28"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=28",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-29": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=29"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=29",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-30": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=30"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=30",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-31": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=31"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=31",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-32": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=32"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=32",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-33": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=33"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=33",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-34": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=34"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=34",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-35": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=35"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=35",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-36": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=36"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=36",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-37": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=37"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=37",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-38": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=38"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=38",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-39": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=39"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=39",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-40": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=40"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=40",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-41": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=41"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=41",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-42": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=42"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=42",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-43": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=43"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=43",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-44": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=44"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=44",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-45": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=45"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=45",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-46": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=46"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=46",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-47": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=47"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=47",
+            "tests/layout/reftests/reftest.list"]
         },
         "reftest-48": {
             "category": "reftest",
-            "extra args": ["--total-chunks=48", "--this-chunk=48"],
+            "extra_args": ["--total-chunks=48", "--this-chunk=48",
+            "tests/layout/reftests/reftest.list"]
         },
         "crashtest-1": {
             "category": "crashtest",

testing/mozharness/configs/android/androidx86.py

@@ -135,11 +135,8 @@ config = {
     "test_suite_definitions": {
         "jsreftest": {
             "category": "reftest",
-            "tests": ["../jsreftest/tests/jstests.list"],
+            "extra_args": ["../jsreftest/tests/jstests.list",
-            "extra_args": [
+                "--extra-profile-file=jsreftest/tests/user.js"]
-                "--suite=jstestbrowser",
-                "--extra-profile-file=jsreftest/tests/user.js"
-            ]
         },
         "mochitest-1": {
             "category": "mochitest",
@@ -155,33 +152,22 @@ config = {
         },
         "reftest-1": {
             "category": "reftest",
-            "extra_args": [
+            "extra_args": ["--total-chunks=3", "--this-chunk=1",
-                "--suite=reftest",
+                "tests/layout/reftests/reftest.list"]
-                "--total-chunks=3",
-                "--this-chunk=1",
-            ],
-            "tests": ["tests/layout/reftests/reftest.list"],
         },
         "reftest-2": {
-            "extra_args": [
+            "category": "reftest",
-                "--suite=reftest",
+            "extra_args": ["--total-chunks=3", "--this-chunk=2",
-                "--total-chunks=3",
+                "tests/layout/reftests/reftest.list"]
-                "--this-chunk=2",
-            ],
-            "tests": ["tests/layout/reftests/reftest.list"],
         },
         "reftest-3": {
-            "extra_args": [
+            "category": "reftest",
-                "--suite=reftest",
+            "extra_args": ["--total-chunks=3", "--this-chunk=3",
-                "--total-chunks=3",
+                "tests/layout/reftests/reftest.list"]
-                "--this-chunk=3",
-            ],
-            "tests": ["tests/layout/reftests/reftest.list"],
         },
         "crashtest": {
             "category": "reftest",
-            "extra_args": ["--suite=crashtest"],
+            "extra_args": ["tests/testing/crashtest/crashtests.list"]
-            "tests": ["tests/testing/crashtest/crashtests.list"]
         },
         "xpcshell": {
             "category": "xpcshell",

testing/mozharness/configs/b2g/emulator_automation_config.py

@@ -74,9 +74,8 @@ config = {
                 "--busybox=%(busybox)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
-                "--suite=crashtest",
+                "tests/testing/crashtest/crashtests.list"
             ],
-            "tests": ["tests/testing/crashtest/crashtests.list",],
             "run_filename": "runreftestb2g.py",
             "testsdir": "reftest"
         },
@@ -95,8 +94,8 @@ config = {
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--extra-profile-file=jsreftest/tests/user.js",
+                "jsreftest/tests/jstests.list"
             ],
-            "tests": ["jsreftest/tests/jstests.list",],
             "run_filename": "remotereftest.py",
             "testsdir": "reftest"
         },
@@ -162,8 +161,8 @@ config = {
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--enable-oop",
+                "tests/layout/reftests/reftest.list"
             ],
-            "tests": ["tests/layout/reftests/reftest.list",],
             "run_filename": "runreftestsb2g.py",
             "testsdir": "reftest"
         },

testing/mozharness/configs/unittests/linux_unittest.py

@@ -2,14 +2,14 @@ import os
 import platform
 
 # OS Specifics
-ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+ABS_WORK_DIR = os.path.join(os.getcwd(), 'build')
 BINARY_PATH = os.path.join(ABS_WORK_DIR, "firefox", "firefox-bin")
 INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
 XPCSHELL_NAME = "xpcshell"
-EXE_SUFFIX = ""
+EXE_SUFFIX = ''
 DISABLE_SCREEN_SAVER = True
 ADJUST_MOUSE_AND_SCREEN = False
-if platform.architecture()[0] == "64bit":
+if platform.architecture()[0] == '64bit':
     TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
     MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
 else:
@@ -20,9 +20,9 @@ else:
 config = {
     "buildbot_json_path": "buildprops.json",
     "exes": {
-        "python": "/tools/buildbot/bin/python",
+        'python': '/tools/buildbot/bin/python',
-        "virtualenv": ["/tools/buildbot/bin/python", "/tools/misc-python/virtualenv.py"],
+        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
-        "tooltool.py": "/tools/tooltool.py",
+        'tooltool.py': "/tools/tooltool.py",
     },
     "find_links": [
         "http://pypi.pvt.build.mozilla.org/pub",
@@ -199,63 +199,35 @@ config = {
     },
     # local reftest suites
     "all_reftest_suites": {
-        "reftest": {
+        "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
-            "options": ["--suite=reftest"],
+        "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
-            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
+        "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
-        },
+        "reftest-ipc": {'env': {'MOZ_OMTC_ENABLED': '1',
-        "crashtest": {
+                                'MOZ_DISABLE_CONTEXT_SHARING_GLX': '1'},
-            "options": ["--suite=crashtest"],
+                        'options': ['--setpref=browser.tabs.remote=true',
-            "tests": ["tests/reftest/tests/testing/crashtest/crashtests.list"]
+                                    '--setpref=browser.tabs.remote.autostart=true',
-        },
+                                    '--setpref=layers.offmainthreadcomposition.testing.enabled=true',
-        "jsreftest": {
+                                    '--setpref=layers.async-pan-zoom.enabled=true',
-            "options":["--extra-profile-file=tests/jsreftest/tests/user.js",
+                                    'tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list']},
-                       "--suite=jstestbrowser"],
+        "reftest-no-accel": ['--setpref=layers.acceleration.force-enabled=disabled',
-            "tests": ["tests/jsreftest/tests/jstests.list"]
+                             'tests/reftest/tests/layout/reftests/reftest.list'],
-        },
+        "crashtest-ipc": {'env': {'MOZ_OMTC_ENABLED': '1',
-        "reftest-ipc": {
+                                  'MOZ_DISABLE_CONTEXT_SHARING_GLX': '1'},
-            "env": {
+                          'options': ['--setpref=browser.tabs.remote=true',
-                "MOZ_OMTC_ENABLED": "1",
+                                      '--setpref=browser.tabs.remote.autostart=true',
-                "MOZ_DISABLE_CONTEXT_SHARING_GLX": "1"
+                                      '--setpref=layers.offmainthreadcomposition.testing.enabled=true',
-            },
+                                      '--setpref=layers.async-pan-zoom.enabled=true',
-            "options": ["--suite=reftest",
+                                      'tests/reftest/tests/testing/crashtest/crashtests.list']},
-                        "--setpref=browser.tabs.remote=true",
-                        "--setpref=browser.tabs.remote.autostart=true",
-                        "--setpref=layers.offmainthreadcomposition.testing.enabled=true",
-                        "--setpref=layers.async-pan-zoom.enabled=true"],
-            "tests": ["tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list"]
-        },
-        "reftest-no-accel": {
-            "options": ["--suite=reftest",
-                        "--setpref=layers.acceleration.force-enabled=disabled"],
-            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]},
-        "crashtest-ipc": {
-            "env": {
-                "MOZ_OMTC_ENABLED": "1",
-                "MOZ_DISABLE_CONTEXT_SHARING_GLX": "1"
-            },
-            "options": ["--suite=crashtest",
-                        "--setpref=browser.tabs.remote=true",
-                        "--setpref=browser.tabs.remote.autostart=true",
-                        "--setpref=layers.offmainthreadcomposition.testing.enabled=true",
-                        "--setpref=layers.async-pan-zoom.enabled=true"],
-            "tests": ["tests/reftest/tests/testing/crashtest/crashtests.list"]
-        },
     },
     "all_xpcshell_suites": {
-        "xpcshell": {
+        "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
-            "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
+                     "%(abs_app_dir)s/" + XPCSHELL_NAME],
-                        "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
+        "xpcshell-addons": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
-            "tests": []
+                            "--tag=addons",
-        },
+                            "%(abs_app_dir)s/" + XPCSHELL_NAME]
-        "xpcshell-addons": {
-            "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
-                        "--tag=addons",
-                        "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
-            "tests": []
-        },
     },
     "all_cppunittest_suites": {
-        "cppunittest": ["tests/cppunittest"]
+        "cppunittest": ['tests/cppunittest']
     },
     "all_gtest_suites": {
         "gtest": []

testing/mozharness/configs/unittests/mac_unittest.py

@@ -166,45 +166,24 @@ config = {
     },
     # local reftest suites
     "all_reftest_suites": {
-        "reftest": {
+        "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
-            'options': ["--suite=reftest"],
+        "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
-            'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
+        "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
-        },
+        "reftest-ipc": ['--setpref=browser.tabs.remote=true',
-        "crashtest": {
-            'options': ["--suite=crashtest"],
-            'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
-        },
-        "jsreftest": {
-            'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
-            'tests': ["tests/jsreftest/tests/jstests.list"]
-        },
-        "reftest-ipc": {
-            'options': ['--suite=reftest',
-                        '--setpref=browser.tabs.remote=true',
                         '--setpref=browser.tabs.remote.autostart=true',
-                        '--setpref=layers.async-pan-zoom.enabled=true'],
+                        '--setpref=layers.async-pan-zoom.enabled=true',
-            'tests': ['tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list']
+                        'tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list'],
-        },
+        "crashtest-ipc": ['--setpref=browser.tabs.remote=true',
-        "crashtest-ipc": {
+                          '--setpref=browser.tabs.remote.autostart=true',
-            'options': ['--suite=crashtest',
+                          '--setpref=layers.async-pan-zoom.enabled=true',
-                        '--setpref=browser.tabs.remote=true',
+                          'tests/reftest/tests/testing/crashtest/crashtests.list'],
-                        '--setpref=browser.tabs.remote.autostart=true',
-                        '--setpref=layers.async-pan-zoom.enabled=true'],
-            'tests': ['tests/reftest/tests/testing/crashtest/crashtests.list']
-        },
     },
     "all_xpcshell_suites": {
-        "xpcshell": {
+        "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
-            'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
+                     "%(abs_app_dir)s/" + XPCSHELL_NAME],
-                        "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
+        "xpcshell-addons": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
-            'tests': []
+                            "--tag=addons",
-        },
+                            "%(abs_app_dir)s/" + XPCSHELL_NAME]
-        "xpcshell-addons": {
-            'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
-                        "--tag=addons",
-                        "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
-            'tests': []
-        },
     },
     "all_cppunittest_suites": {
         "cppunittest": ['tests/cppunittest']

testing/mozharness/configs/unittests/win_unittest.py

@@ -176,56 +176,28 @@ config = {
     },
     # local reftest suites
     "all_reftest_suites": {
-        "reftest": {
+        "reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
-            'options': ["--suite=reftest"],
+        "crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
-            'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
+        "jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
-        },
+        "reftest-ipc": ['--setpref=browser.tabs.remote=true',
-        "crashtest": {
-            'options': ["--suite=crashtest"],
-            'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
-        },
-        "jsreftest": {
-            'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
-            'tests': ["tests/jsreftest/tests/jstests.list"]
-        },
-        "reftest-ipc": {
-            'options': ['--suite=reftest',
-                        '--setpref=browser.tabs.remote=true',
                         '--setpref=browser.tabs.remote.autostart=true',
-                        '--setpref=layers.async-pan-zoom.enabled=true'],
+                        '--setpref=layers.async-pan-zoom.enabled=true',
-            'tests': ['tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list']
+                        'tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list'],
-        },
+        "reftest-no-accel": ["--setpref=gfx.direct2d.disabled=true", "--setpref=layers.acceleration.disabled=true",
-        "reftest-no-accel": {
+                             "tests/reftest/tests/layout/reftests/reftest.list"],
-            "options": ["--suite=reftest",
+        "reftest-omtc": ["--setpref=layers.offmainthreadcomposition.enabled=true",
-                        "--setpref=gfx.direct2d.disabled=true",
+                         "tests/reftest/tests/layout/reftests/reftest.list"],
-                        "--setpref=layers.acceleration.disabled=true"],
+        "crashtest-ipc": ['--setpref=browser.tabs.remote=true',
-            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
+                          '--setpref=browser.tabs.remote.autostart=true',
-        },
+                          '--setpref=layers.async-pan-zoom.enabled=true',
-        "reftest-omtc": {
+                          'tests/reftest/tests/testing/crashtest/crashtests.list'],
-            "options": ["--suite=reftest",
-                        "--setpref=layers.offmainthreadcomposition.enabled=true"],
-            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
-        },
-        "crashtest-ipc": {
-            "options": ["--suite=crashtest",
-                        '--setpref=browser.tabs.remote=true',
-                        '--setpref=browser.tabs.remote.autostart=true',
-                        '--setpref=layers.async-pan-zoom.enabled=true'],
-            "tests": ['tests/reftest/tests/testing/crashtest/crashtests.list'],
-        },
     },
     "all_xpcshell_suites": {
-        "xpcshell": {
+        "xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
-            'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
+                     "%(abs_app_dir)s/" + XPCSHELL_NAME],
-                        "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
+        "xpcshell-addons": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
-            'tests': []
+                            "--tag=addons",
-        },
+                            "%(abs_app_dir)s/" + XPCSHELL_NAME]
-        "xpcshell-addons": {
-            'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
-                        "--tag=addons",
-                        "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
-            'tests': []
-        },
     },
     "all_cppunittest_suites": {
         "cppunittest": ['tests/cppunittest']

testing/mozharness/scripts/android_emulator_unittest.py

@@ -483,15 +483,6 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
                 continue
             cmd.append(arg)
 
-        tests = None
-        if "tests" in self.test_suite_definitions[self.test_suite]:
-            tests = self.test_suite_definitions[self.test_suite]["tests"]
-        elif "tests" in self.config["suite_definitions"][suite_category]:
-            tests = self.config["suite_definitions"][suite_category]["tests"]
-
-        if tests:
-            cmd.extend(tests)
-
         return cmd
 
     def _tooltool_fetch(self, url):

testing/mozharness/scripts/androidx86_emulator_unittest.py

@@ -452,11 +452,6 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
                 continue
             cmd.append(arg)
 
-        if "tests" in self.suite_definitions[suite_name]:
-            cmd.extend(self.suite_definitions[suite_name]["tests"])
-        elif "tests" in self.tree_config["suite_definitions"][suite_category]:
-            cmd.extend(self.tree_config["suite_definitions"][suite_category]["tests"])
-
         return cmd
 
     def preflight_run_tests(self):

testing/mozharness/scripts/b2g_desktop_unittest.py

@@ -154,7 +154,7 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript):
         }
 
         if suite not in self.config["suite_definitions"]:
-            self.fatal("'%s' not defined in the config!" % suite)
+            self.fatal("'%s' not defined in the config!" % suite),
 
         options = self.config["suite_definitions"][suite]["options"]
         if options:
@@ -162,11 +162,6 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript):
                 option = option % str_format_values
                 if not option.endswith('None'):
                     cmd.append(option)
-
-        tests = self.config["suite_definitions"][suite].get("tests")
-        if tests:
-            cmd.extend(tests)
-
         return cmd
 
     def download_and_extract(self):

testing/mozharness/scripts/b2g_emulator_unittest.py

@@ -281,11 +281,6 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
                 option = option % str_format_values
                 if not option.endswith('None'):
                     cmd.append(option)
-
-        tests = self.config["suite_definitions"][suite].get("tests", [])
-        if tests:
-            cmd.extend(tests)
-
         return cmd
 
     def _query_adb(self):
@@ -341,9 +336,8 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
             self.fatal("Don't know how to run --test-suite '%s'!" % suite)
 
         cmd = self._query_abs_base_cmd(suite)
-        cmd = self.append_harness_extra_args(cmd)
-
         cwd = dirs['abs_%s_dir' % suite]
+        cmd = self.append_harness_extra_args(cmd)
 
         # TODO we probably have to move some of the code in
         # scripts/desktop_unittest.py and scripts/marionette.py to

testing/mozharness/scripts/desktop_unittest.py

@@ -588,8 +588,8 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
                 options_list = []
                 env = {}
                 if isinstance(suites[suite], dict):
-                    options_list = suites[suite]['options'] + suites[suite].get("tests", [])
+                    options_list = suites[suite]['options']
-                    env = copy.deepcopy(suites[suite].get('env', {}))
+                    env = copy.deepcopy(suites[suite]['env'])
                 else:
                     options_list = suites[suite]
 

testing/testsuite-targets.mk

@@ -174,7 +174,7 @@ REMOTE_REFTEST = rm -f ./$@.log && $(PYTHON) _tests/reftest/remotereftest.py \
   --dm_trans=$(DM_TRANS) --ignore-window-size \
   --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \
   --httpd-path=_tests/modules \
-  $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log
+  $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) '$(1)' | tee ./$@.log
 
 RUN_REFTEST_B2G = rm -f ./$@.log && $(PYTHON) _tests/reftest/runreftestb2g.py \
   --remote-webserver=10.0.2.2 --b2gpath=${B2G_PATH} --adbpath=${ADB_PATH} \
@@ -209,7 +209,7 @@ reftest-remote:
         echo 'please prepare your host with the environment variable TEST_DEVICE'; \
     else \
         ln -s $(abspath $(topsrcdir)) _tests/reftest/tests; \
-        $(call REMOTE_REFTEST,'tests/$(TEST_PATH)'); \
+        $(call REMOTE_REFTEST,tests/$(TEST_PATH)); \
         $(CHECK_TEST_ERROR); \
     fi
 

testing/xpcshell/Makefile.in

@@ -10,7 +10,6 @@ TEST_HARNESS_FILES := \
   runxpcshelltests.py \
   remotexpcshelltests.py \
   runtestsb2g.py \
-  xpcshellcommandline.py \
   head.js \
   node-spdy \
   moz-spdy \

testing/xpcshell/mach_commands.py

@@ -25,7 +25,8 @@ from mach.decorators import (
     Command,
 )
 
-from xpcshellcommandline import parser_desktop, parser_remote, parser_b2g
+_parser = argparse.ArgumentParser()
+structured.commandline.add_logging_group(_parser)
 
 ADB_NOT_FOUND = '''
 The %s command requires the adb binary to be on your path.
@@ -39,7 +40,6 @@ BUSYBOX_URLS = {
     'x86': 'http://www.busybox.net/downloads/binaries/latest/busybox-i686'
 }
 
-here = os.path.abspath(os.path.dirname(__file__))
 
 if sys.version_info[0] < 3:
     unicode_type = unicode
@@ -55,9 +55,20 @@ class InvalidTestPathError(Exception):
 class XPCShellRunner(MozbuildObject):
     """Run xpcshell tests."""
     def run_suite(self, **kwargs):
-        return self._run_xpcshell_harness(**kwargs)
+        from manifestparser import TestManifest
+        manifest = TestManifest(manifests=[os.path.join(self.topobjdir,
+            '_tests', 'xpcshell', 'xpcshell.ini')])
 
-    def run_test(self, **kwargs):
+        return self._run_xpcshell_harness(manifest=manifest, **kwargs)
+
+    def run_test(self, test_paths, interactive=False,
+                 keep_going=False, sequential=False, shuffle=False,
+                 debugger=None, debuggerArgs=None, debuggerInteractive=None,
+                 jsDebugger=False, jsDebuggerPort=None,
+                 rerun_failures=False, test_objects=None, verbose=False,
+                 log=None, test_tags=None, dump_tests=None,
+                 # ignore parameters from other platforms' options
+                 **kwargs):
         """Runs an individual xpcshell test."""
         from mozbuild.testing import TestResolver
         from manifestparser import TestManifest
@@ -71,15 +82,65 @@ class XPCShellRunner(MozbuildObject):
         if os.path.isdir(src_build_path):
             sys.path.append(src_build_path)
 
-        self.run_suite(**kwargs)
+        if test_paths == 'all':
+            self.run_suite(interactive=interactive,
+                           keep_going=keep_going, shuffle=shuffle, sequential=sequential,
+                           debugger=debugger, debuggerArgs=debuggerArgs,
+                           debuggerInteractive=debuggerInteractive,
+                           jsDebugger=jsDebugger, jsDebuggerPort=jsDebuggerPort,
+                           rerun_failures=rerun_failures,
+                           verbose=verbose, log=log, test_tags=test_tags, dump_tests=dump_tests)
+            return
+        elif test_paths:
+            test_paths = [self._wrap_path_argument(p).relpath() for p in test_paths]
 
+        if test_objects:
+            tests = test_objects
+        else:
+            resolver = self._spawn(TestResolver)
+            tests = list(resolver.resolve_tests(paths=test_paths,
+                flavor='xpcshell'))
+
+        if not tests:
+            raise InvalidTestPathError('We could not find an xpcshell test '
+                'for the passed test path. Please select a path that is '
+                'a test file or is a directory containing xpcshell tests.')
+
+        # Dynamically write out a manifest holding all the discovered tests.
+        manifest = TestManifest()
+        manifest.tests.extend(tests)
+
+        args = {
+            'interactive': interactive,
+            'keep_going': keep_going,
+            'shuffle': shuffle,
+            'sequential': sequential,
+            'debugger': debugger,
+            'debuggerArgs': debuggerArgs,
+            'debuggerInteractive': debuggerInteractive,
+            'jsDebugger': jsDebugger,
+            'jsDebuggerPort': jsDebuggerPort,
+            'rerun_failures': rerun_failures,
+            'manifest': manifest,
+            'verbose': verbose,
+            'log': log,
+            'test_tags': test_tags,
+            'dump_tests': dump_tests,
+        }
+
+        return self._run_xpcshell_harness(**args)
+
+    def _run_xpcshell_harness(self, manifest,
+                              test_path=None, shuffle=False, interactive=False,
+                              keep_going=False, sequential=False,
+                              debugger=None, debuggerArgs=None, debuggerInteractive=None,
+                              jsDebugger=False, jsDebuggerPort=None,
+                              rerun_failures=False, verbose=False, log=None, test_tags=None,
+                              dump_tests=None):
 
-    def _run_xpcshell_harness(self, **kwargs):
         # Obtain a reference to the xpcshell test runner.
         import runxpcshelltests
 
-        log = kwargs.pop("log")
-
         xpcshell = runxpcshelltests.XPCShellTests(log=log)
         self.log_manager.enable_unstructured()
 
@@ -87,41 +148,60 @@ class XPCShellRunner(MozbuildObject):
         modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
         # We want output from the test to be written immediately if we are only
         # running a single test.
-        single_test = (kwargs["testPaths"] is not None or
+        single_test = (test_path is not None or
                        (manifest and len(manifest.test_paths())==1))
-        sequential = kwargs["sequential"] or single_test
+        sequential = sequential or single_test
 
-        if kwargs["xpcshell"] is None:
+        args = {
-            kwargs["xpcshell"] = self.get_binary_path('xpcshell')
+            'manifest': manifest,
+            'xpcshell': self.get_binary_path('xpcshell'),
+            'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
+            'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
+            'interactive': interactive,
+            'keepGoing': keep_going,
+            'logfiles': False,
+            'sequential': sequential,
+            'shuffle': shuffle,
+            'testsRootDir': tests_dir,
+            'testingModulesDir': modules_dir,
+            'profileName': 'firefox',
+            'verbose': verbose or single_test,
+            'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'),
+            'xunitName': 'xpcshell',
+            'pluginsPath': os.path.join(self.distdir, 'plugins'),
+            'debugger': debugger,
+            'debuggerArgs': debuggerArgs,
+            'debuggerInteractive': debuggerInteractive,
+            'jsDebugger': jsDebugger,
+            'jsDebuggerPort': jsDebuggerPort,
+            'test_tags': test_tags,
+            'dump_tests': dump_tests,
+            'utility_path': self.bindir,
+        }
 
-        if kwargs["mozInfo"] is None:
+        if test_path is not None:
-            kwargs["mozInfo"] = os.path.join(self.topobjdir, 'mozinfo.json')
+            args['testPath'] = test_path
 
-        if kwargs["symbolsPath"] is None:
+        # A failure manifest is written by default. If --rerun-failures is
-            kwargs["symbolsPath"] = os.path.join(self.distdir, 'crashreporter-symbols')
+        # specified and a prior failure manifest is found, the prior manifest
-
+        # will be run. A new failure manifest is always written over any
-        if kwargs["logfiles"] is None:
+        # prior failure manifest.
-            kwargs["logfiles"] = False
+        failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
-
+        rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
-        if kwargs["profileName"] is None:
+        if os.path.exists(failure_manifest_path) and rerun_failures:
-            kwargs["profileName"] = "firefox"
+            shutil.move(failure_manifest_path, rerun_manifest_path)
-
+            args['manifest'] = rerun_manifest_path
-        if kwargs["pluginsPath"] is None:
+        elif os.path.exists(failure_manifest_path):
-            kwargs['pluginsPath'] = os.path.join(self.distdir, 'plugins')
+            os.remove(failure_manifest_path)
-
+        elif rerun_failures:
-        if kwargs["utility_path"] is None:
+            print("No failures were found to re-run.")
-            kwargs['utility_path'] = self.bindir
+            return 0
-
+        args['failureManifest'] = failure_manifest_path
-        if kwargs["manifest"] is None:
-            kwargs["manifest"] = os.path.join(tests_dir, "xpcshell.ini")
-
-        if kwargs["failure_manifest"] is None:
-            kwargs["failure_manifest"] = os.path.join(self.statedir, 'xpcshell.failures.ini')
 
         # Python through 2.7.2 has issues with unicode in some of the
         # arguments. Work around that.
         filtered_args = {}
-        for k, v in kwargs.iteritems():
+        for k, v in args.items():
             if isinstance(v, unicode_type):
                 v = v.encode('utf-8')
 
@@ -157,7 +237,12 @@ class AndroidXPCShellRunner(MozbuildObject):
         return dm
 
     """Run Android xpcshell tests."""
-    def run_test(self, **kwargs):
+    def run_test(self,
+                 test_paths, keep_going,
+                 devicemanager, ip, port, remote_test_root, no_setup, local_apk,
+                 test_objects=None, log=None,
+                 # ignore parameters from other platforms' options
+                 **kwargs):
         # TODO Bug 794506 remove once mach integrates with virtualenv.
         build_path = os.path.join(self.topobjdir, 'build')
         if build_path not in sys.path:
@@ -165,53 +250,60 @@ class AndroidXPCShellRunner(MozbuildObject):
 
         import remotexpcshelltests
 
-        dm = self.get_devicemanager(kwargs["dm_trans"], kwargs["deviceIP"], kwargs["devicePort"],
+        dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root)
-                                    kwargs["remoteTestRoot"])
 
-        log = kwargs.pop("log")
+        options = remotexpcshelltests.RemoteXPCShellOptions()
-        self.log_manager.enable_unstructured()
+        options.shuffle = False
-
+        options.sequential = True
-        if kwargs["xpcshell"] is None:
+        options.interactive = False
-            kwargs["xpcshell"] = "xpcshell"
+        options.debugger = None
-
+        options.debuggerArgs = None
-        if not kwargs["objdir"]:
+        options.setup = not no_setup
-            kwargs["objdir"] = self.topobjdir
+        options.keepGoing = keep_going
-
+        options.objdir = self.topobjdir
-        if not kwargs["localLib"]:
+        options.localLib = os.path.join(self.topobjdir, 'dist/fennec')
-            kwargs["localLib"] = os.path.join(self.topobjdir, 'dist/fennec')
+        options.localBin = os.path.join(self.topobjdir, 'dist/bin')
-
+        options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules')
-        if not kwargs["localBin"]:
+        options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
-            kwargs["localBin"] = os.path.join(self.topobjdir, 'dist/bin')
+        options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell.ini')
-
+        options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')
-        if not kwargs["testingModulesDir"]:
+        if local_apk:
-            kwargs["testingModulesDir"] = os.path.join(self.topobjdir, '_tests/modules')
+            options.localAPK = local_apk
-
+        else:
-        if not kwargs["mozInfo"]:
+            for file in os.listdir(os.path.join(options.objdir, "dist")):
-            kwargs["mozInfo"] = os.path.join(self.topobjdir, 'mozinfo.json')
+                if file.endswith(".apk") and file.startswith("fennec"):
-
+                    options.localAPK = os.path.join(options.objdir, "dist")
-        if not kwargs["manifest"]:
+                    options.localAPK = os.path.join(options.localAPK, file)
-            kwargs["manifest"] = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell.ini')
+                    print ("using APK: " + options.localAPK)
-
-        if not kwargs["symbolsPath"]:
-            kwargs["symbolsPath"] = os.path.join(self.distdir, 'crashreporter-symbols')
-
-        if not kwargs["localAPK"]:
-            for file_name in os.listdir(os.path.join(kwargs["objdir"], "dist")):
-                if file_name.endswith(".apk") and file_name.startswith("fennec"):
-                    kwargs["localAPK"] = os.path.join(kwargs["objdir"], "dist", file_name)
-                    print ("using APK: %s" % kwargs["localAPK"])
                     break
             else:
                 raise Exception("You must specify an APK")
 
-        options = argparse.Namespace(**kwargs)
+        if test_paths == 'all':
-        xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, log)
+            testdirs = []
+            options.testPath = None
+            options.verbose = False
+        elif test_objects:
+            if len(test_objects) > 1:
+                print('Warning: only the first test will be used.')
+            testdirs = test_objects[0]['dir_relpath']
+            options.testPath = test_objects[0]['path']
+            options.verbose = True
+        else:
+            if len(test_paths) > 1:
+                print('Warning: only the first test path argument will be used.')
+            testdirs = test_paths[0]
+            options.testPath = test_paths[0]
+            options.verbose = True
 
-        result = xpcshell.runTests(testClass=remotexpcshelltests.RemoteXPCShellTestThread,
+        xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, testdirs, log)
-                                   mobileArgs=xpcshell.mobileArgs,
+
-                                   **vars(options))
+        result = xpcshell.runTests(xpcshell='xpcshell',
+                      testClass=remotexpcshelltests.RemoteXPCShellTestThread,
+                      testdirs=testdirs,
+                      mobileArgs=xpcshell.mobileArgs,
+                      **options.__dict__)
 
-        self.log_manager.disable_unstructured()
 
         return int(not result)
 
@@ -258,66 +350,57 @@ class B2GXPCShellRunner(MozbuildObject):
             f.write(data.read())
         return busybox_path
 
-    def run_test(self, **kwargs):
+    def run_test(self, test_paths, b2g_home=None, busybox=None, device_name=None,
+                 test_objects=None, log=None,
+                 # ignore parameters from other platforms' options
+                 **kwargs):
         try:
             import which
             which.which('adb')
         except which.WhichError:
             # TODO Find adb automatically if it isn't on the path
-            print(ADB_NOT_FOUND % ('mochitest-remote', kwargs["b2g_home"]))
+            print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
             sys.exit(1)
 
+        test_path = None
+        if test_objects:
+            if len(test_objects) > 1:
+                print('Warning: Only the first test will be used.')
+
+            test_path = self._wrap_path_argument(test_objects[0]['path'])
+        elif test_paths:
+            if len(test_paths) > 1:
+                print('Warning: Only the first test path will be used.')
+
+            test_path = self._wrap_path_argument(test_paths[0]).relpath()
+
         import runtestsb2g
+        parser = runtestsb2g.B2GOptions()
+        options, args = parser.parse_args([])
 
-        log = kwargs.pop("log")
+        options.b2g_path = b2g_home
-        self.log_manager.enable_unstructured()
+        options.busybox = busybox or os.environ.get('BUSYBOX')
+        options.localLib = self.bin_dir
+        options.localBin = self.bin_dir
+        options.logdir = self.xpcshell_dir
+        options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell.ini')
+        options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
+        options.objdir = self.topobjdir
+        options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'),
+        options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
+        options.testsRootDir = self.xpcshell_dir
+        options.testPath = test_path
+        options.use_device_libs = True
 
-        if kwargs["xpcshell"] is None:
+        options.emulator = 'arm'
-            kwargs["xpcshell"] = "xpcshell"
+        if device_name.startswith('emulator'):
-        if kwargs["b2g_path"] is None:
+            if 'x86' in device_name:
-            kwargs["b2g_path"] = kwargs["b2g_home"]
+                options.emulator = 'x86'
-        if kwargs["busybox"] is None:
-            kwargs["busybox"] = os.environ.get('BUSYBOX')
-        if kwargs["busybox"] is None:
-            kwargs["busybox"] = self._download_busybox(kwargs["b2g_home"], kwargs["emulator"])
 
-        if kwargs["localLib"] is None:
+        if not options.busybox:
-            kwargs["localLib"] = self.bin_dir
+            options.busybox = self._download_busybox(b2g_home, options.emulator)
-        if kwargs["localBin"] is None:
-            kwargs["localBin"] = self.bin_dir
-        if kwargs["logdir"] is None:
-            kwargs["logdir"] = self.xpcshell_dir
-        if kwargs["manifest"] is None:
-            kwargs["manifest"] = os.path.join(self.xpcshell_dir, 'xpcshell.ini')
-        if kwargs["mozInfo"] is None:
-            kwargs["mozInfo"] = os.path.join(self.topobjdir, 'mozinfo.json')
-        if kwargs["objdir"] is None:
-            kwargs["objdir"] = self.topobjdir
-        if kwargs["symbolsPath"] is None:
-            kwargs["symbolsPath"] = os.path.join(self.distdir, 'crashreporter-symbols')
-        if kwargs["testingModulesDir"] is None:
-            kwargs["testingModulesDir"] = os.path.join(self.tests_dir, 'modules')
-        if kwargs["use_device_libs"] is None:
-            kwargs["use_device_libs"] = True
 
-        if kwargs["device_name"].startswith('emulator') and 'x86' in kwargs["device_name"]:
+        return runtestsb2g.run_remote_xpcshell(parser, options, args, log)
-            kwargs["emulator"] = 'x86'
-
-        parser = parser_b2g()
-        options = argparse.Namespace(**kwargs)
-        rv = runtestsb2g.run_remote_xpcshell(parser, options, log)
-
-        self.log_manager.disable_unstructured()
-        return rv
-
-def get_parser():
-    build_obj = MozbuildObject.from_environment(cwd=here)
-    if conditions.is_android(build_obj):
-        return parser_remote()
-    elif conditions.is_b2g(build_obj):
-        return parser_b2g()
-    else:
-        return parser_desktop()
 
 @CommandProvider
 class MachCommands(MachCommandBase):
@@ -328,10 +411,61 @@ class MachCommands(MachCommandBase):
             setattr(self, attr, getattr(context, attr, None))
 
     @Command('xpcshell-test', category='testing',
-             description='Run XPCOM Shell tests (API direct unit testing)',
+        description='Run XPCOM Shell tests (API direct unit testing)',
-             conditions=[lambda *args: True],
+        conditions=[lambda *args: True],
-             parser=get_parser)
+        parser=_parser)
-
+    @CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',
+        help='Test to run. Can be specified as a single JS file, a directory, '
+             'or omitted. If omitted, the entire test suite is executed.')
+    @CommandArgument('--verbose', '-v', action='store_true',
+        help='Provide full output from each test process.')
+    @CommandArgument("--debugger", default=None, metavar='DEBUGGER',
+                     help = "Run xpcshell under the given debugger.")
+    @CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str,
+                     dest = "debuggerArgs",
+                     help = "pass the given args to the debugger _before_ "
+                            "the application on the command line")
+    @CommandArgument("--debugger-interactive", action = "store_true",
+                     dest = "debuggerInteractive",
+                     help = "prevents the test harness from redirecting "
+                            "stdout and stderr for interactive debuggers")
+    @CommandArgument("--jsdebugger", dest="jsDebugger", action="store_true",
+                     help="Waits for a devtools JS debugger to connect before "
+                          "starting the test.")
+    @CommandArgument("--jsdebugger-port", dest="jsDebuggerPort",
+                     type=int, default=6000,
+                     help="The port to listen on for a debugger connection if "
+                          "--jsdebugger is specified (default=6000).")
+    @CommandArgument('--interactive', '-i', action='store_true',
+        help='Open an xpcshell prompt before running tests.')
+    @CommandArgument('--keep-going', '-k', action='store_true',
+        help='Continue running tests after a SIGINT is received.')
+    @CommandArgument('--sequential', action='store_true',
+        help='Run the tests sequentially.')
+    @CommandArgument('--shuffle', '-s', action='store_true',
+        help='Randomize the execution order of tests.')
+    @CommandArgument('--rerun-failures', action='store_true',
+        help='Reruns failures from last time.')
+    @CommandArgument('--tag', action='append', dest='test_tags',
+        help='Filter out tests that don\'t have the given tag. Can be used '
+             'multiple times in which case the test must contain at least one '
+             'of the given tags.')
+    @CommandArgument('--dump-tests', default=None, type=str, dest='dump_tests',
+        help='Specify path to a filename to dump all the tests that will be run')
+    @CommandArgument('--devicemanager', default='adb', type=str,
+        help='(Android) Type of devicemanager to use for communication: adb or sut')
+    @CommandArgument('--ip', type=str, default=None,
+        help='(Android) IP address of device')
+    @CommandArgument('--port', type=int, default=20701,
+        help='(Android) Port of device')
+    @CommandArgument('--remote_test_root', type=str, default=None,
+        help='(Android) Remote test root such as /mnt/sdcard or /data/local')
+    @CommandArgument('--no-setup', action='store_true',
+        help='(Android) Do not copy files to device')
+    @CommandArgument('--local-apk', type=str, default=None,
+        help='(Android) Use specified Fennec APK')
+    @CommandArgument('--busybox', type=str, default=None,
+        help='(B2G) Path to busybox binary (speeds up installation of tests).')
     def run_xpcshell_test(self, **params):
         from mozbuild.controller.building import BuildDriver
 

testing/xpcshell/remotexpcshelltests.py

@@ -16,8 +16,6 @@ import mozdevice
 import mozfile
 import mozinfo
 
-from xpcshellcommandline import parser_remote
-
 here = os.path.dirname(os.path.abspath(__file__))
 
 def remoteJoin(path1, path2):
@@ -107,7 +105,7 @@ class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
         return (list(sanitize_list(test['head'], 'head')),
                 list(sanitize_list(test['tail'], 'tail')))
 
-    def buildXpcsCmd(self):
+    def buildXpcsCmd(self, testdir):
         # change base class' paths to remote paths and use base class to build command
         self.xpcshell = remoteJoin(self.remoteBinDir, "xpcw")
         self.headJSPath = remoteJoin(self.remoteScriptsDir, 'head.js')
@@ -115,7 +113,7 @@ class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
         self.httpdManifest = remoteJoin(self.remoteComponentsDir, 'httpd.manifest')
         self.testingModulesDir = self.remoteModulesDir
         self.testharnessdir = self.remoteScriptsDir
-        xpcshell.XPCShellTestThread.buildXpcsCmd(self)
+        xpcshell.XPCShellTestThread.buildXpcsCmd(self, testdir)
         # remove "-g <dir> -a <dir>" and add "--greomni <apk>"
         del(self.xpcsCmd[1:5])
         if self.options.localAPK:
@@ -218,7 +216,7 @@ class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
 # via devicemanager.
 class XPCShellRemote(xpcshell.XPCShellTests, object):
 
-    def __init__(self, devmgr, options, log):
+    def __init__(self, devmgr, options, args, log):
         xpcshell.XPCShellTests.__init__(self, log)
 
         # Add Android version (SDK level) to mozinfo so that manifest entries
@@ -519,8 +517,8 @@ class XPCShellRemote(xpcshell.XPCShellTests, object):
             self.device.removeDir(self.remoteMinidumpDir)
         self.device.mkDir(self.remoteMinidumpDir)
 
-    def buildTestList(self, test_tags=None, test_paths=None):
+    def buildTestList(self, test_tags=None):
-        xpcshell.XPCShellTests.buildTestList(self, test_tags=test_tags, test_paths=test_paths)
+        xpcshell.XPCShellTests.buildTestList(self, test_tags=test_tags)
         uniqueTestPaths = set([])
         for test in self.alltests:
             uniqueTestPaths.add(test['here'])
@@ -529,37 +527,90 @@ class XPCShellRemote(xpcshell.XPCShellTests, object):
             remoteScriptDir = remoteJoin(self.remoteScriptsDir, abbrevTestDir)
             self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
 
-def verifyRemoteOptions(parser, options):
+class RemoteXPCShellOptions(xpcshell.XPCShellOptions):
-    if options.localLib is None:
-        if options.localAPK and options.objdir:
-            for path in ['dist/fennec', 'fennec/lib']:
-                options.localLib = os.path.join(options.objdir, path)
-                if os.path.isdir(options.localLib):
-                    break
-            else:
-                parser.error("Couldn't find local library dir, specify --local-lib-dir")
-        elif options.objdir:
-            options.localLib = os.path.join(options.objdir, 'dist/bin')
-        elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
-            # assume tests are being run from a tests.zip
-            options.localLib = os.path.abspath(os.path.join(here, '..', 'bin'))
-        else:
-            parser.error("Couldn't find local library dir, specify --local-lib-dir")
 
-    if options.localBin is None:
+    def __init__(self):
-        if options.objdir:
+        xpcshell.XPCShellOptions.__init__(self)
-            for path in ['dist/bin', 'bin']:
+        defaults = {}
-                options.localBin = os.path.join(options.objdir, path)
+
-                if os.path.isdir(options.localBin):
+        self.add_option("--deviceIP", action="store",
-                    break
+                        type = "string", dest = "deviceIP",
+                        help = "ip address of remote device to test")
+        defaults["deviceIP"] = None
+
+        self.add_option("--devicePort", action="store",
+                        type = "string", dest = "devicePort",
+                        help = "port of remote device to test")
+        defaults["devicePort"] = 20701
+
+        self.add_option("--dm_trans", action="store",
+                        type = "string", dest = "dm_trans",
+                        help = "the transport to use to communicate with device: [adb|sut]; default=sut")
+        defaults["dm_trans"] = "sut"
+
+        self.add_option("--objdir", action="store",
+                        type = "string", dest = "objdir",
+                        help = "local objdir, containing xpcshell binaries")
+        defaults["objdir"] = None
+
+        self.add_option("--apk", action="store",
+                        type = "string", dest = "localAPK",
+                        help = "local path to Fennec APK")
+        defaults["localAPK"] = None
+
+        self.add_option("--noSetup", action="store_false",
+                        dest = "setup",
+                        help = "do not copy any files to device (to be used only if device is already setup)")
+        defaults["setup"] = True
+
+        self.add_option("--local-lib-dir", action="store",
+                        type = "string", dest = "localLib",
+                        help = "local path to library directory")
+        defaults["localLib"] = None
+
+        self.add_option("--local-bin-dir", action="store",
+                        type = "string", dest = "localBin",
+                        help = "local path to bin directory")
+        defaults["localBin"] = None
+
+        self.add_option("--remoteTestRoot", action = "store",
+                    type = "string", dest = "remoteTestRoot",
+                    help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
+        defaults["remoteTestRoot"] = None
+
+        self.set_defaults(**defaults)
+
+    def verifyRemoteOptions(self, options):
+        if options.localLib is None:
+            if options.localAPK and options.objdir:
+                for path in ['dist/fennec', 'fennec/lib']:
+                    options.localLib = os.path.join(options.objdir, path)
+                    if os.path.isdir(options.localLib):
+                        break
+                else:
+                    self.error("Couldn't find local library dir, specify --local-lib-dir")
+            elif options.objdir:
+                options.localLib = os.path.join(options.objdir, 'dist/bin')
+            elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
+                # assume tests are being run from a tests.zip
+                options.localLib = os.path.abspath(os.path.join(here, '..', 'bin'))
             else:
-                parser.error("Couldn't find local binary dir, specify --local-bin-dir")
+                self.error("Couldn't find local library dir, specify --local-lib-dir")
-        elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
+
-            # assume tests are being run from a tests.zip
+        if options.localBin is None:
-            options.localBin = os.path.abspath(os.path.join(here, '..', 'bin'))
+            if options.objdir:
-        else:
+                for path in ['dist/bin', 'bin']:
-            parser.error("Couldn't find local binary dir, specify --local-bin-dir")
+                    options.localBin = os.path.join(options.objdir, path)
-    return options
+                    if os.path.isdir(options.localBin):
+                        break
+                else:
+                    self.error("Couldn't find local binary dir, specify --local-bin-dir")
+            elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
+                # assume tests are being run from a tests.zip
+                options.localBin = os.path.abspath(os.path.join(here, '..', 'bin'))
+            else:
+                self.error("Couldn't find local binary dir, specify --local-bin-dir")
+        return options
 
 class PathMapping:
 
@@ -568,13 +619,14 @@ class PathMapping:
         self.remote = remoteDir
 
 def main():
+
     if sys.version_info < (2,7):
         print >>sys.stderr, "Error: You must use python version 2.7 or newer but less than 3.0"
         sys.exit(1)
 
-    parser = parser_remote()
+    parser = RemoteXPCShellOptions()
     commandline.add_logging_group(parser)
-    options = parser.parse_args()
+    options, args = parser.parse_args()
     if not options.localAPK:
         for file in os.listdir(os.path.join(options.objdir, "dist")):
             if (file.endswith(".apk") and file.startswith("fennec")):
@@ -586,11 +638,16 @@ def main():
             print >>sys.stderr, "Error: please specify an APK"
             sys.exit(1)
 
-    options = verifyRemoteOptions(parser, options)
+    options = parser.verifyRemoteOptions(options)
     log = commandline.setup_logging("Remote XPCShell",
                                     options,
                                     {"tbpl": sys.stdout})
 
+    if len(args) < 1 and options.manifest is None:
+        print >>sys.stderr, """Usage: %s <test dirs>
+             or: %s --manifest=test.manifest """ % (sys.argv[0], sys.argv[0])
+        sys.exit(1)
+
     if options.dm_trans == "adb":
         if options.deviceIP:
             dm = mozdevice.DroidADB(options.deviceIP, options.devicePort, packageName=None, deviceRoot=options.remoteTestRoot)
@@ -606,17 +663,16 @@ def main():
         print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
         sys.exit(1)
 
-    if options.xpcshell is None:
+    xpcsh = XPCShellRemote(dm, options, args, log)
-        options.xpcshell = "xpcshell"
-
-    xpcsh = XPCShellRemote(dm, options, log)
 
     # we don't run concurrent tests on mobile
     options.sequential = True
 
-    if not xpcsh.runTests(testClass=RemoteXPCShellTestThread,
+    if not xpcsh.runTests(xpcshell='xpcshell',
+                          testClass=RemoteXPCShellTestThread,
+                          testdirs=args[0:],
                           mobileArgs=xpcsh.mobileArgs,
-                          **vars(options)):
+                          **options.__dict__):
         sys.exit(1)
 
 

testing/xpcshell/runtestsb2g.py

@@ -9,15 +9,14 @@ import os
 sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))))
 
 import traceback
-import remotexpcshelltests
+from remotexpcshelltests import RemoteXPCShellTestThread, XPCShellRemote, RemoteXPCShellOptions
-from remotexpcshelltests import RemoteXPCShellTestThread, XPCShellRemote
 from mozdevice import devicemanagerADB, DMError
 from mozlog import commandline
 
 DEVICE_TEST_ROOT = '/data/local/tests'
 
+
 from marionette import Marionette
-from xpcshellcommandline import parser_b2g
 
 class B2GXPCShellTestThread(RemoteXPCShellTestThread):
     # Overridden
@@ -76,19 +75,87 @@ class B2GXPCShellRemote(XPCShellRemote):
                 self.env['LD_LIBRARY_PATH'] = '/system/b2g'
                 self.options.use_device_libs = True
 
-def verifyRemoteOptions(parser, options):
+class B2GOptions(RemoteXPCShellOptions):
-    if options.b2g_path is None:
-        parser.error("Need to specify a --b2gpath")
 
-    if options.geckoPath and not options.emulator:
+    def __init__(self):
-        parser.error("You must specify --emulator if you specify --gecko-path")
+        RemoteXPCShellOptions.__init__(self)
+        defaults = {}
 
-    if options.logdir and not options.emulator:
+        self.add_option('--b2gpath', action='store',
-        parser.error("You must specify --emulator if you specify --logdir")
+                        type='string', dest='b2g_path',
-    return remotexpcshelltests.verifyRemoteOptions(parser, options)
+                        help="Path to B2G repo or qemu dir")
+        defaults['b2g_path'] = None
 
-def run_remote_xpcshell(parser, options, log):
+        self.add_option('--emupath', action='store',
-    options = verifyRemoteOptions(parser, options)
+                        type='string', dest='emu_path',
+                        help="Path to emulator folder (if different "
+                                                      "from b2gpath")
+
+        self.add_option('--no-clean', action='store_false',
+                        dest='clean',
+                        help="Do not clean TESTROOT. Saves [lots of] time")
+        defaults['clean'] = True
+
+        defaults['emu_path'] = None
+
+        self.add_option('--emulator', action='store',
+                        type='string', dest='emulator',
+                        help="Architecture of emulator to use: x86 or arm")
+        defaults['emulator'] = None
+
+        self.add_option('--no-window', action='store_true',
+                        dest='no_window',
+                        help="Pass --no-window to the emulator")
+        defaults['no_window'] = False
+
+        self.add_option('--adbpath', action='store',
+                        type='string', dest='adb_path',
+                        help="Path to adb")
+        defaults['adb_path'] = 'adb'
+
+        self.add_option('--address', action='store',
+                        type='string', dest='address',
+                        help="host:port of running Gecko instance to connect to")
+        defaults['address'] = None
+
+        self.add_option('--use-device-libs', action='store_true',
+                        dest='use_device_libs',
+                        help="Don't push .so's")
+        defaults['use_device_libs'] = False
+        self.add_option("--gecko-path", action="store",
+                        type="string", dest="geckoPath",
+                        help="the path to a gecko distribution that should "
+                        "be installed on the emulator prior to test")
+        defaults["geckoPath"] = None
+        self.add_option("--logdir", action="store",
+                        type="string", dest="logdir",
+                        help="directory to store log files")
+        defaults["logdir"] = None
+        self.add_option('--busybox', action='store',
+                        type='string', dest='busybox',
+                        help="Path to busybox binary to install on device")
+        defaults['busybox'] = None
+
+        defaults["remoteTestRoot"] = DEVICE_TEST_ROOT
+        defaults['dm_trans'] = 'adb'
+        defaults['debugger'] = None
+        defaults['debuggerArgs'] = None
+
+        self.set_defaults(**defaults)
+
+    def verifyRemoteOptions(self, options):
+        if options.b2g_path is None:
+            self.error("Need to specify a --b2gpath")
+
+        if options.geckoPath and not options.emulator:
+            self.error("You must specify --emulator if you specify --gecko-path")
+
+        if options.logdir and not options.emulator:
+            self.error("You must specify --emulator if you specify --logdir")
+        return RemoteXPCShellOptions.verifyRemoteOptions(self, options)
+
+def run_remote_xpcshell(parser, options, args, log):
+    options = parser.verifyRemoteOptions(options)
 
     # Create the Marionette instance
     kwargs = {}
@@ -128,18 +195,16 @@ def run_remote_xpcshell(parser, options, log):
 
     if not options.remoteTestRoot:
         options.remoteTestRoot = dm.deviceRoot
-    xpcsh = B2GXPCShellRemote(dm, options, log)
+    xpcsh = B2GXPCShellRemote(dm, options, args, log)
 
     # we don't run concurrent tests on mobile
     options.sequential = True
 
-    if options.xpcshell is None:
-        options.xpcshell = "xpcshell"
-
     try:
-        if not xpcsh.runTests(testClass=B2GXPCShellTestThread,
+        if not xpcsh.runTests(xpcshell='xpcshell', testdirs=args[0:],
-                              mobileArgs=xpcsh.mobileArgs,
+                                 testClass=B2GXPCShellTestThread,
-                              **vars(options)):
+                                 mobileArgs=xpcsh.mobileArgs,
+                                 **options.__dict__):
             sys.exit(1)
     except:
         print "Automation Error: Exception caught while running tests"
@@ -147,13 +212,13 @@ def run_remote_xpcshell(parser, options, log):
         sys.exit(1)
 
 def main():
-    parser = parser_b2g()
+    parser = B2GOptions()
     commandline.add_logging_group(parser)
-    options = parser.parse_args()
+    options, args = parser.parse_args()
     log = commandline.setup_logging("Remote XPCShell",
                                     options,
                                     {"tbpl": sys.stdout})
-    run_remote_xpcshell(parser, options, log)
+    run_remote_xpcshell(parser, options, args, log)
 
 # You usually run this like :
 # python runtestsb2g.py --emulator arm --b2gpath $B2GPATH --manifest $MANIFEST [--xre-path $MOZ_HOST_BIN

testing/xpcshell/runxpcshelltests.py

@@ -23,7 +23,7 @@ import traceback
 from collections import deque, namedtuple
 from distutils import dir_util
 from multiprocessing import cpu_count
-from argparse import ArgumentParser
+from optparse import OptionParser
 from subprocess import Popen, PIPE, STDOUT
 from tempfile import mkdtemp, gettempdir
 from threading import (
@@ -40,9 +40,6 @@ except Exception:
     HAVE_PSUTIL = False
 
 from automation import Automation
-from xpcshellcommandline import parser_desktop
-
-SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
 HARNESS_TIMEOUT = 5 * 60
 
@@ -65,7 +62,7 @@ if os.path.isdir(mozbase):
         sys.path.append(os.path.join(mozbase, package))
 
 from manifestparser import TestManifest
-from manifestparser.filters import chunk_by_slice, tags, pathprefix
+from manifestparser.filters import chunk_by_slice, tags
 from mozlog import commandline
 import mozcrash
 import mozinfo
@@ -103,7 +100,7 @@ def markGotSIGINT(signum, stackFrame):
 
 class XPCShellTestThread(Thread):
     def __init__(self, test_object, event, cleanup_dir_list, retry=True,
-            app_dir_key=None, interactive=False,
+            tests_root_dir=None, app_dir_key=None, interactive=False,
             verbose=False, pStdout=None, pStderr=None, keep_going=False,
             log=None, **kwargs):
         Thread.__init__(self)
@@ -133,6 +130,7 @@ class XPCShellTestThread(Thread):
         self.failureManifest = kwargs.get('failureManifest')
         self.stack_fixer_function = kwargs.get('stack_fixer_function')
 
+        self.tests_root_dir = tests_root_dir
         self.app_dir_key = app_dir_key
         self.interactive = interactive
         self.verbose = verbose
@@ -415,7 +413,7 @@ class XPCShellTestThread(Thread):
         return (list(sanitize_list(headlist, 'head')),
                 list(sanitize_list(taillist, 'tail')))
 
-    def buildXpcsCmd(self):
+    def buildXpcsCmd(self, testdir):
         """
           Load the root head.js file as the first file in our test path, before other head, test, and tail files.
           On a remote system, we overload this to add additional command line arguments, so this gets overloaded.
@@ -617,7 +615,7 @@ class XPCShellTestThread(Thread):
         self.tempDir = self.setupTempDir()
         self.mozInfoJSPath = self.setupMozinfoJS()
 
-        self.buildXpcsCmd()
+        self.buildXpcsCmd(test_dir)
         head_files, tail_files = self.getHeadAndTailFiles(self.test_object)
         cmdH = self.buildCmdHead(head_files, tail_files, self.xpcsCmd)
 
@@ -770,50 +768,30 @@ class XPCShellTests(object):
         self.harness_timeout = HARNESS_TIMEOUT
         self.nodeProc = {}
 
-    def getTestManifest(self, manifest):
+    def buildTestList(self, test_tags=None):
-        if isinstance(manifest, TestManifest):
-            return manifest
-        elif manifest is not None:
-            manifest = os.path.normpath(os.path.abspath(manifest))
-            if os.path.isfile(manifest):
-                return TestManifest([manifest], strict=True)
-            else:
-                ini_path = os.path.join(manifest, "xpcshell.ini")
-        else:
-            ini_path = os.path.join(SCRIPT_DIR, "tests", "xpcshell.ini")
-
-        if os.path.exists(ini_path):
-            return TestManifest([ini_path], strict=True)
-        else:
-            print >> sys.stderr, ("Failed to find manifest at %s; use --manifest "
-                                  "to set path explicitly." % (ini_path,))
-            sys.exit(1)
-
-    def buildTestList(self, test_tags=None, test_paths=None):
         """
           read the xpcshell.ini manifest and set self.alltests to be
           an array of test objects.
 
           if we are chunking tests, it will be done here as well
         """
-
+        if isinstance(self.manifest, TestManifest):
-        if test_paths is None:
+            mp = self.manifest
-            test_paths = []
-
-        if len(test_paths) == 1 and test_paths[0].endswith(".js"):
-            self.singleFile = os.path.basename(test_paths[0])
         else:
-            self.singleFile = None
+            mp = TestManifest(strict=True)
+            if self.manifest is None:
+                for testdir in self.testdirs:
+                    if testdir:
+                        mp.read(os.path.join(testdir, 'xpcshell.ini'))
+            else:
+                mp.read(self.manifest)
 
-        mp = self.getTestManifest(self.manifest)
+        self.buildTestPath()
 
         filters = []
         if test_tags:
             filters.append(tags(test_tags))
 
-        if test_paths:
-            filters.append(pathprefix(test_paths))
-
         if self.singleFile is None and self.totalChunks > 1:
             filters.append(chunk_by_slice(self.thisChunk, self.totalChunks))
         try:
@@ -940,6 +918,31 @@ class XPCShellTests(object):
                 pStderr = STDOUT
         return pStdout, pStderr
 
+    def buildTestPath(self):
+        """
+          If we specifiy a testpath, set the self.testPath variable to be the given directory or file.
+
+          |testPath| will be the optional path only, or |None|.
+          |singleFile| will be the optional test only, or |None|.
+        """
+        self.singleFile = None
+        if self.testPath is not None:
+            if self.testPath.endswith('.js'):
+            # Split into path and file.
+                if self.testPath.find('/') == -1:
+                    # Test only.
+                    self.singleFile = self.testPath
+                else:
+                    # Both path and test.
+                    # Reuse |testPath| temporarily.
+                    self.testPath = self.testPath.rsplit('/', 1)
+                    self.singleFile = self.testPath[1]
+                    self.testPath = self.testPath[0]
+            else:
+                # Path only.
+                # Simply remove optional ending separator.
+                self.testPath = self.testPath.rstrip("/")
+
     def verifyDirPath(self, dirname):
         """
           Simple wrapper to get the absolute path for a given directory name.
@@ -1041,17 +1044,16 @@ class XPCShellTests(object):
             return '%s:%s' % (os.path.basename(test_object['ancestor-manifest']), path)
         return path
 
-    def runTests(self, xpcshell=None, xrePath=None, appPath=None, symbolsPath=None,
+    def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None,
-                 manifest=None, testPaths=None, mobileArgs=None,
+                 manifest=None, testdirs=None, testPath=None, mobileArgs=None,
                  interactive=False, verbose=False, keepGoing=False, logfiles=True,
                  thisChunk=1, totalChunks=1, debugger=None,
                  debuggerArgs=None, debuggerInteractive=False,
                  profileName=None, mozInfo=None, sequential=False, shuffle=False,
-                 testingModulesDir=None, pluginsPath=None,
+                 testsRootDir=None, testingModulesDir=None, pluginsPath=None,
                  testClass=XPCShellTestThread, failureManifest=None,
                  log=None, stream=None, jsDebugger=False, jsDebuggerPort=0,
-                 test_tags=None, dump_tests=None, utility_path=None,
+                 test_tags=None, dump_tests=None, utility_path=None, **otherOptions):
-                 rerun_failures=False, failure_manifest=None, **otherOptions):
         """Run xpcshell tests.
 
         |xpcshell|, is the xpcshell executable to use to run the tests.
@@ -1061,8 +1063,9 @@ class XPCShellTests(object):
           breakpad symbols for processing crashes in tests.
         |manifest|, if provided, is a file containing a list of
           test directories to run.
-        |testPaths|, if provided, is a list of paths to files or directories containing
+        |testdirs|, if provided, is a list of absolute paths of test directories.
-                     tests to run.
+          No-manifest only option.
+        |testPath|, if provided, indicates a single path and/or test to run.
         |pluginsPath|, if provided, custom plugins directory to be returned from
           the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST.
         |interactive|, if set to True, indicates to provide an xpcshell prompt
@@ -1080,6 +1083,8 @@ class XPCShellTests(object):
           directory if running only a subset of tests.
         |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict.
         |shuffle|, if True, execute tests in random order.
+        |testsRootDir|, absolute path to root directory of all tests. This is used
+          by xUnit generation to determine the package name of the tests.
         |testingModulesDir|, if provided, specifies where JS modules reside.
           xpcshell will register a resource handler mapping this path.
         |otherOptions| may be present for the convenience of subclasses
@@ -1087,6 +1092,9 @@ class XPCShellTests(object):
 
         global gotSIGINT
 
+        if testdirs is None:
+            testdirs = []
+
         # Try to guess modules directory.
         # This somewhat grotesque hack allows the buildbot machines to find the
         # modules directory without having to configure the buildbot hosts. This
@@ -1099,16 +1107,6 @@ class XPCShellTests(object):
             if os.path.isdir(possible):
                 testingModulesDir = possible
 
-        if rerun_failures:
-            if os.path.exists(failure_manifest):
-                rerun_manifest = os.path.join(os.path.dirname(failure_manifest), "rerun.ini")
-                shutil.copyfile(failure_manifest, rerun_manifest)
-                os.remove(failure_manifest)
-                manifest = rerun_manifest
-            else:
-                print >> sys.stderr, "No failures were found to re-run."
-                sys.exit(1)
-
         if testingModulesDir:
             # The resource loader expects native paths. Depending on how we were
             # invoked, a UNIX style path may sneak in on Windows. We try to
@@ -1137,6 +1135,8 @@ class XPCShellTests(object):
         self.appPath = appPath
         self.symbolsPath = symbolsPath
         self.manifest = manifest
+        self.testdirs = testdirs
+        self.testPath = testPath
         self.dump_tests = dump_tests
         self.interactive = interactive
         self.verbose = verbose
@@ -1149,7 +1149,11 @@ class XPCShellTests(object):
         self.testingModulesDir = testingModulesDir
         self.pluginsPath = pluginsPath
         self.sequential = sequential
-        self.failure_manifest = failure_manifest
+
+        if not testdirs and not manifest:
+            # nothing to test!
+            self.log.error("Error: No test dirs or test manifest specified!")
+            return False
 
         self.testCount = 0
         self.passCount = 0
@@ -1202,7 +1206,7 @@ class XPCShellTests(object):
 
         pStdout, pStderr = self.getPipes()
 
-        self.buildTestList(test_tags, testPaths)
+        self.buildTestList(test_tags)
         if self.singleFile:
             self.sequential = True
 
@@ -1230,7 +1234,7 @@ class XPCShellTests(object):
             'logfiles': self.logfiles,
             'xpcshell': self.xpcshell,
             'xpcsRunArgs': self.xpcsRunArgs,
-            'failureManifest': self.failure_manifest,
+            'failureManifest': failureManifest,
             'harness_timeout': self.harness_timeout,
             'stack_fixer_function': self.stack_fixer_function,
         }
@@ -1275,10 +1279,13 @@ class XPCShellTests(object):
             if self.singleFile and not path.endswith(self.singleFile):
                 continue
 
+            if self.testPath and path.find(self.testPath) == -1:
+                continue
+
             self.testCount += 1
 
             test = testClass(test_object, self.event, self.cleanup_dir_list,
-                    app_dir_key=appDirKey,
+                    tests_root_dir=testsRootDir, app_dir_key=appDirKey,
                     interactive=interactive,
                     verbose=verbose or test_object.get("verbose") == "true",
                     pStdout=pStdout, pStderr=pStderr,
@@ -1370,7 +1377,7 @@ class XPCShellTests(object):
             self.log.info("Retrying tests that failed when run in parallel.")
         for test_object in self.try_again_list:
             test = testClass(test_object, self.event, self.cleanup_dir_list,
-                    retry=False,
+                    retry=False, tests_root_dir=testsRootDir,
                     app_dir_key=appDirKey, interactive=interactive,
                     verbose=verbose, pStdout=pStdout, pStderr=pStderr,
                     keep_going=keepGoing, log=self.log, mobileArgs=mobileArgs,
@@ -1423,16 +1430,122 @@ class XPCShellTests(object):
         self.log.suite_end()
         return self.failCount == 0
 
+class XPCShellOptions(OptionParser):
+    def __init__(self):
+        """Process command line arguments and call runTests() to do the real work."""
+        OptionParser.__init__(self)
+        self.add_option("--app-path",
+                        type="string", dest="appPath", default=None,
+                        help="application directory (as opposed to XRE directory)")
+        self.add_option("--interactive",
+                        action="store_true", dest="interactive", default=False,
+                        help="don't automatically run tests, drop to an xpcshell prompt")
+        self.add_option("--verbose",
+                        action="store_true", dest="verbose", default=False,
+                        help="always print stdout and stderr from tests")
+        self.add_option("--keep-going",
+                        action="store_true", dest="keepGoing", default=False,
+                        help="continue running tests after test killed with control-C (SIGINT)")
+        self.add_option("--logfiles",
+                        action="store_true", dest="logfiles", default=True,
+                        help="create log files (default, only used to override --no-logfiles)")
+        self.add_option("--dump-tests",
+                        type="string", dest="dump_tests", default=None,
+                        help="Specify path to a filename to dump all the tests that will be run")
+        self.add_option("--manifest",
+                        type="string", dest="manifest", default=None,
+                        help="Manifest of test directories to use")
+        self.add_option("--no-logfiles",
+                        action="store_false", dest="logfiles",
+                        help="don't create log files")
+        self.add_option("--sequential",
+                        action="store_true", dest="sequential", default=False,
+                        help="Run all tests sequentially")
+        self.add_option("--test-path",
+                        type="string", dest="testPath", default=None,
+                        help="single path and/or test filename to test")
+        self.add_option("--tests-root-dir",
+                        type="string", dest="testsRootDir", default=None,
+                        help="absolute path to directory where all tests are located. this is typically $(objdir)/_tests")
+        self.add_option("--testing-modules-dir",
+                        dest="testingModulesDir", default=None,
+                        help="Directory where testing modules are located.")
+        self.add_option("--test-plugin-path",
+                        type="string", dest="pluginsPath", default=None,
+                        help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. "
+                             "By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory "
+                             "to return for NS_APP_PLUGINS_DIR_LIST when queried.")
+        self.add_option("--total-chunks",
+                        type = "int", dest = "totalChunks", default=1,
+                        help = "how many chunks to split the tests up into")
+        self.add_option("--this-chunk",
+                        type = "int", dest = "thisChunk", default=1,
+                        help = "which chunk to run between 1 and --total-chunks")
+        self.add_option("--profile-name",
+                        type = "string", dest="profileName", default=None,
+                        help="name of application profile being tested")
+        self.add_option("--build-info-json",
+                        type = "string", dest="mozInfo", default=None,
+                        help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
+        self.add_option("--shuffle",
+                        action="store_true", dest="shuffle", default=False,
+                        help="Execute tests in random order")
+        self.add_option("--failure-manifest", dest="failureManifest",
+                        action="store",
+                        help="path to file where failure manifest will be written.")
+        self.add_option("--xre-path",
+                        action = "store", type = "string", dest = "xrePath",
+                        # individual scripts will set a sane default
+                        default = None,
+                        help = "absolute path to directory containing XRE (probably xulrunner)")
+        self.add_option("--symbols-path",
+                        action = "store", type = "string", dest = "symbolsPath",
+                        default = None,
+                        help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+        self.add_option("--debugger",
+                        action = "store", dest = "debugger",
+                        help = "use the given debugger to launch the application")
+        self.add_option("--debugger-args",
+                        action = "store", dest = "debuggerArgs",
+                        help = "pass the given args to the debugger _before_ "
+                           "the application on the command line")
+        self.add_option("--debugger-interactive",
+                        action = "store_true", dest = "debuggerInteractive",
+                        help = "prevents the test harness from redirecting "
+                          "stdout and stderr for interactive debuggers")
+        self.add_option("--jsdebugger", dest="jsDebugger", action="store_true",
+                        help="Waits for a devtools JS debugger to connect before "
+                             "starting the test.")
+        self.add_option("--jsdebugger-port", type="int", dest="jsDebuggerPort",
+                        default=6000,
+                        help="The port to listen on for a debugger connection if "
+                             "--jsdebugger is specified.")
+        self.add_option("--tag",
+                        action="append", dest="test_tags",
+                        default=None,
+                        help="filter out tests that don't have the given tag. Can be "
+                             "used multiple times in which case the test must contain "
+                             "at least one of the given tags.")
+        self.add_option("--utility-path",
+                        action="store", dest="utility_path",
+                        default=None,
+                        help="Path to a directory containing utility programs, such "
+                             "as stack fixer scripts.")
 
 def main():
-    parser = parser_desktop()
+    parser = XPCShellOptions()
     commandline.add_logging_group(parser)
-    options = parser.parse_args()
+    options, args = parser.parse_args()
+
 
     log = commandline.setup_logging("XPCShell", options, {"tbpl": sys.stdout})
 
-    if options.xpcshell is None:
+    if len(args) < 2 and options.manifest is None or \
-        print >> sys.stderr, """Must provide path to xpcshell using --xpcshell"""
+       (len(args) < 1 and options.manifest is not None):
+        print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
+              or: %s --manifest=test.manifest <path to xpcshell>""" % (sys.argv[0],
+                                                              sys.argv[0])
+        sys.exit(1)
 
     xpcsh = XPCShellTests(log)
 
@@ -1440,7 +1553,7 @@ def main():
         print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
         sys.exit(1)
 
-    if not xpcsh.runTests(**vars(options)):
+    if not xpcsh.runTests(args[0], testdirs=args[1:], **options.__dict__):
         sys.exit(1)
 
 if __name__ == '__main__':

testing/xpcshell/selftest.py

@@ -422,6 +422,7 @@ tail =
                                           manifest=self.manifest,
                                           mozInfo=mozinfo.info,
                                           shuffle=shuffle,
+                                          testsRootDir=self.tempdir,
                                           verbose=verbose,
                                           sequential=True,
                                           utility_path=self.utility_path),

testing/xpcshell/xpcshellcommandline.py

@@ -1,202 +0,0 @@
-import argparse
-
-def add_common_arguments(parser):
-    parser.add_argument("--app-path",
-                        type=unicode, dest="appPath", default=None,
-                        help="application directory (as opposed to XRE directory)")
-    parser.add_argument("--interactive",
-                        action="store_true", dest="interactive", default=False,
-                        help="don't automatically run tests, drop to an xpcshell prompt")
-    parser.add_argument("--verbose",
-                        action="store_true", dest="verbose", default=False,
-                        help="always print stdout and stderr from tests")
-    parser.add_argument("--keep-going",
-                        action="store_true", dest="keepGoing", default=False,
-                        help="continue running tests after test killed with control-C (SIGINT)")
-    parser.add_argument("--logfiles",
-                        action="store_true", dest="logfiles", default=True,
-                        help="create log files (default, only used to override --no-logfiles)")
-    parser.add_argument("--dump-tests", type=str, dest="dump_tests", default=None,
-                        help="Specify path to a filename to dump all the tests that will be run")
-    parser.add_argument("--manifest",
-                        type=unicode, dest="manifest", default=None,
-                        help="Manifest of test directories to use")
-    parser.add_argument("--no-logfiles",
-                        action="store_false", dest="logfiles",
-                        help="don't create log files")
-    parser.add_argument("--sequential",
-                        action="store_true", dest="sequential", default=False,
-                        help="Run all tests sequentially")
-    parser.add_argument("--testing-modules-dir",
-                        dest="testingModulesDir", default=None,
-                        help="Directory where testing modules are located.")
-    parser.add_argument("--test-plugin-path",
-                        type=str, dest="pluginsPath", default=None,
-                        help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. "
-                        "By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory "
-                        "to return for NS_APP_PLUGINS_DIR_LIST when queried.")
-    parser.add_argument("--total-chunks",
-                        type=int, dest="totalChunks", default=1,
-                        help="how many chunks to split the tests up into")
-    parser.add_argument("--this-chunk",
-                        type=int, dest="thisChunk", default=1,
-                        help="which chunk to run between 1 and --total-chunks")
-    parser.add_argument("--profile-name",
-                        type=str, dest="profileName", default=None,
-                        help="name of application profile being tested")
-    parser.add_argument("--build-info-json",
-                        type=str, dest="mozInfo", default=None,
-                        help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
-    parser.add_argument("--shuffle",
-                        action="store_true", dest="shuffle", default=False,
-                        help="Execute tests in random order")
-    parser.add_argument("--xre-path",
-                        action="store", type=str, dest="xrePath",
-                        # individual scripts will set a sane default
-                        default=None,
-                        help="absolute path to directory containing XRE (probably xulrunner)")
-    parser.add_argument("--symbols-path",
-                        action="store", type=str, dest="symbolsPath",
-                        default=None,
-                        help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
-    parser.add_argument("--debugger",
-                        action="store", dest="debugger",
-                        help="use the given debugger to launch the application")
-    parser.add_argument("--debugger-args",
-                        action="store", dest="debuggerArgs",
-                        help="pass the given args to the debugger _before_ "
-                        "the application on the command line")
-    parser.add_argument("--debugger-interactive",
-                        action="store_true", dest="debuggerInteractive",
-                        help="prevents the test harness from redirecting "
-                        "stdout and stderr for interactive debuggers")
-    parser.add_argument("--jsdebugger", dest="jsDebugger", action="store_true",
-                        help="Waits for a devtools JS debugger to connect before "
-                        "starting the test.")
-    parser.add_argument("--jsdebugger-port", type=int, dest="jsDebuggerPort",
-                        default=6000,
-                        help="The port to listen on for a debugger connection if "
-                        "--jsdebugger is specified.")
-    parser.add_argument("--tag",
-                        action="append", dest="test_tags",
-                        default=None,
-                        help="filter out tests that don't have the given tag. Can be "
-                        "used multiple times in which case the test must contain "
-                        "at least one of the given tags.")
-    parser.add_argument("--utility-path",
-                        action="store", dest="utility_path",
-                        default=None,
-                        help="Path to a directory containing utility programs, such "
-                        "as stack fixer scripts.")
-    parser.add_argument("--xpcshell",
-                        action="store", dest="xpcshell",
-                        default=None,
-                        help="Path to xpcshell binary")
-    # This argument can be just present, or the path to a manifest file. The
-    # just-present case is usually used for mach which can provide a default
-    # path to the failure file from the previous run
-    parser.add_argument("--rerun-failures",
-                        action="store_true",
-                        help="Rerun failures from the previous run, if any")
-    parser.add_argument("--failure-manifest",
-                        action="store",
-                        help="Path to a manifest file from which to rerun failures "
-                        "(with --rerun-failure) or in which to record failed tests")
-    parser.add_argument("testPaths", nargs="*", default=None,
-                        help="Paths of tests to run.")
-
-def add_remote_arguments(parser):
-    parser.add_argument("--deviceIP", action="store", type=str, dest="deviceIP",
-                        help="ip address of remote device to test")
-
-    parser.add_argument("--devicePort", action="store", type=str, dest="devicePort",
-                        default=20701, help="port of remote device to test")
-
-    parser.add_argument("--dm_trans", action="store", type=str, dest="dm_trans",
-                        choices=["adb", "sut"], default="sut",
-                        help="the transport to use to communicate with device: [adb|sut]; default=sut")
-
-    parser.add_argument("--objdir", action="store", type=str, dest="objdir",
-                        help="local objdir, containing xpcshell binaries")
-
-
-    parser.add_argument("--apk", action="store", type=str, dest="localAPK",
-                        help="local path to Fennec APK")
-
-
-    parser.add_argument("--noSetup", action="store_false", dest="setup", default=True,
-                        help="do not copy any files to device (to be used only if device is already setup)")
-
-    parser.add_argument("--local-lib-dir", action="store", type=str, dest="localLib",
-                        help="local path to library directory")
-
-    parser.add_argument("--local-bin-dir", action="store", type=str, dest="localBin",
-                        help="local path to bin directory")
-
-    parser.add_argument("--remoteTestRoot", action="store", type=str, dest="remoteTestRoot",
-                        help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
-
-def add_b2g_arguments(parser):
-    parser.add_argument('--b2gpath', action='store', type=str, dest='b2g_path',
-                        help="Path to B2G repo or qemu dir")
-
-    parser.add_argument('--emupath', action='store', type=str, dest='emu_path',
-                        help="Path to emulator folder (if different "
-                        "from b2gpath")
-
-    parser.add_argument('--no-clean', action='store_false', dest='clean', default=True,
-                        help="Do not clean TESTROOT. Saves [lots of] time")
-
-    parser.add_argument('--emulator', action='store', type=str, dest='emulator',
-                        default="arm", choices=["x86", "arm"],
-                        help="Architecture of emulator to use: x86 or arm")
-
-    parser.add_argument('--no-window', action='store_true', dest='no_window', default=False,
-                        help="Pass --no-window to the emulator")
-
-    parser.add_argument('--adbpath', action='store', type=str, dest='adb_path',
-                        default="adb", help="Path to adb")
-
-    parser.add_argument('--address', action='store', type=str, dest='address',
-                        help="host:port of running Gecko instance to connect to")
-
-    parser.add_argument('--use-device-libs', action='store_true', dest='use_device_libs',
-                        default=None, help="Don't push .so's")
-
-    parser.add_argument("--gecko-path", action="store", type=str, dest="geckoPath",
-                        help="the path to a gecko distribution that should "
-                        "be installed on the emulator prior to test")
-
-    parser.add_argument("--logdir", action="store", type=str, dest="logdir",
-                        help="directory to store log files")
-
-    parser.add_argument('--busybox', action='store', type=str, dest='busybox',
-                        help="Path to busybox binary to install on device")
-
-    parser.set_defaults(remoteTestRoot="/data/local/tests",
-                        dm_trans="adb")
-
-def parser_desktop():
-    parser = argparse.ArgumentParser()
-    add_common_arguments(parser)
-    return parser
-
-def parser_remote():
-    parser = argparse.ArgumentParser()
-    common = parser.add_argument_group("Common Options")
-    add_common_arguments(common)
-    remote = parser.add_argument_group("Remote Options")
-    add_remote_arguments(remote)
-
-    return parser
-
-def parser_b2g():
-    parser = argparse.ArgumentParser()
-    common = parser.add_argument_group("Common Options")
-    add_common_arguments(common)
-    remote = parser.add_argument_group("Remote Options")
-    add_remote_arguments(remote)
-    b2g = parser.add_argument_group("B2G Options")
-    add_b2g_arguments(b2g)
-
-    return parser