diff --git a/testing/config/mozharness/android_arm_4_3_config.py b/testing/config/mozharness/android_arm_4_3_config.py index 42b46df7b544..dbe4e73d210e 100644 --- a/testing/config/mozharness/android_arm_4_3_config.py +++ b/testing/config/mozharness/android_arm_4_3_config.py @@ -7,8 +7,7 @@ config = { "mochitest": { "run_filename": "runtestsremote.py", "testsdir": "mochitest", - "options": ["--autorun", "--close-when-done", "--dm_trans=adb", - "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", + "options": ["--dm_trans=adb", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s", @@ -19,8 +18,7 @@ config = { "mochitest-gl": { "run_filename": "runtestsremote.py", "testsdir": "mochitest", - "options": ["--autorun", "--close-when-done", "--dm_trans=adb", - "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", + "options": ["--dm_trans=adb", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s", @@ -32,16 +30,15 @@ config = { "robocop": { "run_filename": "runtestsremote.py", "testsdir": "mochitest", - "options": ["--autorun", "--close-when-done", "--dm_trans=adb", - "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", + "options": ["--dm_trans=adb", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s", "--quiet", "--log-raw=%(raw_log_file)s", "--total-chunks=4", - "--robocop-path=../..", + "--robocop-apk=../../robocop.apk", "--robocop-ids=fennec_ids.txt", - "--robocop=robocop.ini", + "--robocop-ini=robocop.ini", ], }, "reftest": { diff --git a/testing/config/mozharness/android_arm_config.py b/testing/config/mozharness/android_arm_config.py index 45e0e931362b..40ceeb1e1db5 100644 --- a/testing/config/mozharness/android_arm_config.py +++ b/testing/config/mozharness/android_arm_config.py @@ -7,8 +7,7 @@ config = { "mochitest": { "run_filename": "runtestsremote.py", "testsdir": "mochitest", - "options": ["--autorun", "--close-when-done", "--dm_trans=sut", - "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", + "options": ["--dm_trans=sut", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", @@ -20,8 +19,7 @@ config = { "mochitest-gl": { "run_filename": "runtestsremote.py", "testsdir": "mochitest", - "options": ["--autorun", "--close-when-done", "--dm_trans=sut", - "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", + "options": ["--dm_trans=sut", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", @@ -34,17 +32,16 @@ config = { "robocop": { "run_filename": "runtestsremote.py", "testsdir": "mochitest", - "options": ["--autorun", "--close-when-done", "--dm_trans=sut", - "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", + "options": ["--dm_trans=sut", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s", "--quiet", "--log-raw=%(raw_log_file)s", "--total-chunks=4", - "--robocop-path=../..", + "--robocop-apk=../../robocop.apk", "--robocop-ids=fennec_ids.txt", - "--robocop=robocop.ini", + "--robocop-ini=robocop.ini", ], }, "reftest": { diff --git a/testing/config/mozharness/android_panda_config.py b/testing/config/mozharness/android_panda_config.py index eefbe97757c5..5a21666ad923 100644 --- a/testing/config/mozharness/android_panda_config.py +++ b/testing/config/mozharness/android_panda_config.py @@ -8,7 +8,7 @@ config = { "options": [ "--symbols-path=%(symbols_path)s", "--xre-path=tests/bin", - "--dm_trans=SUT", + "--dm_trans=sut", "--deviceIP=%(device_ip)s", "--localBinDir=../tests/bin", "--apk=%(apk_path)s", @@ -69,12 +69,12 @@ config = { }, "mochitest": { "options": [ + "--dm_trans=sut", "--deviceIP=%(device_ip)s", "--xre-path=../hostutils/xre", "--utility-path=../hostutils/bin", "--certificate-path=certs", "--app=%(app_name)s", - "--console-level=INFO", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", "--symbols-path=%(symbols_path)s", @@ -102,6 +102,7 @@ config = { }, "robocop": { "options": [ + "--dm_trans=sut", "--deviceIP=%(device_ip)s", "--xre-path=../hostutils/xre", "--utility-path=../hostutils/bin", @@ -111,7 +112,7 @@ config = { "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", "--symbols-path=%(symbols_path)s", - "--robocop=mochitest/robocop.ini" + "--robocop-ini=mochitest/robocop.ini" ], "run_filename": "runtestsremote.py", "testsdir": "mochitest" @@ -133,4 +134,4 @@ config = { "testsdir": "xpcshell" } } -} \ No newline at end of file +} diff --git a/testing/config/mozharness/android_x86_config.py b/testing/config/mozharness/android_x86_config.py index 46dad891d50d..4097e96124a2 100644 --- a/testing/config/mozharness/android_x86_config.py +++ b/testing/config/mozharness/android_x86_config.py @@ -6,8 +6,7 @@ config = { "suite_definitions": { "mochitest": { "run_filename": "runtestsremote.py", - "options": ["--autorun", "--close-when-done", "--dm_trans=sut", - "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", + "options": ["--dm_trans=sut", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s", "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s", diff --git a/testing/config/mozharness/b2g_desktop_config.py b/testing/config/mozharness/b2g_desktop_config.py index a5184b720c30..7e17ba66e38b 100644 --- a/testing/config/mozharness/b2g_desktop_config.py +++ b/testing/config/mozharness/b2g_desktop_config.py @@ -6,7 +6,6 @@ config = { "suite_definitions": { "mochitest": { "options": [ - "--console-level=INFO", "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s", "--profile=%(gaia_profile)s", diff --git a/testing/config/mozharness/b2g_emulator_config.py b/testing/config/mozharness/b2g_emulator_config.py index 4701cb1630e4..d93827c6ef8f 100644 --- a/testing/config/mozharness/b2g_emulator_config.py +++ b/testing/config/mozharness/b2g_emulator_config.py @@ -61,7 +61,6 @@ config = { "options": [ "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", - "--console-level=INFO", "--emulator=%(emulator)s", "--logdir=%(logcat_dir)s", "--remote-webserver=%(remote_webserver)s", @@ -82,7 +81,6 @@ config = { "options": [ "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", - "--console-level=INFO", "--emulator=%(emulator)s", "--logdir=%(logcat_dir)s", "--remote-webserver=%(remote_webserver)s", diff --git a/testing/config/mozharness/linux_config.py b/testing/config/mozharness/linux_config.py index 7ebc4cab6ede..b0253001d4cf 100644 --- a/testing/config/mozharness/linux_config.py +++ b/testing/config/mozharness/linux_config.py @@ -30,9 +30,6 @@ config = { "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s", "--certificate-path=tests/certs", - "--autorun", - "--close-when-done", - "--console-level=INFO", "--setpref=webgl.force-enabled=true", "--quiet", "--log-raw=%(raw_log_file)s", diff --git a/testing/config/mozharness/mac_config.py b/testing/config/mozharness/mac_config.py index 6d1b2cbdea0c..9811d93b9312 100644 --- a/testing/config/mozharness/mac_config.py +++ b/testing/config/mozharness/mac_config.py @@ -30,9 +30,6 @@ config = { "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s", "--certificate-path=tests/certs", - "--autorun", - "--close-when-done", - "--console-level=INFO", "--quiet", "--log-raw=%(raw_log_file)s" ], diff --git a/testing/config/mozharness/taskcluster_linux_config.py b/testing/config/mozharness/taskcluster_linux_config.py index 0860181b4c22..753a9c301b37 100644 --- a/testing/config/mozharness/taskcluster_linux_config.py +++ b/testing/config/mozharness/taskcluster_linux_config.py @@ -10,8 +10,7 @@ config = { "mochitest_options": [ "--appname=%(binary_path)s", "--utility-path=tests/bin", "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s", - "--certificate-path=tests/certs", "--autorun", "--close-when-done", - "--console-level=INFO", "--setpref=webgl.force-enabled=true", + "--certificate-path=tests/certs", "--setpref=webgl.force-enabled=true", "--quiet", "--log-raw=%(raw_log_file)s" ], "webapprt_options": [ diff --git a/testing/config/mozharness/windows_config.py b/testing/config/mozharness/windows_config.py index 6d1b2cbdea0c..9811d93b9312 100644 --- a/testing/config/mozharness/windows_config.py +++ b/testing/config/mozharness/windows_config.py @@ -30,9 +30,6 @@ config = { "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s", "--certificate-path=tests/certs", - "--autorun", - "--close-when-done", - "--console-level=INFO", "--quiet", "--log-raw=%(raw_log_file)s" ], diff --git a/testing/mochitest/mach_commands.py b/testing/mochitest/mach_commands.py index e770530dffe2..e75c09acbf05 100644 --- a/testing/mochitest/mach_commands.py +++ b/testing/mochitest/mach_commands.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals -import argparse +from argparse import Namespace import logging import mozpack.path as mozpath import os @@ -24,8 +24,8 @@ from mach.decorators import ( Command, ) +here = os.path.abspath(os.path.dirname(__file__)) -from mozlog import structured ADB_NOT_FOUND = ''' The %s command requires the adb binary to be on your path. @@ -117,18 +117,7 @@ class MochitestRunner(MozbuildObject): 'mochitest') self.bin_dir = os.path.join(self.topobjdir, 'dist', 'bin') - def run_b2g_test( - self, - test_paths=None, - b2g_home=None, - xre_path=None, - total_chunks=None, - this_chunk=None, - no_window=None, - repeat=0, - run_until_failure=False, - chrome=False, - **kwargs): + def run_b2g_test(self, test_paths=None, **kwargs): """Runs a b2g mochitest. test_paths is an enumerable of paths to tests. It can be a relative path @@ -157,13 +146,11 @@ class MochitestRunner(MozbuildObject): ('.py', 'r', imp.PY_SOURCE)) import mochitest - from mochitest_options import B2GOptions - parser = B2GOptions() - options = parser.parse_args([]) + options = Namespace(**kwargs) if test_path: - if chrome: + if options.chrome: test_root_file = mozpath.join( self.mochitest_dir, 'chrome', @@ -180,125 +167,33 @@ class MochitestRunner(MozbuildObject): return 1 options.testPath = test_path - for k, v in kwargs.iteritems(): - setattr(options, k, v) - options.noWindow = no_window - options.totalChunks = total_chunks - options.thisChunk = this_chunk - options.repeat = repeat - options.runUntilFailure = run_until_failure - - options.symbolsPath = os.path.join( - self.distdir, - 'crashreporter-symbols') - - options.consoleLevel = 'INFO' - if conditions.is_b2g_desktop(self): - options.desktop = True - options.app = self.get_binary_path() - if not options.app.endswith('-bin'): - options.app = '%s-bin' % options.app - if not os.path.isfile(options.app): - options.app = options.app[:-len('-bin')] - - return mochitest.run_desktop_mochitests(parser, options) + if options.desktop: + return mochitest.run_desktop_mochitests(options) try: which.which('adb') except which.WhichError: # TODO Find adb automatically if it isn't on the path - print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home)) + print(ADB_NOT_FOUND % ('mochitest-remote', options.b2gPath)) return 1 - options.b2gPath = b2g_home - options.logdir = self.mochitest_dir - options.httpdPath = self.mochitest_dir - options.xrePath = xre_path - options.chrome = chrome - return mochitest.run_remote_mochitests(parser, options) + return mochitest.run_remote_mochitests(options) - def run_desktop_test( - self, - context, - suite=None, - test_paths=None, - debugger=None, - debugger_args=None, - slowscript=False, - screenshot_on_fail=False, - shuffle=False, - closure_behaviour='auto', - rerun_failures=False, - no_autorun=False, - repeat=0, - run_until_failure=False, - slow=False, - chunk_by_dir=0, - chunk_by_runtime=False, - total_chunks=None, - this_chunk=None, - extraPrefs=[], - jsdebugger=False, - debug_on_failure=False, - start_at=None, - end_at=None, - e10s=False, - enable_cpow_warnings=False, - strict_content_sandbox=False, - nested_oop=False, - dmd=False, - dump_output_directory=None, - dump_about_memory_after_test=False, - dump_dmd_after_test=False, - install_extension=None, - quiet=False, - environment=[], - app_override=None, - bisectChunk=None, - runByDir=False, - useTestMediaDevices=False, - timeout=None, - max_timeouts=None, - **kwargs): + def run_desktop_test(self, context, suite=None, test_paths=None, **kwargs): """Runs a mochitest. - test_paths are path to tests. They can be a relative path from the - top source directory, an absolute filename, or a directory containing - test files. - suite is the type of mochitest to run. It can be one of ('plain', 'chrome', 'browser', 'metro', 'a11y', 'jetpack-package', 'jetpack-addon'). - debugger is a program name or path to a binary (presumably a debugger) - to run the test in. e.g. 'gdb' - - debugger_args are the arguments passed to the debugger. - - slowscript is true if the user has requested the SIGSEGV mechanism of - invoking the slow script dialog. - - shuffle is whether test order should be shuffled (defaults to false). - - closure_behaviour denotes whether to keep the browser open after tests - complete. + test_paths are path to tests. They can be a relative path from the + top source directory, an absolute filename, or a directory containing + test files. """ - if rerun_failures and test_paths: - print('Cannot specify both --rerun-failures and a test path.') - return 1 - # Make absolute paths relative before calling os.chdir() below. if test_paths: test_paths = [self._wrap_path_argument( p).relpath() if os.path.isabs(p) else p for p in test_paths] - failure_file_path = os.path.join( - self.statedir, - 'mochitest_failures.json') - - if rerun_failures and not os.path.exists(failure_file_path): - print('No failure file present. Did you run mochitests before?') - return 1 - # runtests.py is ambiguous, so we load the file/module manually. if 'mochitest' not in sys.modules: import imp @@ -321,13 +216,10 @@ class MochitestRunner(MozbuildObject): for handler in remove_handlers: logging.getLogger().removeHandler(handler) - opts = mochitest.MochitestOptions() - options = opts.parse_args([]) + options = Namespace(**kwargs) - options.subsuite = '' flavor = suite - # Need to set the suite options before verifyOptions below. if suite == 'plain': # Don't need additional options for plain. flavor = 'mochitest' @@ -338,6 +230,7 @@ class MochitestRunner(MozbuildObject): flavor = 'browser-chrome' elif suite == 'devtools': options.browserChrome = True + options.subsuite = 'devtools' elif suite == 'jetpack-package': options.jetpackPackage = True elif suite == 'jetpack-addon': @@ -349,70 +242,16 @@ class MochitestRunner(MozbuildObject): options.a11y = True elif suite == 'webapprt-content': options.webapprtContent = True - options.app = self.get_webapp_runtime_path() + if not options.app or options.app == self.get_binary_path(): + options.app = self.get_webapp_runtime_path() elif suite == 'webapprt-chrome': options.webapprtChrome = True - options.app = self.get_webapp_runtime_path() options.browserArgs.append("-test-mode") + if not options.app or options.app == self.get_binary_path(): + options.app = self.get_webapp_runtime_path() else: raise Exception('None or unrecognized mochitest suite type.') - if dmd: - options.dmdPath = self.bin_dir - - options.autorun = not no_autorun - options.closeWhenDone = closure_behaviour != 'open' - options.slowscript = slowscript - options.screenshotOnFail = screenshot_on_fail - options.shuffle = shuffle - options.consoleLevel = 'INFO' - options.repeat = repeat - options.runUntilFailure = run_until_failure - options.runSlower = slow - options.testingModulesDir = os.path.join(self.tests_dir, 'modules') - options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins')) - options.symbolsPath = os.path.join( - self.distdir, - 'crashreporter-symbols') - options.chunkByDir = chunk_by_dir - options.chunkByRuntime = chunk_by_runtime - options.totalChunks = total_chunks - options.thisChunk = this_chunk - options.jsdebugger = jsdebugger - options.debugOnFailure = debug_on_failure - options.startAt = start_at - options.endAt = end_at - options.e10s = e10s - options.enableCPOWWarnings = enable_cpow_warnings - options.strictContentSandbox = strict_content_sandbox - options.nested_oop = nested_oop - options.dumpAboutMemoryAfterTest = dump_about_memory_after_test - options.dumpDMDAfterTest = dump_dmd_after_test - options.dumpOutputDirectory = dump_output_directory - options.quiet = quiet - options.environment = environment - options.extraPrefs = extraPrefs - options.bisectChunk = bisectChunk - options.runByDir = runByDir - options.useTestMediaDevices = useTestMediaDevices - if timeout: - options.timeout = int(timeout) - if max_timeouts: - options.maxTimeouts = int(max_timeouts) - - options.failureFile = failure_file_path - if install_extension is not None: - options.extensionsToInstall = [ - os.path.join( - self.topsrcdir, - install_extension)] - - for k, v in kwargs.iteritems(): - setattr(options, k, v) - - if suite == 'devtools': - options.subsuite = 'devtools' - if test_paths: resolver = self._spawn(TestResolver) @@ -430,368 +269,65 @@ class MochitestRunner(MozbuildObject): manifest = TestManifest() manifest.tests.extend(tests) - if len( - tests) == 1 and closure_behaviour == 'auto' and suite == 'plain': + # XXX why is this such a special case? + if len(tests) == 1 and options.closeWhenDone and suite == 'plain': options.closeWhenDone = False options.manifestFile = manifest - if rerun_failures: - options.testManifest = failure_file_path - - if debugger: - options.debugger = debugger - - if debugger_args: - if options.debugger is None: - print("--debugger-args passed, but no debugger specified.") - return 1 - options.debuggerArgs = debugger_args - - if app_override: - if app_override == "dist": - options.app = self.get_binary_path(where='staged-package') - elif app_override: - options.app = app_override - if options.gmp_path is None: - # Need to fix the location of gmp_fake which might not be - # shipped in the binary - bin_path = self.get_binary_path() - gmp_modules = ( - ('gmp-fake', '1.0'), - ('gmp-clearkey', '0.1'), - ('gmp-fakeopenh264', '1.0') - ) - options.gmp_path = os.pathsep.join( - os.path.join(os.path.dirname(bin_path), *p) - for p in gmp_modules) - - logger_options = { - key: value for key, - value in vars(options).iteritems() if key.startswith('log')} - runner = mochitest.Mochitest(logger_options) - options = opts.verifyOptions(options, runner) - - if options is None: - raise Exception('mochitest option validator failed.') - # We need this to enable colorization of output. self.log_manager.enable_unstructured() - - result = runner.runTests(options) - + result = mochitest.run_test_harness(options) self.log_manager.disable_unstructured() - if runner.message_logger.errors: - result = 1 - runner.message_logger.logger.warning("The following tests failed:") - for error in runner.message_logger.errors: - runner.message_logger.logger.log_raw(error) - - runner.message_logger.finish() - return result - def run_android_test(self, args): + def run_android_test(self, test_path, **kwargs): self.tests_dir = os.path.join(self.topobjdir, '_tests') self.mochitest_dir = os.path.join(self.tests_dir, 'testing', 'mochitest') import imp path = os.path.join(self.mochitest_dir, 'runtestsremote.py') with open(path, 'r') as fh: imp.load_module('runtestsremote', fh, path, - ('.py', 'r', imp.PY_SOURCE)) + ('.py', 'r', imp.PY_SOURCE)) import runtestsremote - sys.exit(runtestsremote.main(args)) + options = Namespace(**kwargs) + if test_path: + options.testPath = test_path -def add_mochitest_general_args(parser): - parser.add_argument( - '--debugger', - '-d', - metavar='DEBUGGER', - help='Debugger binary to run test in. Program name or path.') - - parser.add_argument( - '--debugger-args', - metavar='DEBUGGER_ARGS', - help='Arguments to pass to the debugger.') - - # Bug 933807 introduced JS_DISABLE_SLOW_SCRIPT_SIGNALS to avoid clever - # segfaults induced by the slow-script-detecting logic for Ion/Odin JITted - # code. If we don't pass this, the user will need to periodically type - # "continue" to (safely) resume execution. There are ways to implement - # automatic resuming; see the bug. - parser.add_argument( - '--slowscript', - action='store_true', - help='Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; when not set, recoverable but misleading SIGSEGV instances may occur in Ion/Odin JIT code') - - parser.add_argument( - '--screenshot-on-fail', - action='store_true', - help='Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots.') - - parser.add_argument( - '--shuffle', action='store_true', - help='Shuffle execution order.') - - parser.add_argument( - '--keep-open', - action='store_const', - dest='closure_behaviour', - const='open', - default='auto', - help='Always keep the browser open after tests complete.') - - parser.add_argument( - '--auto-close', - action='store_const', - dest='closure_behaviour', - const='close', - default='auto', - help='Always close the browser after tests complete.') - - parser.add_argument( - '--rerun-failures', - action='store_true', - help='Run only the tests that failed during the last test run.') - - parser.add_argument( - '--no-autorun', - action='store_true', - help='Do not starting running tests automatically.') - - parser.add_argument( - '--repeat', type=int, default=0, - help='Repeat the test the given number of times.') - - parser.add_argument( - "--run-until-failure", - action='store_true', - help='Run tests repeatedly and stops on the first time a test fails. ' - 'Default cap is 30 runs, which can be overwritten ' - 'with the --repeat parameter.') - - parser.add_argument( - '--slow', action='store_true', - help='Delay execution between tests.') - - parser.add_argument( - '--end-at', - type=str, - help='Stop running the test sequence at this test.') - - parser.add_argument( - '--start-at', - type=str, - help='Start running the test sequence at this test.') - - parser.add_argument( - '--chunk-by-dir', - type=int, - help='Group tests together in chunks by this many top directories.') - - parser.add_argument( - '--chunk-by-runtime', - action='store_true', - help="Group tests such that each chunk has roughly the same runtime.") - - parser.add_argument( - '--total-chunks', - type=int, - help='Total number of chunks to split tests into.') - - parser.add_argument( - '--this-chunk', - type=int, - help='If running tests by chunks, the number of the chunk to run.') - - parser.add_argument( - '--debug-on-failure', - action='store_true', - help='Breaks execution and enters the JS debugger on a test failure. ' - 'Should be used together with --jsdebugger.') - - parser.add_argument( - '--setpref', default=[], action='append', - metavar='PREF=VALUE', dest='extraPrefs', - help='defines an extra user preference') - - parser.add_argument( - '--jsdebugger', - action='store_true', - help='Start the browser JS debugger before running the test. Implies --no-autorun.') - - parser.add_argument( - '--e10s', - action='store_true', - help='Run tests with electrolysis preferences and test filtering enabled.') - - parser.add_argument( - '--enable-cpow-warnings', - action='store_true', - help='Run tests with unsafe CPOW usage warnings enabled.') - - parser.add_argument( - '--strict-content-sandbox', - action='store_true', - help='Run tests with a more strict content sandbox (Windows only).') - - parser.add_argument( - '--nested_oop', - action='store_true', - help='Run tests with nested oop preferences and test filtering enabled.') - - parser.add_argument( - '--dmd', action='store_true', - help='Run tests with DMD active.') - - parser.add_argument( - '--dump-about-memory-after-test', - action='store_true', - help='Dump an about:memory log after every test.') - - parser.add_argument( - '--dump-dmd-after-test', action='store_true', - help='Dump a DMD log after every test.') - - parser.add_argument( - '--dump-output-directory', - action='store', - help='Specifies the directory in which to place dumped memory reports.') - - parser.add_argument( - 'test_paths', - default=None, - nargs='*', - metavar='TEST', - help='Test to run. Can be specified as a single file, a ' - 'directory, or omitted. If omitted, the entire test suite is ' - 'executed.') - - parser.add_argument( - '--install-extension', - help='Install given extension before running selected tests. ' - 'Parameter is a path to xpi file.') - - parser.add_argument( - '--quiet', - default=False, - action='store_true', - help='Do not print test log lines unless a failure occurs.') - - parser.add_argument( - '--setenv', - default=[], - action='append', - metavar='NAME=VALUE', - dest='environment', - help="Sets the given variable in the application's environment") - - parser.add_argument( - '--run-by-dir', - default=False, - action='store_true', - dest='runByDir', - help='Run each directory in a single browser instance with a fresh profile.') - - parser.add_argument( - '--bisect-chunk', - type=str, - dest='bisectChunk', - help='Specify the failing test name to find the previous tests that may be causing the failure.') - - parser.add_argument( - '--use-test-media-devices', - default=False, - action='store_true', - dest='useTestMediaDevices', - help='Use test media device drivers for media testing.') - - parser.add_argument( - '--app-override', - default=None, - action='store', - help="Override the default binary used to run tests with the path you provide, e.g. " - " --app-override /usr/bin/firefox . " - "If you have run ./mach package beforehand, you can specify 'dist' to " - "run tests against the distribution bundle's binary.") - - parser.add_argument( - '--timeout', - default=None, - help='The per-test timeout time in seconds (default: 60 seconds)') - - parser.add_argument( - '--max-timeouts', default=None, - help='The maximum number of timeouts permitted before halting testing') - - parser.add_argument( - "--tag", - dest='test_tags', action='append', - help="Filter out tests that don't have the given tag. Can be used " - "multiple times in which case the test must contain at least one " - "of the given tags.") - - parser.add_argument( - "--subsuite", - default=None, - help="Subsuite of tests to run. Unlike tags, subsuites also remove " - "tests from the default set. Only one can be specified at once.") + sys.exit(runtestsremote.run_test_harness(options)) - return parser +# parser -def add_mochitest_b2g_args(parser): - parser.add_argument( - '--busybox', - default=None, - help='Path to busybox binary to install on device') - - parser.add_argument( - '--logdir', default=None, - help='directory to store log files') - - parser.add_argument( - '--profile', default=None, - help='for desktop testing, the path to the \ - gaia profile to use') - - parser.add_argument( - '--gecko-path', default=None, - help='the path to a gecko distribution that should \ - be installed on the emulator prior to test') - - parser.add_argument( - '--no-window', - action='store_true', - default=False, - help='Pass --no-window to the emulator') - - parser.add_argument( - '--sdcard', default="10MB", - help='Define size of sdcard: 1MB, 50MB...etc') - - parser.add_argument( - '--marionette', - default=None, - help='host:port to use when connecting to Marionette') - - return parser +def TestPathArg(func): + test_paths = CommandArgument('test_paths', nargs='*', metavar='TEST', default=None, + help='Test to run. Can be a single test file or a directory of tests to ' + '(run recursively). If omitted, the entire suite is run.') + return test_paths(func) def setup_argument_parser(): - parser = argparse.ArgumentParser() + build_obj = MozbuildObject.from_environment(cwd=here) - general_args = parser.add_argument_group('Mochitest Arguments', - 'Arguments that apply to all versions of mochitest.') - general_args = add_mochitest_general_args(general_args) + build_path = os.path.join(build_obj.topobjdir, 'build') + if build_path not in sys.path: + sys.path.append(build_path) - b2g_args = parser.add_argument_group('B2G Arguments', 'Arguments specific \ - to running mochitest on B2G devices and emulator') - b2g_args = add_mochitest_b2g_args(b2g_args) + mochitest_dir = os.path.join(build_obj.topobjdir, '_tests', 'testing', 'mochitest') - structured.commandline.add_logging_group(parser) - return parser + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + + import imp + path = os.path.join(build_obj.topobjdir, mochitest_dir, 'runtests.py') + with open(path, 'r') as fh: + imp.load_module('mochitest', fh, path, + ('.py', 'r', imp.PY_SOURCE)) + + from mochitest_options import MochitestArgumentParser + + return MochitestArgumentParser() # condition filters @@ -808,6 +344,7 @@ def is_platform_in(*platforms): ' or '.join(platforms)) return is_platform_supported + def verify_host_bin(): # validate MOZ_HOST_BIN environment variables for Android tests MOZ_HOST_BIN = os.environ.get('MOZ_HOST_BIN') @@ -822,13 +359,14 @@ def verify_host_bin(): return 1 return 0 + @CommandProvider class MachCommands(MachCommandBase): def __init__(self, context): MachCommandBase.__init__(self, context) - for attr in ('b2g_home', 'xre_path', 'device_name', 'target_out'): + for attr in ('device_name', 'target_out'): setattr(self, attr, getattr(context, attr, None)) @Command( @@ -837,6 +375,7 @@ class MachCommands(MachCommandBase): conditions=[is_platform_in('firefox', 'mulet', 'b2g', 'b2g_desktop', 'android')], description='Run a plain mochitest (integration test, plain web page).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_plain(self, test_paths, **kwargs): if is_platform_in('firefox', 'mulet')(self): return self.run_mochitest(test_paths, 'plain', **kwargs) @@ -853,13 +392,15 @@ class MachCommands(MachCommandBase): conditions=[is_platform_in('firefox', 'emulator', 'android')], description='Run a chrome mochitest (integration test with some XUL).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_chrome(self, test_paths, **kwargs): + kwargs['chrome'] = True if conditions.is_firefox(self): return self.run_mochitest(test_paths, 'chrome', **kwargs) elif conditions.is_b2g(self) and conditions.is_emulator(self): - return self.run_mochitest_remote(test_paths, chrome=True, **kwargs) + return self.run_mochitest_remote(test_paths, **kwargs) elif conditions.is_android(self): - return self.run_mochitest_android(test_paths, chrome=True, **kwargs) + return self.run_mochitest_android(test_paths, **kwargs) @Command( 'mochitest-browser', @@ -867,6 +408,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run a mochitest with browser chrome (integration test with a standard browser).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_browser(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'browser', **kwargs) @@ -876,6 +418,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run a devtools mochitest with browser chrome (integration test with a standard browser with the devtools frame).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_devtools(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'devtools', **kwargs) @@ -883,6 +426,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run a jetpack package test.', parser=setup_argument_parser) + @TestPathArg def run_mochitest_jetpack_package(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'jetpack-package', **kwargs) @@ -890,6 +434,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run a jetpack addon test.', parser=setup_argument_parser) + @TestPathArg def run_mochitest_jetpack_addon(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'jetpack-addon', **kwargs) @@ -899,6 +444,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run a mochitest with metro browser chrome (tests for Windows touch interface).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_metro(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'metro', **kwargs) @@ -906,6 +452,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run an a11y mochitest (accessibility tests).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_a11y(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'a11y', **kwargs) @@ -915,6 +462,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run a webapprt chrome mochitest (Web App Runtime with the browser chrome).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_webapprt_chrome(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'webapprt-chrome', **kwargs) @@ -924,6 +472,7 @@ class MachCommands(MachCommandBase): conditions=[conditions.is_firefox], description='Run a webapprt content mochitest (Content rendering of the Web App Runtime).', parser=setup_argument_parser) + @TestPathArg def run_mochitest_webapprt_content(self, test_paths, **kwargs): return self.run_mochitest(test_paths, 'webapprt-content', **kwargs) @@ -933,6 +482,7 @@ class MachCommands(MachCommandBase): parser=setup_argument_parser) @CommandArgument('-f', '--flavor', choices=FLAVORS.keys(), help='Only run tests of this flavor.') + @TestPathArg def run_mochitest_general(self, test_paths, flavor=None, test_objects=None, **kwargs): self._preruntest() @@ -1019,12 +569,6 @@ class MachCommands(MachCommandBase): from mozbuild.controller.building import BuildDriver - if self.device_name.startswith('emulator'): - emulator = 'arm' - if 'x86' in self.device_name: - emulator = 'x86' - kwargs['emulator'] = emulator - self._ensure_state_subdir_exists('.') driver = self._spawn(BuildDriver) @@ -1032,8 +576,6 @@ class MachCommands(MachCommandBase): mochitest = self._spawn(MochitestRunner) return mochitest.run_b2g_test( - b2g_home=self.b2g_home, - xre_path=self.xre_path, test_paths=test_paths, **kwargs) @@ -1060,39 +602,28 @@ class MachCommands(MachCommandBase): mochitest = self._spawn(MochitestRunner) return mochitest.run_b2g_test(test_paths=test_paths, **kwargs) - def run_mochitest_android(self, test_paths, chrome=False, **kwargs): + def run_mochitest_android(self, test_paths, **kwargs): host_ret = verify_host_bin() if host_ret != 0: return host_ret - args = [ - '--xre-path=' + os.environ.get('MOZ_HOST_BIN'), - '--dm_trans=adb', - '--deviceIP=', - '--console-level=INFO', - '--app=' + self.substs['ANDROID_PACKAGE_NAME'], - '--log-mach=-', - '--autorun', - '--close-when-done', - '--testing-modules-dir=' + os.path.join(self.topobjdir, '_tests', 'modules'), - ] + test_path = None if test_paths: if len(test_paths) > 1: print('Warning: Only the first test path will be used.') test_path = self._wrap_path_argument(test_paths[0]).relpath() - args.append('--test-path=%s' % test_path) - if chrome: - args.append('--chrome') mochitest = self._spawn(MochitestRunner) - return mochitest.run_android_test(args) + return mochitest.run_android_test(test_path, **kwargs) + @CommandProvider class AndroidCommands(MachCommandBase): @Command('robocop', category='testing', conditions=[conditions.is_android], - description='Run a Robocop test.') + description='Run a Robocop test.', + parser=setup_argument_parser) @CommandArgument( 'test_path', default=None, @@ -1100,37 +631,17 @@ class AndroidCommands(MachCommandBase): metavar='TEST', help='Test to run. Can be specified as a Robocop test name (like "testLoad"), ' 'or omitted. If omitted, the entire test suite is executed.') - def run_robocop(self, test_path): + def run_robocop(self, test_path, **kwargs): host_ret = verify_host_bin() if host_ret != 0: return host_ret - args = [ - '--xre-path=' + os.environ.get('MOZ_HOST_BIN'), - '--dm_trans=adb', - '--deviceIP=', - '--console-level=INFO', - '--app=' + - self.substs['ANDROID_PACKAGE_NAME'], - '--robocop-apk=' + - os.path.join( - self.topobjdir, - 'build', - 'mobile', - 'robocop', - 'robocop-debug.apk'), - '--robocop-ini=' + - os.path.join( - self.topobjdir, - '_tests', - 'testing', - 'mochitest', - 'robocop.ini'), - '--log-mach=-', - ] - - if test_path: - args.append('--test-path=%s' % test_path) + if not kwargs.get('robocopIni'): + kwargs['robocopIni'] = os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'robocop.ini') + if not kwargs.get('robocopApk'): + kwargs['robocopApk'] = os.path.join(self.topobjdir, 'build', 'mobile', + 'robocop', 'robocop-debug.apk') mochitest = self._spawn(MochitestRunner) - return mochitest.run_android_test(args) + return mochitest.run_android_test(test_path, **kwargs) diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py index b2318fcd0dd5..b384e448af1d 100644 --- a/testing/mochitest/mochitest_options.py +++ b/testing/mochitest/mochitest_options.py @@ -2,333 +2,379 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from argparse import ArgumentParser +from abc import ABCMeta, abstractmethod, abstractproperty +from argparse import ArgumentParser, SUPPRESS from urlparse import urlparse -import logging import os import tempfile +from droid import DroidADB, DroidSUT +from mozlog import structured from mozprofile import DEFAULT_PORTS import mozinfo import moznetwork -from automation import Automation here = os.path.abspath(os.path.dirname(__file__)) try: - from mozbuild.base import MozbuildObject + from mozbuild.base import ( + MozbuildObject, + MachCommandConditions as conditions, + ) build_obj = MozbuildObject.from_environment(cwd=here) except ImportError: build_obj = None + conditions = None -__all__ = ["MochitestOptions", "B2GOptions"] VMWARE_RECORDING_HELPER_BASENAME = "vmwarerecordinghelper" -class MochitestOptions(ArgumentParser): - """Usage instructions for runtests.py. - All arguments are optional. - If --chrome is specified, chrome tests will be run instead of web content tests. - If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. - See for details on the logging levels. - """ +class ArgumentContainer(): + __metaclass__ = ABCMeta + + @abstractproperty + def args(self): + pass + + @abstractproperty + def defaults(self): + pass + + @abstractmethod + def validate(self, parser, args, context): + pass + + def get_full_path(self, path, cwd): + """Get an absolute path relative to cwd.""" + return os.path.normpath(os.path.join(cwd, os.path.expanduser(path))) + + +class MochitestArguments(ArgumentContainer): + """General mochitest arguments.""" LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") LEVEL_STRING = ", ".join(LOG_LEVELS) - mochitest_options = [ - [["--close-when-done"], - {"action": "store_true", + + args = [ + [["--keep-open"], + {"action": "store_false", "dest": "closeWhenDone", - "default": False, - "help": "close the application when tests are done running", + "default": True, + "help": "Always keep the browser open after tests complete.", }], [["--appname"], {"dest": "app", "default": None, - "help": "absolute path to application, overriding default", + "help": "Override the default binary used to run tests with the path provided, e.g " + "/usr/bin/firefox. If you have run ./mach package beforehand, you can " + "specify 'dist' to run tests against the distribution bundle's binary.", + "suppress": True, }], [["--utility-path"], {"dest": "utilityPath", "default": build_obj.bindir if build_obj is not None else None, - "help": "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)", + "help": "absolute path to directory containing utility programs " + "(xpcshell, ssltunnel, certutil)", + "suppress": True, }], [["--certificate-path"], {"dest": "certPath", + "default": None, "help": "absolute path to directory containing certificate store to use testing profile", - "default": os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if build_obj is not None else None, + "suppress": True, }], - [["--autorun"], - {"action": "store_true", + [["--no-autorun"], + {"action": "store_false", "dest": "autorun", - "help": "start running tests when the application starts", - "default": False, + "default": True, + "help": "Do not start running tests automatically.", }], [["--timeout"], {"type": int, - "dest": "timeout", - "help": "per-test timeout in seconds", "default": None, + "help": "The per-test timeout in seconds (default: 60 seconds).", + }], + [["--max-timeouts"], + {"type": int, + "dest": "maxTimeouts", + "default": None, + "help": "The maximum number of timeouts permitted before halting testing.", }], [["--total-chunks"], {"type": int, "dest": "totalChunks", - "help": "how many chunks to split the tests up into", + "help": "Total number of chunks to split tests into.", "default": None, }], [["--this-chunk"], {"type": int, "dest": "thisChunk", - "help": "which chunk to run", + "help": "If running tests by chunks, the chunk number to run.", "default": None, }], [["--chunk-by-runtime"], {"action": "store_true", "dest": "chunkByRuntime", - "help": "group tests such that each chunk has roughly the same runtime", + "help": "Group tests such that each chunk has roughly the same runtime.", "default": False, }], [["--chunk-by-dir"], {"type": int, "dest": "chunkByDir", - "help": "group tests together in the same chunk that are in the same top chunkByDir directories", + "help": "Group tests together in the same chunk that are in the same top " + "chunkByDir directories.", "default": 0, }], [["--run-by-dir"], {"action": "store_true", "dest": "runByDir", - "help": "Run each directory in a single browser instance with a fresh profile", + "help": "Run each directory in a single browser instance with a fresh profile.", "default": False, }], [["--shuffle"], - {"dest": "shuffle", - "action": "store_true", - "help": "randomize test order", + {"action": "store_true", + "help": "Shuffle execution order of tests.", "default": False, }], [["--console-level"], {"dest": "consoleLevel", "choices": LOG_LEVELS, - "metavar": "LEVEL", - "help": "one of %s to determine the level of console " - "logging" % LEVEL_STRING, - "default": None, + "default": "INFO", + "help": "One of %s to determine the level of console logging." % LEVEL_STRING, + "suppress": True, }], [["--chrome"], {"action": "store_true", - "dest": "chrome", - "help": "run chrome Mochitests", "default": False, + "help": "Run chrome mochitests.", + "suppress": True, }], [["--ipcplugins"], {"action": "store_true", "dest": "ipcplugins", - "help": "run ipcplugins Mochitests", + "help": "Run ipcplugins mochitests.", "default": False, + "suppress": True, }], [["--test-path"], {"dest": "testPath", - "help": "start in the given directory's tests", "default": "", + "help": "Run the given test or recursively run the given directory of tests.", + # if running from mach, a test_paths arg is exposed instead + "suppress": build_obj is not None, }], [["--bisect-chunk"], {"dest": "bisectChunk", - "help": "Specify the failing test name to find the previous tests that may be causing the failure.", "default": None, + "help": "Specify the failing test name to find the previous tests that may be " + "causing the failure.", }], [["--start-at"], {"dest": "startAt", - "help": "skip over tests until reaching the given test", "default": "", + "help": "Start running the test sequence at this test.", }], [["--end-at"], {"dest": "endAt", - "help": "don't run any tests after the given one", "default": "", + "help": "Stop running the test sequence at this test.", }], [["--browser-chrome"], {"action": "store_true", "dest": "browserChrome", - "help": "run browser chrome Mochitests", "default": False, + "help": "run browser chrome Mochitests", + "suppress": True, }], [["--subsuite"], - {"dest": "subsuite", - "help": "subsuite of tests to run", - "default": None, + {"default": None, + "help": "Subsuite of tests to run. Unlike tags, subsuites also remove tests from " + "the default set. Only one can be specified at once.", }], [["--jetpack-package"], {"action": "store_true", "dest": "jetpackPackage", - "help": "run jetpack package tests", + "help": "Run jetpack package tests.", "default": False, + "suppress": True, }], [["--jetpack-addon"], {"action": "store_true", "dest": "jetpackAddon", - "help": "run jetpack addon tests", + "help": "Run jetpack addon tests.", "default": False, + "suppress": True, }], [["--webapprt-content"], {"action": "store_true", "dest": "webapprtContent", - "help": "run WebappRT content tests", + "help": "Run WebappRT content tests.", "default": False, + "suppress": True, }], [["--webapprt-chrome"], {"action": "store_true", "dest": "webapprtChrome", - "help": "run WebappRT chrome tests", + "help": "Run WebappRT chrome tests.", "default": False, + "suppress": True, }], [["--a11y"], {"action": "store_true", - "dest": "a11y", - "help": "run accessibility Mochitests", + "help": "Run accessibility Mochitests.", "default": False, + "suppress": True, }], [["--setenv"], {"action": "append", "dest": "environment", "metavar": "NAME=VALUE", - "help": "sets the given variable in the application's " - "environment", "default": [], + "help": "Sets the given variable in the application's environment.", }], [["--exclude-extension"], {"action": "append", "dest": "extensionsToExclude", - "help": "excludes the given extension from being installed " - "in the test profile", "default": [], + "help": "Excludes the given extension from being installed in the test profile.", + "suppress": True, }], [["--browser-arg"], {"action": "append", "dest": "browserArgs", - "metavar": "ARG", - "help": "provides an argument to the test application", "default": [], + "help": "Provides an argument to the test application (e.g Firefox).", + "suppress": True, }], [["--leak-threshold"], {"type": int, "dest": "defaultLeakThreshold", - "metavar": "THRESHOLD", - "help": "fail if the number of bytes leaked in default " - "processes through refcounted objects (or bytes " - "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " - "is greater than the given number", "default": 0, + "help": "Fail if the number of bytes leaked in default processes through " + "refcounted objects (or bytes in classes with MOZ_COUNT_CTOR and " + "MOZ_COUNT_DTOR) is greater than the given number.", + "suppress": True, }], [["--fatal-assertions"], {"action": "store_true", "dest": "fatalAssertions", - "help": "abort testing whenever an assertion is hit " - "(requires a debug build to be effective)", "default": False, + "help": "Abort testing whenever an assertion is hit (requires a debug build to " + "be effective).", + "suppress": True, }], [["--extra-profile-file"], {"action": "append", "dest": "extraProfileFiles", - "help": "copy specified files/dirs to testing profile", "default": [], + "help": "Copy specified files/dirs to testing profile. Can be specified more " + "than once.", + "suppress": True, }], [["--install-extension"], {"action": "append", "dest": "extensionsToInstall", - "help": "install the specified extension in the testing profile." - "The extension file's name should be .xpi where is" - "the extension's id as indicated in its install.rdf." - "An optional path can be specified too.", "default": [], + "help": "Install the specified extension in the testing profile. Can be a path " + "to a .xpi file.", }], [["--profile-path"], {"dest": "profilePath", - "help": "Directory where the profile will be stored." - "This directory will be deleted after the tests are finished", "default": None, + "help": "Directory where the profile will be stored. This directory will be " + "deleted after the tests are finished.", + "suppress": True, }], [["--testing-modules-dir"], {"dest": "testingModulesDir", - "help": "Directory where testing-only JS modules are located.", "default": None, + "help": "Directory where testing-only JS modules are located.", + "suppress": True, }], [["--use-vmware-recording"], {"action": "store_true", "dest": "vmwareRecording", - "help": "enables recording while the application is running " - "inside a VMware Workstation 7.0 or later VM", "default": False, + "help": "Enables recording while the application is running inside a VMware " + "Workstation 7.0 or later VM.", + "suppress": True, }], [["--repeat"], {"type": int, - "dest": "repeat", - "metavar": "REPEAT", - "help": "repeats the test or set of tests the given number of times, ie: repeat: 1 will run the test twice.", "default": 0, + "help": "Repeat the tests the given number of times.", }], [["--run-until-failure"], {"action": "store_true", "dest": "runUntilFailure", - "help": "Run tests repeatedly and stops on the first time a test fails. " - "Default cap is 30 runs, which can be overwritten with the --repeat parameter.", "default": False, + "help": "Run tests repeatedly but stop the first time a test fails. Default cap " + "is 30 runs, which can be overridden with the --repeat parameter.", }], [["--manifest"], {"dest": "manifestFile", - "help": ".ini format of tests to run.", "default": None, + "help": "Path to a manifestparser (.ini formatted) manifest of tests to run.", + "suppress": True, }], [["--testrun-manifest-file"], {"dest": "testRunManifestFile", - "help": "Overrides the default filename of the tests.json manifest file that is created from the manifest and used by the test runners to run the tests. Only useful when running multiple test runs simulatenously on the same machine.", "default": 'tests.json', + "help": "Overrides the default filename of the tests.json manifest file that is " + "generated by the harness and used by SimpleTest. Only useful when running " + "multiple test runs simulatenously on the same machine.", + "suppress": True, }], [["--failure-file"], {"dest": "failureFile", - "help": "Filename of the output file where we can store a .json list of failures to be run in the future with --run-only-tests.", "default": None, + "help": "Filename of the output file where we can store a .json list of failures " + "to be run in the future with --run-only-tests.", + "suppress": True, }], [["--run-slower"], {"action": "store_true", "dest": "runSlower", - "help": "Delay execution between test files.", "default": False, + "help": "Delay execution between tests.", }], [["--metro-immersive"], {"action": "store_true", "dest": "immersiveMode", - "help": "launches tests in immersive browser", "default": False, + "help": "Launches tests in an immersive browser.", + "suppress": True, }], [["--httpd-path"], {"dest": "httpdPath", "default": None, - "help": "path to the httpd.js file", + "help": "Path to the httpd.js file.", + "suppress": True, }], [["--setpref"], {"action": "append", + "metavar": "PREF=VALUE", "default": [], "dest": "extraPrefs", - "metavar": "PREF=VALUE", - "help": "defines an extra user preference", + "help": "Defines an extra user preference.", }], [["--jsdebugger"], {"action": "store_true", "default": False, - "dest": "jsdebugger", - "help": "open the browser debugger", + "help": "Start the browser JS debugger before running the test. Implies --no-autorun.", }], [["--debug-on-failure"], {"action": "store_true", "default": False, "dest": "debugOnFailure", - "help": "breaks execution and enters the JS debugger on a test failure. Should be used together with --jsdebugger." + "help": "Breaks execution and enters the JS debugger on a test failure. Should " + "be used together with --jsdebugger." }], [["--e10s"], {"action": "store_true", "default": False, - "dest": "e10s", "help": "Run tests with electrolysis preferences and test filtering enabled.", }], [["--strict-content-sandbox"], @@ -336,17 +382,23 @@ class MochitestOptions(ArgumentParser): "default": False, "dest": "strictContentSandbox", "help": "Run tests with a more strict content sandbox (Windows only).", + "suppress": not mozinfo.isWin, }], [["--nested_oop"], {"action": "store_true", "default": False, - "dest": "nested_oop", "help": "Run tests with nested_oop preferences and test filtering enabled.", }], + [["--dmd"], + {"action": "store_true", + "default": False, + "help": "Run tests with DMD active.", + }], [["--dmd-path"], {"default": None, "dest": "dmdPath", "help": "Specifies the path to the directory containing the shared library for DMD.", + "suppress": True, }], [["--dump-output-directory"], {"default": None, @@ -357,40 +409,41 @@ class MochitestOptions(ArgumentParser): {"action": "store_true", "default": False, "dest": "dumpAboutMemoryAfterTest", - "help": "Produce an about:memory dump after each test in the directory specified " - "by --dump-output-directory." + "help": "Dump an about:memory log after each test in the directory specified " + "by --dump-output-directory.", }], [["--dump-dmd-after-test"], {"action": "store_true", "default": False, "dest": "dumpDMDAfterTest", - "help": "Produce a DMD dump after each test in the directory specified " - "by --dump-output-directory." + "help": "Dump a DMD log after each test in the directory specified " + "by --dump-output-directory.", }], [["--slowscript"], {"action": "store_true", "default": False, - "dest": "slowscript", "help": "Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; " - "when not set, recoverable but misleading SIGSEGV instances " - "may occur in Ion/Odin JIT code." + "when not set, recoverable but misleading SIGSEGV instances " + "may occur in Ion/Odin JIT code.", }], [["--screenshot-on-fail"], {"action": "store_true", "default": False, "dest": "screenshotOnFail", - "help": "Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots." + "help": "Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory " + "for storing the screenshots." }], [["--quiet"], {"action": "store_true", - "default": False, "dest": "quiet", - "help": "Do not print test log lines unless a failure occurs." + "default": False, + "help": "Do not print test log lines unless a failure occurs.", }], [["--pidfile"], {"dest": "pidFile", - "help": "name of the pidfile to generate", "default": "", + "help": "Name of the pidfile to generate.", + "suppress": True, }], [["--use-test-media-devices"], {"action": "store_true", @@ -400,93 +453,121 @@ class MochitestOptions(ArgumentParser): }], [["--gmp-path"], {"default": None, - "dest": "gmp_path", "help": "Path to fake GMP plugin. Will be deduced from the binary if not passed.", + "suppress": True, }], [["--xre-path"], {"dest": "xrePath", "default": None, # individual scripts will set a sane default - "help": "absolute path to directory containing XRE (probably xulrunner)", + "help": "Absolute path to directory containing XRE (probably xulrunner).", + "suppress": True, }], [["--symbols-path"], {"dest": "symbolsPath", "default": None, - "help": "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols", + "help": "Absolute path to directory containing breakpad symbols, or the URL of a " + "zip file containing symbols", + "suppress": True, }], [["--debugger"], - {"dest": "debugger", - "help": "use the given debugger to launch the application", + {"default": None, + "help": "Debugger binary to run tests in. Program name or path.", }], [["--debugger-args"], {"dest": "debuggerArgs", - "help": "pass the given args to the debugger _before_ the application on the command line", + "default": None, + "help": "Arguments to pass to the debugger.", }], [["--debugger-interactive"], {"action": "store_true", "dest": "debuggerInteractive", - "help": "prevents the test harness from redirecting stdout and stderr for interactive debuggers", - }], - [["--max-timeouts"], - {"type": int, - "dest": "maxTimeouts", - "help": "maximum number of timeouts permitted before halting testing", "default": None, + "help": "Prevents the test harness from redirecting stdout and stderr for " + "interactive debuggers.", + "suppress": True, }], [["--tag"], {"action": "append", "dest": "test_tags", "default": None, - "help": "filter out tests that don't have the given tag. Can be " - "used multiple times in which case the test must contain " - "at least one of the given tags.", + "help": "Filter out tests that don't have the given tag. Can be used multiple " + "times in which case the test must contain at least one of the given tags.", }], [["--enable-cpow-warnings"], {"action": "store_true", "dest": "enableCPOWWarnings", - "help": "enable logging of unsafe CPOW usage, which is disabled by default for tests", + "help": "Enable logging of unsafe CPOW usage, which is disabled by default for tests", + "suppress": True, }], ] - def __init__(self, **kwargs): - ArgumentParser.__init__(self, usage=self.__doc__, **kwargs) - for option, value in self.mochitest_options: - # Allocate new lists so references to original don't get mutated. - # allowing multiple uses within a single process. - if "default" in value and isinstance(value["default"], list): - value["default"] = [] - self.add_argument(*option, **value) + defaults = { + # Bug 1065098 - The geckomediaplugin process fails to produce a leak + # log for some reason. + 'ignoreMissingLeaks': ["geckomediaplugin"], - def verifyOptions(self, options, mochitest): - """ verify correct options and cleanup paths """ + # Set server information on the args object + 'webServer': '127.0.0.1', + 'httpPort': DEFAULT_PORTS['http'], + 'sslPort': DEFAULT_PORTS['https'], + 'webSocketPort': '9988', + # The default websocket port is incorrect in mozprofile; it is + # set to the SSL proxy setting. See: + # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517 + # args.webSocketPort = DEFAULT_PORTS['ws'] + } + + def validate(self, parser, options, context): + """Validate generic options.""" # for test manifest parsing. mozinfo.update({"strictContentSandbox": options.strictContentSandbox}) # for test manifest parsing. mozinfo.update({"nested_oop": options.nested_oop}) - if options.app is None: - if build_obj is not None: - options.app = build_obj.get_binary_path() - else: - self.error( - "could not find the application path, --appname must be specified") + # b2g and android don't use 'app' the same way, so skip validation + if parser.app not in ('b2g', 'android'): + if options.app is None: + if build_obj: + options.app = build_obj.get_binary_path() + else: + parser.error( + "could not find the application path, --appname must be specified") + elif options.app == "dist" and build_obj: + options.app = build_obj.get_binary_path(where='staged-package') + + options.app = self.get_full_path(options.app, parser.oldcwd) + if not os.path.exists(options.app): + parser.error("Error: Path {} doesn't exist. Are you executing " + "$objdir/_tests/testing/mochitest/runtests.py?".format( + options.app)) + + if options.gmp_path is None and options.app and build_obj: + # Need to fix the location of gmp_fake which might not be shipped in the binary + gmp_modules = ( + ('gmp-fake', '1.0'), + ('gmp-clearkey', '0.1'), + ('gmp-fakeopenh264', '1.0') + ) + options.gmp_path = os.pathsep.join( + os.path.join(build_obj.bindir, *p) for p in gmp_modules) if options.totalChunks is not None and options.thisChunk is None: - self.error( + parser.error( "thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: - self.error("thisChunk must be between 1 and totalChunks") + parser.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: - self.error( + parser.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided - if options.app != self.get_default('app'): + if options.app != parser.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( @@ -497,58 +578,49 @@ class MochitestOptions(ArgumentParser): # otherwise default to dist/bin options.xrePath = build_obj.bindir else: - self.error( + parser.error( "could not find xre directory, --xre-path must be specified") # allow relative paths - options.xrePath = mochitest.getFullPath(options.xrePath) + options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd) if options.profilePath: - options.profilePath = mochitest.getFullPath(options.profilePath) - options.app = mochitest.getFullPath(options.app) - if options.dmdPath is not None: - options.dmdPath = mochitest.getFullPath(options.dmdPath) + options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) - if not os.path.exists(options.app): - msg = """\ - Error: Path %(app)s doesn't exist. - Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" - self.error(msg % {"app": options.app}) - return None + if options.dmdPath: + options.dmdPath = self.get_full_path(options.dmdPath, parser.oldcwd) + + if options.dmd and not options.dmdPath: + if build_obj: + options.dmdPath = build_obj.bin_dir + else: + parser.error( + "could not find dmd libraries, specify them with --dmd-path") if options.utilityPath: - options.utilityPath = mochitest.getFullPath(options.utilityPath) + options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) if options.certPath: - options.certPath = mochitest.getFullPath(options.certPath) + options.certPath = self.get_full_path(options.certPath, parser.oldcwd) + elif build_obj: + options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') - if options.symbolsPath and len( - urlparse( - options.symbolsPath).scheme) < 2: - options.symbolsPath = mochitest.getFullPath(options.symbolsPath) - - # Set server information on the options object - options.webServer = '127.0.0.1' - options.httpPort = DEFAULT_PORTS['http'] - options.sslPort = DEFAULT_PORTS['https'] - # options.webSocketPort = DEFAULT_PORTS['ws'] - # <- http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l30 - options.webSocketPort = str(9988) - # The default websocket port is incorrect in mozprofile; it is - # set to the SSL proxy setting. See: - # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517 + if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: + options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd) + elif not options.symbolsPath and build_obj: + options.symbolsPath = os.path.join(build_obj.distdir, 'crashreporter-symbols') if options.vmwareRecording: if not mozinfo.isWin: - self.error( + parser.error( "use-vmware-recording is only supported on Windows.") - mochitest.vmwareHelperPath = os.path.join( + options.vmwareHelperPath = os.path.join( options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") - if not os.path.exists(mochitest.vmwareHelperPath): - self.error("%s not found, cannot automate VMware recording." % - mochitest.vmwareHelperPath) + if not os.path.exists(options.vmwareHelperPath): + parser.error("%s not found, cannot automate VMware recording." % + options.vmwareHelperPath) if options.webapprtContent and options.webapprtChrome: - self.error( + parser.error( "Only one of --webapprt-content and --webapprt-chrome may be given.") if options.jsdebugger: @@ -560,20 +632,31 @@ class MochitestOptions(ArgumentParser): options.autorun = False if options.debugOnFailure and not options.jsdebugger: - self.error( - "--debug-on-failure should be used together with --jsdebugger.") + parser.error( + "--debug-on-failure requires --jsdebugger.") + + if options.debuggerArgs and not options.debugger: + parser.error( + "--debugger-args requires --debugger.") - # Try to guess the testing modules directory. - # This somewhat grotesque hack allows the buildbot machines to find the - # modules directory without having to configure the buildbot hosts. This - # code should never be executed in local runs because the build system - # should always set the flag that populates this variable. If buildbot ever - # passes this argument, this code can be deleted. if options.testingModulesDir is None: - possible = os.path.join(here, os.path.pardir, 'modules') + if build_obj: + options.testingModulesDir = os.path.join( + build_obj.topobjdir, '_tests', 'modules') + else: + # Try to guess the testing modules directory. + # This somewhat grotesque hack allows the buildbot machines to find the + # modules directory without having to configure the buildbot hosts. This + # code should never be executed in local runs because the build system + # should always set the flag that populates this variable. If buildbot ever + # passes this argument, this code can be deleted. + possible = os.path.join(here, os.path.pardir, 'modules') - if os.path.isdir(possible): - options.testingModulesDir = possible + if os.path.isdir(possible): + options.testingModulesDir = possible + + if build_obj: + options.extraProfileFiles.append(os.path.join(build_obj.distdir, 'plugins')) # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. @@ -586,8 +669,8 @@ class MochitestOptions(ArgumentParser): options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): - self.error('--testing-modules-dir not a directory: %s' % - options.testingModulesDir) + parser.error('--testing-modules-dir not a directory: %s' % + options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', @@ -597,12 +680,12 @@ class MochitestOptions(ArgumentParser): if options.immersiveMode: if not mozinfo.isWin: - self.error("immersive is only supported on Windows 8 and up.") - mochitest.immersiveHelperPath = os.path.join( + parser.error("immersive is only supported on Windows 8 and up.") + options.immersiveHelperPath = os.path.join( options.utilityPath, "metrotestharness.exe") - if not os.path.exists(mochitest.immersiveHelperPath): - self.error("%s not found, cannot launch immersive tests." % - mochitest.immersiveHelperPath) + if not os.path.exists(options.immersiveHelperPath): + parser.error("%s not found, cannot launch immersive tests." % + options.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: @@ -613,16 +696,16 @@ class MochitestOptions(ArgumentParser): if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): - self.error('--dump-output-directory not a directory: %s' % - options.dumpOutputDirectory) + parser.error('--dump-output-directory not a directory: %s' % + options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: - self.error( + parser.error( '--use-test-media-devices is only supported on Linux currently') for f in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']: if not os.path.isfile(f): - self.error( + parser.error( 'Missing binary %s required for ' '--use-test-media-devices' % f) @@ -638,10 +721,6 @@ class MochitestOptions(ArgumentParser): "geckomediaplugin": 20000, } - # Bug 1065098 - The geckomediaplugin process fails to produce a leak - # log for some reason. - options.ignoreMissingLeaks = ["geckomediaplugin"] - # Bug 1091917 - We exit early in tab processes on Windows, so we don't # get leak logs yet. if mozinfo.isWin: @@ -654,151 +733,191 @@ class MochitestOptions(ArgumentParser): return options -class B2GOptions(MochitestOptions): - b2g_options = [ +class B2GArguments(ArgumentContainer): + """B2G specific arguments.""" + + args = [ [["--b2gpath"], {"dest": "b2gPath", - "help": "path to B2G repo or qemu dir", "default": None, + "help": "Path to B2G repo or QEMU directory.", + "suppress": True, }], [["--desktop"], {"action": "store_true", - "dest": "desktop", - "help": "Run the tests on a B2G desktop build", "default": False, + "help": "Run the tests on a B2G desktop build.", + "suppress": True, }], [["--marionette"], - {"dest": "marionette", + {"default": None, "help": "host:port to use when connecting to Marionette", - "default": None, }], [["--emulator"], - {"dest": "emulator", - "help": "Architecture of emulator to use: x86 or arm", - "default": None, + {"default": None, + "help": "Architecture of emulator to use, x86 or arm", + "suppress": True, }], [["--wifi"], - {"dest": "wifi", + {"default": False, "help": "Devine wifi configuration for on device mochitest", - "default": False, + "suppress": True, }], [["--sdcard"], - {"dest": "sdcard", + {"default": "10MB", "help": "Define size of sdcard: 1MB, 50MB...etc", - "default": "10MB", }], [["--no-window"], {"action": "store_true", "dest": "noWindow", - "help": "Pass --no-window to the emulator", "default": False, + "help": "Pass --no-window to the emulator", }], [["--adbpath"], {"dest": "adbPath", - "help": "path to adb", "default": "adb", + "help": "Path to adb binary.", + "suppress": True, }], [["--deviceIP"], {"dest": "deviceIP", - "help": "ip address of remote device to test", "default": None, + "help": "IP address of remote device to test.", + "suppress": True, }], [["--devicePort"], - {"dest": "devicePort", + {"default": 20701, "help": "port of remote device to test", - "default": 20701, + "suppress": True, }], [["--remote-logfile"], {"dest": "remoteLogFile", - "help": "Name of log file on the device relative to the device root. \ - PLEASE ONLY USE A FILENAME.", "default": None, + "help": "Name of log file on the device relative to the device root. " + "PLEASE ONLY USE A FILENAME.", + "suppress": True, }], [["--remote-webserver"], {"dest": "remoteWebServer", - "help": "ip address where the remote web server is hosted at", "default": None, + "help": "IP address where the remote web server is hosted.", + "suppress": True, }], [["--http-port"], {"dest": "httpPort", - "help": "ip address where the remote web server is hosted at", "default": DEFAULT_PORTS['http'], + "help": "Port used for http on the remote web server.", + "suppress": True, }], [["--ssl-port"], {"dest": "sslPort", - "help": "ip address where the remote web server is hosted at", "default": DEFAULT_PORTS['https'], + "help": "Port used for https on the remote web server.", + "suppress": True, }], [["--gecko-path"], {"dest": "geckoPath", - "help": "the path to a gecko distribution that should \ - be installed on the emulator prior to test", "default": None, + "help": "The path to a gecko distribution that should be installed on the emulator " + "prior to test.", + "suppress": True, }], [["--profile"], {"dest": "profile", - "help": "for desktop testing, the path to the \ - gaia profile to use", "default": None, + "help": "For desktop testing, the path to the gaia profile to use.", }], [["--logdir"], {"dest": "logdir", - "help": "directory to store log files", "default": None, + "help": "Directory to store log files.", }], [['--busybox'], {"dest": 'busybox', - "help": "Path to busybox binary to install on device", "default": None, + "help": "Path to busybox binary to install on device.", }], [['--profile-data-dir'], {"dest": 'profile_data_dir', - "help": "Path to a directory containing preference and other \ - data to be installed into the profile", "default": os.path.join(here, 'profile_data'), + "help": "Path to a directory containing preference and other data to be installed " + "into the profile.", + "suppress": True, }], ] - def __init__(self): - MochitestOptions.__init__(self) - - for option in self.b2g_options: - self.add_argument(*option[0], **option[1]) - - defaults = {} - defaults["logFile"] = "mochitest.log" - defaults["autorun"] = True - defaults["closeWhenDone"] = True - defaults["extensionsToExclude"] = ["specialpowers"] + defaults = { + 'logFile': 'mochitest.log', + 'extensionsToExclude': ['specialpowers'], # See dependencies of bug 1038943. - defaults["defaultLeakThreshold"] = 5536 - self.set_defaults(**defaults) + 'defaultLeakThreshold': 5536, + } + + def validate(self, parser, options, context): + """Validate b2g options.""" + + if options.desktop and not options.app: + if not (build_obj and conditions.is_b2g_desktop(build_obj)): + parser.error( + "--desktop specified, but no b2g desktop build detected! Either " + "build for b2g desktop, or point --appname to a b2g desktop binary.") + elif build_obj and conditions.is_b2g_desktop(build_obj): + options.desktop = True + if not options.app: + options.app = build_obj.get_binary_path() + if not options.app.endswith('-bin'): + options.app = '%s-bin' % options.app + if not os.path.isfile(options.app): + options.app = options.app[:-len('-bin')] - def verifyRemoteOptions(self, options): if options.remoteWebServer is None: if os.name != "nt": options.remoteWebServer = moznetwork.get_ip() else: - self.error( + parser.error( "You must specify a --remote-webserver=") options.webServer = options.remoteWebServer + if not options.b2gPath and hasattr(context, 'b2g_home'): + options.b2gPath = context.b2g_home + + if hasattr(context, 'device_name') and not options.emulator: + if context.device_name.startswith('emulator'): + options.emulator = 'x86' if 'x86' in context.device_name else 'arm' + if options.geckoPath and not options.emulator: - self.error( + parser.error( "You must specify --emulator if you specify --gecko-path") if options.logdir and not options.emulator: - self.error("You must specify --emulator if you specify --logdir") + parser.error("You must specify --emulator if you specify --logdir") + elif not options.logdir and options.emulator and build_obj: + options.logdir = os.path.join( + build_obj.topobjdir, '_tests', 'testing', 'mochitest') + + if hasattr(context, 'xre_path'): + options.xrePath = context.xre_path if not os.path.isdir(options.xrePath): - self.error("--xre-path '%s' is not a directory" % options.xrePath) + parser.error("--xre-path '%s' is not a directory" % options.xrePath) + xpcshell = os.path.join(options.xrePath, 'xpcshell') if not os.access(xpcshell, os.F_OK): - self.error('xpcshell not found at %s' % xpcshell) + parser.error('xpcshell not found at %s' % xpcshell) + if self.elf_arm(xpcshell): - self.error('--xre-path points to an ARM version of xpcshell; it ' - 'should instead point to a version that can run on ' - 'your desktop') + parser.error('--xre-path points to an ARM version of xpcshell; it ' + 'should instead point to a version that can run on ' + 'your desktop') + + if not options.httpdPath and build_obj: + options.httpdPath = os.path.join( + build_obj.topobjdir, '_tests', 'testing', 'mochitest') + + # Bug 1071866 - B2G Mochitests do not always produce a leak log. + options.ignoreMissingLeaks.append("default") + # Bug 1070068 - Leak logging does not work for tab processes on B2G. + options.ignoreMissingLeaks.append("tab") if options.pidFile != "": f = open(options.pidFile, 'w') @@ -807,34 +926,15 @@ class B2GOptions(MochitestOptions): return options - def verifyOptions(self, options, mochitest): - # since we are reusing verifyOptions, it will exit if App is not found - temp = options.app - options.app = __file__ - tempPort = options.httpPort - tempSSL = options.sslPort - tempIP = options.webServer - options = MochitestOptions.verifyOptions(self, options, mochitest) - options.webServer = tempIP - options.app = temp - options.sslPort = tempSSL - options.httpPort = tempPort - - # Bug 1071866 - B2G Mochitests do not always produce a leak log. - options.ignoreMissingLeaks.append("default") - - # Bug 1070068 - Leak logging does not work for tab processes on B2G. - options.ignoreMissingLeaks.append("tab") - - return options - def elf_arm(self, filename): data = open(filename, 'rb').read(20) return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM -class RemoteOptions(MochitestOptions): - remote_options = [ +class AndroidArguments(ArgumentContainer): + """Android specific arguments.""" + + args = [ [["--remote-app-path"], {"dest": "remoteAppPath", "help": "Path to remote executable relative to device root using \ @@ -853,10 +953,10 @@ class RemoteOptions(MochitestOptions): "default": None, }], [["--dm_trans"], - {"dest": "dm_trans", - "default": "sut", - "help": "the transport to use to communicate with device: \ - [adb|sut]; default=sut", + {"choices": ["adb", "sut"], + "default": "adb", + "help": "The transport to use for communication with the device [default: adb].", + "suppress": True, }], [["--devicePort"], {"dest": "devicePort", @@ -869,6 +969,7 @@ class RemoteOptions(MochitestOptions): "default": "fennec", "help": "The executable's name of remote product to test - either \ fennec or firefox, defaults to fennec", + "suppress": True, }], [["--remote-logfile"], {"dest": "remoteLogFile", @@ -885,35 +986,24 @@ class RemoteOptions(MochitestOptions): {"dest": "httpPort", "default": DEFAULT_PORTS['http'], "help": "http port of the remote web server", + "suppress": True, }], [["--ssl-port"], {"dest": "sslPort", "default": DEFAULT_PORTS['https'], "help": "ssl port of the remote web server", + "suppress": True, }], [["--robocop-ini"], {"dest": "robocopIni", "default": "", "help": "name of the .ini file containing the list of tests to run", }], - [["--robocop"], - {"dest": "robocop", - "default": "", - "help": "name of the .ini file containing the list of tests to run. \ - [DEPRECATED- please use --robocop-ini", - }], [["--robocop-apk"], {"dest": "robocopApk", "default": "", "help": "name of the Robocop APK to use for ADB test running", }], - [["--robocop-path"], - {"dest": "robocopPath", - "default": "", - "help": "Path to the folder where robocop.apk is located at. \ - Primarily used for ADB test running. \ - [DEPRECATED- please use --robocop-apk]", - }], [["--robocop-ids"], {"dest": "robocopIds", "default": "", @@ -925,114 +1015,114 @@ class RemoteOptions(MochitestOptions): "default": None, "help": "remote directory to use as test root \ (eg. /mnt/sdcard/tests or /data/local/tests)", + "suppress": True, }], ] - def __init__(self, automation, **kwargs): - self._automation = automation or Automation() - MochitestOptions.__init__(self) + defaults = { + 'dm': None, + 'logFile': 'mochitest.log', + 'utilityPath': None, + } - for option in self.remote_options: - self.add_argument(*option[0], **option[1]) + def validate(self, parser, options, context): + """Validate android options.""" - defaults = {} - defaults["logFile"] = "mochitest.log" - defaults["autorun"] = True - defaults["closeWhenDone"] = True - defaults["utilityPath"] = None - self.set_defaults(**defaults) + if build_obj: + options.log_mach = '-' - def verifyRemoteOptions(self, options, automation): - options_logger = logging.getLogger('MochitestRemote') + if options.dm_trans == "adb": + if options.deviceIP: + options.dm = DroidADB( + options.deviceIP, + options.devicePort, + deviceRoot=options.remoteTestRoot) + elif options.deviceSerial: + options.dm = DroidADB( + None, + None, + deviceSerial=options.deviceSerial, + deviceRoot=options.remoteTestRoot) + else: + options.dm = DroidADB(deviceRoot=options.remoteTestRoot) + elif options.dm_trans == 'sut': + if options.deviceIP is None: + parser.error( + "If --dm_trans = sut, you must provide a device IP") + + options.dm = DroidSUT( + options.deviceIP, + options.devicePort, + deviceRoot=options.remoteTestRoot) if not options.remoteTestRoot: - options.remoteTestRoot = automation._devicemanager.deviceRoot + options.remoteTestRoot = options.dm.deviceRoot if options.remoteWebServer is None: if os.name != "nt": options.remoteWebServer = moznetwork.get_ip() else: - options_logger.error( + parser.error( "you must specify a --remote-webserver=") - return None options.webServer = options.remoteWebServer - if (options.dm_trans == 'sut' and options.deviceIP is None): - options_logger.error( - "If --dm_trans = sut, you must provide a device IP") - return None - - if (options.remoteLogFile is None): + if options.remoteLogFile is None: options.remoteLogFile = options.remoteTestRoot + \ '/logs/mochitest.log' - if (options.remoteLogFile.count('/') < 1): + if options.remoteLogFile.count('/') < 1: options.remoteLogFile = options.remoteTestRoot + \ '/' + options.remoteLogFile - if (options.remoteAppPath and options.app): - options_logger.error( + if options.remoteAppPath and options.app: + parser.error( "You cannot specify both the remoteAppPath and the app setting") - return None - elif (options.remoteAppPath): + elif options.remoteAppPath: options.app = options.remoteTestRoot + "/" + options.remoteAppPath - elif (options.app is None): - # Neither remoteAppPath nor app are set -- error - options_logger.error("You must specify either appPath or app") - return None + elif options.app is None: + if build_obj: + options.app = build_obj.substs['ANDROID_PACKAGE_NAME'] + else: + # Neither remoteAppPath nor app are set -- error + parser.error("You must specify either appPath or app") + + if build_obj and 'MOZ_HOST_BIN' in os.environ: + options.xrePath = os.environ['MOZ_HOST_BIN'] # Only reset the xrePath if it wasn't provided - if (options.xrePath is None): + if options.xrePath is None: options.xrePath = options.utilityPath - if (options.pidFile != ""): + if options.pidFile != "": f = open(options.pidFile, 'w') f.write("%s" % os.getpid()) f.close() - # Robocop specific deprecated options. - if options.robocop: - if options.robocopIni: - options_logger.error( - "can not use deprecated --robocop and replacement --robocop-ini together") - return None - options.robocopIni = options.robocop - del options.robocop - - if options.robocopPath: - if options.robocopApk: - options_logger.error( - "can not use deprecated --robocop-path and replacement --robocop-apk together") - return None - options.robocopApk = os.path.join( - options.robocopPath, - 'robocop.apk') - del options.robocopPath - # Robocop specific options if options.robocopIni != "": if not os.path.exists(options.robocopIni): - options_logger.error( + parser.error( "Unable to find specified robocop .ini manifest '%s'" % options.robocopIni) - return None options.robocopIni = os.path.abspath(options.robocopIni) + if not options.robocopApk and build_obj: + options.robocopApk = os.path.join(build_obj.topobjdir, 'build', 'mobile', + 'robocop', 'robocop-debug.apk') + if options.robocopApk != "": if not os.path.exists(options.robocopApk): - options_logger.error( + parser.error( "Unable to find robocop APK '%s'" % options.robocopApk) - return None options.robocopApk = os.path.abspath(options.robocopApk) if options.robocopIds != "": if not os.path.exists(options.robocopIds): - options_logger.error( + parser.error( "Unable to find specified robocop IDs file '%s'" % options.robocopIds) - return None options.robocopIds = os.path.abspath(options.robocopIds) # allow us to keep original application around for cleanup while @@ -1040,20 +1130,84 @@ class RemoteOptions(MochitestOptions): options.remoteappname = options.app return options - def verifyOptions(self, options, mochitest): - # since we are reusing verifyOptions, it will exit if App is not found - temp = options.app - options.app = __file__ - tempPort = options.httpPort - tempSSL = options.sslPort - tempIP = options.webServer - # We are going to override this option later anyway, just pretend - # like it's not set for verification purposes. - options.dumpOutputDirectory = None - options = MochitestOptions.verifyOptions(self, options, mochitest) - options.webServer = tempIP - options.app = temp - options.sslPort = tempSSL - options.httpPort = tempPort - return options +container_map = { + 'generic': [MochitestArguments], + 'b2g': [MochitestArguments, B2GArguments], + 'android': [MochitestArguments, AndroidArguments], +} + + +class MochitestArgumentParser(ArgumentParser): + """ + Usage instructions for runtests.py. + + All arguments are optional. + If --chrome is specified, chrome tests will be run instead of web content tests. + If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. + See for details on the logging levels. + """ + + _containers = None + context = {} + + def __init__(self, app=None, **kwargs): + ArgumentParser.__init__(self, usage=self.__doc__, conflict_handler='resolve', **kwargs) + + self.oldcwd = os.getcwd() + self.app = app + if not self.app and build_obj: + if conditions.is_android(build_obj): + self.app = 'android' + elif conditions.is_b2g(build_obj): + self.app = 'b2g' + if not self.app: + # platform can't be determined and app wasn't specified explicitly, + # so just use generic arguments and hope for the best + self.app = 'generic' + + if self.app not in container_map: + self.error("Unrecognized app '{}'! Must be one of: {}".format( + self.app, ', '.join(container_map.keys()))) + + defaults = {} + for container in self.containers: + defaults.update(container.defaults) + group = self.add_argument_group(container.__class__.__name__, container.__doc__) + + for cli, kwargs in container.args: + # Allocate new lists so references to original don't get mutated. + # allowing multiple uses within a single process. + if "default" in kwargs and isinstance(kwargs['default'], list): + kwargs["default"] = [] + + if 'suppress' in kwargs: + if kwargs['suppress']: + kwargs['help'] = SUPPRESS + del kwargs['suppress'] + + group.add_argument(*cli, **kwargs) + + self.set_defaults(**defaults) + structured.commandline.add_logging_group(self) + + @property + def containers(self): + if self._containers: + return self._containers + + containers = container_map[self.app] + self._containers = [c() for c in containers] + return self._containers + + def validate(self, args): + for container in self.containers: + args = container.validate(self, args, self.context) + return args + + def parse_args(self, *args, **kwargs): + return self.validate(ArgumentParser.parse_args(self, *args, **kwargs)) + + def parse_known_args(self, *args, **kwargs): + args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs) + return (self.validate(args), remainder) diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index 92e6a780a28e..75640b875559 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -53,13 +53,21 @@ from manifestparser.filters import ( subsuite, tags, ) -from mochitest_options import MochitestOptions +from mochitest_options import MochitestArgumentParser from mozprofile import Profile, Preferences from mozprofile.permissions import ServerLocations from urllib import quote_plus as encodeURIComponent from mozlog.structured.formatters import TbplFormatter from mozlog.structured import commandline +here = os.path.abspath(os.path.dirname(__file__)) + +try: + from mozbuild.base import MozbuildObject + build_obj = MozbuildObject.from_environment(cwd=here) +except ImportError: + build_obj = None + ########################### # Option for NSPR logging # @@ -2595,29 +2603,34 @@ class Mochitest(MochitestUtilsMixin): return dirlist -def main(): +def run_test_harness(options): + logger_options = { + key: value for key, value in vars(options).iteritems() if key.startswith('log')} + runner = Mochitest(logger_options) + result = runner.runTests(options) + + # don't dump failures if running from automation as treeherder already displays them + if build_obj: + if runner.message_logger.errors: + result = 1 + runner.message_logger.logger.warning("The following tests failed:") + for error in runner.message_logger.errors: + runner.message_logger.logger.log_raw(error) + + runner.message_logger.finish() + + return result + + +def cli(args=sys.argv[1:]): # parse command line options - parser = MochitestOptions() - commandline.add_logging_group(parser) - options = parser.parse_args() + parser = MochitestArgumentParser(app='generic') + options = parser.parse_args(args) if options is None: # parsing error sys.exit(1) - logger_options = { - key: value for key, - value in vars(options).iteritems() if key.startswith('log')} - mochitest = Mochitest(logger_options) - options = parser.verifyOptions(options, mochitest) - options.utilityPath = mochitest.getFullPath(options.utilityPath) - options.certPath = mochitest.getFullPath(options.certPath) - if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: - options.symbolsPath = mochitest.getFullPath(options.symbolsPath) - - return_code = mochitest.runTests(options) - mochitest.message_logger.finish() - - sys.exit(return_code) + return run_test_harness(options) if __name__ == "__main__": - main() + sys.exit(cli()) diff --git a/testing/mochitest/runtestsb2g.py b/testing/mochitest/runtestsb2g.py index eacffc52956e..856e073c01f3 100644 --- a/testing/mochitest/runtestsb2g.py +++ b/testing/mochitest/runtestsb2g.py @@ -17,7 +17,7 @@ sys.path.insert(0, here) from automationutils import processLeakLog from runtests import Mochitest from runtests import MochitestUtilsMixin -from mochitest_options import B2GOptions, MochitestOptions +from mochitest_options import MochitestArgumentParser from marionette import Marionette from mozprofile import Profile, Preferences from mozlog import structured @@ -417,7 +417,7 @@ class B2GDesktopMochitest(B2GMochitest, Mochitest): return self.build_profile(options) -def run_remote_mochitests(parser, options): +def run_remote_mochitests(options): # create our Marionette instance marionette_args = { 'adb_path': options.adbPath, @@ -434,7 +434,6 @@ def run_remote_mochitests(parser, options): marionette_args['host'] = host marionette_args['port'] = int(port) - options = parser.verifyRemoteOptions(options) if (options is None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) @@ -446,7 +445,6 @@ def run_remote_mochitests(parser, options): options.xrePath, remote_log_file=options.remoteLogFile) - options = parser.verifyOptions(options, mochitest) if (options is None): sys.exit(1) @@ -469,7 +467,7 @@ def run_remote_mochitests(parser, options): sys.exit(retVal) -def run_desktop_mochitests(parser, options): +def run_desktop_mochitests(options): # create our Marionette instance marionette_args = {} if options.marionette: @@ -486,7 +484,6 @@ def run_desktop_mochitests(parser, options): marionette_args, options, options.profile_data_dir) - options = MochitestOptions.verifyOptions(parser, options, mochitest) if options is None: sys.exit(1) @@ -502,14 +499,13 @@ def run_desktop_mochitests(parser, options): def main(): - parser = B2GOptions() - structured.commandline.add_logging_group(parser) + parser = MochitestArgumentParser(app='b2g') options = parser.parse_args() if options.desktop: - run_desktop_mochitests(parser, options) + run_desktop_mochitests(options) else: - run_remote_mochitests(parser, options) + run_remote_mochitests(options) if __name__ == "__main__": main() diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index 08e84bd40a21..3c3ab549e1ab 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -18,13 +18,11 @@ sys.path.insert( from automation import Automation from remoteautomation import RemoteAutomation, fennecLogcatFilters from runtests import Mochitest, MessageLogger -from mochitest_options import RemoteOptions -from mozlog import structured +from mochitest_options import MochitestArgumentParser from manifestparser import TestManifest from manifestparser.filters import chunk_by_slice import devicemanager -import droid import mozinfo SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) @@ -461,40 +459,16 @@ class MochiRemote(Mochitest): return self._automation.runApp(*args, **kwargs) -def main(args): +def run_test_harness(options): message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) - parser = RemoteOptions(auto) - structured.commandline.add_logging_group(parser) - options = parser.parse_args(args) - - if (options.dm_trans == "adb"): - if (options.deviceIP): - dm = droid.DroidADB( - options.deviceIP, - options.devicePort, - deviceRoot=options.remoteTestRoot) - elif (options.deviceSerial): - dm = droid.DroidADB( - None, - None, - deviceSerial=options.deviceSerial, - deviceRoot=options.remoteTestRoot) - else: - dm = droid.DroidADB(deviceRoot=options.remoteTestRoot) - else: - dm = droid.DroidSUT( - options.deviceIP, - options.devicePort, - deviceRoot=options.remoteTestRoot) - auto.setDeviceManager(dm) - options = parser.verifyRemoteOptions(options, auto) - if options is None: raise ValueError("Invalid options specified, use --help for a list of valid options") + dm = options.dm + auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) log = mochitest.log @@ -508,10 +482,6 @@ def main(args): auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) - options = parser.verifyOptions(options, mochitest) - if (options is None): - return 1 - logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) @@ -738,5 +708,12 @@ def main(args): return retVal +def main(args=sys.argv[1:]): + parser = MochitestArgumentParser(app='android') + options = parser.parse_args(args) + + return run_test_harness(options) + + if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + sys.exit(main()) diff --git a/testing/testsuite-targets.mk b/testing/testsuite-targets.mk index 52b7eb1adcec..d0a5320363e5 100644 --- a/testing/testsuite-targets.mk +++ b/testing/testsuite-targets.mk @@ -29,16 +29,16 @@ endif RUN_MOCHITEST_B2G_DESKTOP = \ rm -f ./$@.log && \ - $(PYTHON) _tests/testing/mochitest/runtestsb2g.py --autorun --close-when-done \ - --console-level=INFO --log-tbpl=./$@.log \ + $(PYTHON) _tests/testing/mochitest/runtestsb2g.py \ + --log-tbpl=./$@.log \ --desktop --profile ${GAIA_PROFILE_DIR} \ --failure-file=$(abspath _tests/testing/mochitest/makefailures.json) \ $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) RUN_MOCHITEST = \ rm -f ./$@.log && \ - $(PYTHON) _tests/testing/mochitest/runtests.py --autorun --close-when-done \ - --console-level=INFO --log-tbpl=./$@.log \ + $(PYTHON) _tests/testing/mochitest/runtests.py \ + --log-tbpl=./$@.log \ --failure-file=$(abspath _tests/testing/mochitest/makefailures.json) \ --testing-modules-dir=$(abspath _tests/modules) \ --extra-profile-file=$(DIST)/plugins \ @@ -46,8 +46,8 @@ RUN_MOCHITEST = \ RERUN_MOCHITEST = \ rm -f ./$@.log && \ - $(PYTHON) _tests/testing/mochitest/runtests.py --autorun --close-when-done \ - --console-level=INFO --log-tbpl=./$@.log \ + $(PYTHON) _tests/testing/mochitest/runtests.py \ + --log-tbpl=./$@.log \ --run-only-tests=makefailures.json \ --testing-modules-dir=$(abspath _tests/modules) \ --extra-profile-file=$(DIST)/plugins \ @@ -55,8 +55,8 @@ RERUN_MOCHITEST = \ RUN_MOCHITEST_REMOTE = \ rm -f ./$@.log && \ - $(PYTHON) _tests/testing/mochitest/runtestsremote.py --autorun --close-when-done \ - --console-level=INFO --log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \ + $(PYTHON) _tests/testing/mochitest/runtestsremote.py \ + --log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \ --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \ --testing-modules-dir=$(abspath _tests/modules) \ $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) @@ -67,7 +67,7 @@ RUN_MOCHITEST_ROBOCOP = \ --robocop-apk=$(DEPTH)/build/mobile/robocop/robocop-debug.apk \ --robocop-ids=$(DEPTH)/mobile/android/base/fennec_ids.txt \ --robocop-ini=_tests/testing/mochitest/robocop.ini \ - --console-level=INFO --log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \ + --log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \ --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \ $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS)