From af6e9daaaad785e933255596e0d55538d0435b0b Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Fri, 26 Jul 2013 14:40:04 -0400 Subject: [PATCH] Bug 865349 - Refactor B2G mochitests off automation.py and onto mozbase, r=jgriffin,ted --- testing/mochitest/Makefile.in | 3 + testing/mochitest/b2g_start_script.js | 54 ++ testing/mochitest/mochitest_options.py | 663 ++++++++++++++++++ testing/mochitest/runtests.py | 918 ++++++++----------------- testing/mochitest/runtestsb2g.py | 601 ++++------------ testing/mochitest/runtestsremote.py | 9 +- 6 files changed, 1130 insertions(+), 1118 deletions(-) create mode 100644 testing/mochitest/b2g_start_script.js create mode 100644 testing/mochitest/mochitest_options.py diff --git a/testing/mochitest/Makefile.in b/testing/mochitest/Makefile.in index 3cdfc89fc81b..bcdd31da577c 100644 --- a/testing/mochitest/Makefile.in +++ b/testing/mochitest/Makefile.in @@ -36,6 +36,7 @@ _SERV_FILES = \ runtestsb2g.py \ runtestsremote.py \ runtestsvmware.py \ + mochitest_options.py \ manifest.webapp \ $(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanager.py \ $(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py \ @@ -62,6 +63,7 @@ _SERV_FILES = \ android.json \ androidx86.json \ b2g.json \ + b2g_start_script.js \ root-ev-tester.crl \ intermediate-ev-tester.crl \ $(NULL) @@ -202,6 +204,7 @@ $(_DEST_DIR): stage-package: $(NSINSTALL) -D $(PKG_STAGE)/mochitest && $(NSINSTALL) -D $(PKG_STAGE)/bin/plugins && $(NSINSTALL) -D $(DIST)/plugins (cd $(DEPTH)/_tests/testing/mochitest/ && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/mochitest && tar -xf -) + @cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/mochitest @(cd $(DIST_BIN) && tar $(TAR_CREATE_FLAGS) - $(TEST_HARNESS_BINS)) | (cd $(PKG_STAGE)/bin && tar -xf -) @(cd $(DIST_BIN)/components && tar $(TAR_CREATE_FLAGS) - $(TEST_HARNESS_COMPONENTS)) | (cd $(PKG_STAGE)/bin/components && tar -xf -) (cd $(topsrcdir)/build/pgo/certs && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/certs && tar -xf -) diff --git a/testing/mochitest/b2g_start_script.js b/testing/mochitest/b2g_start_script.js new file mode 100644 index 000000000000..785f1ad19c65 --- /dev/null +++ b/testing/mochitest/b2g_start_script.js @@ -0,0 +1,54 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +let outOfProcess = __marionetteParams[0] +let mochitestUrl = __marionetteParams[1] + +const CHILD_SCRIPT = "chrome://specialpowers/content/specialpowers.js"; +const CHILD_SCRIPT_API = "chrome://specialpowers/content/specialpowersAPI.js"; +const CHILD_LOGGER_SCRIPT = "chrome://specialpowers/content/MozillaLogger.js"; + +let homescreen = document.getElementById('homescreen'); +let container = homescreen.contentWindow.document.getElementById('test-container'); + +function openWindow(aEvent) { + var popupIframe = aEvent.detail.frameElement; + popupIframe.setAttribute('style', 'position: absolute; left: 0; top: 300px; background: white; '); + + popupIframe.addEventListener('mozbrowserclose', function(e) { + container.parentNode.removeChild(popupIframe); + container.focus(); + }); + + // yes, the popup can call window.open too! + popupIframe.addEventListener('mozbrowseropenwindow', openWindow); + + popupIframe.addEventListener('mozbrowserloadstart', function(e) { + popupIframe.focus(); + }); + + container.parentNode.appendChild(popupIframe); +} +container.addEventListener('mozbrowseropenwindow', openWindow); + +let specialpowers = {}; +let loader = Cc["@mozilla.org/moz/jssubscript-loader;1"].getService(Ci.mozIJSSubScriptLoader); +loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserver.js", specialpowers); +let specialPowersObserver = new specialpowers.SpecialPowersObserver(); +specialPowersObserver.init(); + +let mm = container.QueryInterface(Ci.nsIFrameLoaderOwner).frameLoader.messageManager; +mm.addMessageListener("SPPrefService", specialPowersObserver); +mm.addMessageListener("SPProcessCrashService", specialPowersObserver); +mm.addMessageListener("SPPingService", specialPowersObserver); +mm.addMessageListener("SpecialPowers.Quit", specialPowersObserver); +mm.addMessageListener("SpecialPowers.Focus", specialPowersObserver); +mm.addMessageListener("SPPermissionManager", specialPowersObserver); + +mm.loadFrameScript(CHILD_LOGGER_SCRIPT, true); +mm.loadFrameScript(CHILD_SCRIPT_API, true); +mm.loadFrameScript(CHILD_SCRIPT, true); +specialPowersObserver._isFrameScriptLoaded = true; + +container.src = mochitestUrl; diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py new file mode 100644 index 000000000000..31fba245db74 --- /dev/null +++ b/testing/mochitest/mochitest_options.py @@ -0,0 +1,663 @@ +import optparse +import os +import sys +import tempfile + +from automation import Automation +from automationutils import addCommonOptions, isURL +from mozprofile import DEFAULT_PORTS +import moznetwork + +try: + from mozbuild.base import MozbuildObject + build_obj = MozbuildObject.from_environment() +except ImportError: + build_obj = None + +here = os.path.abspath(os.path.dirname(sys.argv[0])) + +__all__ = ["MochitestOptions", "B2GOptions"] + +VMWARE_RECORDING_HELPER_BASENAME = "vmwarerecordinghelper" + +class MochitestOptions(optparse.OptionParser): + """Usage instructions for runtests.py. + All arguments are optional. + If --chrome is specified, chrome tests will be run instead of web content tests. + If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. + See for details on the logging levels. + """ + + LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") + LEVEL_STRING = ", ".join(LOG_LEVELS) + mochitest_options = [ + [["--close-when-done"], + { "action": "store_true", + "dest": "closeWhenDone", + "default": False, + "help": "close the application when tests are done running", + }], + [["--appname"], + { "action": "store", + "type": "string", + "dest": "app", + "default": build_obj.get_binary_path() if build_obj is not None else None, + "help": "absolute path to application, overriding default", + }], + [["--utility-path"], + { "action": "store", + "type": "string", + "dest": "utilityPath", + "default": build_obj.bindir if build_obj is not None else None, + "help": "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)", + }], + [["--certificate-path"], + { "action": "store", + "type": "string", + "dest": "certPath", + "help": "absolute path to directory containing certificate store to use testing profile", + "default": os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if build_obj is not None else None, + }], + [["--log-file"], + { "action": "store", + "type": "string", + "dest": "logFile", + "metavar": "FILE", + "help": "file to which logging occurs", + "default": "", + }], + [["--autorun"], + { "action": "store_true", + "dest": "autorun", + "help": "start running tests when the application starts", + "default": False, + }], + [["--timeout"], + { "type": "int", + "dest": "timeout", + "help": "per-test timeout in seconds", + "default": None, + }], + [["--total-chunks"], + { "type": "int", + "dest": "totalChunks", + "help": "how many chunks to split the tests up into", + "default": None, + }], + [["--this-chunk"], + { "type": "int", + "dest": "thisChunk", + "help": "which chunk to run", + "default": None, + }], + [["--chunk-by-dir"], + { "type": "int", + "dest": "chunkByDir", + "help": "group tests together in the same chunk that are in the same top chunkByDir directories", + "default": 0, + }], + [["--shuffle"], + { "dest": "shuffle", + "action": "store_true", + "help": "randomize test order", + "default": False, + }], + [["--console-level"], + { "action": "store", + "type": "choice", + "dest": "consoleLevel", + "choices": LOG_LEVELS, + "metavar": "LEVEL", + "help": "one of %s to determine the level of console " + "logging" % LEVEL_STRING, + "default": None, + }], + [["--file-level"], + { "action": "store", + "type": "choice", + "dest": "fileLevel", + "choices": LOG_LEVELS, + "metavar": "LEVEL", + "help": "one of %s to determine the level of file " + "logging if a file has been specified, defaulting " + "to INFO" % LEVEL_STRING, + "default": "INFO", + }], + [["--chrome"], + { "action": "store_true", + "dest": "chrome", + "help": "run chrome Mochitests", + "default": False, + }], + [["--ipcplugins"], + { "action": "store_true", + "dest": "ipcplugins", + "help": "run ipcplugins Mochitests", + "default": False, + }], + [["--test-path"], + { "action": "store", + "type": "string", + "dest": "testPath", + "help": "start in the given directory's tests", + "default": "", + }], + [["--browser-chrome"], + { "action": "store_true", + "dest": "browserChrome", + "help": "run browser chrome Mochitests", + "default": False, + }], + [["--webapprt-content"], + { "action": "store_true", + "dest": "webapprtContent", + "help": "run WebappRT content tests", + "default": False, + }], + [["--webapprt-chrome"], + { "action": "store_true", + "dest": "webapprtChrome", + "help": "run WebappRT chrome tests", + "default": False, + }], + [["--a11y"], + { "action": "store_true", + "dest": "a11y", + "help": "run accessibility Mochitests", + "default": False, + }], + [["--setenv"], + { "action": "append", + "type": "string", + "dest": "environment", + "metavar": "NAME=VALUE", + "help": "sets the given variable in the application's " + "environment", + "default": [], + }], + [["--exclude-extension"], + { "action": "append", + "type": "string", + "dest": "extensionsToExclude", + "help": "excludes the given extension from being installed " + "in the test profile", + "default": [], + }], + [["--browser-arg"], + { "action": "append", + "type": "string", + "dest": "browserArgs", + "metavar": "ARG", + "help": "provides an argument to the test application", + "default": [], + }], + [["--leak-threshold"], + { "action": "store", + "type": "int", + "dest": "leakThreshold", + "metavar": "THRESHOLD", + "help": "fail if the number of bytes leaked through " + "refcounted objects (or bytes in classes with " + "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " + "than the given number", + "default": 0, + }], + [["--fatal-assertions"], + { "action": "store_true", + "dest": "fatalAssertions", + "help": "abort testing whenever an assertion is hit " + "(requires a debug build to be effective)", + "default": False, + }], + [["--extra-profile-file"], + { "action": "append", + "dest": "extraProfileFiles", + "help": "copy specified files/dirs to testing profile", + "default": [], + }], + [["--install-extension"], + { "action": "append", + "dest": "extensionsToInstall", + "help": "install the specified extension in the testing profile." + "The extension file's name should be .xpi where is" + "the extension's id as indicated in its install.rdf." + "An optional path can be specified too.", + "default": [], + }], + [["--profile-path"], + { "action": "store", + "type": "string", + "dest": "profilePath", + "help": "Directory where the profile will be stored." + "This directory will be deleted after the tests are finished", + "default": tempfile.mkdtemp(), + }], + [["--testing-modules-dir"], + { "action": "store", + "type": "string", + "dest": "testingModulesDir", + "help": "Directory where testing-only JS modules are located.", + "default": None, + }], + [["--use-vmware-recording"], + { "action": "store_true", + "dest": "vmwareRecording", + "help": "enables recording while the application is running " + "inside a VMware Workstation 7.0 or later VM", + "default": False, + }], + [["--repeat"], + { "action": "store", + "type": "int", + "dest": "repeat", + "metavar": "REPEAT", + "help": "repeats the test or set of tests the given number of times, ie: repeat: 1 will run the test twice.", + "default": 0, + }], + [["--run-until-failure"], + { "action": "store_true", + "dest": "runUntilFailure", + "help": "Run a test repeatedly and stops on the first time the test fails. " + "Only available when running a single test. Default cap is 30 runs, " + "which can be overwritten with the --repeat parameter.", + "default": False, + }], + [["--run-only-tests"], + { "action": "store", + "type": "string", + "dest": "runOnlyTests", + "help": "JSON list of tests that we only want to run, cannot be specified with --exclude-tests. [DEPRECATED- please use --test-manifest]", + "default": None, + }], + [["--exclude-tests"], + { "action": "store", + "type": "string", + "dest": "excludeTests", + "help": "JSON list of tests that we want to not run, cannot be specified with --run-only-tests. [DEPRECATED- please use --test-manifest]", + "default": None, + }], + [["--test-manifest"], + { "action": "store", + "type": "string", + "dest": "testManifest", + "help": "JSON list of tests to specify 'runtests' and 'excludetests'.", + "default": None, + }], + [["--failure-file"], + { "action": "store", + "type": "string", + "dest": "failureFile", + "help": "Filename of the output file where we can store a .json list of failures to be run in the future with --run-only-tests.", + "default": None, + }], + [["--run-slower"], + { "action": "store_true", + "dest": "runSlower", + "help": "Delay execution between test files.", + "default": False, + }], + [["--metro-immersive"], + { "action": "store_true", + "dest": "immersiveMode", + "help": "launches tests in immersive browser", + "default": False, + }], + [["--httpd-path"], + { "action": "store", + "type": "string", + "dest": "httpdPath", + "default": None, + "help": "path to the httpd.js file", + }], + [["--setpref"], + { "action": "append", + "type": "string", + "default": [], + "dest": "extraPrefs", + "metavar": "PREF=VALUE", + "help": "defines an extra user preference", + }], + ] + + def __init__(self, automation=None, **kwargs): + self._automation = automation or Automation() + optparse.OptionParser.__init__(self, **kwargs) + defaults = {} + + # we want to pass down everything from self._automation.__all__ + addCommonOptions(self, defaults=dict(zip(self._automation.__all__, + [getattr(self._automation, x) for x in self._automation.__all__]))) + + for option in self.mochitest_options: + self.add_option(*option[0], **option[1]) + + self.set_defaults(**defaults) + self.set_usage(self.__doc__) + + def verifyOptions(self, options, mochitest): + """ verify correct options and cleanup paths """ + + if options.totalChunks is not None and options.thisChunk is None: + self.error("thisChunk must be specified when totalChunks is specified") + + if options.totalChunks: + if not 1 <= options.thisChunk <= options.totalChunks: + self.error("thisChunk must be between 1 and totalChunks") + + if options.xrePath is None: + # default xrePath to the app path if not provided + # but only if an app path was explicitly provided + if options.app != self.defaults['app']: + options.xrePath = os.path.dirname(options.app) + elif build_obj is not None: + # otherwise default to dist/bin + options.xrePath = build_obj.bindir + else: + self.error("could not find xre directory, --xre-path must be specified") + + # allow relative paths + options.xrePath = mochitest.getFullPath(options.xrePath) + options.profilePath = mochitest.getFullPath(options.profilePath) + options.app = mochitest.getFullPath(options.app) + + if not os.path.exists(options.app): + msg = """\ + Error: Path %(app)s doesn't exist. + Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" + self.error(msg % {"app": options.app}) + return None + + if options.utilityPath: + options.utilityPath = mochitest.getFullPath(options.utilityPath) + + if options.certPath: + options.certPath = mochitest.getFullPath(options.certPath) + + if options.symbolsPath and not isURL(options.symbolsPath): + options.symbolsPath = mochitest.getFullPath(options.symbolsPath) + + options.webServer = self._automation.DEFAULT_WEB_SERVER + options.httpPort = self._automation.DEFAULT_HTTP_PORT + options.sslPort = self._automation.DEFAULT_SSL_PORT + options.webSocketPort = self._automation.DEFAULT_WEBSOCKET_PORT + + if options.vmwareRecording: + if not self._automation.IS_WIN32: + self.error("use-vmware-recording is only supported on Windows.") + mochitest.vmwareHelperPath = os.path.join( + options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") + if not os.path.exists(mochitest.vmwareHelperPath): + self.error("%s not found, cannot automate VMware recording." % + mochitest.vmwareHelperPath) + + if options.runOnlyTests != None and options.excludeTests != None: + self.error("We can only support --run-only-tests OR --exclude-tests, not both. Please consider using --test-manifest instead.") + + if options.testManifest != None and (options.runOnlyTests != None or options.excludeTests != None): + self.error("Please use --test-manifest only and not --run-only-tests or --exclude-tests.") + + if options.runOnlyTests: + if not os.path.exists(os.path.abspath(options.runOnlyTests)): + self.error("unable to find --run-only-tests file '%s'" % options.runOnlyTests); + options.testManifest = options.runOnlyTests + options.runOnly = True + + if options.excludeTests: + if not os.path.exists(os.path.abspath(options.excludeTests)): + self.error("unable to find --exclude-tests file '%s'" % options.excludeTests); + options.testManifest = options.excludeTests + options.runOnly = False + + if options.webapprtContent and options.webapprtChrome: + self.error("Only one of --webapprt-content and --webapprt-chrome may be given.") + + # Try to guess the testing modules directory. + # This somewhat grotesque hack allows the buildbot machines to find the + # modules directory without having to configure the buildbot hosts. This + # code should never be executed in local runs because the build system + # should always set the flag that populates this variable. If buildbot ever + # passes this argument, this code can be deleted. + if options.testingModulesDir is None: + possible = os.path.join(os.getcwd(), os.path.pardir, 'modules') + + if os.path.isdir(possible): + options.testingModulesDir = possible + + # Even if buildbot is updated, we still want this, as the path we pass in + # to the app must be absolute and have proper slashes. + if options.testingModulesDir is not None: + options.testingModulesDir = os.path.normpath(options.testingModulesDir) + + if not os.path.isabs(options.testingModulesDir): + options.testingModulesDir = os.path.abspath(options.testingModulesDir) + + if not os.path.isdir(options.testingModulesDir): + self.error('--testing-modules-dir not a directory: %s' % + options.testingModulesDir) + + options.testingModulesDir = options.testingModulesDir.replace('\\', '/') + if options.testingModulesDir[-1] != '/': + options.testingModulesDir += '/' + + if options.immersiveMode: + if not self._automation.IS_WIN32: + self.error("immersive is only supported on Windows 8 and up.") + mochitest.immersiveHelperPath = os.path.join( + options.utilityPath, "metrotestharness.exe") + if not os.path.exists(mochitest.immersiveHelperPath): + self.error("%s not found, cannot launch immersive tests." % + mochitest.immersiveHelperPath) + + if options.runUntilFailure: + if not os.path.isfile(os.path.join(mochitest.oldcwd, os.path.dirname(__file__), mochitest.getTestRoot(options), options.testPath)): + self.error("--run-until-failure can only be used together with --test-path specifying a single test.") + if not options.repeat: + options.repeat = 29 + return options + + +class B2GOptions(MochitestOptions): + b2g_options = [ + [["--b2gpath"], + { "action": "store", + "type": "string", + "dest": "b2gPath", + "help": "path to B2G repo or qemu dir", + "default": None, + }], + [["--desktop"], + { "action": "store_true", + "dest": "desktop", + "help": "Run the tests on a B2G desktop build", + "default": False, + }], + [["--marionette"], + { "action": "store", + "type": "string", + "dest": "marionette", + "help": "host:port to use when connecting to Marionette", + "default": None, + }], + [["--emulator"], + { "action": "store", + "type": "string", + "dest": "emulator", + "help": "Architecture of emulator to use: x86 or arm", + "default": None, + }], + [["--sdcard"], + { "action": "store", + "type": "string", + "dest": "sdcard", + "help": "Define size of sdcard: 1MB, 50MB...etc", + "default": "10MB", + }], + [["--no-window"], + { "action": "store_true", + "dest": "noWindow", + "help": "Pass --no-window to the emulator", + "default": False, + }], + [["--adbpath"], + { "action": "store", + "type": "string", + "dest": "adbPath", + "help": "path to adb", + "default": "adb", + }], + [["--deviceIP"], + { "action": "store", + "type": "string", + "dest": "deviceIP", + "help": "ip address of remote device to test", + "default": None, + }], + [["--devicePort"], + { "action": "store", + "type": "string", + "dest": "devicePort", + "help": "port of remote device to test", + "default": 20701, + }], + [["--remote-logfile"], + { "action": "store", + "type": "string", + "dest": "remoteLogFile", + "help": "Name of log file on the device relative to the device root. \ + PLEASE ONLY USE A FILENAME.", + "default" : None, + }], + [["--remote-webserver"], + { "action": "store", + "type": "string", + "dest": "remoteWebServer", + "help": "ip address where the remote web server is hosted at", + "default": None, + }], + [["--http-port"], + { "action": "store", + "type": "string", + "dest": "httpPort", + "help": "ip address where the remote web server is hosted at", + "default": None, + }], + [["--ssl-port"], + { "action": "store", + "type": "string", + "dest": "sslPort", + "help": "ip address where the remote web server is hosted at", + "default": None, + }], + [["--pidfile"], + { "action": "store", + "type": "string", + "dest": "pidFile", + "help": "name of the pidfile to generate", + "default": "", + }], + [["--gecko-path"], + { "action": "store", + "type": "string", + "dest": "geckoPath", + "help": "the path to a gecko distribution that should \ + be installed on the emulator prior to test", + "default": None, + }], + [["--profile"], + { "action": "store", + "type": "string", + "dest": "profile", + "help": "for desktop testing, the path to the \ + gaia profile to use", + "default": None, + }], + [["--logcat-dir"], + { "action": "store", + "type": "string", + "dest": "logcat_dir", + "help": "directory to store logcat dump files", + "default": None, + }], + [['--busybox'], + { "action": 'store', + "type": 'string', + "dest": 'busybox', + "help": "Path to busybox binary to install on device", + "default": None, + }], + [['--profile-data-dir'], + { "action": 'store', + "type": 'string', + "dest": 'profile_data_dir', + "help": "Path to a directory containing preference and other \ + data to be installed into the profile", + "default": os.path.join(here, 'profile_data'), + }], + ] + + def __init__(self): + MochitestOptions.__init__(self) + + for option in self.b2g_options: + self.add_option(*option[0], **option[1]) + + defaults = {} + defaults["httpPort"] = DEFAULT_PORTS['http'] + defaults["sslPort"] = DEFAULT_PORTS['https'] + defaults["remoteTestRoot"] = "/data/local/tests" + defaults["logFile"] = "mochitest.log" + defaults["autorun"] = True + defaults["closeWhenDone"] = True + defaults["testPath"] = "" + defaults["extensionsToExclude"] = ["specialpowers"] + self.set_defaults(**defaults) + + def verifyRemoteOptions(self, options): + if options.remoteWebServer == None: + if os.name != "nt": + options.remoteWebServer = moznetwork.get_ip() + else: + self.error("You must specify a --remote-webserver=") + options.webServer = options.remoteWebServer + + if options.geckoPath and not options.emulator: + self.error("You must specify --emulator if you specify --gecko-path") + + if options.logcat_dir and not options.emulator: + self.error("You must specify --emulator if you specify --logcat-dir") + + if not os.path.isdir(options.xrePath): + self.error("--xre-path '%s' is not a directory" % options.xrePath) + xpcshell = os.path.join(options.xrePath, 'xpcshell') + if not os.access(xpcshell, os.F_OK): + self.error('xpcshell not found at %s' % xpcshell) + if self.elf_arm(xpcshell): + self.error('--xre-path points to an ARM version of xpcshell; it ' + 'should instead point to a version that can run on ' + 'your desktop') + + if options.pidFile != "": + f = open(options.pidFile, 'w') + f.write("%s" % os.getpid()) + f.close() + + return options + + def verifyOptions(self, options, mochitest): + # since we are reusing verifyOptions, it will exit if App is not found + temp = options.app + options.app = sys.argv[0] + tempPort = options.httpPort + tempSSL = options.sslPort + tempIP = options.webServer + options = MochitestOptions.verifyOptions(self, options, mochitest) + options.webServer = tempIP + options.app = temp + options.sslPort = tempSSL + options.httpPort = tempPort + + return options + + def elf_arm(self, filename): + data = open(filename, 'rb').read(20) + return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index a26c3ade513c..b1660cc62257 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -8,394 +8,27 @@ Runs the Mochitest test harness. """ from __future__ import with_statement -from datetime import datetime import optparse import os import os.path import sys import time +import traceback -SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) +SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) sys.path.insert(0, SCRIPT_DIR); import shutil from urllib import quote_plus as encodeURIComponent import urllib2 -import commands from automation import Automation -from automationutils import * -import tempfile +from automationutils import getDebuggerInfo, isURL, processLeakLog +from mochitest_options import MochitestOptions -VMWARE_RECORDING_HELPER_BASENAME = "vmwarerecordinghelper" +import mozinfo +import mozlog -####################### -# COMMANDLINE OPTIONS # -####################### - -class MochitestOptions(optparse.OptionParser): - """Parses Mochitest commandline options.""" - def __init__(self, automation, scriptdir, **kwargs): - self._automation = automation - optparse.OptionParser.__init__(self, **kwargs) - defaults = {} - - # we want to pass down everything from self._automation.__all__ - addCommonOptions(self, defaults=dict(zip(self._automation.__all__, - [getattr(self._automation, x) for x in self._automation.__all__]))) - self._automation.addCommonOptions(self) - - self.add_option("--close-when-done", - action = "store_true", dest = "closeWhenDone", - help = "close the application when tests are done running") - defaults["closeWhenDone"] = False - - self.add_option("--appname", - action = "store", type = "string", dest = "app", - help = "absolute path to application, overriding default") - defaults["app"] = os.path.join(scriptdir, self._automation.DEFAULT_APP) - - self.add_option("--utility-path", - action = "store", type = "string", dest = "utilityPath", - help = "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)") - defaults["utilityPath"] = self._automation.DIST_BIN - - self.add_option("--certificate-path", - action = "store", type = "string", dest = "certPath", - help = "absolute path to directory containing certificate store to use testing profile") - defaults["certPath"] = self._automation.CERTS_SRC_DIR - - self.add_option("--log-file", - action = "store", type = "string", - dest = "logFile", metavar = "FILE", - help = "file to which logging occurs") - defaults["logFile"] = "" - - self.add_option("--autorun", - action = "store_true", dest = "autorun", - help = "start running tests when the application starts") - defaults["autorun"] = False - - self.add_option("--timeout", - type = "int", dest = "timeout", - help = "per-test timeout in seconds") - defaults["timeout"] = None - - self.add_option("--total-chunks", - type = "int", dest = "totalChunks", - help = "how many chunks to split the tests up into") - defaults["totalChunks"] = None - - self.add_option("--this-chunk", - type = "int", dest = "thisChunk", - help = "which chunk to run") - defaults["thisChunk"] = None - - self.add_option("--chunk-by-dir", - type = "int", dest = "chunkByDir", - help = "group tests together in the same chunk that are in the same top chunkByDir directories") - defaults["chunkByDir"] = 0 - - self.add_option("--shuffle", - dest = "shuffle", - action = "store_true", - help = "randomize test order") - defaults["shuffle"] = False - - LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") - LEVEL_STRING = ", ".join(LOG_LEVELS) - - self.add_option("--console-level", - action = "store", type = "choice", dest = "consoleLevel", - choices = LOG_LEVELS, metavar = "LEVEL", - help = "one of %s to determine the level of console " - "logging" % LEVEL_STRING) - defaults["consoleLevel"] = None - - self.add_option("--file-level", - action = "store", type = "choice", dest = "fileLevel", - choices = LOG_LEVELS, metavar = "LEVEL", - help = "one of %s to determine the level of file " - "logging if a file has been specified, defaulting " - "to INFO" % LEVEL_STRING) - defaults["fileLevel"] = "INFO" - - self.add_option("--chrome", - action = "store_true", dest = "chrome", - help = "run chrome Mochitests") - defaults["chrome"] = False - - self.add_option("--ipcplugins", - action = "store_true", dest = "ipcplugins", - help = "run ipcplugins Mochitests") - defaults["ipcplugins"] = False - - self.add_option("--test-path", - action = "store", type = "string", dest = "testPath", - help = "start in the given directory's tests") - defaults["testPath"] = "" - - self.add_option("--browser-chrome", - action = "store_true", dest = "browserChrome", - help = "run browser chrome Mochitests") - defaults["browserChrome"] = False - - self.add_option("--webapprt-content", - action = "store_true", dest = "webapprtContent", - help = "run WebappRT content tests") - defaults["webapprtContent"] = False - - self.add_option("--webapprt-chrome", - action = "store_true", dest = "webapprtChrome", - help = "run WebappRT chrome tests") - defaults["webapprtChrome"] = False - - self.add_option("--a11y", - action = "store_true", dest = "a11y", - help = "run accessibility Mochitests"); - defaults["a11y"] = False - - self.add_option("--setenv", - action = "append", type = "string", - dest = "environment", metavar = "NAME=VALUE", - help = "sets the given variable in the application's " - "environment") - defaults["environment"] = [] - - self.add_option("--exclude-extension", - action = "append", type = "string", - dest = "extensionsToExclude", - help = "excludes the given extension from being installed " - "in the test profile") - defaults["extensionsToExclude"] = [] - - self.add_option("--browser-arg", - action = "append", type = "string", - dest = "browserArgs", metavar = "ARG", - help = "provides an argument to the test application") - defaults["browserArgs"] = [] - - self.add_option("--leak-threshold", - action = "store", type = "int", - dest = "leakThreshold", metavar = "THRESHOLD", - help = "fail if the number of bytes leaked through " - "refcounted objects (or bytes in classes with " - "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " - "than the given number") - defaults["leakThreshold"] = 0 - - self.add_option("--fatal-assertions", - action = "store_true", dest = "fatalAssertions", - help = "abort testing whenever an assertion is hit " - "(requires a debug build to be effective)") - defaults["fatalAssertions"] = False - - self.add_option("--extra-profile-file", - action = "append", dest = "extraProfileFiles", - help = "copy specified files/dirs to testing profile") - defaults["extraProfileFiles"] = [] - - self.add_option("--install-extension", - action = "append", dest = "extensionsToInstall", - help = "install the specified extension in the testing profile." - "The extension file's name should be .xpi where is" - "the extension's id as indicated in its install.rdf." - "An optional path can be specified too.") - defaults["extensionsToInstall"] = [] - - self.add_option("--profile-path", action = "store", - type = "string", dest = "profilePath", - help = "Directory where the profile will be stored." - "This directory will be deleted after the tests are finished") - defaults["profilePath"] = tempfile.mkdtemp() - - self.add_option("--testing-modules-dir", action = "store", - type = "string", dest = "testingModulesDir", - help = "Directory where testing-only JS modules are " - "located.") - defaults["testingModulesDir"] = None - - self.add_option("--use-vmware-recording", - action = "store_true", dest = "vmwareRecording", - help = "enables recording while the application is running " - "inside a VMware Workstation 7.0 or later VM") - defaults["vmwareRecording"] = False - - self.add_option("--repeat", - action = "store", type = "int", - dest = "repeat", metavar = "REPEAT", - help = "repeats the test or set of tests the given number of times, ie: repeat=1 will run the test twice.") - defaults["repeat"] = 0 - - self.add_option("--run-until-failure", - action = "store_true", dest="runUntilFailure", - help = "Run a test repeatedly and stops on the first time the test fails. " - "Only available when running a single test. Default cap is 30 runs, " - "which can be overwritten with the --repeat parameter.") - defaults["runUntilFailure"] = False - - self.add_option("--run-only-tests", - action = "store", type="string", dest = "runOnlyTests", - help = "JSON list of tests that we only want to run, cannot be specified with --exclude-tests. [DEPRECATED- please use --test-manifest]") - defaults["runOnlyTests"] = None - - self.add_option("--exclude-tests", - action = "store", type="string", dest = "excludeTests", - help = "JSON list of tests that we want to not run, cannot be specified with --run-only-tests. [DEPRECATED- please use --test-manifest]") - defaults["excludeTests"] = None - - self.add_option("--test-manifest", - action = "store", type="string", dest = "testManifest", - help = "JSON list of tests to specify 'runtests' and 'excludetests'.") - defaults["testManifest"] = None - - self.add_option("--failure-file", - action = "store", type="string", dest = "failureFile", - help = "Filename of the output file where we can store a .json list of failures to be run in the future with --run-only-tests.") - defaults["failureFile"] = None - - self.add_option("--run-slower", - action = "store_true", dest = "runSlower", - help = "Delay execution between test files.") - defaults["runSlower"] = False - - self.add_option("--metro-immersive", - action = "store_true", dest = "immersiveMode", - help = "launches tests in immersive browser") - defaults["immersiveMode"] = False - - self.add_option("--httpd-path", action = "store", - type = "string", dest = "httpdPath", - help = "path to the httpd.js file") - defaults["httpdPath"] = None - - # -h, --help are automatically handled by OptionParser - - self.set_defaults(**defaults) - - usage = """\ -Usage instructions for runtests.py. -All arguments are optional. -If --chrome is specified, chrome tests will be run instead of web content tests. -If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. -See for details on the logging levels.""" - self.set_usage(usage) - - def verifyOptions(self, options, mochitest): - """ verify correct options and cleanup paths """ - - if options.totalChunks is not None and options.thisChunk is None: - self.error("thisChunk must be specified when totalChunks is specified") - - if options.totalChunks: - if not 1 <= options.thisChunk <= options.totalChunks: - self.error("thisChunk must be between 1 and totalChunks") - - if options.xrePath is None: - # default xrePath to the app path if not provided - # but only if an app path was explicitly provided - if options.app != self.defaults['app']: - options.xrePath = os.path.dirname(options.app) - else: - # otherwise default to dist/bin - options.xrePath = self._automation.DIST_BIN - - # allow relative paths - options.xrePath = mochitest.getFullPath(options.xrePath) - - options.profilePath = mochitest.getFullPath(options.profilePath) - - options.app = mochitest.getFullPath(options.app) - if not os.path.exists(options.app): - msg = """\ - Error: Path %(app)s doesn't exist. - Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" - print msg % {"app": options.app} - return None - - options.utilityPath = mochitest.getFullPath(options.utilityPath) - options.certPath = mochitest.getFullPath(options.certPath) - if options.symbolsPath and not isURL(options.symbolsPath): - options.symbolsPath = mochitest.getFullPath(options.symbolsPath) - - options.webServer = self._automation.DEFAULT_WEB_SERVER - options.httpPort = self._automation.DEFAULT_HTTP_PORT - options.sslPort = self._automation.DEFAULT_SSL_PORT - options.webSocketPort = self._automation.DEFAULT_WEBSOCKET_PORT - - if options.vmwareRecording: - if not self._automation.IS_WIN32: - self.error("use-vmware-recording is only supported on Windows.") - mochitest.vmwareHelperPath = os.path.join( - options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") - if not os.path.exists(mochitest.vmwareHelperPath): - self.error("%s not found, cannot automate VMware recording." % - mochitest.vmwareHelperPath) - - if options.runOnlyTests != None and options.excludeTests != None: - self.error("We can only support --run-only-tests OR --exclude-tests, not both. Please consider using --test-manifest instead.") - - if options.testManifest != None and (options.runOnlyTests != None or options.excludeTests != None): - self.error("Please use --test-manifest only and not --run-only-tests or --exclude-tests.") - - if options.runOnlyTests: - if not os.path.exists(os.path.abspath(options.runOnlyTests)): - self.error("unable to find --run-only-tests file '%s'" % options.runOnlyTests); - options.testManifest = options.runOnlyTests - options.runOnly = True - - if options.excludeTests: - if not os.path.exists(os.path.abspath(options.excludeTests)): - self.error("unable to find --exclude-tests file '%s'" % options.excludeTests); - options.testManifest = options.excludeTests - options.runOnly = False - - if options.webapprtContent and options.webapprtChrome: - self.error("Only one of --webapprt-content and --webapprt-chrome may be given.") - - # Try to guess the testing modules directory. - # This somewhat grotesque hack allows the buildbot machines to find the - # modules directory without having to configure the buildbot hosts. This - # code should never be executed in local runs because the build system - # should always set the flag that populates this variable. If buildbot ever - # passes this argument, this code can be deleted. - if options.testingModulesDir is None: - possible = os.path.join(os.getcwd(), os.path.pardir, 'modules') - - if os.path.isdir(possible): - options.testingModulesDir = possible - - # Even if buildbot is updated, we still want this, as the path we pass in - # to the app must be absolute and have proper slashes. - if options.testingModulesDir is not None: - options.testingModulesDir = os.path.normpath(options.testingModulesDir) - - if not os.path.isabs(options.testingModulesDir): - options.testingModulesDir = os.path.abspath(options.testingModulesDir) - - if not os.path.isdir(options.testingModulesDir): - self.error('--testing-modules-dir not a directory: %s' % - options.testingModulesDir) - - options.testingModulesDir = options.testingModulesDir.replace('\\', '/') - if options.testingModulesDir[-1] != '/': - options.testingModulesDir += '/' - - if options.immersiveMode: - if not self._automation.IS_WIN32: - self.error("immersive is only supported on Windows 8 and up.") - mochitest.immersiveHelperPath = os.path.join( - options.utilityPath, "metrotestharness.exe") - if not os.path.exists(mochitest.immersiveHelperPath): - self.error("%s not found, cannot launch immersive tests." % - mochitest.immersiveHelperPath) - - if options.runUntilFailure: - if not os.path.isfile(os.path.join(mochitest.oldcwd, os.path.dirname(__file__), mochitest.getTestRoot(options), options.testPath)): - self.error("--run-until-failure can only be used together with --test-path specifying a single test.") - if not options.repeat: - options.repeat = 29 - - return options +log = mozlog.getLogger('Mochitest') ####################### @@ -406,24 +39,27 @@ class MochitestServer: "Web server used to serve Mochitests, for closer fidelity to the real web." def __init__(self, automation, options): - self._automation = automation - self._closeWhenDone = options.closeWhenDone - self._utilityPath = options.utilityPath - self._xrePath = options.xrePath - self._profileDir = options.profilePath - self.webServer = options.webServer - self.httpPort = options.httpPort + if isinstance(options, optparse.Values): + options = vars(options) + self._automation = automation or Automation() + self._closeWhenDone = options['closeWhenDone'] + self._utilityPath = options['utilityPath'] + self._xrePath = options['xrePath'] + self._profileDir = options['profilePath'] + self.webServer = options['webServer'] + self.httpPort = options['httpPort'] self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort } - self.testPrefix = "'webapprt_'" if options.webapprtContent else "undefined" - if options.httpdPath: - self._httpdPath = options.httpdPath + self.testPrefix = "'webapprt_'" if options.get('webapprtContent') else "undefined" + + if options.get('httpdPath'): + self._httpdPath = options['httpdPath'] else: self._httpdPath = '.' self._httpdPath = os.path.abspath(self._httpdPath) def start(self): "Run the Mochitest server, returning the process ID of the server." - + env = self._automation.environment(xrePath = self._xrePath) env["XPCOM_DEBUG_BREAK"] = "warn" @@ -433,7 +69,7 @@ class MochitestServer: # features. env["ASAN_OPTIONS"] = "quarantine_size=1:redzone=32" - if self._automation.IS_WIN32: + if mozinfo.isWin: env["PATH"] = env["PATH"] + ";" + self._xrePath args = ["-g", self._xrePath, @@ -446,13 +82,13 @@ class MochitestServer: "-f", "./" + "server.js"] xpcshell = os.path.join(self._utilityPath, - "xpcshell" + self._automation.BIN_SUFFIX) + "xpcshell" + mozinfo.info['bin_suffix']) self._process = self._automation.Process([xpcshell] + args, env = env) pid = self._process.pid if pid < 0: - print "Error starting server." + log.error("Error starting server.") sys.exit(2) - self._automation.log.info("INFO | runtests.py | Server pid: %d", pid) + log.info("runtests.py | Server pid: %d", pid) def ensureReady(self, timeout): assert timeout >= 0 @@ -465,7 +101,7 @@ class MochitestServer: time.sleep(1) i += 1 else: - print "Timed out while waiting for server startup." + log.error("Timed out while waiting for server startup.") self.stop() sys.exit(1) @@ -511,137 +147,48 @@ class WebSocketServer(object): self._process = self._automation.Process(cmd) pid = self._process.pid if pid < 0: - print "Error starting websocket server." + log.error("Error starting websocket server.") sys.exit(2) - self._automation.log.info("INFO | runtests.py | Websocket server pid: %d", pid) + log.info("runtests.py | Websocket server pid: %d", pid) def stop(self): self._process.kill() -class Mochitest(object): +class MochitestUtilsMixin(object): + """ + Class containing some utility functions common to both local and remote + mochitest runners + """ + + # TODO Utility classes are a code smell. This class is temporary + # and should be removed when desktop mochitests are refactored + # on top of mozbase. Each of the functions in here should + # probably live somewhere in mozbase + + oldcwd = os.getcwd() + jarDir = 'mochijar' + # Path to the test script on the server TEST_PATH = "tests" CHROME_PATH = "redirect.html" urlOpts = [] - runSSLTunnel = True - vmwareHelper = None - oldcwd = os.getcwd() - - def __init__(self, automation): - self.automation = automation - - # Max time in seconds to wait for server startup before tests will fail -- if - # this seems big, it's mostly for debug machines where cold startup - # (particularly after a build) takes forever. - if self.automation.IS_DEBUG_BUILD: - self.SERVER_STARTUP_TIMEOUT = 180 - else: - self.SERVER_STARTUP_TIMEOUT = 90 - - self.SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) - os.chdir(self.SCRIPT_DIRECTORY) + def __init__(self): + os.chdir(SCRIPT_DIR) + mozinfo.find_and_update_from_json(SCRIPT_DIR) def getFullPath(self, path): " Get an absolute path relative to self.oldcwd." return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path))) - def buildTestPath(self, options): - """ Build the url path to the specific test harness and test file or directory """ - testHost = "http://mochi.test:8888" - testURL = ("/").join([testHost, self.TEST_PATH, options.testPath]) - if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, options.testPath)) and options.repeat > 0: - testURL = ("/").join([testHost, self.TEST_PATH, os.path.dirname(options.testPath)]) - if options.chrome or options.a11y: - testURL = ("/").join([testHost, self.CHROME_PATH]) - elif options.browserChrome: - testURL = "about:blank" - elif options.ipcplugins: - testURL = ("/").join([testHost, self.TEST_PATH, "dom/plugins/test"]) - return testURL - - def startWebSocketServer(self, options, debuggerInfo): - """ Launch the websocket server """ - if options.webServer != '127.0.0.1': - return - - self.wsserver = WebSocketServer(self.automation, options, - self.SCRIPT_DIRECTORY, debuggerInfo) - self.wsserver.start() - - def stopWebSocketServer(self, options): - if options.webServer != '127.0.0.1': - return - - self.wsserver.stop() - - def startWebServer(self, options): - if options.webServer != '127.0.0.1': - return - - """ Create the webserver and start it up """ - self.server = MochitestServer(self.automation, options) - self.server.start() - - # If we're lucky, the server has fully started by now, and all paths are - # ready, etc. However, xpcshell cold start times suck, at least for debug - # builds. We'll try to connect to the server for awhile, and if we fail, - # we'll try to kill the server and exit with an error. - self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) - - def stopWebServer(self, options): - """ Server's no longer needed, and perhaps more importantly, anything it might - spew to console shouldn't disrupt the leak information table we print next. - """ - if options.webServer != '127.0.0.1': - return - - self.server.stop() - def getLogFilePath(self, logFile): - """ return the log file path relative to the device we are testing on, in most cases + """ return the log file path relative to the device we are testing on, in most cases it will be the full path on the local system """ return self.getFullPath(logFile) - def buildProfile(self, options): - """ create the profile and add optional chrome bits and files if requested """ - if options.browserChrome and options.timeout: - options.extraPrefs.append("testing.browserTestHarness.timeout=%d" % options.timeout) - self.automation.initializeProfile(options.profilePath, - options.extraPrefs, - useServerLocations=True, - prefsPath=os.path.join(self.SCRIPT_DIRECTORY, - 'profile_data', 'prefs_general.js')) - manifest = self.addChromeToProfile(options) - self.copyExtraFilesToProfile(options) - self.installExtensionsToProfile(options) - return manifest - - def buildBrowserEnv(self, options): - """ build the environment variables for the specific test and operating system """ - browserEnv = self.automation.environment(xrePath = options.xrePath) - - # These variables are necessary for correct application startup; change - # via the commandline at your own risk. - browserEnv["XPCOM_DEBUG_BREAK"] = "stack" - - for v in options.environment: - ix = v.find("=") - if ix <= 0: - print "Error: syntax error in --setenv=" + v - return None - browserEnv[v[:ix]] = v[ix + 1:] - - browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file - - if options.fatalAssertions: - browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort" - - return browserEnv - def buildURLOptions(self, options, env): - """ Add test control options from the command line to the url + """ Add test control options from the command line to the url URL parameters to test URL: @@ -654,7 +201,7 @@ class Mochitest(object): timeout -- per-test timeout in seconds repeat -- How many times to repeat the test, ie: repeat=1 will run the test twice. """ - + # allow relative paths for logFile if options.logFile: options.logFile = self.getLogFilePath(options.logFile) @@ -698,6 +245,224 @@ class Mochitest(object): if options.runSlower: self.urlOpts.append("runSlower=true") + def buildTestPath(self, options): + """ Build the url path to the specific test harness and test file or directory """ + testHost = "http://mochi.test:8888" + testURL = ("/").join([testHost, self.TEST_PATH, options.testPath]) + if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, options.testPath)) and options.repeat > 0: + testURL = ("/").join([testHost, self.PLAIN_LOOP_PATH]) + if options.chrome or options.a11y: + testURL = ("/").join([testHost, self.CHROME_PATH]) + elif options.browserChrome: + testURL = "about:blank" + elif options.ipcplugins: + testURL = ("/").join([testHost, self.TEST_PATH, "dom/plugins/test"]) + return testURL + + def startWebSocketServer(self, options, debuggerInfo): + """ Launch the websocket server """ + if options.webServer != '127.0.0.1': + return + + self.wsserver = WebSocketServer(self.automation, options, + SCRIPT_DIR, debuggerInfo) + self.wsserver.start() + + def stopWebSocketServer(self, options): + if options.webServer != '127.0.0.1': + return + + self.wsserver.stop() + + def startWebServer(self, options): + if options.webServer != '127.0.0.1': + return + + """ Create the webserver and start it up """ + self.server = MochitestServer(self.automation, options) + self.server.start() + + # If we're lucky, the server has fully started by now, and all paths are + # ready, etc. However, xpcshell cold start times suck, at least for debug + # builds. We'll try to connect to the server for awhile, and if we fail, + # we'll try to kill the server and exit with an error. + self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) + + def stopWebServer(self, options): + """ Server's no longer needed, and perhaps more importantly, anything it might + spew to console shouldn't disrupt the leak information table we print next. + """ + if options.webServer != '127.0.0.1': + return + + self.server.stop() + + + def copyExtraFilesToProfile(self, options): + "Copy extra files or dirs specified on the command line to the testing profile." + for f in options.extraProfileFiles: + abspath = self.getFullPath(f) + if os.path.isfile(abspath): + shutil.copy2(abspath, options.profilePath) + elif os.path.isdir(abspath): + dest = os.path.join(options.profilePath, os.path.basename(abspath)) + shutil.copytree(abspath, dest) + else: + log.warning("runtests.py | Failed to copy %s to profile", abspath) + continue + + def copyTestsJarToProfile(self, options): + """ copy tests.jar to the profile directory so we can auto register it in the .xul harness """ + testsJarFile = os.path.join(SCRIPT_DIR, "tests.jar") + if not os.path.isfile(testsJarFile): + return False + + shutil.copy2(testsJarFile, options.profilePath) + return True + + def installChromeJar(self, chrome, options): + """ + copy mochijar directory to profile as an extension so we have chrome://mochikit for all harness code + """ + # Write chrome.manifest. + with open(os.path.join(options.profilePath, "extensions", "staged", "mochikit@mozilla.org", "chrome.manifest"), "a") as mfile: + mfile.write(chrome) + + def addChromeToProfile(self, options): + "Adds MochiKit chrome tests to the profile." + + # Create (empty) chrome directory. + chromedir = os.path.join(options.profilePath, "chrome") + os.mkdir(chromedir) + + # Write userChrome.css. + chrome = """ +@namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */ +toolbar, +toolbarpalette { + background-color: rgb(235, 235, 235) !important; +} +toolbar#nav-bar { + background-image: none !important; +} +""" + with open(os.path.join(options.profilePath, "userChrome.css"), "a") as chromeFile: + chromeFile.write(chrome) + + # Call copyTestsJarToProfile(), Write tests.manifest. + manifest = os.path.join(options.profilePath, "tests.manifest") + with open(manifest, "w") as manifestFile: + if self.copyTestsJarToProfile(options): + # Register tests.jar. + manifestFile.write("content mochitests jar:tests.jar!/content/\n"); + else: + # Register chrome directory. + chrometestDir = os.path.abspath(".") + "/" + if self.automation.IS_WIN32: + chrometestDir = "file:///" + chrometestDir.replace("\\", "/") + manifestFile.write("content mochitests %s contentaccessible=yes\n" % chrometestDir) + + if options.testingModulesDir is not None: + manifestFile.write("resource testing-common file:///%s\n" % + options.testingModulesDir) + + # Call installChromeJar(). + if not os.path.isdir(os.path.join(SCRIPT_DIR, self.jarDir)): + log.testFail("invalid setup: missing mochikit extension") + return None + + # Support Firefox (browser), B2G (shell), SeaMonkey (navigator), and Webapp + # Runtime (webapp). + chrome = "" + if options.browserChrome or options.chrome or options.a11y or options.webapprtChrome: + chrome += """ +overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-test-overlay.xul +overlay chrome://browser/content/shell.xul chrome://mochikit/content/browser-test-overlay.xul +overlay chrome://navigator/content/navigator.xul chrome://mochikit/content/browser-test-overlay.xul +overlay chrome://webapprt/content/webapp.xul chrome://mochikit/content/browser-test-overlay.xul +""" + + self.installChromeJar(chrome, options) + return manifest + + + def getExtensionsToInstall(self, options): + "Return a list of extensions to install in the profile" + extensions = options.extensionsToInstall or [] + appDir = options.app[:options.app.rfind(os.sep)] if options.app else options.utilityPath + + extensionDirs = [ + # Extensions distributed with the test harness. + os.path.normpath(os.path.join(SCRIPT_DIR, "extensions")), + ] + if appDir: + # Extensions distributed with the application. + extensionDirs.append(os.path.join(appDir, "distribution", "extensions")) + + for extensionDir in extensionDirs: + if os.path.isdir(extensionDir): + for dirEntry in os.listdir(extensionDir): + if dirEntry not in options.extensionsToExclude: + path = os.path.join(extensionDir, dirEntry) + if os.path.isdir(path) or (os.path.isfile(path) and path.endswith(".xpi")): + extensions.append(path) + + # append mochikit + extensions.append(os.path.join(SCRIPT_DIR, self.jarDir)) + return extensions + + +class Mochitest(MochitestUtilsMixin): + runSSLTunnel = True + vmwareHelper = None + + def __init__(self, automation=None): + super(Mochitest, self).__init__() + self.automation = automation or Automation() + + # Max time in seconds to wait for server startup before tests will fail -- if + # this seems big, it's mostly for debug machines where cold startup + # (particularly after a build) takes forever. + if self.automation.IS_DEBUG_BUILD: + self.SERVER_STARTUP_TIMEOUT = 180 + else: + self.SERVER_STARTUP_TIMEOUT = 90 + + def buildProfile(self, options): + """ create the profile and add optional chrome bits and files if requested """ + if options.browserChrome and options.timeout: + options.extraPrefs.append("testing.browserTestHarness.timeout=%d" % options.timeout) + self.automation.initializeProfile(options.profilePath, + options.extraPrefs, + useServerLocations=True, + prefsPath=os.path.join(SCRIPT_DIR, + 'profile_data', 'prefs_general.js')) + self.copyExtraFilesToProfile(options) + self.installExtensionsToProfile(options) + return self.addChromeToProfile(options) + + def buildBrowserEnv(self, options): + """ build the environment variables for the specific test and operating system """ + browserEnv = self.automation.environment(xrePath = options.xrePath) + + # These variables are necessary for correct application startup; change + # via the commandline at your own risk. + browserEnv["XPCOM_DEBUG_BREAK"] = "stack" + + for v in options.environment: + ix = v.find("=") + if ix <= 0: + log.error("Error: syntax error in --setenv=" + v) + return None + browserEnv[v[:ix]] = v[ix + 1:] + + browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file + + if options.fatalAssertions: + browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort" + + return browserEnv + def cleanup(self, manifest, options): """ remove temporary files and profile """ os.remove(manifest) @@ -709,28 +474,28 @@ class Mochitest(object): from ctypes import cdll self.vmwareHelper = cdll.LoadLibrary(self.vmwareHelperPath) if self.vmwareHelper is None: - self.automation.log.warning("WARNING | runtests.py | Failed to load " - "VMware recording helper") + log.warning("runtests.py | Failed to load " + "VMware recording helper") return - self.automation.log.info("INFO | runtests.py | Starting VMware recording.") + log.info("runtests.py | Starting VMware recording.") try: self.vmwareHelper.StartRecording() except Exception, e: - self.automation.log.warning("WARNING | runtests.py | Failed to start " - "VMware recording: (%s)" % str(e)) + log.warning("runtests.py | Failed to start " + "VMware recording: (%s)" % str(e)) self.vmwareHelper = None def stopVMwareRecording(self): """ stops recording inside VMware VM using the recording helper dll """ assert(self.automation.IS_WIN32) if self.vmwareHelper is not None: - self.automation.log.info("INFO | runtests.py | Stopping VMware " - "recording.") + log.info("runtests.py | Stopping VMware " + "recording.") try: self.vmwareHelper.StopRecording() except Exception, e: - self.automation.log.warning("WARNING | runtests.py | Failed to stop " - "VMware recording: (%s)" % str(e)) + log.warning("runtests.py | Failed to stop " + "VMware recording: (%s)" % str(e)) self.vmwareHelper = None def runTests(self, options, onLaunch=None): @@ -780,7 +545,7 @@ class Mochitest(object): if options.vmwareRecording: self.startVMwareRecording(options); - self.automation.log.info("INFO | runtests.py | Running tests: start.\n") + log.info("runtests.py | Running tests: start.\n") try: status = self.automation.runApp(testURL, browserEnv, options.app, options.profilePath, options.browserArgs, @@ -793,10 +558,11 @@ class Mochitest(object): timeout=timeout, onLaunch=onLaunch) except KeyboardInterrupt: - self.automation.log.info("INFO | runtests.py | Received keyboard interrupt.\n"); + log.info("runtests.py | Received keyboard interrupt.\n"); status = -1 except: - self.automation.log.exception("INFO | runtests.py | Received unexpected exception while running application\n") + traceback.print_exc() + log.error("runtests.py | Received unexpected exception while running application\n") status = 1 if options.vmwareRecording: @@ -806,7 +572,7 @@ class Mochitest(object): self.stopWebSocketServer(options) processLeakLog(self.leak_report_file, options.leakThreshold) - self.automation.log.info("\nINFO | runtests.py | Running tests: end.") + log.info("runtests.py | Running tests: end.") if manifest is not None: self.cleanup(manifest, options) @@ -849,7 +615,7 @@ class Mochitest(object): #TODO: when we upgrade to python 2.6, just use json.dumps(options.__dict__) content = "{" - content += '"testRoot": "%s", ' % (testRoot) + content += '"testRoot": "%s", ' % (testRoot) first = True for opt in options.__dict__.keys(): val = options.__dict__[opt] @@ -878,121 +644,11 @@ class Mochitest(object): return 'chrome' return self.TEST_PATH - def addChromeToProfile(self, options): - "Adds MochiKit chrome tests to the profile." - - # Create (empty) chrome directory. - chromedir = os.path.join(options.profilePath, "chrome") - os.mkdir(chromedir) - - # Write userChrome.css. - chrome = """ -@namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */ -toolbar, -toolbarpalette { - background-color: rgb(235, 235, 235) !important; -} -toolbar#nav-bar { - background-image: none !important; -} -""" - with open(os.path.join(options.profilePath, "userChrome.css"), "a") as chromeFile: - chromeFile.write(chrome) - - # Call copyTestsJarToProfile(), Write tests.manifest. - manifest = os.path.join(options.profilePath, "tests.manifest") - with open(manifest, "w") as manifestFile: - if self.copyTestsJarToProfile(options): - # Register tests.jar. - manifestFile.write("content mochitests jar:tests.jar!/content/\n"); - else: - # Register chrome directory. - chrometestDir = os.path.abspath(".") + "/" - if self.automation.IS_WIN32: - chrometestDir = "file:///" + chrometestDir.replace("\\", "/") - manifestFile.write("content mochitests %s contentaccessible=yes\n" % chrometestDir) - - if options.testingModulesDir is not None: - manifestFile.write("resource testing-common file:///%s\n" % - options.testingModulesDir) - - # Call installChromeJar(). - jarDir = "mochijar" - if not os.path.isdir(os.path.join(self.SCRIPT_DIRECTORY, jarDir)): - self.automation.log.warning("TEST-UNEXPECTED-FAIL | invalid setup: missing mochikit extension") - return None - - # Support Firefox (browser), B2G (shell), SeaMonkey (navigator), and Webapp - # Runtime (webapp). - chrome = "" - if options.browserChrome or options.chrome or options.a11y or options.webapprtChrome: - chrome += """ -overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-test-overlay.xul -overlay chrome://browser/content/shell.xul chrome://mochikit/content/browser-test-overlay.xul -overlay chrome://navigator/content/navigator.xul chrome://mochikit/content/browser-test-overlay.xul -overlay chrome://webapprt/content/webapp.xul chrome://mochikit/content/browser-test-overlay.xul -""" - - self.installChromeJar(jarDir, chrome, options) - return manifest - - def installChromeJar(self, jarDirName, chrome, options): - """ - copy mochijar directory to profile as an extension so we have chrome://mochikit for all harness code - """ - self.automation.installExtension(os.path.join(self.SCRIPT_DIRECTORY, jarDirName), \ - options.profilePath, "mochikit@mozilla.org") - - # Write chrome.manifest. - with open(os.path.join(options.profilePath, "extensions", "staged", "mochikit@mozilla.org", "chrome.manifest"), "a") as mfile: - mfile.write(chrome) - - def copyTestsJarToProfile(self, options): - """ copy tests.jar to the profile directory so we can auto register it in the .xul harness """ - testsJarFile = os.path.join(self.SCRIPT_DIRECTORY, "tests.jar") - if not os.path.isfile(testsJarFile): - return False - - shutil.copy2(testsJarFile, options.profilePath) - return True - - def copyExtraFilesToProfile(self, options): - "Copy extra files or dirs specified on the command line to the testing profile." - for f in options.extraProfileFiles: - abspath = self.getFullPath(f) - if os.path.isfile(abspath): - shutil.copy2(abspath, options.profilePath) - elif os.path.isdir(abspath): - dest = os.path.join(options.profilePath, os.path.basename(abspath)) - shutil.copytree(abspath, dest) - else: - self.automation.log.warning("WARNING | runtests.py | Failed to copy %s to profile", abspath) - continue - - def getExtensionsToInstall(self, options): - "Return a list of extensions to install in the profile" - extensions = options.extensionsToInstall or [] - extensionDirs = [ - # Extensions distributed with the test harness. - os.path.normpath(os.path.join(self.SCRIPT_DIRECTORY, "extensions")), - # Extensions distributed with the application. - os.path.join(options.app[ : options.app.rfind(os.sep)], "distribution", "extensions") - ] - - for extensionDir in extensionDirs: - if os.path.isdir(extensionDir): - for dirEntry in os.listdir(extensionDir): - if dirEntry not in options.extensionsToExclude: - path = os.path.join(extensionDir, dirEntry) - if os.path.isdir(path) or (os.path.isfile(path) and path.endswith(".xpi")): - extensions.append(path) - return extensions - def installExtensionFromPath(self, options, path, extensionID = None): extensionPath = self.getFullPath(path) - self.automation.log.info("INFO | runtests.py | Installing extension at %s to %s." % - (extensionPath, options.profilePath)) + log.info("runtests.py | Installing extension at %s to %s." % + (extensionPath, options.profilePath)) self.automation.installExtension(extensionPath, options.profilePath, extensionID) @@ -1004,7 +660,7 @@ overlay chrome://webapprt/content/webapp.xul chrome://mochikit/content/browser-t def main(): automation = Automation() mochitest = Mochitest(automation) - parser = MochitestOptions(automation, mochitest.SCRIPT_DIRECTORY) + parser = MochitestOptions(automation) options, args = parser.parse_args() options = parser.verifyOptions(options, mochitest) @@ -1016,9 +672,9 @@ def main(): if options.symbolsPath and not isURL(options.symbolsPath): options.symbolsPath = mochitest.getFullPath(options.symbolsPath) - automation.setServerInfo(options.webServer, - options.httpPort, - options.sslPort, + automation.setServerInfo(options.webServer, + options.httpPort, + options.sslPort, options.webSocketPort) sys.exit(mochitest.runTests(options)) diff --git a/testing/mochitest/runtestsb2g.py b/testing/mochitest/runtestsb2g.py index 22ff02357996..6a06c73b3a75 100644 --- a/testing/mochitest/runtestsb2g.py +++ b/testing/mochitest/runtestsb2g.py @@ -2,8 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -import ConfigParser import os +import posixpath import shutil import sys import tempfile @@ -18,25 +18,36 @@ except ImportError: here = os.path.abspath(os.path.dirname(sys.argv[0])) sys.path.insert(0, here) -from automation import Automation -from b2gautomation import B2GRemoteAutomation, B2GDesktopAutomation -from runtests import Mochitest +from runtests import MochitestUtilsMixin from runtests import MochitestOptions from runtests import MochitestServer +from mochitest_options import B2GOptions, MochitestOptions from marionette import Marionette -from mozdevice import DeviceManagerADB, DMError -from mozprofile import Profile, Preferences +from mozdevice import DeviceManagerADB +from mozprofile import Profile, Preferences, DEFAULT_PORTS +from mozrunner import B2GRunner +import mozlog +import mozinfo +import moznetwork -class B2GMochitest(Mochitest): - def __init__(self, automation, OOP=True, profile_data_dir=None, - locations=os.path.join(here, 'server-locations.txt')): - Mochitest.__init__(self, automation) - self.OOP = OOP +log = mozlog.getLogger('Mochitest') + +class B2GMochitest(MochitestUtilsMixin): + def __init__(self, marionette, + out_of_process=True, + profile_data_dir=None, + locations=os.path.join(here, 'server-locations.txt')): + super(B2GMochitest, self).__init__() + self.marionette = marionette + self.out_of_process = out_of_process self.locations = locations self.preferences = [] self.webapps = None + self.test_script = os.path.join(here, 'b2g_start_script.js') + self.test_script_args = [self.out_of_process] + self.product = 'b2g' if profile_data_dir: self.preferences = [os.path.join(profile_data_dir, f) @@ -44,73 +55,19 @@ class B2GMochitest(Mochitest): self.webapps = [os.path.join(profile_data_dir, f) for f in os.listdir(profile_data_dir) if f.startswith('webapp')] - def setupCommonOptions(self, options): - # set the testURL - testURL = self.buildTestPath(options) - if len(self.urlOpts) > 0: - testURL += "?" + "&".join(self.urlOpts) - self.automation.testURL = testURL - - if self.OOP: - OOP_script = """ -let specialpowers = {}; -let loader = Cc["@mozilla.org/moz/jssubscript-loader;1"].getService(Ci.mozIJSSubScriptLoader); -loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserver.js", specialpowers); -let specialPowersObserver = new specialpowers.SpecialPowersObserver(); -specialPowersObserver.init(); - -let mm = container.QueryInterface(Ci.nsIFrameLoaderOwner).frameLoader.messageManager; -mm.addMessageListener("SPPrefService", specialPowersObserver); -mm.addMessageListener("SPProcessCrashService", specialPowersObserver); -mm.addMessageListener("SPPingService", specialPowersObserver); -mm.addMessageListener("SpecialPowers.Quit", specialPowersObserver); -mm.addMessageListener("SpecialPowers.Focus", specialPowersObserver); -mm.addMessageListener("SPPermissionManager", specialPowersObserver); - -mm.loadFrameScript(CHILD_LOGGER_SCRIPT, true); -mm.loadFrameScript(CHILD_SCRIPT_API, true); -mm.loadFrameScript(CHILD_SCRIPT, true); -specialPowersObserver._isFrameScriptLoaded = true; -""" + # mozinfo is populated by the parent class + if mozinfo.info['debug']: + self.SERVER_STARTUP_TIMEOUT = 180 else: - OOP_script = "" + self.SERVER_STARTUP_TIMEOUT = 90 - # Execute this script on start up: loads special powers and sets - # the test-container apps's iframe to the mochitest URL. - self.automation.test_script = """ -const CHILD_SCRIPT = "chrome://specialpowers/content/specialpowers.js"; -const CHILD_SCRIPT_API = "chrome://specialpowers/content/specialpowersAPI.js"; -const CHILD_LOGGER_SCRIPT = "chrome://specialpowers/content/MozillaLogger.js"; + def setup_common_options(self, options): + test_url = self.buildTestPath(options) + if len(self.urlOpts) > 0: + test_url += "?" + "&".join(self.urlOpts) + self.test_script_args.append(test_url) -let homescreen = document.getElementById('homescreen'); -let container = homescreen.contentWindow.document.getElementById('test-container'); - -function openWindow(aEvent) { - var popupIframe = aEvent.detail.frameElement; - popupIframe.setAttribute('style', 'position: absolute; left: 0; top: 300px; background: white; '); - - popupIframe.addEventListener('mozbrowserclose', function(e) { - container.parentNode.removeChild(popupIframe); - container.focus(); - }); - - // yes, the popup can call window.open too! - popupIframe.addEventListener('mozbrowseropenwindow', openWindow); - - popupIframe.addEventListener('mozbrowserloadstart', function(e) { - popupIframe.focus(); - }); - - container.parentNode.appendChild(popupIframe); -} - -container.addEventListener('mozbrowseropenwindow', openWindow); -%s - -container.src = '%s'; -""" % (OOP_script, testURL) - - def buildProfile(self, options): + def build_profile(self, options): # preferences prefs = {} for path in self.preferences: @@ -125,7 +82,7 @@ container.src = '%s'; # interpolate the preferences interpolation = { "server": "%s:%s" % (options.webServer, options.httpPort), - "OOP": "true" if self.OOP else "false" } + "OOP": "true" if self.out_of_process else "false" } prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) @@ -149,274 +106,74 @@ container.src = '%s'; self.copyExtraFilesToProfile(options) return manifest + def run_tests(self, options): + """ Prepare, configure, run tests and cleanup """ -class B2GOptions(MochitestOptions): + self.leak_report_file = os.path.join(options.profilePath, "runtests_leaks.log") + manifest = self.build_profile(options) - def __init__(self, automation, scriptdir, **kwargs): - defaults = {} - MochitestOptions.__init__(self, automation, scriptdir) + self.startWebServer(options) + self.startWebSocketServer(options, None) + self.buildURLOptions(options, {'MOZ_HIDE_RESULTS_TABLE': '1'}) - self.add_option("--b2gpath", action="store", - type="string", dest="b2gPath", - help="path to B2G repo or qemu dir") - defaults["b2gPath"] = None + if options.timeout: + timeout = options.timeout + 30 + elif options.debugger or not options.autorun: + timeout = None + else: + timeout = 330.0 # default JS harness timeout is 300 seconds - self.add_option("--desktop", action="store_true", - dest="desktop", - help="Run the tests on a B2G desktop build") - defaults["desktop"] = False + log.info("runtestsb2g.py | Running tests: start.") + status = 0 + try: + runner_args = { 'profile': self.profile, + 'devicemanager': self._dm, + 'marionette': self.marionette, + 'remote_test_root': self.remote_test_root, + 'test_script': self.test_script, + 'test_script_args': self.test_script_args } + self.runner = B2GRunner(**runner_args) + self.runner.start(outputTimeout=timeout) + self.runner.wait() + except KeyboardInterrupt: + log.info("runtests.py | Received keyboard interrupt.\n"); + status = -1 + except: + traceback.print_exc() + log.error("runtests.py | Received unexpected exception while running application\n") + status = 1 - self.add_option("--marionette", action="store", - type="string", dest="marionette", - help="host:port to use when connecting to Marionette") - defaults["marionette"] = None + self.stopWebServer(options) + self.stopWebSocketServer(options) - self.add_option("--emulator", action="store", - type="string", dest="emulator", - help="Architecture of emulator to use: x86 or arm") - defaults["emulator"] = None + log.info("runtestsb2g.py | Running tests: end.") - self.add_option("--sdcard", action="store", - type="string", dest="sdcard", - help="Define size of sdcard: 1MB, 50MB...etc") - defaults["sdcard"] = "10MB" - - self.add_option("--no-window", action="store_true", - dest="noWindow", - help="Pass --no-window to the emulator") - defaults["noWindow"] = False - - self.add_option("--adbpath", action="store", - type="string", dest="adbPath", - help="path to adb") - defaults["adbPath"] = "adb" - - self.add_option("--deviceIP", action="store", - type="string", dest="deviceIP", - help="ip address of remote device to test") - defaults["deviceIP"] = None - - self.add_option("--devicePort", action="store", - type="string", dest="devicePort", - help="port of remote device to test") - defaults["devicePort"] = 20701 - - self.add_option("--remote-logfile", action="store", - type="string", dest="remoteLogFile", - help="Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.") - defaults["remoteLogFile"] = None - - self.add_option("--remote-webserver", action="store", - type="string", dest="remoteWebServer", - help="ip address where the remote web server is hosted at") - defaults["remoteWebServer"] = None - - self.add_option("--http-port", action="store", - type="string", dest="httpPort", - help="ip address where the remote web server is hosted at") - defaults["httpPort"] = automation.DEFAULT_HTTP_PORT - - self.add_option("--ssl-port", action="store", - type="string", dest="sslPort", - help="ip address where the remote web server is hosted at") - defaults["sslPort"] = automation.DEFAULT_SSL_PORT - - self.add_option("--pidfile", action="store", - type="string", dest="pidFile", - help="name of the pidfile to generate") - defaults["pidFile"] = "" - - self.add_option("--gecko-path", action="store", - type="string", dest="geckoPath", - help="the path to a gecko distribution that should " - "be installed on the emulator prior to test") - defaults["geckoPath"] = None - - self.add_option("--profile", action="store", - type="string", dest="profile", - help="for desktop testing, the path to the " - "gaia profile to use") - defaults["profile"] = None - - self.add_option("--logcat-dir", action="store", - type="string", dest="logcat_dir", - help="directory to store logcat dump files") - defaults["logcat_dir"] = None - - self.add_option('--busybox', action='store', - type='string', dest='busybox', - help="Path to busybox binary to install on device") - defaults['busybox'] = None - self.add_option('--profile-data-dir', action='store', - type='string', dest='profile_data_dir', - help="Path to a directory containing preference and other " - "data to be installed into the profile") - defaults['profile_data_dir'] = os.path.join(here, 'profile_data') - - defaults["remoteTestRoot"] = "/data/local/tests" - defaults["logFile"] = "mochitest.log" - defaults["autorun"] = True - defaults["closeWhenDone"] = True - defaults["testPath"] = "" - defaults["extensionsToExclude"] = ["specialpowers"] - - self.set_defaults(**defaults) - - def verifyRemoteOptions(self, options, automation): - if not options.remoteTestRoot: - options.remoteTestRoot = automation._devicemanager.getDeviceRoot() - productRoot = options.remoteTestRoot + "/" + automation._product - - if options.utilityPath == self._automation.DIST_BIN: - options.utilityPath = productRoot + "/bin" - - if options.remoteWebServer == None: - if os.name != "nt": - options.remoteWebServer = automation.getLanIp() - else: - self.error("You must specify a --remote-webserver=") - options.webServer = options.remoteWebServer - - if options.geckoPath and not options.emulator: - self.error("You must specify --emulator if you specify --gecko-path") - - if options.logcat_dir and not options.emulator: - self.error("You must specify --emulator if you specify --logcat-dir") - - #if not options.emulator and not options.deviceIP: - # print "ERROR: you must provide a device IP" - # return None - - if options.remoteLogFile == None: - options.remoteLogFile = options.remoteTestRoot + '/logs/mochitest.log' - - if options.remoteLogFile.count('/') < 1: - options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile - - # Only reset the xrePath if it wasn't provided - if options.xrePath == None: - options.xrePath = options.utilityPath - - if not os.path.isdir(options.xrePath): - self.error("--xre-path '%s' is not a directory" % options.xrePath) - xpcshell = os.path.join(options.xrePath, 'xpcshell') - if not os.access(xpcshell, os.F_OK): - self.error('xpcshell not found at %s' % xpcshell) - if automation.elf_arm(xpcshell): - self.error('--xre-path points to an ARM version of xpcshell; it ' - 'should instead point to a version that can run on ' - 'your desktop') - - if options.pidFile != "": - f = open(options.pidFile, 'w') - f.write("%s" % os.getpid()) - f.close() - - return options - - def verifyOptions(self, options, mochitest): - # since we are reusing verifyOptions, it will exit if App is not found - temp = options.app - options.app = sys.argv[0] - tempPort = options.httpPort - tempSSL = options.sslPort - tempIP = options.webServer - options = MochitestOptions.verifyOptions(self, options, mochitest) - options.webServer = tempIP - options.app = temp - options.sslPort = tempSSL - options.httpPort = tempPort - - return options - - -class ProfileConfigParser(ConfigParser.RawConfigParser): - """Subclass of RawConfigParser that outputs .ini files in the exact - format expected for profiles.ini, which is slightly different - than the default format. - """ - - def optionxform(self, optionstr): - return optionstr - - def write(self, fp): - if self._defaults: - fp.write("[%s]\n" % ConfigParser.DEFAULTSECT) - for (key, value) in self._defaults.items(): - fp.write("%s=%s\n" % (key, str(value).replace('\n', '\n\t'))) - fp.write("\n") - for section in self._sections: - fp.write("[%s]\n" % section) - for (key, value) in self._sections[section].items(): - if key == "__name__": - continue - if (value is not None) or (self._optcre == self.OPTCRE): - key = "=".join((key, str(value).replace('\n', '\n\t'))) - fp.write("%s\n" % (key)) - fp.write("\n") + if manifest is not None: + self.cleanup(manifest, options) + return status class B2GDeviceMochitest(B2GMochitest): - _automation = None _dm = None - def __init__(self, automation, devmgr, options): - self._automation = automation - B2GMochitest.__init__(self, automation, OOP=True, profile_data_dir=options.profile_data_dir) - self._dm = devmgr - self.runSSLTunnel = False - self.remoteProfile = options.remoteTestRoot + '/profile' - self._automation.setRemoteProfile(self.remoteProfile) - self.remoteLog = options.remoteLogFile - self.localLog = None - self.userJS = '/data/local/user.js' - self.remoteMozillaPath = '/data/b2g/mozilla' - self.bundlesDir = '/system/b2g/distribution/bundles' - self.remoteProfilesIniPath = os.path.join(self.remoteMozillaPath, 'profiles.ini') - self.originalProfilesIni = None + def __init__(self, marionette, devicemanager, profile_data_dir, + local_binary_dir, remote_test_root=None, remote_log_file=None): + B2GMochitest.__init__(self, marionette, out_of_process=True, profile_data_dir=profile_data_dir) + self._dm = devicemanager + self.remote_test_root = remote_test_root or self._dm.getDeviceRoot() + self.remote_profile = posixpath.join(self.remote_test_root, 'profile') + self.remote_log = remote_log_file or posixpath.join(self.remote_test_root, 'log', 'mochitest.log') + self.local_log = None + self.local_binary_dir = local_binary_dir - def copyRemoteFile(self, src, dest): - self._dm._checkCmdAs(['shell', 'dd', 'if=%s' % src, 'of=%s' % dest]) - - def origUserJSExists(self): - return self._dm.fileExists('/data/local/user.js.orig') + if not self._dm.dirExists(posixpath.dirname(self.remote_log)): + self._dm.mkDirs(self.remote_log) def cleanup(self, manifest, options): - if self.localLog: - self._dm.getFile(self.remoteLog, self.localLog) - self._dm.removeFile(self.remoteLog) - - # Delete any bundled extensions - extensionDir = os.path.join(options.profilePath, 'extensions', 'staged') - if os.access(extensionDir, os.F_OK): - for filename in os.listdir(extensionDir): - try: - self._dm._checkCmdAs(['shell', 'rm', '-rf', - os.path.join(self.bundlesDir, filename)]) - except DMError: - pass - - if not options.emulator: - # Remove the test profile - self._dm._checkCmdAs(['shell', 'rm', '-r', self.remoteProfile]) - - if self.origUserJSExists(): - # Restore the original user.js - self._dm.removeFile(self.userJS) - self.copyRemoteFile('%s.orig' % self.userJS, self.userJS) - self._dm.removeFile("%s.orig" % self.userJS) - - if self._dm.fileExists('%s.orig' % self.remoteProfilesIniPath): - # Restore the original profiles.ini - self._dm.removeFile(self.remoteProfilesIniPath) - self.copyRemoteFile('%s.orig' % self.remoteProfilesIniPath, - self.remoteProfilesIniPath) - self._dm.removeFile("%s.orig" % self.remoteProfilesIniPath) - - # We've restored the original profile, so reboot the device so that - # it gets picked up. - self._automation.rebootDevice() + if self.local_log: + self._dm.getFile(self.remote_log, self.local_log) + self._dm.removeFile(self.remote_log) if options.pidFile != "": try: @@ -425,149 +182,49 @@ class B2GDeviceMochitest(B2GMochitest): except: print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % options.pidFile - def findPath(self, paths, filename=None): - for path in paths: - p = path - if filename: - p = os.path.join(p, filename) - if os.path.exists(self.getFullPath(p)): - return path - return None + # stop and clean up the runner + if getattr(self, 'runner', False): + self.runner.cleanup() + self.runner = None def startWebServer(self, options): """ Create the webserver on the host and start it up """ - remoteXrePath = options.xrePath - remoteProfilePath = options.profilePath - remoteUtilityPath = options.utilityPath - localAutomation = Automation() - localAutomation.IS_WIN32 = False - localAutomation.IS_LINUX = False - localAutomation.IS_MAC = False - localAutomation.UNIXISH = False - hostos = sys.platform - if hostos in ['mac', 'darwin']: - localAutomation.IS_MAC = True - elif hostos in ['linux', 'linux2']: - localAutomation.IS_LINUX = True - localAutomation.UNIXISH = True - elif hostos in ['win32', 'win64']: - localAutomation.BIN_SUFFIX = ".exe" - localAutomation.IS_WIN32 = True - - paths = [options.xrePath, - localAutomation.DIST_BIN, - self._automation._product, - os.path.join('..', self._automation._product)] - options.xrePath = self.findPath(paths) - if options.xrePath == None: - print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name) - sys.exit(1) - paths.append("bin") - paths.append(os.path.join("..", "bin")) - - xpcshell = "xpcshell" - if (os.name == "nt"): - xpcshell += ".exe" - - if (options.utilityPath): - paths.insert(0, options.utilityPath) - options.utilityPath = self.findPath(paths, xpcshell) - if options.utilityPath == None: - print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name) - sys.exit(1) - # httpd-path is specified by standard makefile targets and may be specified - # on the command line to select a particular version of httpd.js. If not - # specified, try to select the one from xre.zip, as required in bug 882932. - if not options.httpdPath: - options.httpdPath = os.path.join(options.utilityPath, "components") - - options.profilePath = tempfile.mkdtemp() - self.server = MochitestServer(localAutomation, options) + d = vars(options).copy() + d['xrePath'] = self.local_binary_dir + d['utilityPath'] = self.local_binary_dir + d['profilePath'] = tempfile.mkdtemp() + if d.get('httpdPath') is None: + d['httpdPath'] = os.path.abspath(os.path.join(self.local_binary_dir, 'components')) + self.server = MochitestServer(None, d) self.server.start() if (options.pidFile != ""): f = open(options.pidFile + ".xpcshell.pid", 'w') f.write("%s" % self.server._process.pid) f.close() - self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) - - options.xrePath = remoteXrePath - options.utilityPath = remoteUtilityPath - options.profilePath = remoteProfilePath + self.server.ensureReady(90) def stopWebServer(self, options): if hasattr(self, 'server'): self.server.stop() - def updateProfilesIni(self, profilePath): - # update profiles.ini on the device to point to the test profile - self.originalProfilesIni = tempfile.mktemp() - self._dm.getFile(self.remoteProfilesIniPath, self.originalProfilesIni) - - config = ProfileConfigParser() - config.read(self.originalProfilesIni) - for section in config.sections(): - if 'Profile' in section: - config.set(section, 'IsRelative', 0) - config.set(section, 'Path', profilePath) - - newProfilesIni = tempfile.mktemp() - with open(newProfilesIni, 'wb') as configfile: - config.write(configfile) - - self._dm.pushFile(newProfilesIni, self.remoteProfilesIniPath) - self._dm.pushFile(self.originalProfilesIni, '%s.orig' % self.remoteProfilesIniPath) - - try: - os.remove(newProfilesIni) - os.remove(self.originalProfilesIni) - except: - pass - def buildURLOptions(self, options, env): - self.localLog = options.logFile - options.logFile = self.remoteLog + self.local_log = options.logFile + options.logFile = self.remote_log options.profilePath = self.profile.profile - retVal = Mochitest.buildURLOptions(self, options, env) + retVal = super(B2GDeviceMochitest, self).buildURLOptions(options, env) - self.setupCommonOptions(options) + self.setup_common_options(options) - # Copy the profile to the device. - self._dm._checkCmdAs(['shell', 'rm', '-r', self.remoteProfile]) - try: - self._dm.pushDir(options.profilePath, self.remoteProfile) - except DMError: - print "Automation Error: Unable to copy profile to device." - raise - - # Copy the extensions to the B2G bundles dir. - extensionDir = os.path.join(options.profilePath, 'extensions', 'staged') - # need to write to read-only dir - self._dm._checkCmdAs(['remount']) - for filename in os.listdir(extensionDir): - self._dm._checkCmdAs(['shell', 'rm', '-rf', - os.path.join(self.bundlesDir, filename)]) - try: - self._dm.pushDir(extensionDir, self.bundlesDir) - except DMError: - print "Automation Error: Unable to copy extensions to device." - raise - - # In B2G, user.js is always read from /data/local, not the profile - # directory. Backup the original user.js first so we can restore it. - if not self._dm.fileExists('%s.orig' % self.userJS): - self.copyRemoteFile(self.userJS, '%s.orig' % self.userJS) - self._dm.pushFile(os.path.join(options.profilePath, "user.js"), self.userJS) - self.updateProfilesIni(self.remoteProfile) - options.profilePath = self.remoteProfile - options.logFile = self.localLog + options.profilePath = self.remote_profile + options.logFile = self.local_log return retVal class B2GDesktopMochitest(B2GMochitest): - def __init__(self, automation, options): - B2GMochitest.__init__(self, automation, OOP=False, profile_data_dir=options.profile_data_dir) + def __init__(self, marionette, profile_data_dir): + B2GMochitest.__init__(self, out_of_process=False, profile_data_dir=profile_data_dir) def runMarionetteScript(self, marionette, test_script): assert(marionette.wait_for_port()) @@ -580,14 +237,14 @@ class B2GDesktopMochitest(B2GMochitest): # stdout buffer gets filled (which gets drained only after this # function returns, by waitForFinish), which causes the app to hang. thread = threading.Thread(target=self.runMarionetteScript, - args=(self.automation.marionette, - self.automation.test_script)) + args=(self.marionette, + self.test_script)) thread.start() def buildURLOptions(self, options, env): - retVal = Mochitest.buildURLOptions(self, options, env) + retVal = super(B2GDesktopMochitest, self).buildURLOptions(options, env) - self.setupCommonOptions(options) + self.setup_common_options(options) # Copy the extensions to the B2G bundles dir. extensionDir = os.path.join(options.profilePath, 'extensions', 'staged') @@ -602,12 +259,11 @@ class B2GDesktopMochitest(B2GMochitest): return retVal -def run_remote_mochitests(automation, parser, options): +def run_remote_mochitests(parser, options): # create our Marionette instance kwargs = {} if options.emulator: kwargs['emulator'] = options.emulator - automation.setEmulator(True) if options.noWindow: kwargs['noWindow'] = True if options.geckoPath: @@ -630,8 +286,6 @@ def run_remote_mochitests(automation, parser, options): marionette = Marionette.getMarionetteOrExit(**kwargs) - automation.marionette = marionette - # create the DeviceManager kwargs = {'adbPath': options.adbPath, 'deviceRoot': options.remoteTestRoot} @@ -639,28 +293,23 @@ def run_remote_mochitests(automation, parser, options): kwargs.update({'host': options.deviceIP, 'port': options.devicePort}) dm = DeviceManagerADB(**kwargs) - automation.setDeviceManager(dm) - options = parser.verifyRemoteOptions(options, automation) + options = parser.verifyRemoteOptions(options) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) - automation.setProduct("b2g") - - mochitest = B2GDeviceMochitest(automation, dm, options) + mochitest = B2GDeviceMochitest(marionette, dm, options.profile_data_dir, options.xrePath, + remote_test_root=options.remoteTestRoot, + remote_log_file=options.remoteLogFile) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) - logParent = os.path.dirname(options.remoteLogFile) - dm.mkDir(logParent) - automation.setRemoteLog(options.remoteLogFile) - automation.setServerInfo(options.webServer, options.httpPort, options.sslPort) retVal = 1 try: mochitest.cleanup(None, options) - retVal = mochitest.runTests(options) + retVal = mochitest.run_tests(options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() @@ -674,10 +323,7 @@ def run_remote_mochitests(automation, parser, options): sys.exit(retVal) - def run_desktop_mochitests(parser, options): - automation = B2GDesktopAutomation() - # create our Marionette instance kwargs = {} if options.marionette: @@ -685,9 +331,8 @@ def run_desktop_mochitests(parser, options): kwargs['host'] = host kwargs['port'] = int(port) marionette = Marionette.getMarionetteOrExit(**kwargs) - automation.marionette = marionette - mochitest = B2GDesktopMochitest(automation, options) + mochitest = B2GDesktopMochitest(marionette, options.profile_data_dir) # b2g desktop builds don't always have a b2g-bin file if options.app[-4:] == '-bin': @@ -700,24 +345,16 @@ def run_desktop_mochitests(parser, options): if options.desktop and not options.profile: raise Exception("must specify --profile when specifying --desktop") - automation.setServerInfo(options.webServer, - options.httpPort, - options.sslPort, - options.webSocketPort) - sys.exit(mochitest.runTests(options, - onLaunch=mochitest.startTests)) - + sys.exit(mochitest.runTests(options, onLaunch=mochitest.startTests)) def main(): - scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) - automation = B2GRemoteAutomation(None, "fennec") - parser = B2GOptions(automation, scriptdir) + parser = B2GOptions() options, args = parser.parse_args() if options.desktop: run_desktop_mochitests(parser, options) else: - run_remote_mochitests(automation, parser, options) + run_remote_mochitests(parser, options) if __name__ == "__main__": main() diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index db98bb57e54b..c05051a65689 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -17,8 +17,8 @@ sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]) from automation import Automation from remoteautomation import RemoteAutomation, fennecLogcatFilters from runtests import Mochitest -from runtests import MochitestOptions from runtests import MochitestServer +from mochitest_options import MochitestOptions import devicemanager import droid @@ -26,9 +26,9 @@ import manifestparser class RemoteOptions(MochitestOptions): - def __init__(self, automation, scriptdir, **kwargs): + def __init__(self, automation, **kwargs): defaults = {} - MochitestOptions.__init__(self, automation, scriptdir) + MochitestOptions.__init__(self, automation) self.add_option("--remote-app-path", action="store", type = "string", dest = "remoteAppPath", @@ -519,9 +519,8 @@ class MochiRemote(Mochitest): def main(): - scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) auto = RemoteAutomation(None, "fennec") - parser = RemoteOptions(auto, scriptdir) + parser = RemoteOptions(auto) options, args = parser.parse_args() if (options.dm_trans == "adb"):