Backout a5b75feea6dd (bug 826111) on suspicion of breaking marionette on linux on a CLOSED TREE

This commit is contained in:
Ed Morley 2013-01-15 11:04:25 +00:00
parent ccee4c2c0e
commit 9820f29c6a
6 changed files with 178 additions and 389 deletions

View File

@ -274,7 +274,7 @@ class Automation(object):
cursor.execute("PRAGMA user_version=3");
# SQL copied from nsPermissionManager.cpp
cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
cursor.execute("""CREATE TABLE moz_hosts (
id INTEGER PRIMARY KEY,
host TEXT,
type TEXT,
@ -285,24 +285,25 @@ class Automation(object):
isInBrowserElement INTEGER)""")
# Insert desired permissions
c = 0
for perm in permissions.keys():
for host,allow in permissions[perm]:
cursor.execute("INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0)",
(host, perm, 1 if allow else 2))
c += 1
cursor.execute("INSERT INTO moz_hosts values(?, ?, ?, ?, 0, 0, 0, 0)",
(c, host, perm, 1 if allow else 2))
# Commit and close
permDB.commit()
cursor.close()
def setupTestApps(self, profileDir, apps):
webappJSONTemplate = Template(""""$id": {
webappJSONTemplate = Template(""""$name": {
"origin": "$origin",
"installOrigin": "$origin",
"receipt": null,
"installTime": 132333986000,
"manifestURL": "$manifestURL",
"localId": $localId,
"id": "$id",
"appStatus": $appStatus,
"csp": "$csp"
}""")
@ -332,73 +333,16 @@ class Automation(object):
# Create webapps/webapps.json
webappsDir = os.path.join(profileDir, "webapps")
if not os.access(webappsDir, os.F_OK):
os.mkdir(webappsDir)
os.mkdir(webappsDir);
lineRe = re.compile(r'(.*?)"(.*?)": (.*)')
webappsJSONFilename = os.path.join(webappsDir, "webapps.json")
webappsJSON = []
if os.access(webappsJSONFilename, os.F_OK):
# If there is an existing webapps.json file (which will be the case for
# b2g), we parse the data in the existing file before appending test
# test apps to it.
startId = 1
webappsJSONFile = open(webappsJSONFilename, "r")
contents = webappsJSONFile.read()
for app_content in contents.split('},'):
app = {}
# ghetto json parser needed due to lack of json/simplejson on test slaves
for line in app_content.split('\n'):
m = lineRe.match(line)
if m:
value = m.groups()[2]
# remove any trailing commas
if value[-1:] == ',':
value = value[:-1]
# set the app name from a line that looks like this:
# "name.gaiamobile.org": {
if value == '{':
app['id'] = m.groups()[1]
# parse string, None, bool and int types
elif value[0:1] == '"':
app[m.groups()[1]] = value[1:-1]
elif value == "null":
app[m.groups()[1]] = None
elif value == "true":
app[m.groups()[1]] = True
elif value == "false":
app[m.groups()[1]] = False
else:
app[m.groups()[1]] = int(value)
if app:
webappsJSON.append(app)
webappsJSONFile.close()
startId = 1
for app in webappsJSON:
if app['localId'] >= startId:
startId = app['localId'] + 1
if not app.get('csp'):
app['csp'] = ''
if not app.get('appStatus'):
app['appStatus'] = 3
for localId, app in enumerate(apps):
app['localId'] = localId + startId # localId must be from 1..N
if not app.get('id'):
app['id'] = app['name']
webappsJSON.append(app)
app['localId'] = localId + 1 # Has to be 1..n
webappsJSON.append(webappJSONTemplate.substitute(app))
webappsJSON = '{\n' + ',\n'.join(webappsJSON) + '\n}\n'
contents = []
for app in webappsJSON:
contents.append(webappJSONTemplate.substitute(app))
contents = '{\n' + ',\n'.join(contents) + '\n}\n'
webappsJSONFile = open(webappsJSONFilename, "w")
webappsJSONFile.write(contents)
webappsJSONFile = open(os.path.join(webappsDir, "webapps.json"), "a")
webappsJSONFile.write(webappsJSON)
webappsJSONFile.close()
# Create manifest file for each app.
@ -412,18 +356,12 @@ class Automation(object):
manifestFile.write(manifest)
manifestFile.close()
def initializeProfile(self, profileDir, extraPrefs=[],
useServerLocations=False,
initialProfile=None):
def initializeProfile(self, profileDir, extraPrefs = [], useServerLocations = False):
" Sets up the standard testing profile."
prefs = []
# Start with a clean slate.
shutil.rmtree(profileDir, True)
if initialProfile:
shutil.copytree(initialProfile, profileDir)
else:
os.mkdir(profileDir)
# Set up permissions database
@ -1095,7 +1033,7 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t
runSSLTunnel = False, utilityPath = None,
xrePath = None, certPath = None,
debuggerInfo = None, symbolsPath = None,
timeout = -1, maxTime = None, onLaunch = None):
timeout = -1, maxTime = None):
"""
Run the app, log the duration it took to execute, return the status code.
Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
@ -1152,11 +1090,6 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t
stderr = subprocess.STDOUT)
self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
if onLaunch is not None:
# Allow callers to specify an onLaunch callback to be fired after the
# app is launched.
onLaunch()
status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath)
self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))

View File

@ -111,15 +111,10 @@ class B2GRemoteAutomation(Automation):
print "WARNING: unable to remove directory: %s" % (dumpDir)
return crashed
def initializeProfile(self, profileDir, extraPrefs=[],
useServerLocations=False,
initialProfile=None):
def initializeProfile(self, profileDir, extraPrefs = [], useServerLocations = False):
# add b2g specific prefs
extraPrefs.extend(["browser.manifestURL='dummy (bug 772307)'"])
return Automation.initializeProfile(self, profileDir,
extraPrefs,
useServerLocations,
initialProfile)
return Automation.initializeProfile(self, profileDir, extraPrefs, useServerLocations)
def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
# if remote profile is specified, use that instead
@ -346,26 +341,3 @@ class B2GRemoteAutomation(Automation):
def kill(self):
# this should never happen
raise Exception("'kill' called on B2GInstance")
class B2GDesktopAutomation(Automation):
def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
""" build the application command line """
cmd = os.path.abspath(app)
args = []
if debuggerInfo:
args.extend(debuggerInfo["args"])
args.append(cmd)
cmd = os.path.abspath(debuggerInfo["path"])
if self.IS_MAC:
args.append("-foreground")
profileDirectory = profileDir + "/"
args.extend(("-profile", profileDirectory))
args.extend(extraArgs)
return cmd, args

View File

@ -2,6 +2,9 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/
import datetime
import socket
import time
from mozrunner import Runner
@ -28,3 +31,23 @@ class GeckoInstance(object):
def close(self):
self.runner.stop()
self.runner.cleanup()
def wait_for_port(self, timeout=3000):
assert(self.marionette_port)
starttime = datetime.datetime.now()
while datetime.datetime.now() - starttime < datetime.timedelta(seconds=timeout):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.marionette_host, self.marionette_port))
data = sock.recv(16)
print "closing socket"
sock.close()
if '"from"' in data:
print "got data"
time.sleep(5)
return True
except:
import traceback
print traceback.format_exc()
time.sleep(1)
return False

View File

@ -2,10 +2,8 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import datetime
import socket
import sys
import time
import traceback
from client import MarionetteClient
@ -129,7 +127,7 @@ class Marionette(object):
self.instance = GeckoInstance(host=self.host, port=self.port,
bin=self.bin, profile=self.profile)
self.instance.start()
assert(self.wait_for_port())
assert(self.instance.wait_for_port())
if emulator:
self.emulator = Emulator(homedir=homedir,
@ -198,22 +196,6 @@ class Marionette(object):
# flagging the error.
sys.exit()
def wait_for_port(self, timeout=3000):
starttime = datetime.datetime.now()
while datetime.datetime.now() - starttime < datetime.timedelta(seconds=timeout):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host, self.port))
data = sock.recv(16)
sock.close()
if '"from"' in data:
time.sleep(1)
return True
except socket.error:
pass
time.sleep(1)
return False
def _send_message(self, command, response_key, **kwargs):
if not self.session and command not in ('newSession', 'getStatus'):
raise MarionetteException(message="Please start a session")

View File

@ -563,9 +563,7 @@ class Mochitest(object):
def buildProfile(self, options):
""" create the profile and add optional chrome bits and files if requested """
self.automation.initializeProfile(options.profilePath,
options.extraPrefs,
useServerLocations=True)
self.automation.initializeProfile(options.profilePath, options.extraPrefs, useServerLocations = True)
manifest = self.addChromeToProfile(options)
self.copyExtraFilesToProfile(options)
self.installExtensionsToProfile(options)
@ -684,7 +682,7 @@ class Mochitest(object):
"VMware recording: (%s)" % str(e))
self.vmwareHelper = None
def runTests(self, options, onLaunch=None):
def runTests(self, options):
""" Prepare, configure, run tests and cleanup """
debuggerInfo = getDebuggerInfo(self.oldcwd, options.debugger, options.debuggerArgs,
options.debuggerInteractive);
@ -737,8 +735,7 @@ class Mochitest(object):
certPath=options.certPath,
debuggerInfo=debuggerInfo,
symbolsPath=options.symbolsPath,
timeout=timeout,
onLaunch=onLaunch)
timeout = timeout)
except KeyboardInterrupt:
self.automation.log.info("INFO | runtests.py | Received keyboard interrupt.\n");
status = -1

View File

@ -4,16 +4,14 @@
import ConfigParser
import os
import shutil
import sys
import tempfile
import threading
import traceback
sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))))
from automation import Automation
from b2gautomation import B2GRemoteAutomation, B2GDesktopAutomation
from b2gautomation import B2GRemoteAutomation
from runtests import Mochitest
from runtests import MochitestOptions
from runtests import MochitestServer
@ -24,71 +22,6 @@ import devicemanagerADB
from marionette import Marionette
class B2GMochitestMixin(object):
def setupCommonOptions(self, options, OOP=True):
# set the testURL
testURL = self.buildTestPath(options)
if len(self.urlOpts) > 0:
testURL += "?" + "&".join(self.urlOpts)
self.automation.testURL = testURL
if OOP:
OOP_pref = "true"
OOP_script = """
let specialpowers = {};
let loader = Cc["@mozilla.org/moz/jssubscript-loader;1"].getService(Ci.mozIJSSubScriptLoader);
loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserver.js", specialpowers);
let specialPowersObserver = new specialpowers.SpecialPowersObserver();
specialPowersObserver.init();
let mm = container.QueryInterface(Ci.nsIFrameLoaderOwner).frameLoader.messageManager;
mm.addMessageListener("SPPrefService", specialPowersObserver);
mm.addMessageListener("SPProcessCrashService", specialPowersObserver);
mm.addMessageListener("SPPingService", specialPowersObserver);
mm.addMessageListener("SpecialPowers.Quit", specialPowersObserver);
mm.addMessageListener("SPPermissionManager", specialPowersObserver);
mm.loadFrameScript(CHILD_LOGGER_SCRIPT, true);
mm.loadFrameScript(CHILD_SCRIPT_API, true);
mm.loadFrameScript(CHILD_SCRIPT, true);
specialPowersObserver._isFrameScriptLoaded = true;
"""
else:
OOP_pref = "false"
OOP_script = ""
# Execute this script on start up: loads special powers and sets
# the test-container apps's iframe to the mochitest URL.
self.automation.test_script = """
const CHILD_SCRIPT = "chrome://specialpowers/content/specialpowers.js";
const CHILD_SCRIPT_API = "chrome://specialpowers/content/specialpowersAPI.js";
const CHILD_LOGGER_SCRIPT = "chrome://specialpowers/content/MozillaLogger.js";
let homescreen = document.getElementById('homescreen');
let container = homescreen.contentWindow.document.getElementById('test-container');
container.setAttribute('mozapp', 'http://mochi.test:8888/manifest.webapp');
%s
container.src = '%s';
""" % (OOP_script, testURL)
# Set extra prefs for B2G.
f = open(os.path.join(options.profilePath, "user.js"), "a")
f.write("""
user_pref("browser.homescreenURL","app://test-container.gaiamobile.org/index.html");
user_pref("browser.manifestURL","app://test-container.gaiamobile.org/manifest.webapp");
user_pref("dom.mozBrowserFramesEnabled", %s);
user_pref("dom.ipc.tabs.disabled", false);
user_pref("dom.ipc.browser_frames.oop_by_default", false);
user_pref("dom.mozBrowserFramesWhitelist","app://test-container.gaiamobile.org,http://mochi.test:8888");
user_pref("marionette.loadearly", true);
user_pref("marionette.force-local", true);
""" % OOP_pref)
f.close()
class B2GOptions(MochitestOptions):
def __init__(self, automation, scriptdir, **kwargs):
@ -100,11 +33,6 @@ class B2GOptions(MochitestOptions):
help = "path to B2G repo or qemu dir")
defaults["b2gPath"] = None
self.add_option("--desktop", action="store_true",
dest="desktop",
help="Run the tests on a B2G desktop build")
defaults["desktop"] = False
self.add_option("--marionette", action="store",
type = "string", dest = "marionette",
help = "host:port to use when connecting to Marionette")
@ -170,18 +98,10 @@ class B2GOptions(MochitestOptions):
help="the path to a gecko distribution that should "
"be installed on the emulator prior to test")
defaults["geckoPath"] = None
self.add_option("--profile", action="store",
type="string", dest="profile",
help="for desktop testing, the path to the "
"gaia profile to use")
defaults["profile"] = None
self.add_option("--logcat-dir", action="store",
type="string", dest="logcat_dir",
help="directory to store logcat dump files")
defaults["logcat_dir"] = None
self.add_option('--busybox', action='store',
type='string', dest='busybox',
help="Path to busybox binary to install on device")
@ -280,7 +200,7 @@ class ProfileConfigParser(ConfigParser.RawConfigParser):
fp.write("\n")
class B2GMochitest(Mochitest, B2GMochitestMixin):
class B2GMochitest(Mochitest):
_automation = None
_dm = None
@ -465,7 +385,57 @@ class B2GMochitest(Mochitest, B2GMochitestMixin):
options.profilePath = self.localProfile
retVal = Mochitest.buildURLOptions(self, options, env)
self.setupCommonOptions(options)
# set the testURL
testURL = self.buildTestPath(options)
if len(self.urlOpts) > 0:
testURL += "?" + "&".join(self.urlOpts)
self._automation.testURL = testURL
# execute this script on start up.
# loads special powers and sets the test-container
# apps's iframe to the mochitest URL.
self._automation.test_script = """
const CHILD_SCRIPT = "chrome://specialpowers/content/specialpowers.js";
const CHILD_SCRIPT_API = "chrome://specialpowers/content/specialpowersAPI.js";
const CHILD_LOGGER_SCRIPT = "chrome://specialpowers/content/MozillaLogger.js";
let homescreen = document.getElementById('homescreen');
let container = homescreen.contentWindow.document.getElementById('test-container');
container.setAttribute('mozapp', 'http://mochi.test:8888/manifest.webapp');
let specialpowers = {};
let loader = Cc["@mozilla.org/moz/jssubscript-loader;1"].getService(Ci.mozIJSSubScriptLoader);
loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserver.js", specialpowers);
let specialPowersObserver = new specialpowers.SpecialPowersObserver();
specialPowersObserver.init();
let mm = container.QueryInterface(Ci.nsIFrameLoaderOwner).frameLoader.messageManager;
mm.addMessageListener("SPPrefService", specialPowersObserver);
mm.addMessageListener("SPProcessCrashService", specialPowersObserver);
mm.addMessageListener("SPPingService", specialPowersObserver);
mm.addMessageListener("SpecialPowers.Quit", specialPowersObserver);
mm.addMessageListener("SPPermissionManager", specialPowersObserver);
mm.loadFrameScript(CHILD_LOGGER_SCRIPT, true);
mm.loadFrameScript(CHILD_SCRIPT_API, true);
mm.loadFrameScript(CHILD_SCRIPT, true);
specialPowersObserver._isFrameScriptLoaded = true;
container.src = '%s';
""" % testURL
# Set extra prefs for B2G.
f = open(os.path.join(options.profilePath, "user.js"), "a")
f.write("""
user_pref("browser.homescreenURL","app://test-container.gaiamobile.org/index.html");
user_pref("browser.manifestURL","app://test-container.gaiamobile.org/manifest.webapp");
user_pref("dom.mozBrowserFramesEnabled", true);
user_pref("dom.ipc.tabs.disabled", false);
user_pref("dom.ipc.browser_frames.oop_by_default", false);
user_pref("dom.mozBrowserFramesWhitelist","app://test-container.gaiamobile.org,http://mochi.test:8888");
user_pref("marionette.loadearly", true);
""")
f.close()
# Copy the profile to the device.
self._dm._checkCmdAs(['shell', 'rm', '-r', self.remoteProfile])
@ -499,61 +469,17 @@ class B2GMochitest(Mochitest, B2GMochitestMixin):
return retVal
class B2GDesktopMochitest(Mochitest, B2GMochitestMixin):
def main():
scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
auto = B2GRemoteAutomation(None, "fennec")
parser = B2GOptions(auto, scriptdir)
options, args = parser.parse_args()
def __init__(self, automation):
#self._automation = automation
Mochitest.__init__(self, automation)
def runMarionetteScript(self, marionette, test_script):
assert(marionette.wait_for_port())
marionette.start_session()
marionette.set_context(marionette.CONTEXT_CHROME)
marionette.execute_script(test_script)
def startTests(self):
# This is run in a separate thread because otherwise, the app's
# stdout buffer gets filled (which gets drained only after this
# function returns, by waitForFinish), which causes the app to hang.
thread = threading.Thread(target=self.runMarionetteScript,
args=(self.automation.marionette,
self.automation.test_script))
thread.start()
def buildURLOptions(self, options, env):
retVal = Mochitest.buildURLOptions(self, options, env)
self.setupCommonOptions(options, OOP=False)
# Copy the extensions to the B2G bundles dir.
extensionDir = os.path.join(options.profilePath, 'extensions', 'staged')
bundlesDir = os.path.join(os.path.dirname(options.app),
'distribution', 'bundles')
for filename in os.listdir(extensionDir):
shutil.rmtree(os.path.join(bundlesDir, filename), True)
shutil.copytree(os.path.join(extensionDir, filename),
os.path.join(bundlesDir, filename))
return retVal
def buildProfile(self, options):
self.automation.initializeProfile(options.profilePath,
options.extraPrefs,
useServerLocations=True,
initialProfile=options.profile)
manifest = self.addChromeToProfile(options)
self.copyExtraFilesToProfile(options)
self.installExtensionsToProfile(options)
return manifest
def run_remote_mochitests(automation, parser, options):
# create our Marionette instance
kwargs = {}
if options.emulator:
kwargs['emulator'] = options.emulator
automation.setEmulator(True)
auto.setEmulator(True)
if options.noWindow:
kwargs['noWindow'] = True
if options.geckoPath:
@ -574,7 +500,7 @@ def run_remote_mochitests(automation, parser, options):
marionette = Marionette.getMarionetteOrExit(**kwargs)
automation.marionette = marionette
auto.marionette = marionette
# create the DeviceManager
kwargs = {'adbPath': options.adbPath,
@ -583,15 +509,15 @@ def run_remote_mochitests(automation, parser, options):
kwargs.update({'host': options.deviceIP,
'port': options.devicePort})
dm = devicemanagerADB.DeviceManagerADB(**kwargs)
automation.setDeviceManager(dm)
options = parser.verifyRemoteOptions(options, automation)
auto.setDeviceManager(dm)
options = parser.verifyRemoteOptions(options, auto)
if (options == None):
print "ERROR: Invalid options specified, use --help for a list of valid options"
sys.exit(1)
automation.setProduct("b2g")
auto.setProduct("b2g")
mochitest = B2GMochitest(automation, dm, options)
mochitest = B2GMochitest(auto, dm, options)
options = parser.verifyOptions(options, mochitest)
if (options == None):
@ -599,8 +525,8 @@ def run_remote_mochitests(automation, parser, options):
logParent = os.path.dirname(options.remoteLogFile)
dm.mkDir(logParent)
automation.setRemoteLog(options.remoteLogFile)
automation.setServerInfo(options.webServer, options.httpPort, options.sslPort)
auto.setRemoteLog(options.remoteLogFile)
auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)
retVal = 1
try:
mochitest.cleanup(None, options)
@ -618,50 +544,6 @@ def run_remote_mochitests(automation, parser, options):
sys.exit(retVal)
def run_desktop_mochitests(parser, options):
automation = B2GDesktopAutomation()
# create our Marionette instance
kwargs = {}
if options.marionette:
host, port = options.marionette.split(':')
kwargs['host'] = host
kwargs['port'] = int(port)
marionette = Marionette.getMarionetteOrExit(**kwargs)
automation.marionette = marionette
mochitest = B2GDesktopMochitest(automation)
# b2g desktop builds don't always have a b2g-bin file
if options.app[-4:] == '-bin':
options.app = options.app[:-4]
options = MochitestOptions.verifyOptions(parser, options, mochitest)
if options == None:
sys.exit(1)
if options.desktop and not options.profile:
raise Exception("must specify --profile when specifying --desktop")
automation.setServerInfo(options.webServer,
options.httpPort,
options.sslPort,
options.webSocketPort)
sys.exit(mochitest.runTests(options,
onLaunch=mochitest.startTests))
def main():
scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
automation = B2GRemoteAutomation(None, "fennec")
parser = B2GOptions(automation, scriptdir)
options, args = parser.parse_args()
if options.desktop:
run_desktop_mochitests(parser, options)
else:
run_remote_mochitests(automation, parser, options)
if __name__ == "__main__":
main()