mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-28 20:55:39 +00:00
8f32d32df0
Reftests hackily create a second argument parser and set defaults on it to get around the fact that the mach command sends the harness a dict rather than a Namespace object. This is bad because: 1. It's much less hacky to just create a Namespace object directly (rather than making a second parser) 2. Most other mach commands actually *do* return a Namespace object, reftest is the odd one out here. So this patch makes the reftest mach commands convert to the Namespace object, which allows us to get rid of the hacks in the reftest harness. This also does some light refactoring of the entry points to the reftest harness so make it more consistent with mochitest and xpcshell. MozReview-Commit-ID: 5HMfm64wmK2 --HG-- extra : rebase_source : 7e9c0fc64e3aeced7662a9df13f94b1b5d751bc8
413 lines
16 KiB
Python
413 lines
16 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import sys
|
|
import os
|
|
import time
|
|
import tempfile
|
|
import traceback
|
|
import urllib2
|
|
|
|
import mozdevice
|
|
import mozinfo
|
|
from automation import Automation
|
|
from remoteautomation import RemoteAutomation, fennecLogcatFilters
|
|
|
|
from output import OutputHandler
|
|
from runreftest import RefTest, ReftestResolver
|
|
import reftestcommandline
|
|
|
|
# We need to know our current directory so that we can serve our test files from it.
|
|
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
|
|
|
|
|
|
class RemoteReftestResolver(ReftestResolver):
|
|
def absManifestPath(self, path):
|
|
script_abs_path = os.path.join(SCRIPT_DIRECTORY, path)
|
|
if os.path.exists(script_abs_path):
|
|
rv = script_abs_path
|
|
elif os.path.exists(os.path.abspath(path)):
|
|
rv = os.path.abspath(path)
|
|
else:
|
|
print >> sys.stderr, "Could not find manifest %s" % script_abs_path
|
|
sys.exit(1)
|
|
return os.path.normpath(rv)
|
|
|
|
def manifestURL(self, options, path):
|
|
# Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
|
|
# It's possible for this url to have a leading "..", but reftest.js will fix that up
|
|
relPath = os.path.relpath(path, SCRIPT_DIRECTORY)
|
|
return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath)
|
|
|
|
|
|
class ReftestServer:
|
|
""" Web server used to serve Reftests, for closer fidelity to the real web.
|
|
It is virtually identical to the server used in mochitest and will only
|
|
be used for running reftests remotely.
|
|
Bug 581257 has been filed to refactor this wrapper around httpd.js into
|
|
it's own class and use it in both remote and non-remote testing. """
|
|
|
|
def __init__(self, automation, options, scriptDir):
|
|
self.automation = automation
|
|
self._utilityPath = options.utilityPath
|
|
self._xrePath = options.xrePath
|
|
self._profileDir = options.serverProfilePath
|
|
self.webServer = options.remoteWebServer
|
|
self.httpPort = options.httpPort
|
|
self.scriptDir = scriptDir
|
|
self.pidFile = options.pidFile
|
|
self._httpdPath = os.path.abspath(options.httpdPath)
|
|
self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort }
|
|
|
|
def start(self):
|
|
"Run the Refest server, returning the process ID of the server."
|
|
|
|
env = self.automation.environment(xrePath = self._xrePath)
|
|
env["XPCOM_DEBUG_BREAK"] = "warn"
|
|
if self.automation.IS_WIN32:
|
|
env["PATH"] = env["PATH"] + ";" + self._xrePath
|
|
|
|
args = ["-g", self._xrePath,
|
|
"-v", "170",
|
|
"-f", os.path.join(self._httpdPath, "httpd.js"),
|
|
"-e", "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = '%(port)s'; const _SERVER_ADDR ='%(server)s';" %
|
|
{"profile" : self._profileDir.replace('\\', '\\\\'), "port" : self.httpPort, "server" : self.webServer },
|
|
"-f", os.path.join(self.scriptDir, "server.js")]
|
|
|
|
xpcshell = os.path.join(self._utilityPath,
|
|
"xpcshell" + self.automation.BIN_SUFFIX)
|
|
|
|
if not os.access(xpcshell, os.F_OK):
|
|
raise Exception('xpcshell not found at %s' % xpcshell)
|
|
if self.automation.elf_arm(xpcshell):
|
|
raise Exception('xpcshell at %s is an ARM binary; please use '
|
|
'the --utility-path argument to specify the path '
|
|
'to a desktop version.' % xpcshell)
|
|
|
|
self._process = self.automation.Process([xpcshell] + args, env = env)
|
|
pid = self._process.pid
|
|
if pid < 0:
|
|
print "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server."
|
|
return 2
|
|
self.automation.log.info("INFO | remotereftests.py | Server pid: %d", pid)
|
|
|
|
if (self.pidFile != ""):
|
|
f = open(self.pidFile + ".xpcshell.pid", 'w')
|
|
f.write("%s" % pid)
|
|
f.close()
|
|
|
|
def ensureReady(self, timeout):
|
|
assert timeout >= 0
|
|
|
|
aliveFile = os.path.join(self._profileDir, "server_alive.txt")
|
|
i = 0
|
|
while i < timeout:
|
|
if os.path.exists(aliveFile):
|
|
break
|
|
time.sleep(1)
|
|
i += 1
|
|
else:
|
|
print "TEST-UNEXPECTED-FAIL | remotereftests.py | Timed out while waiting for server startup."
|
|
self.stop()
|
|
return 1
|
|
|
|
def stop(self):
|
|
if hasattr(self, '_process'):
|
|
try:
|
|
c = urllib2.urlopen(self.shutdownURL)
|
|
c.read()
|
|
c.close()
|
|
|
|
rtncode = self._process.poll()
|
|
if (rtncode == None):
|
|
self._process.terminate()
|
|
except:
|
|
self._process.kill()
|
|
|
|
class RemoteReftest(RefTest):
|
|
use_marionette = False
|
|
remoteApp = ''
|
|
resolver_cls = RemoteReftestResolver
|
|
|
|
def __init__(self, automation, devicemanager, options, scriptDir):
|
|
RefTest.__init__(self)
|
|
self.automation = automation
|
|
self._devicemanager = devicemanager
|
|
self.scriptDir = scriptDir
|
|
self.remoteApp = options.app
|
|
self.remoteProfile = options.remoteProfile
|
|
self.remoteTestRoot = options.remoteTestRoot
|
|
self.remoteLogFile = options.remoteLogFile
|
|
self.localLogName = options.localLogName
|
|
self.pidFile = options.pidFile
|
|
if self.automation.IS_DEBUG_BUILD:
|
|
self.SERVER_STARTUP_TIMEOUT = 180
|
|
else:
|
|
self.SERVER_STARTUP_TIMEOUT = 90
|
|
self.automation.deleteANRs()
|
|
self.automation.deleteTombstones()
|
|
|
|
self._populate_logger(options)
|
|
outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath)
|
|
# RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest
|
|
# MessageLogger object to re-use this code path.
|
|
outputHandler.write = outputHandler.__call__
|
|
self.automation._processArgs['messageLogger'] = outputHandler
|
|
|
|
def findPath(self, paths, filename = None):
|
|
for path in paths:
|
|
p = path
|
|
if filename:
|
|
p = os.path.join(p, filename)
|
|
if os.path.exists(self.getFullPath(p)):
|
|
return path
|
|
return None
|
|
|
|
def startWebServer(self, options):
|
|
""" Create the webserver on the host and start it up """
|
|
remoteXrePath = options.xrePath
|
|
remoteUtilityPath = options.utilityPath
|
|
localAutomation = Automation()
|
|
localAutomation.IS_WIN32 = False
|
|
localAutomation.IS_LINUX = False
|
|
localAutomation.IS_MAC = False
|
|
localAutomation.UNIXISH = False
|
|
hostos = sys.platform
|
|
if (hostos == 'mac' or hostos == 'darwin'):
|
|
localAutomation.IS_MAC = True
|
|
elif (hostos == 'linux' or hostos == 'linux2'):
|
|
localAutomation.IS_LINUX = True
|
|
localAutomation.UNIXISH = True
|
|
elif (hostos == 'win32' or hostos == 'win64'):
|
|
localAutomation.BIN_SUFFIX = ".exe"
|
|
localAutomation.IS_WIN32 = True
|
|
|
|
paths = [options.xrePath, localAutomation.DIST_BIN, self.automation._product, os.path.join('..', self.automation._product)]
|
|
options.xrePath = self.findPath(paths)
|
|
if options.xrePath == None:
|
|
print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
|
|
return 1
|
|
paths.append("bin")
|
|
paths.append(os.path.join("..", "bin"))
|
|
|
|
xpcshell = "xpcshell"
|
|
if (os.name == "nt"):
|
|
xpcshell += ".exe"
|
|
|
|
if (options.utilityPath):
|
|
paths.insert(0, options.utilityPath)
|
|
options.utilityPath = self.findPath(paths, xpcshell)
|
|
if options.utilityPath == None:
|
|
print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
|
|
return 1
|
|
|
|
options.serverProfilePath = tempfile.mkdtemp()
|
|
self.server = ReftestServer(localAutomation, options, self.scriptDir)
|
|
retVal = self.server.start()
|
|
if retVal:
|
|
return retVal
|
|
retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
|
|
if retVal:
|
|
return retVal
|
|
|
|
options.xrePath = remoteXrePath
|
|
options.utilityPath = remoteUtilityPath
|
|
return 0
|
|
|
|
def stopWebServer(self, options):
|
|
self.server.stop()
|
|
|
|
def createReftestProfile(self, options, manifest):
|
|
profile = RefTest.createReftestProfile(self,
|
|
options,
|
|
manifest,
|
|
server=options.remoteWebServer,
|
|
port=options.httpPort)
|
|
profileDir = profile.profile
|
|
|
|
prefs = {}
|
|
prefs["app.update.url.android"] = ""
|
|
prefs["browser.firstrun.show.localepicker"] = False
|
|
prefs["reftest.remote"] = True
|
|
prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
|
|
|
|
prefs["layout.css.devPixelsPerPx"] = "1.0"
|
|
# Because Fennec is a little wacky (see bug 1156817) we need to load the
|
|
# reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport.
|
|
prefs["apz.allow_zooming"] = False
|
|
|
|
# Set the extra prefs.
|
|
profile.set_preferences(prefs)
|
|
|
|
try:
|
|
self._devicemanager.pushDir(profileDir, options.remoteProfile)
|
|
self._devicemanager.chmodDir(options.remoteProfile)
|
|
except mozdevice.DMError:
|
|
print "Automation Error: Failed to copy profiledir to device"
|
|
raise
|
|
|
|
return profile
|
|
|
|
def copyExtraFilesToProfile(self, options, profile):
|
|
profileDir = profile.profile
|
|
RefTest.copyExtraFilesToProfile(self, options, profile)
|
|
try:
|
|
self._devicemanager.pushDir(profileDir, options.remoteProfile)
|
|
self._devicemanager.chmodDir(options.remoteProfile)
|
|
except mozdevice.DMError:
|
|
print "Automation Error: Failed to copy extra files to device"
|
|
raise
|
|
|
|
def printDeviceInfo(self, printLogcat=False):
|
|
try:
|
|
if printLogcat:
|
|
logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
|
|
print ''.join(logcat)
|
|
print "Device info:"
|
|
devinfo = self._devicemanager.getInfo()
|
|
for category in devinfo:
|
|
if type(devinfo[category]) is list:
|
|
print " %s:" % category
|
|
for item in devinfo[category]:
|
|
print " %s" % item
|
|
else:
|
|
print " %s: %s" % (category, devinfo[category])
|
|
print "Test root: %s" % self._devicemanager.deviceRoot
|
|
except mozdevice.DMError:
|
|
print "WARNING: Error getting device information"
|
|
|
|
def environment(self, **kwargs):
|
|
return self.automation.environment(**kwargs)
|
|
|
|
def buildBrowserEnv(self, options, profileDir):
|
|
browserEnv = RefTest.buildBrowserEnv(self, options, profileDir)
|
|
# remove desktop environment not used on device
|
|
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
|
|
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
|
|
return browserEnv
|
|
|
|
def runApp(self, profile, binary, cmdargs, env,
|
|
timeout=None, debuggerInfo=None,
|
|
symbolsPath=None, options=None,
|
|
valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None):
|
|
status = self.automation.runApp(None, env,
|
|
binary,
|
|
profile.profile,
|
|
cmdargs,
|
|
utilityPath=options.utilityPath,
|
|
xrePath=options.xrePath,
|
|
debuggerInfo=debuggerInfo,
|
|
symbolsPath=symbolsPath,
|
|
timeout=timeout)
|
|
return status
|
|
|
|
def cleanup(self, profileDir):
|
|
# Pull results back from device
|
|
if self.remoteLogFile and \
|
|
self._devicemanager.fileExists(self.remoteLogFile):
|
|
self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
|
|
else:
|
|
print "WARNING: Unable to retrieve log file (%s) from remote " \
|
|
"device" % self.remoteLogFile
|
|
self._devicemanager.removeDir(self.remoteProfile)
|
|
self._devicemanager.removeDir(self.remoteTestRoot)
|
|
RefTest.cleanup(self, profileDir)
|
|
if (self.pidFile != ""):
|
|
try:
|
|
os.remove(self.pidFile)
|
|
os.remove(self.pidFile + ".xpcshell.pid")
|
|
except:
|
|
print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
|
|
|
|
|
|
def run_test_harness(parser, options):
|
|
if options.dm_trans == 'sut' and options.deviceIP == None:
|
|
print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option"
|
|
return 1
|
|
|
|
dm_args = {
|
|
'deviceRoot': options.remoteTestRoot,
|
|
'host': options.deviceIP,
|
|
'port': options.devicePort,
|
|
}
|
|
|
|
dm_cls = mozdevice.DroidSUT
|
|
if options.dm_trans == 'adb':
|
|
dm_args['adbPath'] = options.adb_path
|
|
if not dm_args['host']:
|
|
dm_args['deviceSerial'] = options.deviceSerial
|
|
dm_cls = mozdevice.DroidADB
|
|
|
|
try:
|
|
dm = dm_cls(**dm_args)
|
|
except mozdevice.DMError:
|
|
traceback.print_exc()
|
|
print "Automation Error: exception while initializing devicemanager. Most likely the device is not in a testable state."
|
|
return 1
|
|
|
|
automation = RemoteAutomation(None)
|
|
automation.setDeviceManager(dm)
|
|
|
|
if options.remoteProductName:
|
|
automation.setProduct(options.remoteProductName)
|
|
|
|
# Set up the defaults and ensure options are set
|
|
parser.validate_remote(options, automation)
|
|
|
|
# Check that Firefox is installed
|
|
expected = options.app.split('/')[-1]
|
|
installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected])
|
|
if expected not in installed:
|
|
print "%s is not installed on this device" % expected
|
|
return 1
|
|
|
|
automation.setAppName(options.app)
|
|
automation.setRemoteProfile(options.remoteProfile)
|
|
automation.setRemoteLog(options.remoteLogFile)
|
|
reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY)
|
|
parser.validate(options, reftest)
|
|
|
|
if mozinfo.info['debug']:
|
|
print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout
|
|
options.timeout = 600
|
|
|
|
# Hack in a symbolic link for jsreftest
|
|
os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")))
|
|
|
|
# Start the webserver
|
|
retVal = reftest.startWebServer(options)
|
|
if retVal:
|
|
return retVal
|
|
|
|
procName = options.app.split('/')[-1]
|
|
if (dm.processExist(procName)):
|
|
dm.killProcess(procName)
|
|
|
|
if options.printDeviceInfo:
|
|
reftest.printDeviceInfo()
|
|
|
|
#an example manifest name to use on the cli
|
|
# manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list"
|
|
retVal = 0
|
|
try:
|
|
dm.recordLogcat()
|
|
retVal = reftest.runTests(options.tests, options)
|
|
except:
|
|
print "Automation Error: Exception caught while running tests"
|
|
traceback.print_exc()
|
|
retVal = 1
|
|
|
|
reftest.stopWebServer(options)
|
|
|
|
if options.printDeviceInfo:
|
|
reftest.printDeviceInfo(printLogcat=True)
|
|
|
|
return retVal
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = reftestcommandline.RemoteArgumentsParser()
|
|
options = parser.parse_args()
|
|
sys.exit(run_test_harness(parser, options))
|