mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-25 13:51:41 +00:00
Bug 552300 - 'Use VMware VMs to run mochitests, optionally record and repeat until they fail.' r=ted.
This commit is contained in:
Ben Turner
parent
1890899acf
commit
cf6512babc
@ -62,6 +62,7 @@ _SERV_FILES = \
|
||||
runtests.py \
|
||||
automation.py \
|
||||
runtestsremote.py \
|
||||
runtestsvmware.py \
|
||||
$(topsrcdir)/build/mobile/devicemanager.py \
|
||||
$(topsrcdir)/build/automationutils.py \
|
||||
$(topsrcdir)/build/poster.zip \
|
||||
|
||||
@ -272,7 +272,7 @@ See <http://mochikit.com/doc/html/MochiKit/Logging.html> for details on the logg
|
||||
mochitest.vmwareHelperPath = os.path.join(
|
||||
options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll")
|
||||
if not os.path.exists(mochitest.vmwareHelperPath):
|
||||
self.error("%s not found, cannot automate VMWare recording." %
|
||||
self.error("%s not found, cannot automate VMware recording." %
|
||||
mochitest.vmwareHelperPath)
|
||||
|
||||
return options
|
||||
@ -504,34 +504,34 @@ class Mochitest(object):
|
||||
os.remove(manifest)
|
||||
shutil.rmtree(options.profilePath)
|
||||
|
||||
def startVMWareRecording(self, options):
|
||||
def startVMwareRecording(self, options):
|
||||
""" starts recording inside VMware VM using the recording helper dll """
|
||||
assert(self.automation.IS_WIN32)
|
||||
from ctypes import cdll
|
||||
self.vmwareHelper = cdll.LoadLibrary(self.vmwareHelperPath)
|
||||
if self.vmwareHelper is None:
|
||||
self.automation.log.warning("WARNING | runtests.py | Failed to load "
|
||||
"VMWare recording helper")
|
||||
"VMware recording helper")
|
||||
return
|
||||
self.automation.log.info("INFO | runtests.py | Starting VMWare recording.")
|
||||
self.automation.log.info("INFO | runtests.py | Starting VMware recording.")
|
||||
try:
|
||||
self.vmwareHelper.StartRecording()
|
||||
except Exception, e:
|
||||
self.automation.log.warning("WARNING | runtests.py | Failed to start "
|
||||
"VMWare recording: (%s)" % str(e))
|
||||
"VMware recording: (%s)" % str(e))
|
||||
self.vmwareHelper = None
|
||||
|
||||
def stopVMWareRecording(self):
|
||||
def stopVMwareRecording(self):
|
||||
""" stops recording inside VMware VM using the recording helper dll """
|
||||
assert(self.automation.IS_WIN32)
|
||||
if self.vmwareHelper is not None:
|
||||
self.automation.log.info("INFO | runtests.py | Stopping VMWare "
|
||||
self.automation.log.info("INFO | runtests.py | Stopping VMware "
|
||||
"recording.")
|
||||
try:
|
||||
self.vmwareHelper.StopRecording()
|
||||
except Exception, e:
|
||||
self.automation.log.warning("WARNING | runtests.py | Failed to stop "
|
||||
"VMWare recording: (%s)" % str(e))
|
||||
"VMware recording: (%s)" % str(e))
|
||||
self.vmwareHelper = None
|
||||
|
||||
def runTests(self, options):
|
||||
@ -570,7 +570,7 @@ class Mochitest(object):
|
||||
timeout = 330.0 # default JS harness timeout is 300 seconds
|
||||
|
||||
if options.vmwareRecording:
|
||||
self.startVMWareRecording(options);
|
||||
self.startVMwareRecording(options);
|
||||
|
||||
self.automation.log.info("INFO | runtests.py | Running tests: start.\n")
|
||||
status = self.automation.runApp(testURL, browserEnv, options.app,
|
||||
@ -584,7 +584,7 @@ class Mochitest(object):
|
||||
timeout = timeout)
|
||||
|
||||
if options.vmwareRecording:
|
||||
self.stopVMWareRecording();
|
||||
self.stopVMwareRecording();
|
||||
|
||||
self.stopWebServer(options)
|
||||
processLeakLog(self.leak_report_file, options.leakThreshold)
|
||||
|
||||
429
testing/mochitest/runtestsvmware.py
Normal file
429
testing/mochitest/runtestsvmware.py
Normal file
@ -0,0 +1,429 @@
|
||||
#
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
|
||||
#
|
||||
# The contents of this file are subject to the Mozilla Public License Version
|
||||
# 1.1 (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
# http://www.mozilla.org/MPL/
|
||||
#
|
||||
# Software distributed under the License is distributed on an "AS IS" basis,
|
||||
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
||||
# for the specific language governing rights and limitations under the
|
||||
# License.
|
||||
#
|
||||
# The Original Code is VMware Mochitest Runner.
|
||||
#
|
||||
# The Initial Developer of the Original Code is
|
||||
# Mozilla Foundation.
|
||||
# Portions created by the Initial Developer are Copyright (C) 2010
|
||||
# the Initial Developer. All Rights Reserved.
|
||||
#
|
||||
# Contributor(s):
|
||||
# Ben Turner <bent.mozilla@gmail.com>
|
||||
#
|
||||
# Alternatively, the contents of this file may be used under the terms of
|
||||
# either the GNU General Public License Version 2 or later (the "GPL"), or
|
||||
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
|
||||
# in which case the provisions of the GPL or the LGPL are applicable instead
|
||||
# of those above. If you wish to allow use of your version of this file only
|
||||
# under the terms of either the GPL or the LGPL, and not to allow others to
|
||||
# use your version of this file under the terms of the MPL, indicate your
|
||||
# decision by deleting the provisions above and replace them with the notice
|
||||
# and other provisions required by the GPL or the LGPL. If you do not delete
|
||||
# the provisions above, a recipient may use your version of this file under
|
||||
# the terms of any one of the MPL, the GPL or the LGPL.
|
||||
#
|
||||
# ***** END LICENSE BLOCK *****
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import types
|
||||
from optparse import OptionValueError
|
||||
from subprocess import PIPE
|
||||
from time import sleep
|
||||
from tempfile import mkstemp
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.realpath(
|
||||
os.path.dirname(sys.argv[0]))))
|
||||
|
||||
from automation import Automation
|
||||
from runtests import Mochitest, MochitestOptions
|
||||
|
||||
class VMwareOptions(MochitestOptions):
|
||||
def __init__(self, automation, mochitest, **kwargs):
|
||||
defaults = {}
|
||||
MochitestOptions.__init__(self, automation, mochitest.SCRIPT_DIRECTORY)
|
||||
|
||||
def checkPathCallback(option, opt_str, value, parser):
|
||||
path = mochitest.getFullPath(value)
|
||||
if not os.path.exists(path):
|
||||
raise OptionValueError("Path %s does not exist for %s option"
|
||||
% (path, opt_str))
|
||||
setattr(parser.values, option.dest, path)
|
||||
|
||||
self.add_option("--with-vmware-vm",
|
||||
action = "callback", type = "string", dest = "vmx",
|
||||
callback = checkPathCallback,
|
||||
help = "launches the given VM and runs mochitests inside")
|
||||
defaults["vmx"] = None
|
||||
|
||||
self.add_option("--with-vmrun-executable",
|
||||
action = "callback", type = "string", dest = "vmrun",
|
||||
callback = checkPathCallback,
|
||||
help = "specifies the vmrun.exe to use for VMware control")
|
||||
defaults["vmrun"] = None
|
||||
|
||||
self.add_option("--shutdown-vm-when-done",
|
||||
action = "store_true", dest = "shutdownVM",
|
||||
help = "shuts down the VM when mochitests complete")
|
||||
defaults["shutdownVM"] = False
|
||||
|
||||
self.add_option("--repeat-until-failure",
|
||||
action = "store_true", dest = "repeatUntilFailure",
|
||||
help = "Runs tests continuously until failure")
|
||||
defaults["repeatUntilFailure"] = False
|
||||
|
||||
self.set_defaults(**defaults)
|
||||
|
||||
class VMwareMochitest(Mochitest):
|
||||
_pathFixRegEx = re.compile(r'^[cC](\:[\\\/]+)')
|
||||
|
||||
def convertHostPathsToGuestPaths(self, string):
|
||||
""" converts a path on the host machine to a path on the guest machine """
|
||||
# XXXbent Lame!
|
||||
return self._pathFixRegEx.sub(r'z\1', string)
|
||||
|
||||
def prepareGuestArguments(self, parser, options):
|
||||
""" returns an array of command line arguments needed to replicate the
|
||||
current set of options in the guest """
|
||||
args = []
|
||||
for key in options.__dict__.keys():
|
||||
# Don't send these args to the vm test runner!
|
||||
if key == "vmrun" or key == "vmx" or key == "repeatUntilFailure":
|
||||
continue
|
||||
|
||||
value = options.__dict__[key]
|
||||
valueType = type(value)
|
||||
|
||||
# Find the option in the parser's list.
|
||||
option = None
|
||||
for index in range(len(parser.option_list)):
|
||||
if str(parser.option_list[index].dest) == key:
|
||||
option = parser.option_list[index]
|
||||
break
|
||||
if not option:
|
||||
continue
|
||||
|
||||
# No need to pass args on the command line if they're just going to set
|
||||
# default values. The exception is list values... For some reason the
|
||||
# option parser modifies the defaults as well as the values when using the
|
||||
# "append" action.
|
||||
if value == parser.defaults[option.dest]:
|
||||
if valueType == types.StringType and \
|
||||
value == self.convertHostPathsToGuestPaths(value):
|
||||
continue
|
||||
if valueType != types.ListType:
|
||||
continue
|
||||
|
||||
def getArgString(arg, option):
|
||||
if option.action == "store_true" or option.action == "store_false":
|
||||
return str(option)
|
||||
return "%s=%s" % (str(option),
|
||||
self.convertHostPathsToGuestPaths(str(arg)))
|
||||
|
||||
if valueType == types.ListType:
|
||||
# Expand lists into separate args.
|
||||
for item in value:
|
||||
args.append(getArgString(item, option))
|
||||
else:
|
||||
args.append(getArgString(value, option))
|
||||
|
||||
return tuple(args)
|
||||
|
||||
def launchVM(self, options):
|
||||
""" launches the VM and enables shared folders """
|
||||
# Launch VM first.
|
||||
self.automation.log.info("INFO | runtests.py | Launching the VM.")
|
||||
(result, stdout) = self.runVMCommand(self.vmrunargs + ("start", self.vmx))
|
||||
if result:
|
||||
return result
|
||||
|
||||
# Make sure that shared folders are enabled.
|
||||
self.automation.log.info("INFO | runtests.py | Enabling shared folders in "
|
||||
"the VM.")
|
||||
(result, stdout) = self.runVMCommand(self.vmrunargs + \
|
||||
("enableSharedFolders", self.vmx))
|
||||
if result:
|
||||
return result
|
||||
|
||||
def shutdownVM(self):
|
||||
""" shuts down the VM """
|
||||
self.automation.log.info("INFO | runtests.py | Shutting down the VM.")
|
||||
command = self.vmrunargs + ("runProgramInGuest", self.vmx,
|
||||
"c:\\windows\\system32\\shutdown.exe", "/s", "/t", "1")
|
||||
(result, stdout) = self.runVMCommand(command)
|
||||
return result
|
||||
|
||||
def runVMCommand(self, command, expectedErrors=[], silent=False):
|
||||
""" runs a command in the VM using the vmrun.exe helper """
|
||||
commandString = ""
|
||||
for part in command:
|
||||
commandString += str(part) + " "
|
||||
if not silent:
|
||||
self.automation.log.info("INFO | runtests.py | Running command: %s"
|
||||
% commandString)
|
||||
|
||||
commonErrors = ["Error: Invalid user name or password for the guest OS",
|
||||
"Unable to connect to host."]
|
||||
expectedErrors.extend(commonErrors)
|
||||
|
||||
# VMware can't run commands until the VM has fully loaded so keep running
|
||||
# this command in a loop until it succeeds or we try 100 times.
|
||||
errorString = ""
|
||||
for i in range(100):
|
||||
process = Automation.Process(command, stdout=PIPE)
|
||||
result = process.wait()
|
||||
if result == 0:
|
||||
break
|
||||
|
||||
for line in process.stdout.readlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
errorString = line
|
||||
break
|
||||
|
||||
expected = False
|
||||
for error in expectedErrors:
|
||||
if errorString.startswith(error):
|
||||
expected = True
|
||||
|
||||
if not expected:
|
||||
self.automation.log.warning("WARNING | runtests.py | Command \"%s\" "
|
||||
"failed with result %d, : %s"
|
||||
% (commandString, result, errorString))
|
||||
break
|
||||
|
||||
if not silent:
|
||||
self.automation.log.info("INFO | runtests.py | Running command again.")
|
||||
|
||||
return (result, process.stdout.readlines())
|
||||
|
||||
def monitorVMExecution(self, appname, logfilepath):
|
||||
""" monitors test execution in the VM. Waits for the test process to start,
|
||||
then watches the log file for test failures and checks the status of the
|
||||
process to catch crashes. Returns True if mochitests ran successfully.
|
||||
"""
|
||||
success = True
|
||||
|
||||
self.automation.log.info("INFO | runtests.py | Waiting for test process to "
|
||||
"start.")
|
||||
|
||||
listProcessesCommand = self.vmrunargs + ("listProcessesInGuest", self.vmx)
|
||||
expectedErrors = [ "Error: The virtual machine is not powered on" ]
|
||||
|
||||
running = False
|
||||
for i in range(100):
|
||||
(result, stdout) = self.runVMCommand(listProcessesCommand, expectedErrors,
|
||||
silent=True)
|
||||
if result:
|
||||
self.automation.log.warning("WARNING | runtests.py | Failed to get "
|
||||
"list of processes in VM!")
|
||||
return False
|
||||
for line in stdout:
|
||||
line = line.strip()
|
||||
if line.find(appname) != -1:
|
||||
running = True
|
||||
break
|
||||
if running:
|
||||
break
|
||||
sleep(1)
|
||||
|
||||
self.automation.log.info("INFO | runtests.py | Found test process, "
|
||||
"monitoring log.")
|
||||
|
||||
completed = False
|
||||
nextLine = 0
|
||||
while running:
|
||||
log = open(logfilepath, "rb")
|
||||
lines = log.readlines()
|
||||
if len(lines) > nextLine:
|
||||
linesToPrint = lines[nextLine:]
|
||||
for line in linesToPrint:
|
||||
line = line.strip()
|
||||
if line.find("INFO SimpleTest FINISHED") != -1:
|
||||
completed = True
|
||||
continue
|
||||
if line.find("ERROR TEST-UNEXPECTED-FAIL") != -1:
|
||||
self.automation.log.info("INFO | runtests.py | Detected test "
|
||||
"failure: \"%s\"" % line)
|
||||
success = False
|
||||
nextLine = len(lines)
|
||||
log.close()
|
||||
|
||||
(result, stdout) = self.runVMCommand(listProcessesCommand, expectedErrors,
|
||||
silent=True)
|
||||
if result:
|
||||
self.automation.log.warning("WARNING | runtests.py | Failed to get "
|
||||
"list of processes in VM!")
|
||||
return False
|
||||
|
||||
stillRunning = False
|
||||
for line in stdout:
|
||||
line = line.strip()
|
||||
if line.find(appname) != -1:
|
||||
stillRunning = True
|
||||
break
|
||||
if stillRunning:
|
||||
sleep(5)
|
||||
else:
|
||||
if not completed:
|
||||
self.automation.log.info("INFO | runtests.py | Test process exited "
|
||||
"without finishing tests, maybe crashed.")
|
||||
success = False
|
||||
running = stillRunning
|
||||
|
||||
return success
|
||||
|
||||
def getCurentSnapshotList(self):
|
||||
""" gets a list of snapshots from the VM """
|
||||
(result, stdout) = self.runVMCommand(self.vmrunargs + ("listSnapshots",
|
||||
self.vmx))
|
||||
snapshots = []
|
||||
if result != 0:
|
||||
self.automation.log.warning("WARNING | runtests.py | Failed to get list "
|
||||
"of snapshots in VM!")
|
||||
return snapshots
|
||||
for line in stdout:
|
||||
if line.startswith("Total snapshots:"):
|
||||
continue
|
||||
snapshots.append(line.strip())
|
||||
return snapshots
|
||||
|
||||
def runTests(self, parser, options):
|
||||
""" runs mochitests in the VM """
|
||||
# Base args that must always be passed to vmrun.
|
||||
self.vmrunargs = (options.vmrun, "-T", "ws", "-gu", "Replay", "-gp",
|
||||
"mozilla")
|
||||
self.vmrun = options.vmrun
|
||||
self.vmx = options.vmx
|
||||
|
||||
result = self.launchVM(options)
|
||||
if result:
|
||||
return result
|
||||
|
||||
if options.vmwareRecording:
|
||||
snapshots = self.getCurentSnapshotList()
|
||||
|
||||
def innerRun():
|
||||
""" subset of the function that must run every time if we're running until
|
||||
failure """
|
||||
# Make a new shared file for the log file.
|
||||
(logfile, logfilepath) = mkstemp(suffix=".log")
|
||||
os.close(logfile)
|
||||
# Get args to pass to VM process. Make sure we autorun and autoclose.
|
||||
options.autorun = True
|
||||
options.closeWhenDone = True
|
||||
options.logFile = logfilepath
|
||||
self.automation.log.info("INFO | runtests.py | Determining guest "
|
||||
"arguments.")
|
||||
runtestsArgs = self.prepareGuestArguments(parser, options)
|
||||
runtestsPath = self.convertHostPathsToGuestPaths(self.SCRIPT_DIRECTORY)
|
||||
runtestsPath = os.path.join(runtestsPath, "runtests.py")
|
||||
runtestsCommand = self.vmrunargs + ("runProgramInGuest", self.vmx,
|
||||
"-activeWindow", "-interactive", "-noWait",
|
||||
"c:\\mozilla-build\\python25\\python.exe",
|
||||
runtestsPath) + runtestsArgs
|
||||
expectedErrors = [ "Unable to connect to host.",
|
||||
"Error: The virtual machine is not powered on" ]
|
||||
self.automation.log.info("INFO | runtests.py | Launching guest test "
|
||||
"runner.")
|
||||
(result, stdout) = self.runVMCommand(runtestsCommand, expectedErrors)
|
||||
if result:
|
||||
return (result, False)
|
||||
self.automation.log.info("INFO | runtests.py | Waiting for guest test "
|
||||
"runner to complete.")
|
||||
mochitestsSucceeded = self.monitorVMExecution(
|
||||
os.path.basename(options.app), logfilepath)
|
||||
if mochitestsSucceeded:
|
||||
self.automation.log.info("INFO | runtests.py | Guest tests passed!")
|
||||
else:
|
||||
self.automation.log.info("INFO | runtests.py | Guest tests failed.")
|
||||
if mochitestsSucceeded and options.vmwareRecording:
|
||||
newSnapshots = self.getCurentSnapshotList()
|
||||
if len(newSnapshots) > len(snapshots):
|
||||
self.automation.log.info("INFO | runtests.py | Removing last "
|
||||
"recording.")
|
||||
(result, stdout) = self.runVMCommand(self.vmrunargs + \
|
||||
("deleteSnapshot", self.vmx,
|
||||
newSnapshots[-1]))
|
||||
self.automation.log.info("INFO | runtests.py | Removing guest log file.")
|
||||
for i in range(30):
|
||||
try:
|
||||
os.remove(logfilepath)
|
||||
break
|
||||
except:
|
||||
sleep(1)
|
||||
self.automation.log.warning("WARNING | runtests.py | Couldn't remove "
|
||||
"guest log file, trying again.")
|
||||
return (result, mochitestsSucceeded)
|
||||
|
||||
if options.repeatUntilFailure:
|
||||
succeeded = True
|
||||
result = 0
|
||||
count = 1
|
||||
while result == 0 and succeeded:
|
||||
self.automation.log.info("INFO | runtests.py | Beginning mochitest run "
|
||||
"(%d)." % count)
|
||||
count += 1
|
||||
(result, succeeded) = innerRun()
|
||||
else:
|
||||
self.automation.log.info("INFO | runtests.py | Beginning mochitest run.")
|
||||
(result, succeeded) = innerRun()
|
||||
|
||||
if not succeeded and options.vmwareRecording:
|
||||
newSnapshots = self.getCurentSnapshotList()
|
||||
if len(newSnapshots) > len(snapshots):
|
||||
self.automation.log.info("INFO | runtests.py | Failed recording saved "
|
||||
"as '%s'." % newSnapshots[-1])
|
||||
|
||||
if result:
|
||||
return result
|
||||
|
||||
if options.shutdownVM:
|
||||
result = self.shutdownVM()
|
||||
if result:
|
||||
return result
|
||||
|
||||
return 0
|
||||
|
||||
def main():
|
||||
automation = Automation()
|
||||
mochitest = VMwareMochitest(automation)
|
||||
|
||||
parser = VMwareOptions(automation, mochitest)
|
||||
options, args = parser.parse_args()
|
||||
options = parser.verifyOptions(options, mochitest)
|
||||
if (options == None):
|
||||
sys.exit(1)
|
||||
|
||||
if options.vmx is None:
|
||||
parser.error("A virtual machine must be specified with " +
|
||||
"--with-vmware-vm")
|
||||
|
||||
if options.vmrun is None:
|
||||
options.vmrun = os.path.join("c:\\", "Program Files", "VMware",
|
||||
"VMware VIX", "vmrun.exe")
|
||||
if not os.path.exists(options.vmrun):
|
||||
options.vmrun = os.path.join("c:\\", "Program Files (x86)", "VMware",
|
||||
"VMware VIX", "vmrun.exe")
|
||||
if not os.path.exists(options.vmrun):
|
||||
parser.error("Could not locate vmrun.exe, use --with-vmrun-executable" +
|
||||
" to identify its location")
|
||||
|
||||
sys.exit(mochitest.runTests(parser, options))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user