Bug 749011 - Add support for running mochitest-plain on B2G, r=jmaher

--HG--
extra : rebase_source : 4c260e11c51c5015d2ded66f4b8cfc94a9969298

build/mobile/b2gautomation.py

@@ -0,0 +1,232 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import automationutils
+import os
+import re
+import socket
+import shutil
+import sys
+import tempfile
+import time
+
+from automation import Automation
+from devicemanager import DeviceManager, NetworkTools
+
+class B2GRemoteAutomation(Automation):
+    _devicemanager = None
+
+    def __init__(self, deviceManager, appName='', remoteLog=None,
+                 marionette=None):
+        self._devicemanager = deviceManager
+        self._appName = appName
+        self._remoteProfile = None
+        self._remoteLog = remoteLog
+        self.marionette = marionette
+
+        # Default our product to b2g
+        self._product = "b2g"
+        Automation.__init__(self)
+
+    def setDeviceManager(self, deviceManager):
+        self._devicemanager = deviceManager
+
+    def setAppName(self, appName):
+        self._appName = appName
+
+    def setRemoteProfile(self, remoteProfile):
+        self._remoteProfile = remoteProfile
+
+    def setProduct(self, product):
+        self._product = product
+
+    def setRemoteLog(self, logfile):
+        self._remoteLog = logfile
+
+    # Set up what we need for the remote environment
+    def environment(self, env=None, xrePath=None, crashreporter=True):
+        # Because we are running remote, we don't want to mimic the local env
+        # so no copying of os.environ
+        if env is None:
+            env = {}
+
+        return env
+
+    def checkForCrashes(self, directory, symbolsPath):
+        # XXX: This will have to be updated after crash reporting on b2g
+        # is in place.
+        dumpDir = tempfile.mkdtemp()
+        self._devicemanager.getDirectory(self._remoteProfile + '/minidumps/', dumpDir)
+        automationutils.checkForCrashes(dumpDir, symbolsPath, self.lastTestSeen)
+        try:
+          shutil.rmtree(dumpDir)
+        except:
+          print "WARNING: unable to remove directory: %s" % (dumpDir)
+
+    def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
+        # if remote profile is specified, use that instead
+        if (self._remoteProfile):
+            profileDir = self._remoteProfile
+
+        cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)
+
+        return app, args
+
+    def getLanIp(self):
+        nettools = NetworkTools()
+        return nettools.getLanIp()
+
+    def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime,
+                      debuggerInfo, symbolsPath, logger):
+        """ Wait for mochitest to finish (as evidenced by a signature string
+            in logcat), or for a given amount of time to elapse with no
+            output.
+        """
+        timeout = timeout or 120
+
+        didTimeout = False
+
+        done = time.time() + timeout
+        while True:
+            currentlog = proc.stdout
+            if currentlog:
+                done = time.time() + timeout
+                print currentlog
+                if 'INFO SimpleTest FINISHED' in currentlog:
+                    return 0
+            else:
+                if time.time() > done:
+                    self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed "
+                                  "out after %d seconds with no output",
+                                  self.lastTestSeen, int(timeout))
+                    return 1
+
+    def getDeviceStatus(self, serial=None):
+        # Get the current status of the device.  If we know the device
+        # serial number, we look for that, otherwise we use the (presumably
+        # only) device shown in 'adb devices'.
+        serial = serial or self._devicemanager.deviceSerial
+        status = 'unknown'
+
+        for line in self._devicemanager.runCmd(['devices']).stdout.readlines():
+            result =  re.match('(.*?)\t(.*)', line)
+            if result:
+                thisSerial = result.group(1)
+                if not serial or thisSerial == serial:
+                    serial = thisSerial
+                    status = result.group(2)
+
+        return (serial, status)
+
+    def rebootDevice(self):
+        # find device's current status and serial number
+        serial, status = self.getDeviceStatus()
+
+        # reboot!
+        self._devicemanager.checkCmd(['reboot'])
+
+        # wait for device to come back to previous status
+        print 'waiting for device to come back online after reboot'
+        start = time.time()
+        rserial, rstatus = self.getDeviceStatus(serial)
+        while rstatus != 'device':
+            if time.time() - start > 120:
+                # device hasn't come back online in 2 minutes, something's wrong
+                raise Exception("Device %s (status: %s) not back online after reboot" % (serial, rstatus))
+            time.sleep(5)
+            rserial, rstatus = self.getDeviceStatus(serial)
+        print 'device:', serial, 'status:', rstatus
+
+    def Process(self, cmd, stdout=None, stderr=None, env=None, cwd=None):
+        # On a desktop or fennec run, the Process method invokes a gecko
+        # process in which to run mochitests.  For B2G, we simply
+        # reboot the device (which was configured with a test profile
+        # already), wait for B2G to start up, and then navigate to the
+        # test url using Marionette.  There doesn't seem to be any way
+        # to pass env variables into the B2G process, but this doesn't 
+        # seem to matter.
+
+        instance = self.B2GInstance(self._devicemanager)
+
+        # reboot device so it starts up with the mochitest profile
+        # XXX:  We could potentially use 'stop b2g' + 'start b2g' to achieve
+        # a similar effect; will see which is more stable while attempting
+        # to bring up the continuous integration.
+        self.rebootDevice()
+
+        # Infrequently, gecko comes up before networking does, so wait a little
+        # bit to give the network time to become available.
+        # XXX:  need a more robust mechanism for this
+        time.sleep(20)
+
+        # Set up port forwarding again for Marionette, since any that
+        # existed previously got wiped out by the reboot.
+        self._devicemanager.checkCmd(['forward',
+                                      'tcp:%s' % self.marionette.port,
+                                      'tcp:%s' % self.marionette.port])
+
+        # start a marionette session
+        session = self.marionette.start_session()
+        if 'b2g' not in session:
+            raise Exception("bad session value %s returned by start_session" % session)
+
+        # start the tests by navigating to the mochitest url
+        self.marionette.execute_script("window.location.href='%s';" % self.testURL)
+
+        return instance
+
+    # be careful here as this inner class doesn't have access to outer class members
+    class B2GInstance(object):
+        """Represents a B2G instance running on a device, and exposes
+           some process-like methods/properties that are expected by the
+           automation.
+        """
+
+        def __init__(self, dm):
+            self.dm = dm
+            self.lastloglines = []
+
+        @property
+        def pid(self):
+            # a dummy value to make the automation happy
+            return 0
+
+        @property
+        def stdout(self):
+            # Return any part of logcat output that wasn't returned
+            # previously.  This is done by fetching about the last 50
+            # lines of the log (logcat -t 50 generally fetches 50-58 lines),
+            # and comparing against the previous set to find new lines.
+            t = self.dm.runCmd(['logcat', '-t', '50']).stdout.read()
+            if t == None: return ''
+
+            t = t.strip('\n').strip()
+            loglines = t.split('\n')
+            line_index = 0
+
+            # If there are more than 50 lines, we skip the first 20; this
+            # is because the number of lines returned
+            # by logcat -t 50 varies (usually between 50 and 58).
+            log_index = 20 if len(loglines) > 50 else 0
+
+            for index, line in enumerate(loglines[log_index:]):
+                line_index = index + log_index + 1
+                try:
+                    self.lastloglines.index(line)
+                except ValueError:
+                    break
+
+            newoutput = '\n'.join(loglines[line_index:])
+            self.lastloglines = loglines
+
+            return newoutput
+
+        def wait(self, timeout = None):
+            # this should never happen
+            raise Exception("'wait' called on B2GInstance")
+
+        def kill(self):
+            # this should never happen
+            raise Exception("'kill' called on B2GInstance")
+

testing/mochitest/Makefile.in

@@ -79,6 +79,7 @@ include $(topsrcdir)/build/automation-build.mk
 _SERV_FILES = 	\
 		runtests.py \
 		automation.py \
+		runtestsb2g.py \
 		runtestsremote.py \
 		runtestsvmware.py \
 		$(topsrcdir)/build/mobile/devicemanager.py \
@@ -87,6 +88,7 @@ _SERV_FILES = 	\
 		$(topsrcdir)/build/automationutils.py \
 		$(topsrcdir)/build/manifestparser.py \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
+		$(topsrcdir)/build/mobile/b2gautomation.py \
 		gen_template.pl \
 		server.js \
 		harness-overlay.xul \

testing/mochitest/runtestsb2g.py

@@ -0,0 +1,412 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import ConfigParser
+import os
+import re
+import sys
+import tempfile
+import time
+import urllib
+
+sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))))
+
+from automation import Automation
+from b2gautomation import B2GRemoteAutomation
+from runtests import Mochitest
+from runtests import MochitestOptions
+from runtests import MochitestServer
+
+import devicemanagerADB
+import manifestparser
+
+from marionette import Marionette
+
+
+class B2GOptions(MochitestOptions):
+
+    def __init__(self, automation, scriptdir, **kwargs):
+        defaults = {}
+        MochitestOptions.__init__(self, automation, scriptdir)
+
+        self.add_option("--b2gpath", action="store",
+                    type = "string", dest = "b2gPath",
+                    help = "path to B2G repo or qemu dir")
+        defaults["b2gPath"] = None
+
+        self.add_option("--marionette", action="store",
+                    type = "string", dest = "marionette",
+                    help = "host:port to use when connecting to Marionette")
+        defaults["marionette"] = None
+
+        self.add_option("--emulator", action="store_true",
+                    dest = "emulator",
+                    help = "True if using a b2g emulator")
+        defaults["emulator"] = False
+
+        self.add_option("--adbpath", action="store",
+                    type = "string", dest = "adbPath",
+                    help = "path to adb")
+        defaults["adbPath"] = "adb"
+
+        self.add_option("--deviceIP", action="store",
+                    type = "string", dest = "deviceIP",
+                    help = "ip address of remote device to test")
+        defaults["deviceIP"] = None
+
+        self.add_option("--devicePort", action="store",
+                    type = "string", dest = "devicePort",
+                    help = "port of remote device to test")
+        defaults["devicePort"] = 20701
+
+        self.add_option("--remote-logfile", action="store",
+                    type = "string", dest = "remoteLogFile",
+                    help = "Name of log file on the device relative to the device root.  PLEASE ONLY USE A FILENAME.")
+        defaults["remoteLogFile"] = None
+
+        self.add_option("--remote-webserver", action = "store",
+                    type = "string", dest = "remoteWebServer",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["remoteWebServer"] = None
+
+        self.add_option("--http-port", action = "store",
+                    type = "string", dest = "httpPort",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
+
+        self.add_option("--ssl-port", action = "store",
+                    type = "string", dest = "sslPort",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["sslPort"] = automation.DEFAULT_SSL_PORT
+
+        self.add_option("--pidfile", action = "store",
+                    type = "string", dest = "pidFile",
+                    help = "name of the pidfile to generate")
+        defaults["pidFile"] = ""
+
+        defaults["remoteTestRoot"] = None
+        defaults["logFile"] = "mochitest.log"
+        defaults["autorun"] = True
+        defaults["closeWhenDone"] = True
+        defaults["testPath"] = ""
+
+        self.set_defaults(**defaults)
+
+    def verifyRemoteOptions(self, options, automation):
+        options.remoteTestRoot = automation._devicemanager.getDeviceRoot()
+        productRoot = options.remoteTestRoot + "/" + automation._product
+
+        if options.utilityPath == self._automation.DIST_BIN:
+            options.utilityPath = productRoot + "/bin"
+
+        if options.remoteWebServer == None:
+            if os.name != "nt":
+                options.remoteWebServer = automation.getLanIp()
+            else:
+                print "ERROR: you must specify a --remote-webserver=<ip address>\n"
+                return None
+
+        options.webServer = options.remoteWebServer
+
+        #if not options.emulator and not options.deviceIP:
+        #    print "ERROR: you must provide a device IP"
+        #    return None
+
+        if options.remoteLogFile == None:
+            options.remoteLogFile = options.remoteTestRoot + '/logs/mochitest.log'
+
+        if options.remoteLogFile.count('/') < 1:
+            options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
+
+        # Only reset the xrePath if it wasn't provided
+        if options.xrePath == None:
+            options.xrePath = options.utilityPath
+
+        if options.pidFile != "":
+            f = open(options.pidFile, 'w')
+            f.write("%s" % os.getpid())
+            f.close()
+
+        return options
+
+    def verifyOptions(self, options, mochitest):
+        # since we are reusing verifyOptions, it will exit if App is not found
+        temp = options.app
+        options.app = sys.argv[0]
+        tempPort = options.httpPort
+        tempSSL = options.sslPort
+        tempIP = options.webServer
+        options = MochitestOptions.verifyOptions(self, options, mochitest)
+        options.webServer = tempIP
+        options.app = temp
+        options.sslPort = tempSSL
+        options.httpPort = tempPort
+
+        return options 
+
+
+class ProfileConfigParser(ConfigParser.RawConfigParser):
+    """Subclass of RawConfigParser that outputs .ini files in the exact
+       format expected for profiles.ini, which is slightly different
+       than the default format.
+    """
+
+    def optionxform(self, optionstr):
+        return optionstr
+
+    def write(self, fp):
+        if self._defaults:
+            fp.write("[%s]\n" % ConfigParser.DEFAULTSECT)
+            for (key, value) in self._defaults.items():
+                fp.write("%s=%s\n" % (key, str(value).replace('\n', '\n\t')))
+            fp.write("\n")
+        for section in self._sections:
+            fp.write("[%s]\n" % section)
+            for (key, value) in self._sections[section].items():
+                if key == "__name__":
+                    continue
+                if (value is not None) or (self._optcre == self.OPTCRE):
+                    key = "=".join((key, str(value).replace('\n', '\n\t')))
+                fp.write("%s\n" % (key))
+            fp.write("\n")
+
+
+class B2GMochitest(Mochitest):
+
+    _automation = None
+    _dm = None
+    localProfile = None
+
+    def __init__(self, automation, devmgr, options):
+        self._automation = automation
+        Mochitest.__init__(self, self._automation)
+        self._dm = devmgr
+        self.runSSLTunnel = False
+        self.remoteProfile = options.remoteTestRoot + '/profile'
+        self._automation.setRemoteProfile(self.remoteProfile)
+        self.remoteLog = options.remoteLogFile
+        self.remoteProfilesIniPath = '/data/b2g/mozilla/profiles.ini'
+        self.originalProfilesIni = None
+
+    def cleanup(self, manifest, options):
+        self._dm.getFile(self.remoteLog, self.localLog)
+        self._dm.removeFile(self.remoteLog)
+        self._dm.removeDir(self.remoteProfile)
+
+        if self.originalProfilesIni:
+            try:
+                self.restoreProfilesIni()
+                os.remove(self.originalProfilesIni)
+            except:
+                pass
+
+        if options.pidFile != "":
+            try:
+                os.remove(options.pidFile)
+                os.remove(options.pidFile + ".xpcshell.pid")
+            except:
+                print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % options.pidFile
+
+        # We've restored the original profile, so reboot the device so that
+        # it gets picked up.
+        self._automation.rebootDevice()
+
+    def findPath(self, paths, filename = None):
+        for path in paths:
+            p = path
+            if filename:
+                p = os.path.join(p, filename)
+            if os.path.exists(self.getFullPath(p)):
+                return path
+        return None
+
+    def startWebServer(self, options):
+        """ Create the webserver on the host and start it up """
+        remoteXrePath = options.xrePath
+        remoteProfilePath = options.profilePath
+        remoteUtilityPath = options.utilityPath
+        localAutomation = Automation()
+        localAutomation.IS_WIN32 = False
+        localAutomation.IS_LINUX = False
+        localAutomation.IS_MAC = False
+        localAutomation.UNIXISH = False
+        hostos = sys.platform
+        if hostos in ['mac', 'darwin']:
+            localAutomation.IS_MAC = True
+        elif hostos in ['linux', 'linux2']:
+            localAutomation.IS_LINUX = True
+            localAutomation.UNIXISH = True
+        elif hostos in ['win32', 'win64']:
+            localAutomation.BIN_SUFFIX = ".exe"
+            localAutomation.IS_WIN32 = True
+
+        paths = [options.xrePath,
+                 localAutomation.DIST_BIN,
+                 self._automation._product,
+                 os.path.join('..', self._automation._product)]
+        options.xrePath = self.findPath(paths)
+        if options.xrePath == None:
+            print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
+            sys.exit(1)
+        paths.append("bin")
+        paths.append(os.path.join("..", "bin"))
+
+        xpcshell = "xpcshell"
+        if (os.name == "nt"):
+            xpcshell += ".exe"
+
+        if (options.utilityPath):
+            paths.insert(0, options.utilityPath)
+        options.utilityPath = self.findPath(paths, xpcshell)
+        if options.utilityPath == None:
+            print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
+            sys.exit(1)
+
+        options.profilePath = tempfile.mkdtemp()
+        self.server = MochitestServer(localAutomation, options)
+        self.server.start()
+
+        if (options.pidFile != ""):
+            f = open(options.pidFile + ".xpcshell.pid", 'w')
+            f.write("%s" % self.server._process.pid)
+            f.close()
+        self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
+
+        options.xrePath = remoteXrePath
+        options.utilityPath = remoteUtilityPath
+        options.profilePath = remoteProfilePath
+
+    def stopWebServer(self, options):
+        self.server.stop()
+
+    def buildProfile(self, options):
+        if self.localProfile:
+            options.profilePath = self.localProfile
+        print 'buildProfile', repr(options)
+        manifest = Mochitest.buildProfile(self, options)
+        self.localProfile = options.profilePath
+
+        # Profile isn't actually copied to device until
+        # buildURLOptions is called.
+
+        options.profilePath = self.remoteProfile
+        return manifest
+
+    def restoreProfilesIni(self):
+        # restore profiles.ini on the device to its previous state
+        if not self.originalProfilesIni or not os.access(self.originalProfilesIni, os.F_OK):
+            raise DMError('Unable to install original profiles.ini; file not found: %s',
+                          self.originalProfilesIni)
+
+        self._dm.pushFile(self.originalProfilesIni, self.remoteProfilesIniPath)
+
+    def updateProfilesIni(self, profilePath):
+        # update profiles.ini on the device to point to the test profile
+        self.originalProfilesIni = tempfile.mktemp()
+        self._dm.getFile(self.remoteProfilesIniPath, self.originalProfilesIni)
+
+        config = ProfileConfigParser()
+        config.read(self.originalProfilesIni)
+        for section in config.sections():
+            if 'Profile' in section:
+                config.set(section, 'IsRelative', 0)
+                config.set(section, 'Path', profilePath)
+
+        newProfilesIni = tempfile.mktemp()
+        with open(newProfilesIni, 'wb') as configfile:
+            config.write(configfile)
+
+        self._dm.pushFile(newProfilesIni, self.remoteProfilesIniPath)
+
+    def buildURLOptions(self, options, env):
+        self.localLog = options.logFile
+        options.logFile = self.remoteLog
+        options.profilePath = self.localProfile
+        retVal = Mochitest.buildURLOptions(self, options, env)
+
+        # set the testURL
+        testURL = self.buildTestPath(options)
+        if len(self.urlOpts) > 0:
+            testURL += "?" + "&".join(self.urlOpts)
+        self._automation.testURL = testURL
+
+        # Set the B2G homepage as a static local page, since wi-fi generally
+        # isn't available as soon as the device boots.
+        f = open(os.path.join(options.profilePath, "user.js"), "a")
+        f.write('user_pref("browser.homescreenURL", "data:text/html,mochitest-plain should start soon");\n')
+        f.close()
+
+        self._dm.removeDir(self.remoteProfile)
+        if self._dm.pushDir(options.profilePath, self.remoteProfile) == None:
+            raise devicemanager.FileError("Unable to copy profile to device.")
+
+        self.updateProfilesIni(self.remoteProfile)
+
+        options.profilePath = self.remoteProfile
+        options.logFile = self.localLog
+        return retVal
+
+
+def main():
+    scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+    auto = B2GRemoteAutomation(None, "fennec")
+    parser = B2GOptions(auto, scriptdir)
+    options, args = parser.parse_args()
+
+    # create our Marionette instance
+    kwargs = {'emulator': options.emulator}
+    if options.b2gPath:
+        kwargs['homedir'] = options.b2gPath
+    if options.marionette:
+        host,port = options.marionette.split(':')
+        kwargs['host'] = host
+        kwargs['port'] = int(port)
+    marionette = Marionette(**kwargs)
+
+    auto.marionette = marionette
+
+    # create the DeviceManager
+    kwargs = {'adbPath': options.adbPath}
+    if options.deviceIP:
+        kwargs.update({'host': options.deviceIP,
+                       'port': options.devicePort})
+    dm = devicemanagerADB.DeviceManagerADB(**kwargs)
+
+    auto.setDeviceManager(dm)
+    options = parser.verifyRemoteOptions(options, auto)
+    if (options == None):
+        print "ERROR: Invalid options specified, use --help for a list of valid options"
+        sys.exit(1)
+
+    auto.setProduct("b2g")
+
+    mochitest = B2GMochitest(auto, dm, options)
+
+    options = parser.verifyOptions(options, mochitest)
+    if (options == None):
+        sys.exit(1)
+
+    logParent = os.path.dirname(options.remoteLogFile)
+    dm.mkDir(logParent);
+    auto.setRemoteLog(options.remoteLogFile)
+    auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)
+
+    retVal = 1
+    try:
+        retVal = mochitest.runTests(options)
+    except:
+        print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running tests." % sys.exc_info()[1]
+        mochitest.stopWebServer(options)
+        mochitest.stopWebSocketServer(options)
+        try:
+            mochitest.cleanup(None, options)
+        except:
+            pass
+            sys.exit(1)
+
+    sys.exit(retVal)
+
+if __name__ == "__main__":
+    main()
+