Merge mozilla-central into mozilla-inbound

.gitignore

@@ -19,6 +19,7 @@ ID
 /config.cache
 /config.log
 /.clang_complete
+/mach.ini
 
 # Empty marker file that's generated when we check out NSS
 security/manager/.nss.checkout

.hgignore

@@ -18,6 +18,7 @@
 ^config\.cache$
 ^config\.log$
 ^\.clang_complete
+^mach.ini$
 
 # Empty marker file that's generated when we check out NSS
 ^security/manager/\.nss\.checkout$

mach

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import platform
+import sys
+
+# Ensure we are running Python 2.7+. We put this check here so we generate a
+# user-friendly error message rather than a cryptic stack trace on module
+# import.
+if sys.version_info[0] == 2 and sys.version_info[1] < 7:
+    print('Python 2.7 or above is required to run mach.')
+    print('You are running', platform.python_version())
+    sys.exit(1)
+
+# TODO Bug 794506 Integrate with the in-tree virtualenv configuration.
+SEARCH_PATHS = [
+    'python/mach',
+    'python/mozbuild',
+    'build',
+    'build/pymake',
+    'python/blessings',
+    'python/psutil',
+    'python/which',
+    'other-licenses/ply',
+    'xpcom/idl-parser',
+    'testing/xpcshell',
+    'testing/mozbase/mozprocess',
+    'testing/mozbase/mozinfo',
+]
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+
+try:
+    import mach.main
+except ImportError:
+    SEARCH_PATHS.reverse()
+    sys.path[0:0] = [os.path.join(our_dir, path) for path in SEARCH_PATHS]
+
+    import mach.main
+
+# All of the code is in a module because EVERYTHING IS A LIBRARY.
+mach = mach.main.Mach(our_dir)
+mach.run(sys.argv[1:])

python/mach/README.rst

@@ -0,0 +1,103 @@
+The mach Driver
+===============
+
+The *mach* driver is the command line interface (CLI) to the source tree.
+
+The *mach* driver is invoked by running the *mach* script or from
+instantiating the *Mach* class from the *mach.main* module.
+
+Implementing mach Commands
+--------------------------
+
+The *mach* driver follows the convention of popular tools like Git,
+Subversion, and Mercurial and provides a common driver for multiple
+sub-commands.
+
+Modules inside *mach* typically contain 1 or more classes which
+inherit from *mach.base.ArgumentProvider*. Modules that inherit from
+this class are hooked up to the *mach* CLI driver. So, to add a new
+sub-command/action to *mach*, one simply needs to create a new class in
+the *mach* package which inherits from *ArgumentProvider*.
+
+Currently, you also need to hook up some plumbing in
+*mach.main.Mach*. In the future, we hope to have automatic detection
+of submodules.
+
+Your command class performs the role of configuring the *mach* frontend
+argument parser as well as providing the methods invoked if a command is
+requested. These methods will take the user-supplied input, do something
+(likely by calling a backend function in a separate module), then format
+output to the terminal.
+
+The plumbing to hook up the arguments to the *mach* driver involves
+light magic. At *mach* invocation time, the driver creates a new
+*argparse* instance. For each registered class that provides commands,
+it calls the *populate_argparse* static method, passing it the parser
+instance.
+
+Your class's *populate_argparse* function should register sub-commands
+with the parser.
+
+For example, say you want to provide the *doitall* command. e.g. *mach
+doitall*. You would create the module *mach.doitall* and this
+module would contain the following class:
+
+    from mach.base import ArgumentProvider
+
+    class DoItAll(ArgumentProvider):
+        def run(self, more=False):
+            print 'I did it!'
+
+        @staticmethod
+        def populate_argparse(parser):
+            # Create the parser to handle the sub-command.
+            p = parser.add_parser('doitall', help='Do it all!')
+
+            p.add_argument('more', action='store_true', default=False,
+                help='Do more!')
+
+            # Tell driver that the handler for this sub-command is the
+            # method *run* on the class *DoItAll*.
+            p.set_defaults(cls=DoItAll, method='run')
+
+The most important line here is the call to *set_defaults*.
+Specifically, the *cls* and *method* parameters, which tell the driver
+which class to instantiate and which method to execute if this command
+is requested.
+
+The specified method will receive all arguments parsed from the command.
+It is important that you use named - not positional - arguments for your
+handler functions or things will blow up. This is because the mach driver
+is using the ``**kwargs`` notation to call the defined method.
+
+In the future, we may provide additional syntactical sugar to make all
+this easier. For example, we may provide decorators on methods to hook
+up commands and handlers.
+
+Minimizing Code in Mach
+-----------------------
+
+Mach is just a frontend. Therefore, code in this package should pertain to
+one of 3 areas:
+
+1. Obtaining user input (parsing arguments, prompting, etc)
+2. Calling into some other Python package
+3. Formatting output
+
+Mach should not contain core logic pertaining to the desired task. If you
+find yourself needing to invent some new functionality, you should implement
+it as a generic package outside of mach and then write a mach shim to call
+into it. There are many advantages to this approach, including reusability
+outside of mach (others may want to write other frontends) and easier testing
+(it is easier to test generic libraries than code that interacts with the
+command line or terminal).
+
+Keeping Frontend Modules Small
+------------------------------
+
+The frontend modules providing mach commands are currently all loaded when
+the mach CLI driver starts. Therefore, there is potential for *import bloat*.
+
+We want the CLI driver to load quickly. So, please delay load external modules
+until they are actually required. In other words, don't use a global
+*import* when you can import from inside a specific command's handler.

python/mach/mach/base.py

@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+
+class ArgumentProvider(object):
+    """Base class for classes wishing to provide CLI arguments to mach."""
+
+    @staticmethod
+    def populate_argparse(parser):
+        raise Exception("populate_argparse not implemented.")

python/mach/mach/main.py

@@ -0,0 +1,197 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides functionality for the command-line build tool
+# (mach). It is packaged as a module because everything is a library.
+
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import sys
+
+from mozbuild.base import BuildConfig
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+# Import sub-command modules
+# TODO Bug 794509 do this via auto-discovery. Update README once this is
+# done.
+from mach.settings import Settings
+from mach.testing import Testing
+
+# Classes inheriting from ArgumentProvider that provide commands.
+HANDLERS = [
+    Settings,
+    Testing,
+]
+
+# Classes inheriting from ConfigProvider that provide settings.
+# TODO this should come from auto-discovery somehow.
+SETTINGS_PROVIDERS = [
+    BuildConfig,
+]
+
+# Settings for argument parser that don't get proxied to sub-module. i.e. these
+# are things consumed by the driver itself.
+CONSUMED_ARGUMENTS = [
+    'settings_file',
+    'verbose',
+    'logfile',
+    'log_interval',
+    'action',
+    'cls',
+    'method',
+    'func',
+]
+
+class Mach(object):
+    """Contains code for the command-line `mach` interface."""
+
+    USAGE = """%(prog)s subcommand [arguments]
+
+mach provides an interface to performing common developer tasks. You specify
+an action/sub-command and it performs it.
+
+Some common actions are:
+
+    %(prog)s help      Show full help, including the list of all commands.
+    %(prog)s test      Run tests.
+
+To see more help for a specific action, run:
+
+  %(prog)s <command> --help
+"""
+
+    def __init__(self, cwd):
+        assert os.path.isdir(cwd)
+
+        self.cwd = cwd
+        self.log_manager = LoggingManager()
+        self.logger = logging.getLogger(__name__)
+        self.settings = ConfigSettings()
+
+        self.log_manager.register_structured_logger(self.logger)
+
+    def run(self, argv):
+        """Runs mach with arguments provided from the command line."""
+        parser = self.get_argument_parser()
+
+        if not len(argv):
+            # We don't register the usage until here because if it is globally
+            # registered, argparse always prints it. This is not desired when
+            # running with --help.
+            parser.usage = Mach.USAGE
+            parser.print_usage()
+            return 0
+
+        if argv[0] == 'help':
+            parser.print_help()
+            return 0
+
+        args = parser.parse_args(argv)
+
+        # Add JSON logging to a file if requested.
+        if args.logfile:
+            self.log_manager.add_json_handler(args.logfile)
+
+        # Up the logging level if requested.
+        log_level = logging.INFO
+        if args.verbose:
+            log_level = logging.DEBUG
+
+        # Always enable terminal logging. The log manager figures out if we are
+        # actually in a TTY or are a pipe and does the right thing.
+        self.log_manager.add_terminal_logging(level=log_level,
+            write_interval=args.log_interval)
+
+        self.load_settings(args)
+        conf = BuildConfig(self.settings)
+
+        stripped = {k: getattr(args, k) for k in vars(args) if k not in
+            CONSUMED_ARGUMENTS}
+
+        # If the action is associated with a class, instantiate and run it.
+        # All classes must be Base-derived and take the expected argument list.
+        if hasattr(args, 'cls'):
+            cls = getattr(args, 'cls')
+            instance = cls(self.cwd, self.settings, self.log_manager)
+            fn = getattr(instance, getattr(args, 'method'))
+
+        # If the action is associated with a function, call it.
+        elif hasattr(args, 'func'):
+            fn = getattr(args, 'func')
+        else:
+            raise Exception('Dispatch configuration error in module.')
+
+        fn(**stripped)
+
+    def log(self, level, action, params, format_str):
+        """Helper method to record a structured log event."""
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def load_settings(self, args):
+        """Determine which settings files apply and load them.
+
+        Currently, we only support loading settings from a single file.
+        Ideally, we support loading from multiple files. This is supported by
+        the ConfigSettings API. However, that API currently doesn't track where
+        individual values come from, so if we load from multiple sources then
+        save, we effectively do a full copy. We don't want this. Until
+        ConfigSettings does the right thing, we shouldn't expose multi-file
+        loading.
+
+        We look for a settings file in the following locations. The first one
+        found wins:
+
+          1) Command line argument
+          2) Environment variable
+          3) Default path
+        """
+        for provider in SETTINGS_PROVIDERS:
+            provider.register_settings()
+            self.settings.register_provider(provider)
+
+        p = os.path.join(self.cwd, 'mach.ini')
+
+        if args.settings_file:
+            p = args.settings_file
+        elif 'MACH_SETTINGS_FILE' in os.environ:
+            p = os.environ['MACH_SETTINGS_FILE']
+
+        self.settings.load_file(p)
+
+        return os.path.exists(p)
+
+    def get_argument_parser(self):
+        """Returns an argument parser for the command-line interface."""
+
+        parser = argparse.ArgumentParser()
+
+        settings_group = parser.add_argument_group('Settings')
+        settings_group.add_argument('--settings', dest='settings_file',
+            metavar='FILENAME', help='Path to settings file.')
+
+        logging_group = parser.add_argument_group('Logging')
+        logging_group.add_argument('-v', '--verbose', dest='verbose',
+            action='store_true', default=False,
+            help='Print verbose output.')
+        logging_group.add_argument('-l', '--log-file', dest='logfile',
+            metavar='FILENAME', type=argparse.FileType('ab'),
+            help='Filename to write log data to.')
+        logging_group.add_argument('--log-interval', dest='log_interval',
+            action='store_true', default=False,
+            help='Prefix log line with interval from last message rather '
+                'than relative time. Note that this is NOT execution time '
+                'if there are parallel operations.')
+
+        subparser = parser.add_subparsers(dest='action')
+
+        # Register argument action providers with us.
+        for cls in HANDLERS:
+            cls.populate_argparse(subparser)
+
+        return parser

python/mach/mach/settings.py

@@ -0,0 +1,51 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+from textwrap import TextWrapper
+
+from mozbuild.base import MozbuildObject
+from mach.base import ArgumentProvider
+
+class Settings(MozbuildObject, ArgumentProvider):
+    """Interact with settings for mach.
+
+    Currently, we only provide functionality to view what settings are
+    available. In the future, this module will be used to modify settings, help
+    people create configs via a wizard, etc.
+    """
+    def list_settings(self):
+        """List available settings in a concise list."""
+        for section in sorted(self.settings):
+            for option in sorted(self.settings[section]):
+                short, full = self.settings.option_help(section, option)
+                print('%s.%s -- %s' % (section, option, short))
+
+    def create(self):
+        """Create an empty settings file with full documentation."""
+        wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ')
+
+        for section in sorted(self.settings):
+            print('[%s]' % section)
+            print('')
+
+            for option in sorted(self.settings[section]):
+                short, full = self.settings.option_help(section, option)
+
+                print(wrapper.fill(full))
+                print(';%s =' % option)
+                print('')
+
+    @staticmethod
+    def populate_argparse(parser):
+        lst = parser.add_parser('settings-list',
+            help='Show available config settings.')
+
+        lst.set_defaults(cls=Settings, method='list_settings')
+
+        create = parser.add_parser('settings-create',
+            help='Print a new settings file with usage info.')
+
+        create.set_defaults(cls=Settings, method='create')

python/mach/mach/terminal.py

@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This file contains code for interacting with terminals.
+
+All the terminal interaction code is consolidated so the complexity can be in
+one place, away from code that is commonly looked at.
+"""
+
+from __future__ import print_function, unicode_literals
+
+import logging
+import sys
+
+
+class LoggingHandler(logging.Handler):
+    """Custom logging handler that works with terminal window dressing.
+
+    This is alternative terminal logging handler which contains smarts for
+    emitting terminal control characters properly. Currently, it has generic
+    support for "footer" elements at the bottom of the screen. Functionality
+    can be added when needed.
+    """
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+        self.fh = sys.stdout
+        self.footer = None
+
+    def flush(self):
+        self.acquire()
+
+        try:
+            self.fh.flush()
+        finally:
+            self.release()
+
+    def emit(self, record):
+        msg = self.format(record)
+
+        if self.footer:
+            self.footer.clear()
+
+        self.fh.write(msg)
+        self.fh.write('\n')
+
+        if self.footer:
+            self.footer.draw()
+
+        # If we don't flush, the footer may not get drawn.
+        self.flush()
+
+
+class TerminalFooter(object):
+    """Represents something drawn on the bottom of a terminal."""
+    def __init__(self, terminal):
+        self.t = terminal
+        self.fh = sys.stdout
+
+    def _clear_lines(self, n):
+        for i in xrange(n):
+            self.fh.write(self.t.move_x(0))
+            self.fh.write(self.t.clear_eol())
+            self.fh.write(self.t.move_up())
+
+        self.fh.write(self.t.move_down())
+        self.fh.write(self.t.move_x(0))
+
+    def clear(self):
+        raise Exception('clear() must be implemented.')
+
+    def draw(self):
+        raise Exception('draw() must be implemented.')
+

python/mach/mach/testing.py

@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from mach.base import ArgumentProvider
+from mozbuild.base import MozbuildObject
+
+
+generic_help = 'Test to run. Can be specified as a single JS file, a ' +\
+'directory, or omitted. If omitted, the entire test suite is executed.'
+
+
+class Testing(MozbuildObject, ArgumentProvider):
+    """Provides commands for running tests."""
+
+    def run_suite(self, suite):
+        from mozbuild.testing.suite import Suite
+
+        s = self._spawn(Suite)
+        s.run_suite(suite)
+
+    def run_mochitest(self, test_file, flavor):
+        from mozbuild.testing.mochitest import MochitestRunner
+
+        mochitest = self._spawn(MochitestRunner)
+        mochitest.run_mochitest_test(test_file, flavor)
+
+    def run_xpcshell_test(self, **params):
+        from mozbuild.testing.xpcshell import XPCShellRunner
+
+        xpcshell = self._spawn(XPCShellRunner)
+        xpcshell.run_test(**params)
+
+    @staticmethod
+    def populate_argparse(parser):
+        # Whole suites.
+        group = parser.add_parser('test', help="Perform tests.")
+
+        suites = set(['xpcshell', 'mochitest-plain', 'mochitest-chrome',
+            'mochitest-browser', 'all'])
+
+        group.add_argument('suite', default='all', choices=suites, nargs='?',
+            help="Test suite to run.")
+
+        group.set_defaults(cls=Testing, method='run_suite', suite='all')
+
+        mochitest_plain = parser.add_parser('mochitest-plain',
+            help='Run a plain mochitest.')
+        mochitest_plain.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        mochitest_plain.set_defaults(cls=Testing, method='run_mochitest',
+            flavor='plain')
+
+        mochitest_chrome = parser.add_parser('mochitest-chrome',
+            help='Run a chrome mochitest.')
+        mochitest_chrome.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        mochitest_chrome.set_defaults(cls=Testing, method='run_mochitest',
+            flavor='chrome')
+
+        mochitest_browser = parser.add_parser('mochitest-browser',
+            help='Run a mochitest with browser chrome.')
+        mochitest_browser.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        mochitest_browser.set_defaults(cls=Testing, method='run_mochitest',
+            flavor='browser')
+
+        xpcshell = parser.add_parser('xpcshell-test',
+            help="Run an individual xpcshell test.")
+
+        xpcshell.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        xpcshell.add_argument('--debug', '-d', action='store_true',
+            help='Run test in debugger.')
+
+        xpcshell.set_defaults(cls=Testing, method='run_xpcshell_test')

python/mach/setup.py

@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup
+
+VERSION = '0.1'
+
+setup(
+    name='mach',
+    description='CLI frontend to mozilla-central.',
+    license='MPL 2.0',
+    packages=['mach'],
+    version=VERSION
+)
+

python/mozbuild/README.rst

@@ -12,6 +12,7 @@ Modules Overview
   includes managing compiler warnings.
 * mozbuild.logging -- Defines mozbuild's logging infrastructure.
   mozbuild uses a structured logging backend.
+* mozbuild.testing -- Interfaces for running tests.
 
 Structured Logging
 ==================

python/mozbuild/mozbuild/base.py

@@ -0,0 +1,363 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import logging
+import multiprocessing
+import os
+import pymake.parser
+import shlex
+import subprocess
+import which
+
+from mozprocess.processhandler import ProcessHandlerMixin
+from pymake.data import Makefile
+from tempfile import TemporaryFile
+
+from mozbuild.config import ConfigProvider
+from mozbuild.config import PositiveIntegerType
+
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if 'SHELL' in os.environ:
+    _current_shell = os.environ['SHELL']
+elif 'MOZILLABUILD' in os.environ:
+    _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+elif 'COMSPEC' in os.environ:
+    _current_shell = os.environ['COMSPEC']
+else:
+    raise Exception('Could not detect environment shell!')
+
+_in_msys = False
+
+if os.environ.get('MSYSTEM', None) == 'MINGW32':
+    _in_msys = True
+
+    if not _current_shell.lower().endswith('.exe'):
+        _current_shell += '.exe'
+
+
+class MozbuildObject(object):
+    """Base class providing basic functionality useful to many modules.
+
+    Modules in this package typically require common functionality such as
+    accessing the current config, getting the location of the source directory,
+    running processes, etc. This classes provides that functionality. Other
+    modules can inherit from this class to obtain this functionality easily.
+    """
+    def __init__(self, topsrcdir, settings, log_manager, topobjdir=None):
+        """Create a new Mozbuild object instance.
+
+        Instances are bound to a source directory, a ConfigSettings instance,
+        and a LogManager instance. The topobjdir may be passed in as well. If
+        it isn't, it will be calculated from the active mozconfig.
+        """
+        self.topsrcdir = topsrcdir
+        self.settings = settings
+        self.config = BuildConfig(settings)
+        self.logger = logging.getLogger(__name__)
+        self.log_manager = log_manager
+
+        self._config_guess_output = None
+        self._make = None
+        self._topobjdir = topobjdir
+
+    @property
+    def topobjdir(self):
+        if self._topobjdir is None:
+            self._load_mozconfig()
+
+        if self._topobjdir is None:
+            self._topobjdir = 'obj-%s' % self._config_guess
+
+        return self._topobjdir
+
+    @property
+    def distdir(self):
+        return os.path.join(self.topobjdir, 'dist')
+
+    @property
+    def bindir(self):
+        return os.path.join(self.topobjdir, 'dist', 'bin')
+
+    @property
+    def statedir(self):
+        return os.path.join(self.topobjdir, '.mozbuild')
+
+    def log(self, level, action, params, format_str):
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def _load_mozconfig(self, path=None):
+        # The mozconfig loader outputs a make file. We parse and load this make
+        # file with pymake and evaluate it in a context similar to client.mk.
+
+        loader = os.path.join(self.topsrcdir, 'build', 'autoconf',
+            'mozconfig2client-mk')
+
+        # os.environ from a library function is somewhat evil. But, mozconfig
+        # files are tightly coupled with the environment by definition. In the
+        # future, perhaps we'll have a more sanitized environment for mozconfig
+        # execution.
+        env = dict(os.environ)
+        if path is not None:
+            env['MOZCONFIG'] = path
+
+        env['CONFIG_GUESS'] = self._config_guess
+
+        output = subprocess.check_output([loader, self.topsrcdir],
+            stderr=subprocess.PIPE, cwd=self.topsrcdir, env=env)
+
+        # The output is make syntax. We parse this in a specialized make
+        # context.
+        statements = pymake.parser.parsestring(output, 'mozconfig')
+
+        makefile = Makefile(workdir=self.topsrcdir, env={
+            'TOPSRCDIR': self.topsrcdir,
+            'CONFIG_GUESS': self._config_guess})
+
+        statements.execute(makefile)
+
+        def get_value(name):
+            exp = makefile.variables.get(name)[2]
+
+            return exp.resolvestr(makefile, makefile.variables)
+
+        for name, flavor, source, value in makefile.variables:
+            # We only care about variables that came from the parsed mozconfig.
+            if source != pymake.data.Variables.SOURCE_MAKEFILE:
+                continue
+
+            # Ignore some pymake built-ins.
+            if name in ('.PYMAKE', 'MAKELEVEL', 'MAKEFLAGS'):
+                continue
+
+            if name == 'MOZ_OBJDIR':
+                self._topobjdir = get_value(name)
+
+            # If we want to extract other variables defined by mozconfig, here
+            # is where we'd do it.
+
+    @property
+    def _config_guess(self):
+        if self._config_guess_output is None:
+            p = os.path.join(self.topsrcdir, 'build', 'autoconf',
+                'config.guess')
+            self._config_guess_output = subprocess.check_output([p],
+                cwd=self.topsrcdir).strip()
+
+        return self._config_guess_output
+
+    def _ensure_objdir_exists(self):
+        if os.path.isdir(self.statedir):
+            return
+
+        os.makedirs(self.statedir)
+
+    def _ensure_state_subdir_exists(self, subdir):
+        path = os.path.join(self.statedir, subdir)
+
+        if os.path.isdir(path):
+            return
+
+        os.makedirs(path)
+
+    def _get_state_filename(self, filename, subdir=None):
+        path = self.statedir
+
+        if subdir:
+            path = os.path.join(path, subdir)
+
+        return os.path.join(path, filename)
+
+    def _get_srcdir_path(self, path):
+        """Convert a relative path in the source directory to a full path."""
+        return os.path.join(self.topsrcdir, path)
+
+    def _get_objdir_path(self, path):
+        """Convert a relative path in the object directory to a full path."""
+        return os.path.join(self.topobjdir, path)
+
+    def _run_make(self, directory=None, filename=None, target=None, log=True,
+            srcdir=False, allow_parallel=True, line_handler=None, env=None,
+            ignore_errors=False):
+        """Invoke make.
+
+        directory -- Relative directory to look for Makefile in.
+        filename -- Explicit makefile to run.
+        target -- Makefile target(s) to make. Can be a string or iterable of
+            strings.
+        srcdir -- If True, invoke make from the source directory tree.
+            Otherwise, make will be invoked from the object directory.
+        """
+        self._ensure_objdir_exists()
+
+        args = [self._make_path]
+
+        if directory:
+            args.extend(['-C', directory])
+
+        if filename:
+            args.extend(['-f', filename])
+
+        if allow_parallel:
+            args.append('-j%d' % self.settings.build.threads)
+
+        if ignore_errors:
+            args.append('-k')
+
+        # Silent mode by default.
+        args.append('-s')
+
+        # Print entering/leaving directory messages. Some consumers look at
+        # these to measure progress. Ideally, we'd do everything with pymake
+        # and use hooks in its API. Unfortunately, it doesn't provide that
+        # feature... yet.
+        args.append('-w')
+
+        if isinstance(target, list):
+            args.extend(target)
+        elif target:
+            args.append(target)
+
+        fn = self._run_command_in_objdir
+
+        if srcdir:
+            fn = self._run_command_in_srcdir
+
+        params = {
+            'args': args,
+            'line_handler': line_handler,
+            'explicit_env': env,
+            'log_level': logging.INFO,
+            'require_unix_environment': True,
+            'ignore_errors': ignore_errors,
+        }
+
+        if log:
+            params['log_name'] = 'make'
+
+        fn(**params)
+
+    @property
+    def _make_path(self):
+        if self._make is None:
+            if self._is_windows():
+                self._make = os.path.join(self.topsrcdir, 'build', 'pymake',
+                    'make.py')
+
+            else:
+                for test in ['gmake', 'make']:
+                    try:
+                        self._make = which.which(test)
+                        break
+                    except which.WhichError:
+                        continue
+
+        if self._make is None:
+            raise Exception('Could not find suitable make binary!')
+
+        return self._make
+
+    def _run_command_in_srcdir(self, **args):
+        self._run_command(cwd=self.topsrcdir, **args)
+
+    def _run_command_in_objdir(self, **args):
+        self._run_command(cwd=self.topobjdir, **args)
+
+    def _run_command(self, args=None, cwd=None, append_env=None,
+        explicit_env=None, log_name=None, log_level=logging.INFO,
+        line_handler=None, require_unix_environment=False,
+        ignore_errors=False):
+        """Runs a single command to completion.
+
+        Takes a list of arguments to run where the first item is the
+        executable. Runs the command in the specified directory and
+        with optional environment variables.
+
+        append_env -- Dict of environment variables to append to the current
+            set of environment variables.
+        explicit_env -- Dict of environment variables to set for the new
+            process. Any existing environment variables will be ignored.
+
+        require_unix_environment if True will ensure the command is executed
+        within a UNIX environment. Basically, if we are on Windows, it will
+        execute the command via an appropriate UNIX-like shell.
+        """
+        assert isinstance(args, list) and len(args)
+
+        if require_unix_environment and _in_msys:
+            # Always munge Windows-style into Unix style for the command.
+            prog = args[0].replace('\\', '/')
+
+            # PyMake removes the C: prefix. But, things seem to work here
+            # without it. Not sure what that's about.
+
+            # We run everything through the msys shell. We need to use
+            # '-c' and pass all the arguments as one argument because that is
+            # how sh works.
+            cline = subprocess.list2cmdline([prog] + args[1:])
+            args = [_current_shell, '-c', cline]
+
+        self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
+
+        def handleLine(line):
+            if line_handler:
+                line_handler(line)
+
+            if not log_name:
+                return
+
+            self.log(log_level, log_name, {'line': line.strip()}, '{line}')
+
+        use_env = {}
+        if explicit_env:
+            use_env = explicit_env
+        else:
+            use_env.update(os.environ)
+
+            if append_env:
+                use_env.update(env)
+
+        p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+            processOutputLine=[handleLine], universal_newlines=True)
+        p.run()
+        p.processOutput()
+        status = p.wait()
+
+        if status != 0 and not ignore_errors:
+            raise Exception('Process executed with non-0 exit code: %s' % args)
+
+    def _is_windows(self):
+        return os.name in ('nt', 'ce')
+
+    def _spawn(self, cls):
+        """Create a new MozbuildObject-derived class instance from ourselves.
+
+        This is used as a convenience method to create other
+        MozbuildObject-derived class instances. It can only be used on
+        classes that have the same constructor arguments as us.
+        """
+
+        return cls(self.topsrcdir, self.settings, self.log_manager,
+            topobjdir=self.topobjdir)
+
+
+class BuildConfig(ConfigProvider):
+    """The configuration for mozbuild."""
+
+    def __init__(self, settings):
+        self.settings = settings
+
+    @classmethod
+    def _register_settings(cls):
+        def register(section, option, type_cls, **kwargs):
+            cls.register_setting(section, option, type_cls, domain='mozbuild',
+                **kwargs)
+
+        register('build', 'threads', PositiveIntegerType,
+            default=multiprocessing.cpu_count())

python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po

@@ -0,0 +1,8 @@
+msgid "build.threads.short"
+msgstr "Thread Count"
+
+msgid "build.threads.full"
+msgstr "The number of threads to use when performing CPU intensive tasks. "
+       "This constrols the level of parallelization. The default value is "
+       "the number of cores in your machine."
+

python/mozbuild/mozbuild/test/test_base.py

@@ -0,0 +1,61 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+from tempfile import NamedTemporaryFile
+
+from mozbuild.base import BuildConfig
+from mozbuild.base import MozbuildObject
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+
+curdir = os.path.dirname(__file__)
+topsrcdir = os.path.normpath(os.path.join(curdir, '..', '..', '..', '..'))
+log_manager = LoggingManager()
+
+
+class TestBuildConfig(unittest.TestCase):
+    def test_basic(self):
+        c = ConfigSettings()
+        c.register_provider(BuildConfig)
+
+        c.build.threads = 6
+
+
+class TestMozbuildObject(unittest.TestCase):
+    def get_base(self):
+        settings = ConfigSettings()
+        settings.register_provider(BuildConfig)
+
+        return MozbuildObject(topsrcdir, settings, log_manager)
+
+    def test_mozconfig_parsing(self):
+        with NamedTemporaryFile(mode='wt') as mozconfig:
+            mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir')
+            mozconfig.flush()
+
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            base = self.get_base()
+            base._load_mozconfig()
+
+            self.assertEqual(base.topobjdir, '%s/some-objdir' % topsrcdir)
+
+        del os.environ['MOZCONFIG']
+
+    def test_objdir_config_guess(self):
+        base = self.get_base()
+
+        with NamedTemporaryFile() as mozconfig:
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            self.assertIsNotNone(base.topobjdir)
+            self.assertEqual(len(base.topobjdir.split()), 1)
+
+        del os.environ['MOZCONFIG']

python/mozbuild/mozbuild/testing/mochitest.py

@@ -0,0 +1,82 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+
+from mozbuild.base import MozbuildObject
+
+
+class MochitestRunner(MozbuildObject):
+    """Easily run mochitests.
+
+    This currently contains just the basics for running mochitests. We may want
+    to hook up result parsing, etc.
+    """
+    def run_plain_suite(self):
+        """Runs all plain mochitests."""
+        # TODO hook up Python harness runner.
+        self._run_make(directory='.', target='mochitest-plain')
+
+    def run_chrome_suite(self):
+        """Runs all chrome mochitests."""
+        # TODO hook up Python harness runner.
+        self._run_make(directory='.', target='mochitest-chrome')
+
+    def run_browser_chrome_suite(self):
+        """Runs browser chrome mochitests."""
+        # TODO hook up Python harness runner.
+        self._run_make(directory='.', target='mochitest-browser-chrome')
+
+    def run_all(self):
+        self.run_plain_suite()
+        self.run_chrome_suite()
+        self.run_browser_chrome_suite()
+
+    def run_mochitest_test(self, test_file=None, suite=None):
+        """Runs a mochitest.
+
+        test_file is a path to a test file. It can be a relative path from the
+        top source directory, an absolute filename, or a directory containing
+        test files.
+
+        suite is the type of mochitest to run. It can be one of ('plain',
+        'chrome', 'browser').
+        """
+        if test_file is None:
+            raise Exception('test_file must be defined.')
+
+        parsed = self._parse_test_path(test_file)
+
+        # TODO hook up harness via native Python
+        target = None
+        if suite == 'plain':
+            target = 'mochitest-plain'
+        elif suite == 'chrome':
+            target = 'mochitest-chrome'
+        elif suite == 'browser':
+            target = 'mochitest-browser-chrome'
+        else:
+            raise Exception('None or unrecognized mochitest suite type.')
+
+        env = {'TEST_PATH': parsed['normalized']}
+
+        self._run_make(directory='.', target=target, env=env)
+
+    def _parse_test_path(self, test_path):
+        is_dir = os.path.isdir(test_path)
+
+        if is_dir and not test_path.endswith(os.path.sep):
+            test_path += os.path.sep
+
+        normalized = test_path
+
+        if test_path.startswith(self.topsrcdir):
+            normalized = test_path[len(self.topsrcdir):]
+
+        return {
+            'normalized': normalized,
+            'is_dir': is_dir,
+        }

python/mozbuild/mozbuild/testing/suite.py

@@ -0,0 +1,49 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from mozbuild.base import MozbuildObject
+from mozbuild.testing.xpcshell import XPCShellRunner
+from mozbuild.testing.mochitest import MochitestRunner
+
+
+class Suite(MozbuildObject):
+    def run_suite(self, suite):
+        """Run a named test suite.
+
+        Recognized names are:
+
+          all - All test suites
+          mochitest-plain - Plain mochitests
+          mochitest-chrome - mochitests with chrome
+          mochitest-browser - mochitests with browser chrome
+          xpcshell - xpcshell tests
+
+        TODO support for other test suite types.
+        """
+
+        xpcshell = self._spawn(XPCShellRunner)
+        mochitest = self._spawn(MochitestRunner)
+
+        if suite == 'all':
+            xpcshell.run_suite()
+            mochitest.run_plain_suite()
+            mochitest.run_chrome_suite()
+            mochitest.run_browser_chrome_suite()
+            return
+
+        m = {
+            'xpcshell': xpcshell.run_suite,
+            'mochitest-plain': mochitest.run_plain_suite,
+            'mochitest-chrome': mochitest.run_chrome_suite,
+            'mochitest-browser': mochitest.run_browser_chrome_suite,
+        }
+
+        method = m.get(suite, None)
+
+        if method is None:
+            raise Exception('Unknown test suite: %s' % suite)
+
+        return method()

python/mozbuild/mozbuild/testing/xpcshell.py

@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This modules contains code for interacting with xpcshell tests.
+
+from __future__ import unicode_literals
+
+import os.path
+
+from StringIO import StringIO
+
+from mozbuild.base import MozbuildObject
+
+
+class XPCShellRunner(MozbuildObject):
+    """Run xpcshell tests."""
+    def run_suite(self):
+        # TODO hook up to harness runner and support things like shuffle,
+        # proper progress updates, etc.
+        self._run_make(directory='.', target='xpcshell-tests')
+
+    def run_test(self, test_file, debug=False):
+        """Runs an individual xpcshell test."""
+        if test_file == 'all':
+            self.run_suite()
+            return
+
+        # dirname() gets confused if there isn't a trailing slash.
+        if os.path.isdir(test_file) and not test_file.endswith(os.path.sep):
+            test_file += os.path.sep
+
+        relative_dir = test_file
+
+        if test_file.startswith(self.topsrcdir):
+            relative_dir = test_file[len(self.topsrcdir):]
+
+        test_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell',
+                os.path.dirname(relative_dir))
+
+        args = {
+            'debug': debug,
+            'test_dirs': [test_dir],
+        }
+
+        if os.path.isfile(test_file):
+            args['test_path'] = os.path.basename(test_file)
+
+        self._run_xpcshell_harness(**args)
+
+    def _run_xpcshell_harness(self, test_dirs=None, manifest=None,
+        test_path=None, debug=False):
+
+        # Obtain a reference to the xpcshell test runner.
+        import runxpcshelltests
+
+        dummy_log = StringIO()
+        xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log)
+        self.log_manager.enable_unstructured()
+
+        tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell')
+        modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
+
+        args = {
+            'xpcshell': os.path.join(self.bindir, 'xpcshell'),
+            'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
+            'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
+            'logfiles': False,
+            'testsRootDir': tests_dir,
+            'testingModulesDir': modules_dir,
+            'profileName': 'firefox',
+            'verbose': test_path is not None,
+        }
+
+        if manifest is not None:
+            args['manifest'] = manifest
+        elif test_dirs is not None:
+            if isinstance(test_dirs, list):
+                args['testdirs'] = test_dirs
+            else:
+                args['testdirs'] = [test_dirs]
+        else:
+            raise Exception('One of test_dirs or manifest must be provided.')
+
+        if test_path is not None:
+            args['testPath'] = test_path
+
+        # TODO do something with result.
+        xpcshell.runTests(**args)
+
+        self.log_manager.disable_unstructured()