diff --git a/aclocal.m4 b/aclocal.m4 index 43a5816ed0f2..52398cf6224c 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -26,6 +26,7 @@ builtin(include, build/autoconf/arch.m4)dnl builtin(include, build/autoconf/android.m4)dnl builtin(include, build/autoconf/zlib.m4)dnl builtin(include, build/autoconf/linux.m4)dnl +builtin(include, build/autoconf/python-virtualenv.m4)dnl MOZ_PROG_CHECKMSYS() diff --git a/build/ConfigStatus.py b/build/ConfigStatus.py index 1e13afd120a6..dddebe8164f7 100644 --- a/build/ConfigStatus.py +++ b/build/ConfigStatus.py @@ -11,6 +11,8 @@ from optparse import OptionParser import sys, re, os, posixpath, ntpath import errno from StringIO import StringIO +from os.path import relpath + # Standalone js doesn't have virtualenv. sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'config')) from Preprocessor import Preprocessor @@ -21,29 +23,6 @@ def log(string): if verbose: print >>sys.stderr, string -# We need relpath, but it is introduced in python 2.6 -# http://docs.python.org/library/os.path.html -def my_relpath(path, start): - """ - Return a relative version of a path - from /usr/lib/python2.6/posixpath.py - """ - - if not path: - raise ValueError("no path specified") - - start_list = os.path.abspath(start).split(os.path.sep) - path_list = os.path.abspath(path).split(os.path.sep) - - # Work out how much of the filepath is shared by start and path. - i = len(os.path.commonprefix([start_list, path_list])) - - rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return os.curdir - return os.path.join(*rel_list) - -relpath = getattr(os.path, "relpath", my_relpath) def ensureParentDir(file): '''Ensures the directory parent to the given file exists''' diff --git a/build/autoconf/python-virtualenv.m4 b/build/autoconf/python-virtualenv.m4 new file mode 100644 index 000000000000..34780ed068c8 --- /dev/null +++ b/build/autoconf/python-virtualenv.m4 @@ -0,0 +1,77 @@ +dnl This Source Code Form is subject to the terms of the Mozilla Public +dnl License, v. 2.0. If a copy of the MPL was not distributed with this +dnl file, You can obtain one at http://mozilla.org/MPL/2.0/. + +AC_DEFUN([MOZ_PYTHON], +[ + +dnl We honor the Python path defined in an environment variable. This is used +dnl to pass the virtualenv's Python from the main configure to SpiderMonkey's +dnl configure, for example. +if test -z "$PYTHON"; then + MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python) + if test -z "$PYTHON"; then + AC_MSG_ERROR([python was not found in \$PATH]) + fi +else + AC_MSG_RESULT([Using Python from environment variable \$PYTHON]) +fi + +_virtualenv_topsrcdir= +_virtualenv_populate_path= + +dnl If this is a mozilla-central, we'll find the virtualenv in the top +dnl source directory. If this is a SpiderMonkey build, we assume we're at +dnl js/src and try to find the virtualenv from the mozilla-central root. +for base in $MOZILLA_CENTRAL_PATH $_topsrcdir $_topsrcdir/../..; do + possible=$base/build/virtualenv/populate_virtualenv.py + + if test -e $possible; then + _virtualenv_topsrcdir=$base + _virtualenv_populate_path=$possible + break + fi +done + +if test -z $_virtualenv_populate_path; then + AC_MSG_ERROR([Unable to find Virtualenv population script. In order +to build, you will need mozilla-central's virtualenv. + +If you are building from a mozilla-central checkout, you should never see this +message. If you are building from a source archive, the source archive was +likely not created properly (it is missing the virtualenv files). + +If you have a copy of mozilla-central available, define the +MOZILLA_CENTRAL_PATH environment variable to the top source directory of +mozilla-central and relaunch configure.]) + +fi + +if test -z $DONT_POPULATE_VIRTUALENV; then + AC_MSG_RESULT([Creating Python environment]) + dnl This verifies our Python version is sane and ensures the Python + dnl virtualenv is present and up to date. It sanitizes the environment + dnl for us, so we don't need to clean anything out. + $PYTHON $_virtualenv_populate_path \ + $_virtualenv_topsrcdir $MOZ_BUILD_ROOT/_virtualenv || exit 1 + + case "$host_os" in + mingw*) + PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe + ;; + *) + PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python + ;; + esac +fi + +AC_SUBST(PYTHON) + +AC_MSG_CHECKING([Python environment is Mozilla virtualenv]) +$PYTHON -c "import mozbuild.base" +if test "$?" != 0; then + AC_MSG_ERROR([Python environment does not appear to be sane.]) +fi +AC_MSG_RESULT([yes]) +]) + diff --git a/build/valgrind/cross-architecture.sup b/build/valgrind/cross-architecture.sup index b9266f33ed4a..6f7cf0246bf8 100644 --- a/build/valgrind/cross-architecture.sup +++ b/build/valgrind/cross-architecture.sup @@ -254,6 +254,14 @@ fun:_ZN2js15ArgumentsObject14createExpectedEP9JSContextPNS_10StackFrameE ... } +{ + Bug 812423 + Memcheck:Leak + fun:malloc + fun:_ZN2js15ArgumentsObject6createI13CopyFrameArgsEEPS0_P9JSContextN2JS6HandleIP8JSScriptEENS7_IP10JSFunctionEEjRT_ + fun:_ZN2js15ArgumentsObject14createExpectedEP9JSContextNS_16AbstractFramePtrE + ... +} { Bug 823782 Memcheck:Leak diff --git a/configure.in b/configure.in index 5c6f375638e7..4c5f5e92626a 100644 --- a/configure.in +++ b/configure.in @@ -147,29 +147,7 @@ else touch $_objdir/CLOBBER fi -MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python) -if test -z "$PYTHON"; then - AC_MSG_ERROR([python was not found in \$PATH]) -fi - -AC_MSG_RESULT([Creating Python environment]) -dnl This verifies our Python version is sane and ensures the Python -dnl virtualenv is present and up to date. It sanitizes the environment -dnl for us, so we don't need to clean anything out. -$PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \ - $_topsrcdir $MOZ_BUILD_ROOT/_virtualenv || exit 1 - -dnl Create a virtualenv where we can install local Python packages -case "$host_os" in -mingw*) - PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe - ;; -*) - PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python - ;; -esac - -AC_SUBST(PYTHON) +MOZ_PYTHON MOZ_DEFAULT_COMPILER @@ -9347,6 +9325,9 @@ export MOZ_NATIVE_ZLIB export MOZ_ZLIB_CFLAGS export MOZ_ZLIB_LIBS export MOZ_APP_NAME +export DONT_POPULATE_VIRTUALENV=1 +export PYTHON +export MOZILLA_CENTRAL_PATH=$_topsrcdir export STLPORT_CPPFLAGS export STLPORT_LDFLAGS export STLPORT_LIBS diff --git a/js/src/aclocal.m4 b/js/src/aclocal.m4 index 7d6b28dcf098..afa827187503 100644 --- a/js/src/aclocal.m4 +++ b/js/src/aclocal.m4 @@ -25,5 +25,6 @@ builtin(include, build/autoconf/arch.m4)dnl builtin(include, build/autoconf/android.m4)dnl builtin(include, build/autoconf/zlib.m4)dnl builtin(include, build/autoconf/linux.m4)dnl +builtin(include, build/autoconf/python-virtualenv.m4)dnl MOZ_PROG_CHECKMSYS() diff --git a/js/src/build/ConfigStatus.py b/js/src/build/ConfigStatus.py index 1e13afd120a6..dddebe8164f7 100644 --- a/js/src/build/ConfigStatus.py +++ b/js/src/build/ConfigStatus.py @@ -11,6 +11,8 @@ from optparse import OptionParser import sys, re, os, posixpath, ntpath import errno from StringIO import StringIO +from os.path import relpath + # Standalone js doesn't have virtualenv. sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'config')) from Preprocessor import Preprocessor @@ -21,29 +23,6 @@ def log(string): if verbose: print >>sys.stderr, string -# We need relpath, but it is introduced in python 2.6 -# http://docs.python.org/library/os.path.html -def my_relpath(path, start): - """ - Return a relative version of a path - from /usr/lib/python2.6/posixpath.py - """ - - if not path: - raise ValueError("no path specified") - - start_list = os.path.abspath(start).split(os.path.sep) - path_list = os.path.abspath(path).split(os.path.sep) - - # Work out how much of the filepath is shared by start and path. - i = len(os.path.commonprefix([start_list, path_list])) - - rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return os.curdir - return os.path.join(*rel_list) - -relpath = getattr(os.path, "relpath", my_relpath) def ensureParentDir(file): '''Ensures the directory parent to the given file exists''' diff --git a/js/src/build/autoconf/python-virtualenv.m4 b/js/src/build/autoconf/python-virtualenv.m4 new file mode 100644 index 000000000000..34780ed068c8 --- /dev/null +++ b/js/src/build/autoconf/python-virtualenv.m4 @@ -0,0 +1,77 @@ +dnl This Source Code Form is subject to the terms of the Mozilla Public +dnl License, v. 2.0. If a copy of the MPL was not distributed with this +dnl file, You can obtain one at http://mozilla.org/MPL/2.0/. + +AC_DEFUN([MOZ_PYTHON], +[ + +dnl We honor the Python path defined in an environment variable. This is used +dnl to pass the virtualenv's Python from the main configure to SpiderMonkey's +dnl configure, for example. +if test -z "$PYTHON"; then + MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python) + if test -z "$PYTHON"; then + AC_MSG_ERROR([python was not found in \$PATH]) + fi +else + AC_MSG_RESULT([Using Python from environment variable \$PYTHON]) +fi + +_virtualenv_topsrcdir= +_virtualenv_populate_path= + +dnl If this is a mozilla-central, we'll find the virtualenv in the top +dnl source directory. If this is a SpiderMonkey build, we assume we're at +dnl js/src and try to find the virtualenv from the mozilla-central root. +for base in $MOZILLA_CENTRAL_PATH $_topsrcdir $_topsrcdir/../..; do + possible=$base/build/virtualenv/populate_virtualenv.py + + if test -e $possible; then + _virtualenv_topsrcdir=$base + _virtualenv_populate_path=$possible + break + fi +done + +if test -z $_virtualenv_populate_path; then + AC_MSG_ERROR([Unable to find Virtualenv population script. In order +to build, you will need mozilla-central's virtualenv. + +If you are building from a mozilla-central checkout, you should never see this +message. If you are building from a source archive, the source archive was +likely not created properly (it is missing the virtualenv files). + +If you have a copy of mozilla-central available, define the +MOZILLA_CENTRAL_PATH environment variable to the top source directory of +mozilla-central and relaunch configure.]) + +fi + +if test -z $DONT_POPULATE_VIRTUALENV; then + AC_MSG_RESULT([Creating Python environment]) + dnl This verifies our Python version is sane and ensures the Python + dnl virtualenv is present and up to date. It sanitizes the environment + dnl for us, so we don't need to clean anything out. + $PYTHON $_virtualenv_populate_path \ + $_virtualenv_topsrcdir $MOZ_BUILD_ROOT/_virtualenv || exit 1 + + case "$host_os" in + mingw*) + PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe + ;; + *) + PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python + ;; + esac +fi + +AC_SUBST(PYTHON) + +AC_MSG_CHECKING([Python environment is Mozilla virtualenv]) +$PYTHON -c "import mozbuild.base" +if test "$?" != 0; then + AC_MSG_ERROR([Python environment does not appear to be sane.]) +fi +AC_MSG_RESULT([yes]) +]) + diff --git a/js/src/configure.in b/js/src/configure.in index 1e34bf1240cb..94db6d422a65 100644 --- a/js/src/configure.in +++ b/js/src/configure.in @@ -55,7 +55,6 @@ NSPR_VERSION=4 dnl Set the minimum version of toolkit libs used by mozilla dnl ======================================================== PERL_VERSION=5.006 -PYTHON_VERSION=2.7 WINDRES_VERSION=2.14.90 W32API_VERSION=3.14 @@ -665,10 +664,7 @@ else AC_MSG_RESULT([yes]) fi -MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python) -if test -z "$PYTHON"; then - AC_MSG_ERROR([python was not found in \$PATH]) -fi +MOZ_PYTHON if test -z "$COMPILE_ENVIRONMENT"; then NSINSTALL_BIN='$(PYTHON) $(topsrcdir)/config/nsinstall.py' @@ -1463,17 +1459,6 @@ case "$host" in ;; esac -dnl We require version 2.7 of Python to build. -AC_MSG_CHECKING([for Python version >= $PYTHON_VERSION but not 3.x]) -changequote(,) -$PYTHON -c "import sys; sys.exit(sys.version[:3] < sys.argv[1] or sys.version[:2] != '2.')" $PYTHON_VERSION -_python_res=$? -changequote([,]) -if test "$_python_res" != 0; then - AC_MSG_ERROR([Python $PYTHON_VERSION or higher (but not Python 3.x) is required.]) -fi -AC_MSG_RESULT([yes]) - dnl Check for using a custom implementation dnl ======================================================== AC_MSG_CHECKING(for custom implementation) @@ -4268,7 +4253,6 @@ dnl win32 options AC_SUBST(MOZ_MAPINFO) AC_SUBST(MOZ_BROWSE_INFO) AC_SUBST(MOZ_TOOLS_DIR) -AC_SUBST(PYTHON) dnl Echo the CFLAGS to remove extra whitespace. CFLAGS=`echo \ diff --git a/mach b/mach index 936228e3f0ad..658be941ce3a 100755 --- a/mach +++ b/mach @@ -42,6 +42,7 @@ MACH_MODULES = [ 'python/mozboot/mozboot/mach_commands.py', 'python/mozbuild/mozbuild/config.py', 'python/mozbuild/mozbuild/mach_commands.py', + 'python/mozbuild/mozbuild/frontend/mach_commands.py', 'testing/mochitest/mach_commands.py', 'testing/xpcshell/mach_commands.py', ] diff --git a/python/Makefile.in b/python/Makefile.in index 473d91c7f556..1f98cc9cca07 100644 --- a/python/Makefile.in +++ b/python/Makefile.in @@ -12,6 +12,7 @@ include $(DEPTH)/config/autoconf.mk test_dirs := \ mozbuild/mozbuild/test \ mozbuild/mozbuild/test/compilation \ + mozbuild/mozbuild/test/frontend \ $(NULL) PYTHON_UNIT_TESTS := $(foreach dir,$(test_dirs),$(wildcard $(srcdir)/$(dir)/*.py)) diff --git a/python/mozbuild/README.rst b/python/mozbuild/README.rst index bd621fae3550..94bb0cac6442 100644 --- a/python/mozbuild/README.rst +++ b/python/mozbuild/README.rst @@ -10,4 +10,30 @@ Modules Overview * mozbuild.compilation -- Functionality related to compiling. This includes managing compiler warnings. +* mozbuild.frontend -- Functionality for reading build frontend files + (what defines the build system) and converting them to data structures + which are fed into build backends to produce backend configurations. + +Overview +======== + +The build system consists of frontend files that define what to do. They +say things like "compile X" "copy Y." + +The mozbuild.frontend package contains code for reading these frontend +files and converting them to static data structures. The set of produced +static data structures for the tree constitute the current build +configuration. + +There exist entities called build backends. From a high level, build +backends consume the build configuration and do something with it. They +typically produce tool-specific files such as make files which can be used +to build the tree. + +Builders are entities that build the tree. They typically have high +cohesion with a specific build backend. + +Piecing it all together, we have frontend files that are parsed into data +structures. These data structures are fed into a build backend. The output +from build backends is used by builders to build the tree. diff --git a/python/mozbuild/TODO b/python/mozbuild/TODO new file mode 100644 index 000000000000..4f519f9dd791 --- /dev/null +++ b/python/mozbuild/TODO @@ -0,0 +1,3 @@ +dom/imptests Makefile.in's are autogenerated. See +dom/imptests/writeMakefile.py and bug 782651. We will need to update +writeMakefile.py to produce mozbuild files. diff --git a/python/mozbuild/mozbuild/frontend/README.rst b/python/mozbuild/mozbuild/frontend/README.rst new file mode 100644 index 000000000000..c66efa1587b9 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/README.rst @@ -0,0 +1,137 @@ +================= +mozbuild.frontend +================= + +The mozbuild.frontend package is of sufficient importance and complexity +to warrant its own README file. If you are looking for documentation on +how the build system gets started, you've come to the right place. + +Overview +======== + +Tree metadata (including the build system) is defined by a collection of +files in the source tree called *mozbuild* files. These typically are files +named *moz.build*. But, the actual name can vary. + +Each *mozbuild* file defines basic metadata about the part of the tree +(typically directory scope) it resides in. This includes build system +configuration, such as the list of C++ files to compile or headers to install +or libraries to link together. + +*mozbuild* files are actually Python scripts. However, their execution +is governed by special rules. This will be explained later. + +Once a *mozbuild* file has executed, it is converted into a set of static +data structures. + +The set of all data structures from all relevant *mozbuild* files +constitute all of the metadata from the tree. + +How *mozbuild* Files Work +========================= + +As stated above, *mozbuild* files are actually Python scripts. However, +their behavior is very different from what you would expect if you executed +the file using the standard Python interpreter from the command line. + +There are two properties that make execution of *mozbuild* files special: + +1. They are evaluated in a sandbox which exposes a limited subset of Python +2. There is a special set of global variables which hold the output from + execution. + +The limited subset of Python is actually an extremely limited subset. +Only a few built-ins are exposed. These include *True*, *False*, and +*None*. Global functions like *import*, *print*, and *open* aren't defined. +Without these, *mozbuild* files can do very little. This is by design. + +The side-effects of the execution of a *mozbuild* file are used to define +the build configuration. Specifically, variables set during the execution +of a *mozbuild* file are examined and their values are used to populate +data structures. + +The enforced convention is that all UPPERCASE names inside a sandbox are +reserved and it is the value of these variables post-execution that is +examined. Furthermore, the set of allowed UPPERCASE variable names and +their types is statically defined. If you attempt to reference or assign +to an UPPERCASE variable name that isn't known to the build system or +attempt to assign a value of the wrong type (e.g. a string when it wants a +list), an error will be raised during execution of the *mozbuild* file. +This strictness is to ensure that assignment to all UPPERCASE variables +actually does something. If things weren't this way, *mozbuild* files +might think they were doing something but in reality wouldn't be. We don't +want to create false promises, so we validate behavior strictly. + +If a variable is not UPPERCASE, you can do anything you want with it, +provided it isn't a function or other built-in. In other words, normal +Python rules apply. + +All of the logic for loading and evaluating *mozbuild* files is in the +*reader* module. Of specific interest is the *MozbuildSandbox* class. The +*BuildReader* class is also important, as it is in charge of +instantiating *MozbuildSandbox* instances and traversing a tree of linked +*mozbuild* files. Unless you are a core component of the build system, +*BuildReader* is probably the only class you care about in this module. + +The set of variables and functions *exported* to the sandbox is defined by +the *sandbox_symbols* module. These data structures are actually used to +populate MozbuildSandbox instances. And, there are tests to ensure that the +sandbox doesn't add new symbols without those symbols being added to the +module. And, since the module contains documentation, this ensures the +documentation is up to date (at least in terms of symbol membership). + +How Sandboxes are Converted into Data Structures +================================================ + +The output of a *mozbuild* file execution is essentially a dict of all +the special UPPERCASE variables populated during its execution. While these +dicts are data structures, they aren't the final data structures that +represent the build configuration. + +We feed the *mozbuild* execution output (actually *reader.MozbuildSandbox* +instances) into a *TreeMetadataEmitter* class instance. This class is +defined in the *emitter* module. *TreeMetadataEmitter* converts the +*MozbuildSandbox* instances into instances of the *TreeMetadata*-derived +classes from the *data* module. + +All the classes in the *data* module define a domain-specific +component of the tree metadata, including build configuration. File compilation +and IDL generation are separate classes, for example. The only thing these +classes have in common is that they inherit from *TreeMetadata*, which is +merely an abstract base class. + +The set of all emitted *TreeMetadata* instances (converted from executed +*mozbuild* files) constitutes the aggregate tree metadata. This is the +the authoritative definition of the build system, etc and is what's used by +all downstream consumers, such as build backends. There is no monolithic +class or data structure. Instead, the tree metadata is modeled as a collection +of *TreeMetadata* instances. + +There is no defined mapping between the number of +*MozbuildSandbox*/*moz.build* instances and *TreeMetadata* instances. +Some *mozbuild* files will emit only 1 *TreeMetadata* instance. Some +will emit 7. Some may even emit 0! + +The purpose of this *emitter* layer between the raw *mozbuild* execution +result and *TreeMetadata* is to facilitate additional normalization and +verification of the output. There are multiple downstream consumers of +this data and there is common functionality shared between them. An +abstraction layer that provides high-level filtering is a useful feature. +Thus *TreeMetadataEmitter* exists. + +Other Notes +=========== + +*reader.BuildReader* and *emitter.TreeMetadataEmitter* have a nice +stream-based API courtesy of generators. When you hook them up properly, +*TreeMetadata* instances can be consumed before all *mozbuild* files have +been read. This means that errors down the pipe can trigger before all +upstream tasks (such as executing and converting) are complete. This should +reduce the turnaround time in the event of errors. This likely translates to +a more rapid pace for implementing backends, which require lots of iterative +runs through the entire system. + +Lots of code in this sub-module is applicable to other systems, not just +Mozilla's. However, some of the code is tightly coupled. If there is a will +to extract the generic bits for re-use in other projects, that can and should +be done. diff --git a/python/mozbuild/mozbuild/frontend/__init__.py b/python/mozbuild/mozbuild/frontend/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py new file mode 100644 index 000000000000..299bf5b7b1d9 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +r"""Data structures representing Mozilla's source tree. + +The frontend files are parsed into static data structures. These data +structures are defined in this module. + +All data structures of interest are children of the TreeMetadata class. + +Logic for populating these data structures is not defined in this class. +Instead, what we have here are dumb container classes. The emitter module +contains the code for converting executed mozbuild files into these data +structures. +""" + +from __future__ import unicode_literals + +from collections import OrderedDict + + +class TreeMetadata(object): + """Base class for all data being captured.""" + + +class SandboxDerived(TreeMetadata): + """Build object derived from a single MozbuildSandbox instance. + + It holds fields common to all sandboxes. This class is likely never + instantiated directly but is instead derived from. + """ + + __slots__ = ( + 'objdir', + 'relativedir', + 'srcdir', + 'topobjdir', + 'topsrcdir', + ) + + def __init__(self, sandbox): + self.topsrcdir = sandbox['TOPSRCDIR'] + self.topobjdir = sandbox['TOPOBJDIR'] + + self.relativedir = sandbox['RELATIVEDIR'] + self.srcdir = sandbox['SRCDIR'] + self.objdir = sandbox['OBJDIR'] + + +class DirectoryTraversal(SandboxDerived): + """Describes how directory traversal for building should work. + + This build object is likely only of interest to the recursive make backend. + Other build backends should (ideally) not attempt to mimic the behavior of + the recursive make backend. The only reason this exists is to support the + existing recursive make backend while the transition to mozbuild frontend + files is complete and we move to a more optimal build backend. + + Fields in this class correspond to similarly named variables in the + frontend files. + """ + __slots__ = ( + 'dirs', + 'parallel_dirs', + 'tool_dirs', + 'test_dirs', + 'test_tool_dirs', + 'tier_dirs', + 'tier_static_dirs', + ) + + def __init__(self, sandbox): + SandboxDerived.__init__(self, sandbox) + + self.dirs = [] + self.parallel_dirs = [] + self.tool_dirs = [] + self.test_dirs = [] + self.test_tool_dirs = [] + self.tier_dirs = OrderedDict() + self.tier_static_dirs = OrderedDict() diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py new file mode 100644 index 000000000000..bd1753bd50f1 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from .data import DirectoryTraversal +from .reader import MozbuildSandbox + + +class TreeMetadataEmitter(object): + """Converts the executed mozbuild files into data structures. + + This is a bridge between reader.py and data.py. It takes what was read by + reader.BuildReader and converts it into the classes defined in the data + module. + """ + + def __init__(self, config): + self.config = config + + def emit(self, output): + """Convert the BuildReader output into data structures. + + The return value from BuildReader.read_topsrcdir() (a generator) is + typically fed into this function. + """ + for out in output: + if isinstance(out, MozbuildSandbox): + for o in self.emit_from_sandbox(out): + yield o + else: + raise Exception('Unhandled output type: %s' % out) + + def emit_from_sandbox(self, sandbox): + """Convert a MozbuildSandbox to tree metadata objects. + + This is a generator of mozbuild.frontend.data.SandboxDerived instances. + """ + + # We always emit a directory traversal descriptor. This is needed by + # the recursive make backend. + for o in self._emit_directory_traversal_from_sandbox(sandbox): yield o + + def _emit_directory_traversal_from_sandbox(self, sandbox): + o = DirectoryTraversal(sandbox) + o.dirs = sandbox.get('DIRS', []) + o.parallel_dirs = sandbox.get('PARALLEL_DIRS', []) + o.tool_dirs = sandbox.get('TOOL_DIRS', []) + o.test_dirs = sandbox.get('TEST_DIRS', []) + o.test_tool_dirs = sandbox.get('TEST_TOOL_DIRS', []) + + if 'TIERS' in sandbox: + for tier in sandbox['TIERS']: + o.tier_dirs[tier] = sandbox['TIERS'][tier]['regular'] + o.tier_static_dirs[tier] = sandbox['TIERS'][tier]['static'] + + yield o + diff --git a/python/mozbuild/mozbuild/frontend/mach_commands.py b/python/mozbuild/mozbuild/frontend/mach_commands.py new file mode 100644 index 000000000000..9674a14d65f0 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/mach_commands.py @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, unicode_literals + +import textwrap + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command +) + +from mozbuild.frontend.sandbox_symbols import ( + FUNCTIONS, + SPECIAL_VARIABLES, + VARIABLES, + doc_to_paragraphs, +) + + +def get_doc(doc): + """Split documentation into summary line and everything else.""" + paragraphs = doc_to_paragraphs(doc) + + summary = paragraphs[0] + extra = paragraphs[1:] + + return summary, extra + +def print_extra(extra): + """Prints the 'everything else' part of documentation intelligently.""" + for para in extra: + for line in textwrap.wrap(para): + print(line) + + print('') + + if not len(extra): + print('') + + +@CommandProvider +class MozbuildFileCommands(object): + @Command('mozbuild-reference', + help='View reference documentation on mozbuild files.') + @CommandArgument('symbol', default=None, nargs='*', + help='Symbol to view help on. If not specified, all will be shown.') + @CommandArgument('--name-only', '-n', default=False, action='store_true', + help='Print symbol names only.') + def reference(self, symbol, name_only=False): + if name_only: + for s in sorted(VARIABLES.keys()): + print(s) + + for s in sorted(FUNCTIONS.keys()): + print(s) + + for s in sorted(SPECIAL_VARIABLES.keys()): + print(s) + + return 0 + + if len(symbol): + for s in symbol: + if s in VARIABLES: + self.variable_reference(s) + continue + elif s in FUNCTIONS: + self.function_reference(s) + continue + elif s in SPECIAL_VARIABLES: + self.special_reference(s) + continue + + print('Could not find symbol: %s' % s) + return 1 + + return 0 + + print('=========') + print('VARIABLES') + print('=========') + print('') + print('This section lists all the variables that may be set ') + print('in moz.build files.') + print('') + + for v in sorted(VARIABLES.keys()): + self.variable_reference(v) + + print('=========') + print('FUNCTIONS') + print('=========') + print('') + print('This section lists all the functions that may be called ') + print('in moz.build files.') + print('') + + for f in sorted(FUNCTIONS.keys()): + self.function_reference(f) + + print('=================') + print('SPECIAL VARIABLES') + print('=================') + print('') + + for v in sorted(SPECIAL_VARIABLES.keys()): + self.special_reference(v) + + return 0 + + def variable_reference(self, v): + typ, default, doc = VARIABLES[v] + + print(v) + print('=' * len(v)) + print('') + + summary, extra = get_doc(doc) + + print(summary) + print('') + print('Type: %s' % typ.__name__) + print('Default Value: %s' % default) + print('') + print_extra(extra) + + def function_reference(self, f): + attr, args, doc = FUNCTIONS[f] + + print(f) + print('=' * len(f)) + print('') + + summary, extra = get_doc(doc) + + print(summary) + print('') + + arg_types = [] + + for t in args: + if isinstance(t, list): + inner_types = [t2.__name__ for t2 in t] + arg_types.append(' | ' .join(inner_types)) + continue + + arg_types.append(t.__name__) + + arg_s = '(%s)' % ', '.join(arg_types) + + print('Arguments: %s' % arg_s) + print('') + print_extra(extra) + + def special_reference(self, v): + typ, doc = SPECIAL_VARIABLES[v] + + print(v) + print('=' * len(v)) + print('') + + summary, extra = get_doc(doc) + + print(summary) + print('') + print('Type: %s' % typ.__name__) + print('') + print_extra(extra) diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py new file mode 100644 index 000000000000..dd567b71075b --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -0,0 +1,575 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains code for reading metadata from the build system into +# data structures. + +r"""Read build frontend files into data structures. + +In terms of code architecture, the main interface is BuildReader. BuildReader +starts with a root mozbuild file. It creates a new execution environment for +this file, which is represented by the Sandbox class. The Sandbox class is what +defines what is allowed to execute in an individual mozbuild file. The Sandbox +consists of a local and global namespace, which are modeled by the +LocalNamespace and GlobalNamespace classes, respectively. The global namespace +contains all of the takeaway information from the execution. The local +namespace is for throwaway local variables and its contents are discarded after +execution. + +The BuildReader contains basic logic for traversing a tree of mozbuild files. +It does this by examining specific variables populated during execution. +""" + +from __future__ import print_function, unicode_literals + +import logging +import os +import sys +import traceback +import types + +from io import StringIO + +from mozbuild.util import ( + ReadOnlyDefaultDict, + ReadOnlyDict, +) + +from .sandbox import ( + SandboxExecutionError, + SandboxLoadError, + Sandbox, +) + +from .sandbox_symbols import ( + FUNCTIONS, + VARIABLES, +) + + +if sys.version_info.major == 2: + text_type = unicode + type_type = types.TypeType +else: + text_type = str + type_type = type + +def log(logger, level, action, params, formatter): + logger.log(level, formatter, extra={'action': action, 'params': params}) + + +class MozbuildSandbox(Sandbox): + """Implementation of a Sandbox tailored for mozbuild files. + + We expose a few useful functions and expose the set of variables defining + Mozilla's build system. + """ + def __init__(self, config, path): + """Create an empty mozbuild Sandbox. + + config is a ConfigStatus instance (the output of configure). path is + the path of the main mozbuild file that is being executed. It is used + to compute encountered relative paths. + """ + Sandbox.__init__(self, allowed_variables=VARIABLES) + + self.config = config + + topobjdir = os.path.abspath(config.topobjdir) + + # This may not always hold true. If we ever have autogenerated mozbuild + # files in topobjdir, we'll need to change this. + assert os.path.normpath(path).startswith(os.path.normpath(config.topsrcdir)) + assert not os.path.normpath(path).startswith(os.path.normpath(topobjdir)) + + relpath = os.path.relpath(path, config.topsrcdir).replace(os.sep, '/') + reldir = os.path.dirname(relpath) + + with self._globals.allow_all_writes() as d: + d['TOPSRCDIR'] = config.topsrcdir + d['TOPOBJDIR'] = topobjdir + d['RELATIVEDIR'] = reldir + d['SRCDIR'] = os.path.join(config.topsrcdir, reldir).replace(os.sep, '/').rstrip('/') + d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/') + + d['CONFIG'] = ReadOnlyDefaultDict(config.substs, + global_default=None) + + # Register functions. + for name, func in FUNCTIONS.items(): + d[name] = getattr(self, func[0]) + + self._normalized_topsrcdir = os.path.normpath(config.topsrcdir) + + def exec_file(self, path, filesystem_absolute=False): + """Override exec_file to normalize paths and restrict file loading. + + If the path is absolute, behavior is governed by filesystem_absolute. + If filesystem_absolute is True, the path is interpreted as absolute on + the actual filesystem. If it is false, the path is treated as absolute + within the current topsrcdir. + + If the path is not absolute, it will be treated as relative to the + currently executing file. If there is no currently executing file, it + will be treated as relative to topsrcdir. + + Paths will be rejected if they do not fall under topsrcdir. + """ + if os.path.isabs(path): + if not filesystem_absolute: + path = os.path.normpath(os.path.join(self.config.topsrcdir, + path[1:])) + + else: + if len(self._execution_stack): + path = os.path.normpath(os.path.join( + os.path.dirname(self._execution_stack[-1]), + path)) + else: + path = os.path.normpath(os.path.join( + self.config.topsrcdir, path)) + + # realpath() is needed for true security. But, this isn't for security + # protection, so it is omitted. + normalized_path = os.path.normpath(path) + if not normalized_path.startswith(self._normalized_topsrcdir): + raise SandboxLoadError(list(self._execution_stack), + sys.exc_info()[2], illegal_path=path) + + Sandbox.exec_file(self, path) + + def _add_tier_directory(self, tier, reldir, static=False): + """Register a tier directory with the build.""" + if isinstance(reldir, text_type): + reldir = [reldir] + + if not tier in self['TIERS']: + self['TIERS'][tier] = { + 'regular': [], + 'static': [], + } + + key = 'static' if static else 'regular' + + for path in reldir: + if path in self['TIERS'][tier][key]: + raise Exception('Directory has already been registered with ' + 'tier: %s' % path) + + self['TIERS'][tier][key].append(path) + + def _include(self, path): + """Include and exec another file within the context of this one.""" + + # exec_file() handles normalization and verification of the path. + self.exec_file(path) + + +class SandboxValidationError(Exception): + """Represents an error encountered when validating sandbox results.""" + pass + + +class BuildReaderError(Exception): + """Represents errors encountered during BuildReader execution. + + The main purpose of this class is to facilitate user-actionable error + messages. Execution errors should say: + + - Why they failed + - Where they failed + - What can be done to prevent the error + + A lot of the code in this class should arguably be inside sandbox.py. + However, extraction is somewhat difficult given the additions + MozbuildSandbox has over Sandbox (e.g. the concept of included files - + which affect error messages, of course). + """ + def __init__(self, file_stack, trace, sandbox_exec_error=None, + sandbox_load_error=None, validation_error=None, other_error=None): + + self.file_stack = file_stack + self.trace = trace + self.sandbox_exec = sandbox_exec_error + self.sandbox_load = sandbox_load_error + self.validation_error = validation_error + self.other = other_error + + @property + def main_file(self): + return self.file_stack[-1] + + @property + def actual_file(self): + # We report the file that called out to the file that couldn't load. + if self.sandbox_load is not None: + if len(self.sandbox_load.file_stack) > 1: + return self.sandbox_load.file_stack[-2] + + if len(self.file_stack) > 1: + return self.file_stack[-2] + + if self.sandbox_error is not None and \ + len(self.sandbox_error.file_stack): + return self.sandbox_error.file_stack[-1] + + return self.file_stack[-1] + + @property + def sandbox_error(self): + return self.sandbox_exec or self.sandbox_load + + def __str__(self): + s = StringIO() + + delim = '=' * 30 + s.write('%s\nERROR PROCESSING MOZBUILD FILE\n%s\n\n' % (delim, delim)) + + s.write('The error occurred while processing the following file:\n') + s.write('\n') + s.write(' %s\n' % self.actual_file) + s.write('\n') + + if self.actual_file != self.main_file and not self.sandbox_load: + s.write('This file was included as part of processing:\n') + s.write('\n') + s.write(' %s\n' % self.main_file) + s.write('\n') + + if self.sandbox_error is not None: + self._print_sandbox_error(s) + elif self.validation_error is not None: + s.write('The error occurred when validating the result of ') + s.write('the execution. The reported error is:\n') + s.write('\n') + s.write(' %s\n' % self.validation_error.message) + s.write('\n') + else: + s.write('The error appears to be part of the %s ' % __name__) + s.write('Python module itself! It is possible you have stumbled ') + s.write('across a legitimate bug.\n') + s.write('\n') + + for l in traceback.format_exception(type(self.other), self.other, + self.trace): + s.write(unicode(l)) + + return s.getvalue() + + def _print_sandbox_error(self, s): + # Try to find the frame of the executed code. + script_frame = None + for frame in traceback.extract_tb(self.sandbox_error.trace): + if frame[0] == self.actual_file: + script_frame = frame + + # Reset if we enter a new execution context. This prevents errors + # in this module from being attributes to a script. + elif frame[0] == __file__ and frame[2] == 'exec_source': + script_frame = None + + if script_frame is not None: + s.write('The error was triggered on line %d ' % script_frame[1]) + s.write('of this file:\n') + s.write('\n') + s.write(' %s\n' % script_frame[3]) + s.write('\n') + + if self.sandbox_load is not None: + self._print_sandbox_load_error(s) + return + + self._print_sandbox_exec_error(s) + + def _print_sandbox_load_error(self, s): + assert self.sandbox_load is not None + + if self.sandbox_load.illegal_path is not None: + s.write('The underlying problem is an illegal file access. ') + s.write('This is likely due to trying to access a file ') + s.write('outside of the top source directory.\n') + s.write('\n') + s.write('The path whose access was denied is:\n') + s.write('\n') + s.write(' %s\n' % self.sandbox_load.illegal_path) + s.write('\n') + s.write('Modify the script to not access this file and ') + s.write('try again.\n') + return + + if self.sandbox_load.read_error is not None: + if not os.path.exists(self.sandbox_load.read_error): + s.write('The underlying problem is we referenced a path ') + s.write('that does not exist. That path is:\n') + s.write('\n') + s.write(' %s\n' % self.sandbox_load.read_error) + s.write('\n') + s.write('Either create the file if it needs to exist or ') + s.write('do not reference it.\n') + else: + s.write('The underlying problem is a referenced path could ') + s.write('not be read. The trouble path is:\n') + s.write('\n') + s.write(' %s\n' % self.sandbox_load.read_error) + s.write('\n') + s.write('It is possible the path is not correct. Is it ') + s.write('pointing to a directory? It could also be a file ') + s.write('permissions issue. Ensure that the file is ') + s.write('readable.\n') + + return + + # This module is buggy if you see this. + raise AssertionError('SandboxLoadError with unhandled properties!') + + def _print_sandbox_exec_error(self, s): + assert self.sandbox_exec is not None + + inner = self.sandbox_exec.exc_value + + if isinstance(inner, SyntaxError): + s.write('The underlying problem is a Python syntax error ') + s.write('on line %d:\n' % inner.lineno) + s.write('\n') + s.write(' %s\n' % inner.text) + s.write((' ' * (inner.offset + 4)) + '^\n') + s.write('\n') + s.write('Fix the syntax error and try again.\n') + return + + if isinstance(inner, KeyError): + self._print_keyerror(inner, s) + elif isinstance(inner, ValueError): + self._print_valueerror(inner, s) + else: + self._print_exception(inner, s) + + def _print_keyerror(self, inner, s): + if inner.args[0] not in ('global_ns', 'local_ns'): + self._print_exception(unner, s) + return + + if inner.args[0] == 'global_ns': + verb = None + if inner.args[1] == 'get_unknown': + verb = 'read' + elif inner.args[1] == 'set_unknown': + verb = 'write' + else: + raise AssertionError('Unhandled global_ns: %s' % inner.args[1]) + + s.write('The underlying problem is an attempt to %s ' % verb) + s.write('a reserved UPPERCASE variable that does not exist.\n') + s.write('\n') + s.write('The variable %s causing the error is:\n' % verb) + s.write('\n') + s.write(' %s\n' % inner.args[2]) + s.write('\n') + s.write('Please change the file to not use this variable.\n') + s.write('\n') + s.write('For reference, the set of valid variables is:\n') + s.write('\n') + s.write(', '.join(sorted(VARIABLES.keys())) + '\n') + return + + s.write('The underlying problem is a reference to an undefined ') + s.write('local variable:\n') + s.write('\n') + s.write(' %s\n' % inner.args[2]) + s.write('\n') + s.write('Please change the file to not reference undefined ') + s.write('variables and try again.\n') + + def _print_valueerror(self, inner, s): + if inner.args[0] not in ('global_ns', 'local_ns'): + self._print_exception(inner, s) + return + + assert inner.args[1] == 'set_type' + + s.write('The underlying problem is an attempt to write an illegal ') + s.write('value to a special variable.\n') + s.write('\n') + s.write('The variable whose value was rejected is:\n') + s.write('\n') + s.write(' %s' % inner.args[2]) + s.write('\n') + s.write('The value being written to it was of the following type:\n') + s.write('\n') + s.write(' %s\n' % type(inner.args[3]).__name__) + s.write('\n') + s.write('This variable expects the following type(s):\n') + s.write('\n') + if type(inner.args[4]) == type_type: + s.write(' %s\n' % inner.args[4].__name__) + else: + for t in inner.args[4]: + s.write( ' %s\n' % t.__name__) + s.write('\n') + s.write('Change the file to write a value of the appropriate type ') + s.write('and try again.\n') + + def _print_exception(self, e, s): + s.write('An error was encountered as part of executing the file ') + s.write('itself. The error appears to be the fault of the script.\n') + s.write('\n') + s.write('The error as reported by Python is:\n') + s.write('\n') + s.write(' %s\n' % traceback.format_exception_only(type(e), e)) + + +class BuildReader(object): + """Read a tree of mozbuild files into data structures. + + This is where the build system starts. You give it a tree configuration + (the output of configuration) and it executes the moz.build files and + collects the data they define. + """ + + def __init__(self, config): + self.config = config + self.topsrcdir = config.topsrcdir + + self._log = logging.getLogger(__name__) + self._read_files = set() + self._normalized_topsrcdir = os.path.normpath(config.topsrcdir) + self._execution_stack = [] + + def read_topsrcdir(self): + """Read the tree of mozconfig files into a data structure. + + This starts with the tree's top-most mozbuild file and descends into + all linked mozbuild files until all relevant files have been evaluated. + + This is a generator of Sandbox instances. As each mozbuild file is + read, a new Sandbox is created. Each created Sandbox is returned. + """ + path = os.path.join(self.topsrcdir, 'moz.build') + return self.read_mozbuild(path, read_tiers=True, + filesystem_absolute=True) + + def read_mozbuild(self, path, read_tiers=False, filesystem_absolute=False, + descend=True): + """Read and process a mozbuild file, descending into children. + + This starts with a single mozbuild file, executes it, and descends into + other referenced files per our traversal logic. + + The traversal logic is to iterate over the *DIRS variables, treating + each element as a relative directory path. For each encountered + directory, we will open the moz.build file located in that + directory in a new Sandbox and process it. + + If read_tiers is True (it should only be True for the top-level + mozbuild file in a project), the TIERS variable will be used for + traversal as well. + + If descend is True (the default), we will descend into child + directories and files per variable values. + + Traversal is performed depth first (for no particular reason). + """ + self._execution_stack.append(path) + try: + for s in self._read_mozbuild(path, read_tiers=read_tiers, + filesystem_absolute=filesystem_absolute, descend=descend): + yield s + + except BuildReaderError as bre: + raise bre + + except SandboxExecutionError as se: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], sandbox_exec_error=se) + + except SandboxLoadError as sle: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], sandbox_load_error=sle) + + except SandboxValidationError as ve: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], validation_error=ve) + + except Exception as e: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], other_error=e) + + def _read_mozbuild(self, path, read_tiers, filesystem_absolute, descend): + path = os.path.normpath(path) + log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path}, + 'Reading file: {path}') + + if path in self._read_files: + log(self._log, logging.WARNING, 'read_already', {'path': path}, + 'File already read. Skipping: {path}') + return + + self._read_files.add(path) + + sandbox = MozbuildSandbox(self.config, path) + sandbox.exec_file(path, filesystem_absolute=filesystem_absolute) + yield sandbox + + # Traverse into referenced files. + + # We first collect directories populated in variables. + dir_vars = ['DIRS', 'PARALLEL_DIRS', 'TOOL_DIRS'] + + if self.config.substs.get('ENABLE_TESTS', False) == '1': + dir_vars.extend(['TEST_DIRS', 'TEST_TOOL_DIRS']) + + # It's very tempting to use a set here. Unfortunately, the recursive + # make backend needs order preserved. Once we autogenerate all backend + # files, we should be able to convert this to a set. + dirs = [] + for var in dir_vars: + if not var in sandbox: + continue + + for d in sandbox[var]: + if d in dirs: + raise SandboxValidationError( + 'Directory (%s) registered multiple times in %s' % ( + d, var)) + + dirs.append(d) + + # We also have tiers whose members are directories. + if 'TIERS' in sandbox: + if not read_tiers: + raise SandboxValidationError( + 'TIERS defined but it should not be') + + for tier, values in sandbox['TIERS'].items(): + for var in ('regular', 'static'): + for d in values[var]: + if d in dirs: + raise SandboxValidationError( + 'Tier directory (%s) registered multiple ' + 'times in %s' % (d, tier)) + dirs.append(d) + + curdir = os.path.dirname(path) + for relpath in dirs: + child_path = os.path.join(curdir, relpath, 'moz.build') + + # Ensure we don't break out of the topsrcdir. We don't do realpath + # because it isn't necessary. If there are symlinks in the srcdir, + # that's not our problem. We're not a hosted application: we don't + # need to worry about security too much. + child_path = os.path.normpath(child_path) + if not child_path.startswith(self._normalized_topsrcdir): + raise SandboxValidationError( + 'Attempting to process file outside of topsrcdir: %s' % + child_path) + + if not descend: + continue + + for res in self.read_mozbuild(child_path, read_tiers=False, + filesystem_absolute=True): + yield res + + self._execution_stack.pop() + diff --git a/python/mozbuild/mozbuild/frontend/sandbox.py b/python/mozbuild/mozbuild/frontend/sandbox.py new file mode 100644 index 000000000000..c94a6cea6939 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/sandbox.py @@ -0,0 +1,348 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +r"""Python sandbox implementation for build files. + +This module contains classes for Python sandboxes that execute in a +highly-controlled environment. + +The main class is `Sandbox`. This provides an execution environment for Python +code. + +The behavior inside sandboxes is mostly regulated by the `GlobalNamespace` and +`LocalNamespace` classes. These represent the global and local namespaces in +the sandbox, respectively. + +Code in this module takes a different approach to exception handling compared +to what you'd see elsewhere in Python. Arguments to built-in exceptions like +KeyError are machine parseable. This machine-friendly data is used to present +user-friendly error messages in the case of errors. +""" + +from __future__ import unicode_literals + +import copy +import os +import sys + +from contextlib import contextmanager + +from mozbuild.util import ( + ReadOnlyDefaultDict, + ReadOnlyDict, +) + + +class GlobalNamespace(dict): + """Represents the globals namespace in a sandbox. + + This is a highly specialized dictionary employing light magic. + + At the crux we have the concept of a restricted keys set. Only very + specific keys may be retrieved or mutated. The rules are as follows: + + - The '__builtins__' key is hardcoded and is read-only. + - The set of variables that can be assigned or accessed during + execution is passed into the constructor. + + When variables are assigned to, we verify assignment is allowed. Assignment + is allowed if the variable is known (set defined at constructor time) and + if the value being assigned is the expected type (also defined at + constructor time). + + When variables are read, we first try to read the existing value. If a + value is not found and it is defined in the allowed variables set, we + return the default value for it. We don't assign default values until + they are accessed because this makes debugging the end-result much + simpler. Instead of a data structure with lots of empty/default values, + you have a data structure with only the values that were read or touched. + + Instantiators of this class are given a backdoor to perform setting of + arbitrary values. e.g. + + ns = GlobalNamespace() + with ns.allow_all_writes(): + ns['foo'] = True + + ns['bar'] = True # KeyError raised. + """ + + # The default set of builtins. + BUILTINS = ReadOnlyDict({ + # Only real Python built-ins should go here. + 'None': None, + 'False': False, + 'True': True, + }) + + def __init__(self, allowed_variables=None, builtins=None): + """Create a new global namespace having specific variables. + + allowed_variables is a dict of the variables that can be queried and + mutated. Keys in this dict are the strings representing keys in this + namespace which are valid. Values are tuples of type, default value, + and a docstring describing the purpose of the variable. + + builtins is the value to use for the special __builtins__ key. If not + defined, the BUILTINS constant attached to this class is used. The + __builtins__ object is read-only. + """ + builtins = builtins or self.BUILTINS + + assert isinstance(builtins, ReadOnlyDict) + + dict.__init__(self, {'__builtins__': builtins}) + + self._allowed_variables = allowed_variables or {} + + # We need to record this because it gets swallowed as part of + # evaluation. + self.last_name_error = None + + self._allow_all_writes = False + + def __getitem__(self, name): + try: + return dict.__getitem__(self, name) + except KeyError: + pass + + # The variable isn't present yet. Fall back to VARIABLES. + default = self._allowed_variables.get(name, None) + if default is None: + self.last_name_error = KeyError('global_ns', 'get_unknown', name) + raise self.last_name_error + + dict.__setitem__(self, name, copy.deepcopy(default[1])) + return dict.__getitem__(self, name) + + def __setitem__(self, name, value): + if self._allow_all_writes: + dict.__setitem__(self, name, value) + return + + # We don't need to check for name.isupper() here because LocalNamespace + # only sends variables our way if isupper() is True. + default = self._allowed_variables.get(name, None) + + if default is None: + self.last_name_error = KeyError('global_ns', 'set_unknown', name, + value) + raise self.last_name_error + + if not isinstance(value, default[0]): + self.last_name_error = ValueError('global_ns', 'set_type', name, + value, default[0]) + raise self.last_name_error + + dict.__setitem__(self, name, value) + + @contextmanager + def allow_all_writes(self): + """Allow any variable to be written to this instance. + + This is used as a context manager. When activated, all writes + (__setitem__ calls) are allowed. When the context manager is exited, + the instance goes back to its default behavior of only allowing + whitelisted mutations. + """ + self._allow_all_writes = True + yield self + self._allow_all_writes = False + + +class LocalNamespace(dict): + """Represents the locals namespace in a Sandbox. + + This behaves like a dict except with some additional behavior tailored + to our sandbox execution model. + + Under normal rules of exec(), doing things like += could have interesting + consequences. Keep in mind that a += is really a read, followed by the + creation of a new variable, followed by a write. If the read came from the + global namespace, then the write would go to the local namespace, resulting + in fragmentation. This is not desired. + + LocalNamespace proxies reads and writes for global-looking variables + (read: UPPERCASE) to the global namespace. This means that attempting to + read or write an unknown variable results in exceptions raised from the + GlobalNamespace. + """ + def __init__(self, global_ns): + """Create a local namespace associated with a GlobalNamespace.""" + dict.__init__({}) + + self._globals = global_ns + self.last_name_error = None + + def __getitem__(self, name): + if name.isupper(): + return self._globals[name] + + return dict.__getitem__(self, name) + + def __setitem__(self, name, value): + if name.isupper(): + self._globals[name] = value + return + + dict.__setitem__(self, name, value) + + +class SandboxError(Exception): + def __init__(self, file_stack): + self.file_stack = file_stack + + +class SandboxExecutionError(SandboxError): + """Represents errors encountered during execution of a Sandbox. + + This is a simple container exception. It's purpose is to capture state + so something else can report on it. + """ + def __init__(self, file_stack, exc_type, exc_value, trace): + SandboxError.__init__(self, file_stack) + + self.exc_type = exc_type + self.exc_value = exc_value + self.trace = trace + + +class SandboxLoadError(SandboxError): + """Represents errors encountered when loading a file for execution. + + This exception represents errors in a Sandbox that occurred as part of + loading a file. The error could have occurred in the course of executing + a file. If so, the file_stack will be non-empty and the file that caused + the load will be on top of the stack. + """ + def __init__(self, file_stack, trace, illegal_path=None, read_error=None): + SandboxError.__init__(self, file_stack) + + self.trace = trace + self.illegal_path = illegal_path + self.read_error = read_error + + +class Sandbox(object): + """Represents a sandbox for executing Python code. + + This class both provides a sandbox for execution of a single mozbuild + frontend file as well as an interface to the results of that execution. + + Sandbox is effectively a glorified wrapper around compile() + exec(). You + point it at some Python code and it executes it. The main difference from + executing Python code like normal is that the executed code is very limited + in what it can do: the sandbox only exposes a very limited set of Python + functionality. Only specific types and functions are available. This + prevents executed code from doing things like import modules, open files, + etc. + + Sandboxes are bound to a mozconfig instance. These objects are produced by + the output of configure. + + Sandbox instances can be accessed like dictionaries to facilitate result + retrieval. e.g. foo = sandbox['FOO']. Direct assignment is not allowed. + + Each sandbox has associated with it a GlobalNamespace and LocalNamespace. + Only data stored in the GlobalNamespace is retrievable via the dict + interface. This is because the local namespace should be irrelevant: it + should only contain throwaway variables. + """ + def __init__(self, allowed_variables=None, builtins=None): + """Initialize a Sandbox ready for execution. + + The arguments are proxied to GlobalNamespace.__init__. + """ + self._globals = GlobalNamespace(allowed_variables=allowed_variables, + builtins=builtins) + self._locals = LocalNamespace(self._globals) + self._execution_stack = [] + + def exec_file(self, path): + """Execute code at a path in the sandbox. + + The path must be absolute. + """ + assert os.path.isabs(path) + + source = None + + try: + with open(path, 'rt') as fd: + source = fd.read() + except Exception as e: + raise SandboxLoadError(list(self._execution_stack), + sys.exc_info()[2], read_error=path) + + self.exec_source(source, path) + + def exec_source(self, source, path): + """Execute Python code within a string. + + The passed string should contain Python code to be executed. The string + will be compiled and executed. + + You should almost always go through exec_file() because exec_source() + does not perform extra path normalization. This can cause relative + paths to behave weirdly. + """ + self._execution_stack.append(path) + + # We don't have to worry about bytecode generation here because we are + # too low-level for that. However, we could add bytecode generation via + # the marshall module if parsing performance were ever an issue. + + try: + # compile() inherits the __future__ from the module by default. We + # do want Unicode literals. + code = compile(source, path, 'exec') + exec(code, self._globals, self._locals) + except SandboxError as e: + raise e + except NameError as e: + # A NameError is raised when a local or global could not be found. + # The original KeyError has been dropped by the interpreter. + # However, we should have it cached in our namespace instances! + + # Unless a script is doing something wonky like catching NameError + # itself (that would be silly), if there is an exception on the + # global namespace, that's our error. + actual = e + + if self._globals.last_name_error is not None: + actual = self._globals.last_name_error + elif self._locals.last_name_error is not None: + actual = self._locals.last_name_error + + raise SandboxExecutionError(list(self._execution_stack), + type(actual), actual, sys.exc_info()[2]) + + except Exception as e: + # Need to copy the stack otherwise we get a reference and that is + # mutated during the finally. + exc = sys.exc_info() + raise SandboxExecutionError(list(self._execution_stack), exc[0], + exc[1], exc[2]) + finally: + self._execution_stack.pop() + + # Dict interface proxies reads to global namespace. + def __len__(self): + return len(self._globals) + + def __getitem__(self, name): + return self._globals[name] + + def __iter__(self): + return iter(self._globals) + + def iterkeys(self): + return self.__iter__() + + def __contains__(self, key): + return key in self._globals + + def get(self, key, default=None): + return self._globals.get(key, default) diff --git a/python/mozbuild/mozbuild/frontend/sandbox_symbols.py b/python/mozbuild/mozbuild/frontend/sandbox_symbols.py new file mode 100644 index 000000000000..5cb6219cc45f --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/sandbox_symbols.py @@ -0,0 +1,249 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +###################################################################### +# DO NOT UPDATE THIS FILE WITHOUT SIGN-OFF FROM A BUILD MODULE PEER. # +###################################################################### + +r"""Defines the global config variables. + +This module contains data structures defining the global symbols that have +special meaning in the frontend files for the build system. + +If you are looking for the absolute authority on what the global namespace in +the Sandbox consists of, you've come to the right place. +""" + +from __future__ import unicode_literals + +from collections import OrderedDict + + +def doc_to_paragraphs(doc): + """Take a documentation string and converts it to paragraphs. + + This normalizes the inline strings in VARIABLES and elsewhere in this file. + + It returns a list of paragraphs. It is up to the caller to insert newlines + or to wrap long lines (e.g. by using textwrap.wrap()). + """ + lines = [line.strip() for line in doc.split('\n')] + + paragraphs = [] + current = [] + for line in lines: + if not len(line): + if len(current): + paragraphs.append(' '.join(current)) + current = [] + + continue + + current.append(line) + + if len(current): + paragraphs.append(' '.join(current)) + + return paragraphs + + +# This defines the set of mutable global variables. +# +# Each variable is a tuple of: +# +# (type, default_value, docs) +# +VARIABLES = { + # Variables controlling reading of other frontend files. + 'DIRS': (list, [], + """Child directories to descend into looking for build frontend files. + + This works similarly to the DIRS variable in make files. Each str value + in the list is the name of a child directory. When this file is done + parsing, the build reader will descend into each listed directory and + read the frontend file there. If there is no frontend file, an error + is raised. + + Values are relative paths. They can be multiple directory levels + above or below. Use ".." for parent directories and "/" for path + delimiters. + """), + + 'PARALLEL_DIRS': (list, [], + """A parallel version of DIRS. + + Ideally this variable does not exist. It is provided so a transition + from recursive makefiles can be made. Once the build system has been + converted to not use Makefile's for the build frontend, this will + likely go away. + """), + + 'TOOL_DIRS': (list, [], + """Like DIRS but for tools. + + Tools are for pieces of the build system that aren't required to + produce a working binary (in theory). They provide things like test + code and utilities. + """), + + 'TEST_DIRS': (list, [], + """Like DIRS but only for directories that contain test-only code. + + If tests are not enabled, this variable will be ignored. + + This variable may go away once the transition away from Makefiles is + complete. + """), + + 'TEST_TOOL_DIRS': (list, [], + """TOOL_DIRS that is only executed if tests are enabled. + """), + + + 'TIERS': (OrderedDict, OrderedDict(), + """Defines directories constituting the tier traversal mechanism. + + The recursive make backend iteration is organized into tiers. There are + major tiers (keys in this dict) that correspond roughly to applications + or libraries being built. e.g. base, nspr, js, platform, app. Within + each tier are phases like export, libs, and tools. The recursive make + backend iterates over each phase in the first tier then proceeds to the + next tier until all tiers are exhausted. + + Tiers are a way of working around deficiencies in recursive make. These + will probably disappear once we no longer rely on recursive make for + the build backend. They will likely be replaced by DIRS. + + This variable is typically not populated directly. Instead, it is + populated by calling add_tier_dir(). + """), +} + +# The set of functions exposed to the sandbox. +# +# Each entry is a tuple of: +# +# (method attribute, (argument types), docs) +# +# The first element is an attribute on Sandbox that should be a function type. +# +FUNCTIONS = { + 'include': ('_include', (str,), + """Include another mozbuild file in the context of this one. + + This is similar to a #include in C languages. The filename passed to + the function will be read and its contents will be evaluated within the + context of the calling file. + + If a relative path is given, it is evaluated as relative to the file + currently being processed. If there is a chain of multiple include(), + the relative path computation is from the most recent/active file. + + If an absolute path is given, it is evaluated from TOPSRCDIR. In other + words, include('/foo') references the path TOPSRCDIR + '/foo'. + + Example usage + ------------- + + # Include "sibling.build" from the current directory. + include('sibling.build') + + # Include "foo.build" from a path within the top source directory. + include('/elsewhere/foo.build') + """), + + 'add_tier_dir': ('_add_tier_directory', (str, [str, list], bool), + """Register a directory for tier traversal. + + This is the preferred way to populate the TIERS variable. + + Tiers are how the build system is organized. The build process is + divided into major phases called tiers. The most important tiers are + "platform" and "apps." The platform tier builds the Gecko platform + (typically outputting libxul). The apps tier builds the configured + application (browser, mobile/android, b2g, etc). + + This function is typically only called by the main moz.build file or a + file directly included by the main moz.build file. An error will be + raised if it is called when it shouldn't be. + + An error will also occur if you attempt to add the same directory to + the same tier multiple times. + + Example usage + ------------- + + # Register a single directory with the 'platform' tier. + add_tier_dir('platform', 'xul') + + # Register multiple directories with the 'app' tier. + add_tier_dir('app', ['components', 'base']) + + # Register a directory as having static content (no dependencies). + add_tier_dir('base', 'foo', static=True) + """), + +} + +# Special variables. These complement VARIABLES. +SPECIAL_VARIABLES = { + 'TOPSRCDIR': (str, + """Constant defining the top source directory. + + The top source directory is the parent directory containing the source + code and all build files. It is typically the root directory of a + cloned repository. + """), + + 'TOPOBJDIR': (str, + """Constant defining the top object directory. + + The top object directory is the parent directory which will contain + the output of the build. This is commonly referred to as "the object + directory." + """), + + 'RELATIVEDIR': (str, + """Constant defining the relative path of this file. + + The relative path is from TOPSRCDIR. This is defined as relative to the + main file being executed, regardless of whether additional files have + been included using include(). + """), + + 'SRCDIR': (str, + """Constant defining the source directory of this file. + + This is the path inside TOPSRCDIR where this file is located. It is the + same as TOPSRCDIR + RELATIVEDIR. + """), + + 'OBJDIR': (str, + """The path to the object directory for this file. + + Is is the same as TOPOBJDIR + RELATIVEDIR. + """), + + 'CONFIG': (dict, + """Dictionary containing the current configuration variables. + + All the variables defined by the configuration system are available + through this object. e.g. ENABLE_TESTS, CFLAGS, etc. + + Values in this container are read-only. Attempts at changing values + will result in a run-time error. + + Access to an unknown variable will return None. + """), + + '__builtins__': (dict, + """Exposes Python built-in types. + + The set of exposed Python built-ins is currently: + + True + False + None + """), +} diff --git a/python/mozbuild/mozbuild/test/common.py b/python/mozbuild/mozbuild/test/common.py new file mode 100644 index 000000000000..05d34fd573e7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/common.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os + +from mach.logging import LoggingManager + + +# By including this module, tests get structured logging. +log_manager = LoggingManager() +log_manager.add_terminal_logging() + +# mozconfig is not a reusable type (it's actually a module) so, we +# have to mock it. +class MockConfig(object): + def __init__(self, topsrcdir='/path/to/topsrcdir'): + self.topsrcdir = topsrcdir + self.topobjdir = '/path/to/topobjdir' + + self.substs = { + 'MOZ_FOO': 'foo', + 'MOZ_BAR': 'bar', + 'MOZ_TRUE': '1', + 'MOZ_FALSE': '', + } + + def child_path(self, p): + return os.path.join(self.topsrcdir, p) diff --git a/python/mozbuild/mozbuild/test/frontend/__init__.py b/python/mozbuild/mozbuild/test/frontend/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-basic/included.build b/python/mozbuild/mozbuild/test/frontend/data/include-basic/included.build new file mode 100644 index 000000000000..bb492a24211b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-basic/included.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += ['bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-basic/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-basic/moz.build new file mode 100644 index 000000000000..b8301f3f324b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-basic/moz.build @@ -0,0 +1,6 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] + +include('included.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-1.build b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-1.build new file mode 100644 index 000000000000..a6a0fd8ea6a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-1.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('included-2.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-2.build b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-2.build new file mode 100644 index 000000000000..9bfc65481d4c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-2.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +ILLEGAL = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/moz.build new file mode 100644 index 000000000000..6dfb33d92ccb --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('included-1.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-missing/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-missing/moz.build new file mode 100644 index 000000000000..f7fc25465842 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-missing/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('missing.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-outside-topsrcdir/relative.build b/python/mozbuild/mozbuild/test/frontend/data/include-outside-topsrcdir/relative.build new file mode 100644 index 000000000000..f8084f0dd2cf --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-outside-topsrcdir/relative.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../moz.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child.build new file mode 100644 index 000000000000..446207081acc --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../parent.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child2.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child2.build new file mode 100644 index 000000000000..618a75ed0ebb --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child2.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('grandchild/grandchild.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/grandchild/grandchild.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/grandchild/grandchild.build new file mode 100644 index 000000000000..4d721fde4b04 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/grandchild/grandchild.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../../parent.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/parent.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/parent.build new file mode 100644 index 000000000000..a2ed3fa4901a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/parent.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/moz.build new file mode 100644 index 000000000000..806a44ccd1c7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('/sibling.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/sibling.build b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/sibling.build new file mode 100644 index 000000000000..a2ed3fa4901a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/sibling.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-bad-dir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-bad-dir/moz.build new file mode 100644 index 000000000000..a2ed3fa4901a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-bad-dir/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-basic/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-basic/moz.build new file mode 100644 index 000000000000..9bfc65481d4c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-basic/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +ILLEGAL = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/child.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/child.build new file mode 100644 index 000000000000..9bfc65481d4c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/child.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +ILLEGAL = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/moz.build new file mode 100644 index 000000000000..8d3ad9d61815 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('child.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-missing-include/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-missing-include/moz.build new file mode 100644 index 000000000000..f7fc25465842 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-missing-include/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('missing.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-outside-topsrcdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-outside-topsrcdir/moz.build new file mode 100644 index 000000000000..535dd66e7d27 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-outside-topsrcdir/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../include-basic/moz.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-read-unknown-global/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-read-unknown-global/moz.build new file mode 100644 index 000000000000..62b30187d14f --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-read-unknown-global/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +l = FOO diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-repeated-dir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-repeated-dir/moz.build new file mode 100644 index 000000000000..bdd2c5cf8b15 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-repeated-dir/moz.build @@ -0,0 +1,6 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] + +DIRS += ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-script-error/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-script-error/moz.build new file mode 100644 index 000000000000..93bc9f7ea12a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-script-error/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +foo = True + None diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build new file mode 100644 index 000000000000..d39fac0f6786 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +foo = diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-bad-value/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-bad-value/moz.build new file mode 100644 index 000000000000..187ffeaa5fae --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-bad-value/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = 'dir' diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-unknown-global/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-unknown-global/moz.build new file mode 100644 index 000000000000..89a94205f554 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-unknown-global/moz.build @@ -0,0 +1,6 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['dir1', 'dir2'] + +FOO = 'bar' diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/moz.build new file mode 100644 index 000000000000..6ad52854e0e8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/moz.build @@ -0,0 +1,8 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += ['regular'] +PARALLEL_DIRS = ['parallel'] +TEST_DIRS = ['test'] +TEST_TOOL_DIRS = ['test_tool'] +TOOL_DIRS = ['tool'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/parallel/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/parallel/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/regular/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/regular/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/test/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/test/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/test_tool/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/test_tool/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/tool/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/tool/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-outside-topsrcdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-outside-topsrcdir/moz.build new file mode 100644 index 000000000000..55f786dd545d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-outside-topsrcdir/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../../foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/bar/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/foo/moz.build new file mode 100644 index 000000000000..03636952d8b8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/foo/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/moz.build new file mode 100644 index 000000000000..a2ed3fa4901a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/bar/moz.build new file mode 100644 index 000000000000..eaaedcc6bfc0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/bar/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/foo/moz.build new file mode 100644 index 000000000000..03636952d8b8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/foo/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/moz.build new file mode 100644 index 000000000000..e75a1d58e7ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo', 'bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/bar/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/biz/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/biz/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/moz.build new file mode 100644 index 000000000000..8ace80025622 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/moz.build @@ -0,0 +1 @@ +DIRS = ['biz'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/moz.build new file mode 100644 index 000000000000..e75a1d58e7ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo', 'bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-fails-in-subdir/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-fails-in-subdir/foo/moz.build new file mode 100644 index 000000000000..c0aebdd02c0e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-fails-in-subdir/foo/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +add_tier_dir('illegal', 'IRRELEVANT') diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-fails-in-subdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-fails-in-subdir/moz.build new file mode 100644 index 000000000000..a2ed3fa4901a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-fails-in-subdir/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/bar/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/baz/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/baz/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/foo/biz/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/foo/biz/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/foo/moz.build new file mode 100644 index 000000000000..0c0155d21f50 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/foo/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['biz'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/foo_static/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/foo_static/moz.build new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/moz.build new file mode 100644 index 000000000000..b319fba114e6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-tier-simple/moz.build @@ -0,0 +1,8 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +add_tier_dir('t1', 'foo') +add_tier_dir('t1', 'foo_static', static=True) + +add_tier_dir('t2', 'bar') +add_tier_dir('t3', 'baz', static=True) diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py new file mode 100644 index 000000000000..064f41a4be57 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import unittest + +from mozbuild.frontend.data import DirectoryTraversal +from mozbuild.frontend.emitter import TreeMetadataEmitter +from mozbuild.frontend.reader import BuildReader + +from mozbuild.test.common import MockConfig + + +data_path = os.path.abspath(os.path.dirname(__file__)) +data_path = os.path.join(data_path, 'data') + + +class TestEmitterBasic(unittest.TestCase): + def reader(self, name): + config = MockConfig(os.path.join(data_path, name)) + config.substs['ENABLE_TESTS'] = '1' + + return BuildReader(config) + + def test_dirs_traversal_simple(self): + reader = self.reader('traversal-simple') + emitter = TreeMetadataEmitter(reader.config) + + objs = list(emitter.emit(reader.read_topsrcdir())) + + self.assertEqual(len(objs), 4) + + for o in objs: + self.assertIsInstance(o, DirectoryTraversal) + self.assertEqual(o.parallel_dirs, []) + self.assertEqual(o.tool_dirs, []) + self.assertEqual(o.test_dirs, []) + self.assertEqual(o.test_tool_dirs, []) + self.assertEqual(len(o.tier_dirs), 0) + self.assertEqual(len(o.tier_static_dirs), 0) + + reldirs = [o.relativedir for o in objs] + self.assertEqual(reldirs, ['', 'foo', 'foo/biz', 'bar']) + + dirs = [o.dirs for o in objs] + self.assertEqual(dirs, [['foo', 'bar'], ['biz'], [], []]) + + def test_traversal_all_vars(self): + reader = self.reader('traversal-all-vars') + emitter = TreeMetadataEmitter(reader.config) + + objs = list(emitter.emit(reader.read_topsrcdir())) + self.assertEqual(len(objs), 6) + + for o in objs: + self.assertIsInstance(o, DirectoryTraversal) + + reldirs = set([o.relativedir for o in objs]) + self.assertEqual(reldirs, set(['', 'parallel', 'regular', 'test', + 'test_tool', 'tool'])) + + for o in objs: + reldir = o.relativedir + + if reldir == '': + self.assertEqual(o.dirs, ['regular']) + self.assertEqual(o.parallel_dirs, ['parallel']) + self.assertEqual(o.test_dirs, ['test']) + self.assertEqual(o.test_tool_dirs, ['test_tool']) + self.assertEqual(o.tool_dirs, ['tool']) + + def test_tier_simple(self): + reader = self.reader('traversal-tier-simple') + emitter = TreeMetadataEmitter(reader.config) + + objs = list(emitter.emit(reader.read_topsrcdir())) + self.assertEqual(len(objs), 6) + + reldirs = [o.relativedir for o in objs] + self.assertEqual(reldirs, ['', 'foo', 'foo/biz', 'foo_static', 'bar', + 'baz']) + diff --git a/python/mozbuild/mozbuild/test/frontend/test_namespaces.py b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py new file mode 100644 index 000000000000..a04fb71a45fb --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import unittest + +from mozunit import main + +from mozbuild.frontend.sandbox import ( + GlobalNamespace, + LocalNamespace, +) + +from mozbuild.frontend.sandbox_symbols import VARIABLES + + +class TestGlobalNamespace(unittest.TestCase): + def test_builtins(self): + ns = GlobalNamespace() + + self.assertIn('__builtins__', ns) + self.assertEqual(ns['__builtins__']['True'], True) + + def test_key_rejection(self): + # Lowercase keys should be rejected during normal operation. + ns = GlobalNamespace(allowed_variables=VARIABLES) + + with self.assertRaises(KeyError) as ke: + ns['foo'] = True + + e = ke.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_unknown') + self.assertEqual(e[2], 'foo') + self.assertTrue(e[3]) + + # Unknown uppercase keys should be rejected. + with self.assertRaises(KeyError) as ke: + ns['FOO'] = True + + e = ke.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_unknown') + self.assertEqual(e[2], 'FOO') + self.assertTrue(e[3]) + + def test_allowed_set(self): + self.assertIn('DIRS', VARIABLES) + + ns = GlobalNamespace(allowed_variables=VARIABLES) + + ns['DIRS'] = ['foo'] + self.assertEqual(ns['DIRS'], ['foo']) + + def test_value_checking(self): + ns = GlobalNamespace(allowed_variables=VARIABLES) + + # Setting to a non-allowed type should not work. + with self.assertRaises(ValueError) as ve: + ns['DIRS'] = True + + e = ve.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_type') + self.assertEqual(e[2], 'DIRS') + self.assertTrue(e[3]) + self.assertEqual(e[4], list) + + def test_allow_all_writes(self): + ns = GlobalNamespace(allowed_variables=VARIABLES) + + with ns.allow_all_writes() as d: + d['foo'] = True + self.assertTrue(d['foo']) + + with self.assertRaises(KeyError) as ke: + ns['foo'] = False + + self.assertEqual(ke.exception.args[1], 'set_unknown') + + self.assertTrue(d['foo']) + + def test_key_checking(self): + # Checking for existence of a key should not populate the key if it + # doesn't exist. + g = GlobalNamespace(allowed_variables=VARIABLES) + + self.assertFalse('DIRS' in g) + self.assertFalse('DIRS' in g) + + +class TestLocalNamespace(unittest.TestCase): + def test_locals(self): + g = GlobalNamespace(allowed_variables=VARIABLES) + l = LocalNamespace(g) + + l['foo'] = ['foo'] + self.assertEqual(l['foo'], ['foo']) + + l['foo'] += ['bar'] + self.assertEqual(l['foo'], ['foo', 'bar']) + + def test_global_proxy_reads(self): + g = GlobalNamespace(allowed_variables=VARIABLES) + g['DIRS'] = ['foo'] + + l = LocalNamespace(g) + + self.assertEqual(l['DIRS'], g['DIRS']) + + # Reads to missing UPPERCASE vars should result in KeyError. + with self.assertRaises(KeyError) as ke: + v = l['FOO'] + + e = ke.exception + self.assertEqual(e.args[0], 'global_ns') + self.assertEqual(e.args[1], 'get_unknown') + + def test_global_proxy_writes(self): + g = GlobalNamespace(allowed_variables=VARIABLES) + l = LocalNamespace(g) + + l['DIRS'] = ['foo'] + + self.assertEqual(l['DIRS'], ['foo']) + self.assertEqual(g['DIRS'], ['foo']) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py new file mode 100644 index 000000000000..b0b3e921013b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import sys +import unittest + +from mozunit import main + +from mozbuild.frontend.reader import BuildReaderError +from mozbuild.frontend.reader import BuildReader + +from mozbuild.test.common import MockConfig + + +if sys.version_info.major == 2: + text_type = 'unicode' +else: + text_type = 'str' + +data_path = os.path.abspath(os.path.dirname(__file__)) +data_path = os.path.join(data_path, 'data') + + +class TestBuildReader(unittest.TestCase): + def config(self, name): + path = os.path.join(data_path, name) + + return MockConfig(path) + + def reader(self, name, enable_tests=False): + config = self.config(name) + + if enable_tests: + config.substs['ENABLE_TESTS'] = '1' + + return BuildReader(config) + + def file_path(self, name, *args): + return os.path.join(data_path, name, *args) + + def test_dirs_traversal_simple(self): + reader = self.reader('traversal-simple') + + sandboxes = list(reader.read_topsrcdir()) + + self.assertEqual(len(sandboxes), 4) + + def test_dirs_traversal_no_descend(self): + reader = self.reader('traversal-simple') + + path = os.path.join(reader.topsrcdir, 'moz.build') + self.assertTrue(os.path.exists(path)) + + sandboxes = list(reader.read_mozbuild(path, + filesystem_absolute=True, descend=False)) + + self.assertEqual(len(sandboxes), 1) + + def test_dirs_traversal_all_variables(self): + reader = self.reader('traversal-all-vars', enable_tests=True) + + sandboxes = list(reader.read_topsrcdir()) + self.assertEqual(len(sandboxes), 6) + + def test_tiers_traversal(self): + reader = self.reader('traversal-tier-simple') + + sandboxes = list(reader.read_topsrcdir()) + self.assertEqual(len(sandboxes), 6) + + def test_tier_subdir(self): + # add_tier_dir() should fail when not in the top directory. + reader = self.reader('traversal-tier-fails-in-subdir') + + with self.assertRaises(Exception): + list(reader.read_topsrcdir()) + + def test_relative_dirs(self): + # Ensure relative directories are traversed. + reader = self.reader('traversal-relative-dirs') + + sandboxes = list(reader.read_topsrcdir()) + self.assertEqual(len(sandboxes), 3) + + def test_repeated_dirs_ignored(self): + # Ensure repeated directories are ignored. + reader = self.reader('traversal-repeated-dirs') + + sandboxes = list(reader.read_topsrcdir()) + self.assertEqual(len(sandboxes), 3) + + def test_outside_topsrcdir(self): + # References to directories outside the topsrcdir should fail. + reader = self.reader('traversal-outside-topsrcdir') + + with self.assertRaises(Exception): + list(reader.read_topsrcdir()) + + def test_error_basic(self): + reader = self.reader('reader-error-basic') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertEqual(e.actual_file, self.file_path('reader-error-basic', + 'moz.build')) + + self.assertIn('The error occurred while processing the', str(e)) + + def test_error_included_from(self): + reader = self.reader('reader-error-included-from') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertEqual(e.actual_file, + self.file_path('reader-error-included-from', 'child.build')) + self.assertEqual(e.main_file, + self.file_path('reader-error-included-from', 'moz.build')) + + self.assertIn('This file was included as part of processing', str(e)) + + def test_error_syntax_error(self): + reader = self.reader('reader-error-syntax') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('Python syntax error on line 4', str(e)) + self.assertIn(' foo =', str(e)) + self.assertIn(' ^', str(e)) + + def test_error_read_unknown_global(self): + reader = self.reader('reader-error-read-unknown-global') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error was triggered on line 4', str(e)) + self.assertIn('The underlying problem is an attempt to read', str(e)) + self.assertIn(' FOO', str(e)) + + def test_error_write_unknown_global(self): + reader = self.reader('reader-error-write-unknown-global') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error was triggered on line 6', str(e)) + self.assertIn('The underlying problem is an attempt to write', str(e)) + self.assertIn(' FOO', str(e)) + + def test_error_write_bad_value(self): + reader = self.reader('reader-error-write-bad-value') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error was triggered on line 4', str(e)) + self.assertIn('is an attempt to write an illegal value to a special', + str(e)) + + self.assertIn('variable whose value was rejected is:\n\n DIRS', + str(e)) + + self.assertIn('written to it was of the following type:\n\n %s' % text_type, + str(e)) + + self.assertIn('expects the following type(s):\n\n list', str(e)) + + def test_error_illegal_path(self): + reader = self.reader('reader-error-outside-topsrcdir') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The underlying problem is an illegal file access', + str(e)) + + def test_error_missing_include_path(self): + reader = self.reader('reader-error-missing-include') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('we referenced a path that does not exist', str(e)) + + def test_error_script_error(self): + reader = self.reader('reader-error-script-error') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error appears to be the fault of the script', + str(e)) + self.assertIn(' ["TypeError: unsupported operand', str(e)) + + def test_error_bad_dir(self): + reader = self.reader('reader-error-bad-dir') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('we referenced a path that does not exist', str(e)) + + def test_error_repeated_dir(self): + reader = self.reader('reader-error-repeated-dir') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('Directory (foo) registered multiple times in DIRS', + str(e)) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py new file mode 100644 index 000000000000..d205495e3bce --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py @@ -0,0 +1,272 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import shutil +import unittest + +from mozunit import main + +from mozbuild.frontend.reader import MozbuildSandbox + +from mozbuild.frontend.sandbox import ( + SandboxExecutionError, + SandboxLoadError, +) + +from mozbuild.frontend.sandbox_symbols import ( + FUNCTIONS, + SPECIAL_VARIABLES, + VARIABLES, +) + +from mozbuild.test.common import MockConfig + + +test_data_path = os.path.abspath(os.path.dirname(__file__)) +test_data_path = os.path.join(test_data_path, 'data') + + +class TestSandbox(unittest.TestCase): + def sandbox(self, relpath='moz.build', data_path=None): + config = None + + if data_path is not None: + config = MockConfig(os.path.join(test_data_path, data_path)) + else: + config = MockConfig() + + return MozbuildSandbox(config, config.child_path(relpath)) + + def test_default_state(self): + sandbox = self.sandbox() + config = sandbox.config + + self.assertEqual(sandbox['TOPSRCDIR'], config.topsrcdir) + self.assertEqual(sandbox['TOPOBJDIR'], + os.path.abspath(config.topobjdir)) + self.assertEqual(sandbox['RELATIVEDIR'], '') + self.assertEqual(sandbox['SRCDIR'], config.topsrcdir) + self.assertEqual(sandbox['OBJDIR'], + os.path.abspath(config.topobjdir).replace(os.sep, '/')) + + def test_symbol_presence(self): + # Ensure no discrepancies between the master symbol table and what's in + # the sandbox. + sandbox = self.sandbox() + + all_symbols = set() + all_symbols |= set(FUNCTIONS.keys()) + all_symbols |= set(SPECIAL_VARIABLES.keys()) + + for symbol in sandbox: + self.assertIn(symbol, all_symbols) + all_symbols.remove(symbol) + + self.assertEqual(len(all_symbols), 0) + + def test_path_calculation(self): + sandbox = self.sandbox('foo/bar/moz.build') + config = sandbox.config + + self.assertEqual(sandbox['RELATIVEDIR'], 'foo/bar') + self.assertEqual(sandbox['SRCDIR'], '/'.join([config.topsrcdir, + 'foo/bar'])) + self.assertEqual(sandbox['OBJDIR'], + os.path.abspath('/'.join([config.topobjdir, 'foo/bar'])).replace(os.sep, '/')) + + def test_config_access(self): + sandbox = self.sandbox() + config = sandbox.config + + self.assertIn('CONFIG', sandbox) + self.assertEqual(sandbox['CONFIG']['MOZ_TRUE'], '1') + self.assertEqual(sandbox['CONFIG']['MOZ_FOO'], config.substs['MOZ_FOO']) + + # Access to an undefined substitution should return None. + self.assertNotIn('MISSING', sandbox['CONFIG']) + self.assertIsNone(sandbox['CONFIG']['MISSING']) + + # Should shouldn't be allowed to assign to the config. + with self.assertRaises(Exception): + sandbox['CONFIG']['FOO'] = '' + + def test_dict_interface(self): + sandbox = self.sandbox() + config = sandbox.config + + self.assertFalse('foo' in sandbox) + self.assertFalse('FOO' in sandbox) + + self.assertTrue(sandbox.get('foo', True)) + self.assertEqual(sandbox.get('TOPSRCDIR'), config.topsrcdir) + self.assertGreater(len(sandbox), 6) + + for key in sandbox: + continue + + for key in sandbox.iterkeys(): + continue + + def test_exec_source_success(self): + sandbox = self.sandbox() + + sandbox.exec_source('foo = True', 'foo.py') + + self.assertNotIn('foo', sandbox) + + def test_exec_compile_error(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('2f23;k;asfj', 'foo.py') + + self.assertEqual(se.exception.file_stack, ['foo.py']) + self.assertIsInstance(se.exception.exc_value, SyntaxError) + + def test_exec_import_denied(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('import sys', 'import.py') + + self.assertIsInstance(se.exception, SandboxExecutionError) + self.assertEqual(se.exception.exc_type, ImportError) + + def test_exec_source_multiple(self): + sandbox = self.sandbox() + + sandbox.exec_source('DIRS = ["foo"]', 'foo.py') + sandbox.exec_source('DIRS = ["bar"]', 'foo.py') + + self.assertEqual(sandbox['DIRS'], ['bar']) + + def test_exec_source_illegal_key_set(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('ILLEGAL = True', 'foo.py') + + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + e = se.exception.exc_value + self.assertEqual(e.args[0], 'global_ns') + self.assertEqual(e.args[1], 'set_unknown') + + def test_add_tier_dir_regular_str(self): + sandbox = self.sandbox() + + sandbox.exec_source('add_tier_dir("t1", "foo")', 'foo.py') + + self.assertEqual(sandbox['TIERS']['t1'], + {'regular': ['foo'], 'static': []}) + + def test_add_tier_dir_regular_list(self): + sandbox = self.sandbox() + + sandbox.exec_source('add_tier_dir("t1", ["foo", "bar"])', 'foo.py') + + self.assertEqual(sandbox['TIERS']['t1'], + {'regular': ['foo', 'bar'], 'static': []}) + + def test_add_tier_dir_static(self): + sandbox = self.sandbox() + + sandbox.exec_source('add_tier_dir("t1", "foo", static=True)', 'foo.py') + + self.assertEqual(sandbox['TIERS']['t1'], + {'regular': [], 'static': ['foo']}) + + def test_tier_order(self): + sandbox = self.sandbox() + + source = ''' +add_tier_dir('t1', 'foo') +add_tier_dir('t1', 'bar') +add_tier_dir('t2', 'baz', static=True) +add_tier_dir('t3', 'biz') +add_tier_dir('t1', 'bat', static=True) +''' + + sandbox.exec_source(source, 'foo.py') + + self.assertEqual([k for k in sandbox['TIERS'].keys()], ['t1', 't2', 't3']) + + def test_tier_multiple_registration(self): + sandbox = self.sandbox() + + sandbox.exec_source('add_tier_dir("t1", "foo")', 'foo.py') + + with self.assertRaises(SandboxExecutionError): + sandbox.exec_source('add_tier_dir("t1", "foo")', 'foo.py') + + def test_include_basic(self): + sandbox = self.sandbox(data_path='include-basic') + + sandbox.exec_file('moz.build') + + self.assertEqual(sandbox['DIRS'], ['foo', 'bar']) + + def test_include_outside_topsrcdir(self): + sandbox = self.sandbox(data_path='include-outside-topsrcdir') + + with self.assertRaises(SandboxLoadError) as se: + sandbox.exec_file('relative.build') + + expected = os.path.join(test_data_path, 'moz.build') + self.assertEqual(se.exception.illegal_path, expected) + + def test_include_error_stack(self): + # Ensure the path stack is reported properly in exceptions. + sandbox = self.sandbox(data_path='include-file-stack') + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_file('moz.build') + + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + args = e.exc_value.args + self.assertEqual(args[0], 'global_ns') + self.assertEqual(args[1], 'set_unknown') + self.assertEqual(args[2], 'ILLEGAL') + + expected_stack = [os.path.join(sandbox.config.topsrcdir, p) for p in [ + 'moz.build', 'included-1.build', 'included-2.build']] + + self.assertEqual(e.file_stack, expected_stack) + + def test_include_missing(self): + sandbox = self.sandbox(data_path='include-missing') + + with self.assertRaises(SandboxLoadError) as sle: + sandbox.exec_file('moz.build') + + self.assertIsNotNone(sle.exception.read_error) + + def test_include_relative_from_child_dir(self): + # A relative path from a subdirectory should be relative from that + # child directory. + sandbox = self.sandbox(data_path='include-relative-from-child') + sandbox.exec_file('child/child.build') + self.assertEqual(sandbox['DIRS'], ['foo']) + + sandbox = self.sandbox(data_path='include-relative-from-child') + sandbox.exec_file('child/child2.build') + self.assertEqual(sandbox['DIRS'], ['foo']) + + def test_include_topsrcdir_relative(self): + # An absolute path for include() is relative to topsrcdir. + + sandbox = self.sandbox(data_path='include-topsrcdir-relative') + sandbox.exec_file('moz.build') + + self.assertEqual(sandbox['DIRS'], ['foo']) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_sandbox_symbols.py b/python/mozbuild/mozbuild/test/frontend/test_sandbox_symbols.py new file mode 100644 index 000000000000..f2b46dcfdda8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox_symbols.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from mozunit import main + +from mozbuild.frontend.sandbox_symbols import ( + FUNCTIONS, + SPECIAL_VARIABLES, + VARIABLES, +) + + +class TestSymbols(unittest.TestCase): + def _verify_doc(self, doc): + # Documentation should be of the format: + # """SUMMARY LINE + # + # EXTRA PARAGRAPHS + # """ + + self.assertNotIn('\r', doc) + + lines = doc.split('\n') + + # No trailing whitespace. + for line in lines[0:-1]: + self.assertEqual(line, line.rstrip()) + + self.assertGreater(len(lines), 0) + self.assertGreater(len(lines[0].strip()), 0) + + # Last line should be empty. + self.assertEqual(lines[-1].strip(), '') + + def test_documentation_formatting(self): + for typ, default, doc in VARIABLES.values(): + self._verify_doc(doc) + + for attr, args, doc in FUNCTIONS.values(): + self._verify_doc(doc) + + for typ, doc in SPECIAL_VARIABLES.values(): + self._verify_doc(doc) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_containers.py b/python/mozbuild/mozbuild/test/test_containers.py new file mode 100644 index 000000000000..f2a1e2da793b --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_containers.py @@ -0,0 +1,104 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from mozunit import main + +from mozbuild.util import ( + DefaultOnReadDict, + ReadOnlyDefaultDict, + ReadOnlyDict, +) + +class TestReadOnlyDict(unittest.TestCase): + def test_basic(self): + original = {'foo': 1, 'bar': 2} + + test = ReadOnlyDict(original) + + self.assertEqual(original, test) + self.assertEqual(test['foo'], 1) + + with self.assertRaises(KeyError): + value = test['missing'] + + with self.assertRaises(Exception): + test['baz'] = True + +class TestDefaultOnReadDict(unittest.TestCase): + def test_no_defaults(self): + original = {'foo': 1, 'bar': 2} + + test = DefaultOnReadDict(original) + self.assertEqual(original, test) + + with self.assertRaises(KeyError): + value = test['missing'] + + test['foo'] = 5 + self.assertEqual(test['foo'], 5) + + def test_dict_defaults(self): + original = {'foo': 1, 'bar': 2} + + test = DefaultOnReadDict(original, defaults={'baz': 3}) + + self.assertEqual(original, test) + self.assertEqual(test['baz'], 3) + + with self.assertRaises(KeyError): + value = test['missing'] + + test['baz'] = 4 + self.assertEqual(test['baz'], 4) + + def test_global_default(self): + original = {'foo': 1} + + test = DefaultOnReadDict(original, defaults={'bar': 2}, + global_default=10) + + self.assertEqual(original, test) + self.assertEqual(test['foo'], 1) + + self.assertEqual(test['bar'], 2) + self.assertEqual(test['baz'], 10) + + test['bar'] = 3 + test['baz'] = 12 + test['other'] = 11 + + self.assertEqual(test['bar'], 3) + self.assertEqual(test['baz'], 12) + self.assertEqual(test['other'], 11) + + +class TestReadOnlyDefaultDict(unittest.TestCase): + def test_simple(self): + original = {'foo': 1, 'bar': 2} + + test = ReadOnlyDefaultDict(original) + + self.assertEqual(original, test) + + self.assertEqual(test['foo'], 1) + + with self.assertRaises(KeyError): + value = test['missing'] + + def test_assignment(self): + test = ReadOnlyDefaultDict({}) + + with self.assertRaises(Exception): + test['foo'] = True + + def test_defaults(self): + test = ReadOnlyDefaultDict({}, defaults={'foo': 1}) + + self.assertEqual(test['foo'], 1) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index e8fcfc4c4e46..51e52bb7f7a1 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -7,6 +7,7 @@ from __future__ import unicode_literals +import copy import hashlib @@ -27,3 +28,59 @@ def hash_file(path): h.update(data) return h.hexdigest() + + +class ReadOnlyDict(dict): + """A read-only dictionary.""" + def __init__(self, d): + dict.__init__(self, d) + + def __setitem__(self, name, value): + raise Exception('Object does not support assignment.') + + +class undefined_default(object): + """Represents an undefined argument value that isn't None.""" + + +undefined = undefined_default() + + +class DefaultOnReadDict(dict): + """A dictionary that returns default values for missing keys on read.""" + + def __init__(self, d, defaults=None, global_default=undefined): + """Create an instance from an iterable with defaults. + + The first argument is fed into the dict constructor. + + defaults is a dict mapping keys to their default values. + + global_default is the default value for *all* missing keys. If it isn't + specified, no default value for keys not in defaults will be used and + IndexError will be raised on access. + """ + dict.__init__(self, d) + + self._defaults = defaults or {} + self._global_default = global_default + + def __getitem__(self, k): + try: + return dict.__getitem__(self, k) + except: + pass + + if k in self._defaults: + dict.__setitem__(self, k, copy.deepcopy(self._defaults[k])) + elif self._global_default != undefined: + dict.__setitem__(self, k, copy.deepcopy(self._global_default)) + + return dict.__getitem__(self, k) + + +class ReadOnlyDefaultDict(DefaultOnReadDict, ReadOnlyDict): + """A read-only dictionary that supports default values on retrieval.""" + def __init__(self, d, defaults=None, global_default=undefined): + DefaultOnReadDict.__init__(self, d, defaults, global_default) +