add a --bootstrap mode for configure.py

Instead of bootstrapping through a separate script, instead make
configure.py able to either generate a build.ninja *or* just execute
all the computed commands to build a ninja binary.

configure.py

@@ -24,14 +24,90 @@ from __future__ import print_function
 from optparse import OptionParser
 import os
 import pipes
+import string
+import subprocess
 import sys
 import platform_helper
 sys.path.insert(0, 'misc')
 
 import ninja_syntax
 
+
+class Bootstrap:
+    """API shim for ninja_syntax.Writer that instead runs the commands.
+
+    Used to bootstrap Ninja from scratch.  In --bootstrap mode this
+    class is used to execute all the commands to build an executable.
+    It also proxies all calls to an underlying ninja_syntax.Writer, to
+    behave like non-bootstrap mode.
+    """
+    def __init__(self, writer):
+        self.writer = writer
+        # Map of variable name => expanded variable value.
+        self.vars = {}
+        # Map of rule name => dict of rule attributes.
+        self.rules = {
+            'phony': {}
+        }
+
+    def comment(self, text):
+        return self.writer.comment(text)
+
+    def newline(self):
+        return self.writer.newline()
+
+    def variable(self, key, val):
+        self.vars[key] = self._expand(val)
+        return self.writer.variable(key, val)
+
+    def rule(self, name, **kwargs):
+        self.rules[name] = kwargs
+        return self.writer.rule(name, **kwargs)
+
+    def build(self, outputs, rule, inputs=None, **kwargs):
+        ruleattr = self.rules[rule]
+        cmd = ruleattr.get('command')
+        if cmd is None:  # A phony rule, for example.
+            return
+
+        # Implement just enough of Ninja variable expansion etc. to
+        # make the bootstrap build work.
+        local_vars = {
+            'in': self._expand_paths(inputs),
+            'out': self._expand_paths(outputs)
+        }
+        for key, val in kwargs.get('variables', []):
+            local_vars[key] = ' '.join(ninja_syntax.as_list(val))
+
+        self._run_command(self._expand(cmd, local_vars))
+
+        return self.writer.build(outputs, rule, inputs, **kwargs)
+
+    def default(self, paths):
+        return self.writer.default(paths)
+
+    def _expand_paths(self, paths):
+        """Expand $vars in an array of paths, e.g. from a 'build' block."""
+        paths = ninja_syntax.as_list(paths)
+        return ' '.join(map(self._expand, paths))
+
+    def _expand(self, str, local_vars={}):
+        """Expand $vars in a string."""
+        return ninja_syntax.expand(str, self.vars, local_vars)
+
+    def _run_command(self, cmdline):
+        """Run a subcommand, quietly.  Prints the full command on error."""
+        try:
+            subprocess.check_call(cmdline, shell=True)
+        except subprocess.CalledProcessError, e:
+            print('when running: ', cmdline)
+            raise
+
+
 parser = OptionParser()
 profilers = ['gmon', 'pprof']
+parser.add_option('--bootstrap', action='store_true',
+                  help='bootstrap a ninja binary from nothing')
 parser.add_option('--platform',
                   help='target platform (' +
                        '/'.join(platform_helper.platforms()) + ')',
@@ -64,8 +140,20 @@ else:
     host = platform
 
 BUILD_FILENAME = 'build.ninja'
-buildfile = open(BUILD_FILENAME, 'w')
-n = ninja_syntax.Writer(buildfile)
+ninja_writer = ninja_syntax.Writer(open(BUILD_FILENAME, 'w'))
+n = ninja_writer
+
+if options.bootstrap:
+    # Make the build directory.
+    try:
+        os.mkdir('build')
+    except OSError:
+        pass
+    # Wrap ninja_writer with the Bootstrapper, which also executes the
+    # commands.
+    print('bootstrapping ninja...')
+    n = Bootstrap(n)
+
 n.comment('This file is used to build ninja itself.')
 n.comment('It is generated by ' + os.path.basename(__file__) + '.')
 n.newline()
@@ -74,7 +162,10 @@ n.variable('ninja_required_version', '1.3')
 n.newline()
 
 n.comment('The arguments passed to configure.py, for rerunning it.')
-n.variable('configure_args', ' '.join(sys.argv[1:]))
+configure_args = sys.argv[1:]
+if '--bootstrap' in configure_args:
+    configure_args.remove('--bootstrap')
+n.variable('configure_args', ' '.join(configure_args))
 env_keys = set(['CXX', 'AR', 'CFLAGS', 'LDFLAGS'])
 configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys)
 if configure_env:
@@ -114,7 +205,8 @@ else:
     n.variable('ar', configure_env.get('AR', 'ar'))
 
 if platform.is_msvc():
-    cflags = ['/nologo',  # Don't print startup banner.
+    cflags = ['/showIncludes',
+              '/nologo',  # Don't print startup banner.
               '/Zi',  # Create pdb with debug info.
               '/W4',  # Highest warning level.
               '/WX',  # Warnings as errors.
@@ -129,6 +221,10 @@ if platform.is_msvc():
               '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS',
               '/D_VARIADIC_MAX=10',
               '/DNINJA_PYTHON="%s"' % options.with_python]
+    if options.bootstrap:
+        # In bootstrap mode, we have no ninja process to catch /showIncludes
+        # output.
+        cflags.remove('/showIncludes')
     if platform.msvc_needs_fs():
         cflags.append('/FS')
     ldflags = ['/DEBUG', '/libpath:$builddir']
@@ -200,9 +296,10 @@ n.newline()
 
 if platform.is_msvc():
     n.rule('cxx',
-        command='$cxx /showIncludes $cflags -c $in /Fo$out',
+        command='$cxx $cflags -c $in /Fo$out',
         description='CXX $out',
-        deps='msvc')
+        deps='msvc'  # /showIncludes is included in $cflags.
+    )
 else:
     n.rule('cxx',
         command='$cxx -MMD -MT $out -MF $out.d $cflags -c $in -o $out',
@@ -252,7 +349,6 @@ if have_browse:
 
 n.comment('the depfile parser and ninja lexers are generated using re2c.')
 def has_re2c():
-    import subprocess
     try:
         proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE)
         return int(proc.communicate()[0], 10) >= 1103
@@ -321,6 +417,12 @@ ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib,
 n.newline()
 all_targets += ninja
 
+if options.bootstrap:
+    # We've built the ninja binary.  Don't run any more commands
+    # through the bootstrap executor, but continue writing the
+    # build.ninja file.
+    n = ninja_writer
+
 n.comment('Tests all build into ninja_test executable.')
 
 test_libs = libs
@@ -434,4 +536,16 @@ if host.is_linux():
 
 n.build('all', 'phony', all_targets)
 
+n.close()
 print('wrote %s.' % BUILD_FILENAME)
+
+if options.bootstrap:
+    print('bootstrap complete.  rebuilding...')
+    if platform.is_windows():
+        bootstrap_exe = 'ninja.bootstrap.exe'
+        if os.path.exists(bootstrap_exe):
+            os.unlink(bootstrap_exe)
+        os.rename('ninja.exe', bootstrap_exe)
+        subprocess.check_call('ninja.bootstrap.exe', shell=True)
+    else:
+        subprocess.check_call('./ninja', shell=True)

misc/ninja_syntax.py

@@ -60,16 +60,16 @@ class Writer(object):
 
     def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
               variables=None):
-        outputs = self._as_list(outputs)
+        outputs = as_list(outputs)
         out_outputs = [escape_path(x) for x in outputs]
-        all_inputs = [escape_path(x) for x in self._as_list(inputs)]
+        all_inputs = [escape_path(x) for x in as_list(inputs)]
 
         if implicit:
-            implicit = [escape_path(x) for x in self._as_list(implicit)]
+            implicit = [escape_path(x) for x in as_list(implicit)]
             all_inputs.append('|')
             all_inputs.extend(implicit)
         if order_only:
-            order_only = [escape_path(x) for x in self._as_list(order_only)]
+            order_only = [escape_path(x) for x in as_list(order_only)]
             all_inputs.append('||')
             all_inputs.extend(order_only)
 
@@ -94,7 +94,7 @@ class Writer(object):
         self._line('subninja %s' % path)
 
     def default(self, paths):
-        self._line('default %s' % ' '.join(self._as_list(paths)))
+        self._line('default %s' % ' '.join(as_list(paths)))
 
     def _count_dollars_before_index(self, s, i):
         """Returns the number of '$' characters right in front of s[i]."""
@@ -141,12 +141,16 @@ class Writer(object):
 
         self.output.write(leading_space + text + '\n')
 
-    def _as_list(self, input):
-        if input is None:
-            return []
-        if isinstance(input, list):
-            return input
-        return [input]
+    def close(self):
+        self.output.close()
+
+
+def as_list(input):
+    if input is None:
+        return []
+    if isinstance(input, list):
+        return input
+    return [input]
 
 
 def escape(string):