gecko-dev/build/subconfigure.py
Chris Manchester a2b572cc91 Bug 1466401 - Re-run configure in js/src based on dependencies generated by python configure. r=mshal
MozReview-Commit-ID: 3ueBBHiux3M

--HG--
extra : rebase_source : 78b42537f73a9688cc3d4befc9f7f7cd3b2db0c2
2018-06-07 15:50:06 -07:00

428 lines
15 KiB
Python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This script is used to capture the content of config.status-generated
# files and subsequently restore their timestamp if they haven't changed.
import argparse
import errno
import os
import re
import subprocess
import sys
import pickle
import mozpack.path as mozpath
class File(object):
def __init__(self, path):
self._path = path
self._content = open(path, 'rb').read()
stat = os.stat(path)
self._times = (stat.st_atime, stat.st_mtime)
@property
def path(self):
return self._path
@property
def mtime(self):
return self._times[1]
@property
def modified(self):
'''Returns whether the file was modified since the instance was
created. Result is memoized.'''
if hasattr(self, '_modified'):
return self._modified
modified = True
if os.path.exists(self._path):
if open(self._path, 'rb').read() == self._content:
modified = False
self._modified = modified
return modified
def update_time(self):
'''If the file hasn't changed since the instance was created,
restore its old modification time.'''
if not self.modified:
os.utime(self._path, self._times)
# As defined in the various sub-configures in the tree
PRECIOUS_VARS = set([
'build_alias',
'host_alias',
'target_alias',
'CC',
'CFLAGS',
'LDFLAGS',
'LIBS',
'CPPFLAGS',
'CPP',
'CCC',
'CXXFLAGS',
'CXX',
'CCASFLAGS',
'CCAS',
])
CONFIGURE_DATA = 'configure.pkl'
# Autoconf, in some of the sub-configures used in the tree, likes to error
# out when "precious" variables change in value. The solution it gives to
# straighten things is to either run make distclean or remove config.cache.
# There's no reason not to do the latter automatically instead of failing,
# doing the cleanup (which, on buildbots means a full clobber), and
# restarting from scratch.
def maybe_clear_cache(data):
env = dict(data['env'])
for kind in ('target', 'host', 'build'):
arg = data[kind]
if arg is not None:
env['%s_alias' % kind] = arg
# configure can take variables assignments in its arguments, and that
# overrides whatever is in the environment.
for arg in data['args']:
if arg[:1] != '-' and '=' in arg:
key, value = arg.split('=', 1)
env[key] = value
comment = re.compile(r'^\s+#')
cache = {}
with open(data['cache-file']) as f:
for line in f:
if not comment.match(line) and '=' in line:
key, value = line.rstrip(os.linesep).split('=', 1)
# If the value is quoted, unquote it
if value[:1] == "'":
value = value[1:-1].replace("'\\''", "'")
cache[key] = value
for precious in PRECIOUS_VARS:
# If there is no entry at all for that precious variable, then
# its value is not precious for that particular configure.
if 'ac_cv_env_%s_set' % precious not in cache:
continue
is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set'
value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None
if value != env.get(precious):
print('Removing %s because of %s value change from:' % (data['cache-file'], precious))
print(' %s' % (value if value is not None else 'undefined'))
print('to:')
print(' %s' % env.get(precious, 'undefined'))
os.remove(data['cache-file'])
return True
return False
def split_template(s):
"""Given a "file:template" string, returns "file", "template". If the string
is of the form "file" (without a template), returns "file", "file.in"."""
if ':' in s:
return s.split(':', 1)
return s, '%s.in' % s
def get_config_files(data):
# config.status in js/src never contains the output we try to scan here.
if data['relobjdir'] == 'js/src':
return [], []
config_status = mozpath.join(data['objdir'], 'config.status')
if not os.path.exists(config_status):
return [], []
config_files = []
command_files = []
# Scan the config.status output for information about configuration files
# it generates.
config_status_output = subprocess.check_output(
[data['shell'], '-c', '%s --help' % config_status],
stderr=subprocess.STDOUT).splitlines()
state = None
for line in config_status_output:
if line.startswith('Configuration') and line.endswith(':'):
if line.endswith('commands:'):
state = 'commands'
else:
state = 'config'
elif not line.strip():
state = None
elif state:
for f, t in (split_template(couple) for couple in line.split()):
f = mozpath.join(data['objdir'], f)
t = mozpath.join(data['srcdir'], t)
if state == 'commands':
command_files.append(f)
else:
config_files.append((f, t))
return config_files, command_files
def prepare(srcdir, objdir, shell, args):
parser = argparse.ArgumentParser()
parser.add_argument('--target', type=str)
parser.add_argument('--host', type=str)
parser.add_argument('--build', type=str)
parser.add_argument('--cache-file', type=str)
# The --srcdir argument is simply ignored. It's a useless autoconf feature
# that we don't support well anyways. This makes it stripped from `others`
# and allows to skip setting it when calling the subconfigure (configure
# will take it from the configure path anyways).
parser.add_argument('--srcdir', type=str)
data_file = os.path.join(objdir, CONFIGURE_DATA)
previous_args = None
if os.path.exists(data_file):
with open(data_file, 'rb') as f:
data = pickle.load(f)
previous_args = data['args']
# Msys likes to break environment variables and command line arguments,
# so read those from stdin, as they are passed from the configure script
# when necessary (on windows).
input = sys.stdin.read()
if input:
data = {a: b for [a, b] in eval(input)}
environ = {a: b for a, b in data['env']}
# These environment variables as passed from old-configure may contain
# posix-style paths, which will not be meaningful to the js
# subconfigure, which runs as a native python process, so use their
# values from the environment. In the case of autoconf implemented
# subconfigures, Msys will re-convert them properly.
for var in ('HOME', 'TERM', 'PATH', 'TMPDIR', 'TMP',
'TEMP', 'INCLUDE'):
if var in environ and var in os.environ:
environ[var] = os.environ[var]
args = data['args']
else:
environ = os.environ
args, others = parser.parse_known_args(args)
data = {
'target': args.target,
'host': args.host,
'build': args.build,
'args': others,
'shell': shell,
'srcdir': srcdir,
'env': environ,
}
if args.cache_file:
data['cache-file'] = mozpath.normpath(mozpath.join(os.getcwd(),
args.cache_file))
else:
data['cache-file'] = mozpath.join(objdir, 'config.cache')
if previous_args is not None:
data['previous-args'] = previous_args
try:
os.makedirs(objdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(data_file, 'wb') as f:
pickle.dump(data, f)
def prefix_lines(text, prefix):
return ''.join('%s> %s' % (prefix, line) for line in text.splitlines(True))
def execute_and_prefix(*args, **kwargs):
prefix = kwargs['prefix']
del kwargs['prefix']
proc = subprocess.Popen(*args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, **kwargs)
while True:
line = proc.stdout.readline()
if not line:
break
print(prefix_lines(line.rstrip(), prefix))
sys.stdout.flush()
return proc.wait()
def run(objdir):
ret = 0
with open(os.path.join(objdir, CONFIGURE_DATA), 'rb') as f:
data = pickle.load(f)
data['objdir'] = objdir
relobjdir = data['relobjdir'] = os.path.relpath(objdir, os.getcwd())
cache_file = data['cache-file']
cleared_cache = True
if os.path.exists(cache_file):
cleared_cache = maybe_clear_cache(data)
config_files, command_files = get_config_files(data)
contents = []
for f, t in config_files:
contents.append(File(f))
# AC_CONFIG_COMMANDS actually only registers tags, not file names
# but most commands are tagged with the file name they create.
# However, a few don't, or are tagged with a directory name (and their
# command is just to create that directory)
for f in command_files:
if os.path.isfile(f):
contents.append(File(f))
# Only run configure if one of the following is true:
# - config.status doesn't exist
# - config.status is older than an input to configure
# - the configure arguments changed
# - the environment changed in a way that requires a cache clear.
configure = mozpath.join(data['srcdir'], 'old-configure')
config_status_path = mozpath.join(objdir, 'config.status')
skip_configure = True
if not os.path.exists(config_status_path):
skip_configure = False
config_status = None
else:
config_status = File(config_status_path)
config_status_deps = mozpath.join(objdir, 'config_status_deps.in')
if not os.path.exists(config_status_deps):
skip_configure = False
else:
with open(config_status_deps, 'r') as fh:
dep_files = fh.read().splitlines() + [configure]
if (any(not os.path.exists(f) or (config_status.mtime < os.path.getmtime(f))
for f in dep_files) or
data.get('previous-args', data['args']) != data['args'] or
cleared_cache):
skip_configure = False
if not skip_configure:
if mozpath.normsep(relobjdir) == 'js/src':
# Because configure is a shell script calling a python script
# calling a shell script, on Windows, with msys screwing the
# environment, we lose the benefits from our own efforts in this
# script to get past the msys problems. So manually call the python
# script instead, so that we don't do a native->msys transition
# here. Then the python configure will still have the right
# environment when calling the shell configure.
command = [
sys.executable,
os.path.join(os.path.dirname(__file__), '..', 'configure.py'),
'--enable-project=js',
]
data['env']['OLD_CONFIGURE'] = os.path.join(
os.path.dirname(configure), 'old-configure')
else:
command = [data['shell'], configure]
for kind in ('target', 'build', 'host'):
if data.get(kind) is not None:
command += ['--%s=%s' % (kind, data[kind])]
command += data['args']
command += ['--cache-file=%s' % cache_file]
# Pass --no-create to configure so that it doesn't run config.status.
# We're going to run it ourselves.
command += ['--no-create']
print(prefix_lines('configuring', relobjdir))
print(prefix_lines('running %s' % ' '.join(command[:-1]), relobjdir))
sys.stdout.flush()
returncode = execute_and_prefix(command, cwd=objdir, env=data['env'],
prefix=relobjdir)
if returncode:
return returncode
# Only run config.status if one of the following is true:
# - config.status changed or did not exist
# - one of the templates for config files is newer than the corresponding
# config file.
skip_config_status = True
if mozpath.normsep(relobjdir) == 'js/src':
# Running config.status in js/src actually does nothing, so we just
# skip it.
pass
elif not config_status or config_status.modified:
# If config.status doesn't exist after configure (because it's not
# an autoconf configure), skip it.
if os.path.exists(config_status_path):
skip_config_status = False
else:
# config.status changed or was created, so we need to update the
# list of config and command files.
config_files, command_files = get_config_files(data)
for f, t in config_files:
if not os.path.exists(t) or \
os.path.getmtime(f) < os.path.getmtime(t):
skip_config_status = False
if not skip_config_status:
if skip_configure:
print(prefix_lines('running config.status', relobjdir))
sys.stdout.flush()
ret = execute_and_prefix([data['shell'], '-c', './config.status'],
cwd=objdir, env=data['env'], prefix=relobjdir)
for f in contents:
f.update_time()
return ret
def subconfigure(args):
parser = argparse.ArgumentParser()
parser.add_argument('--list', type=str,
help='File containing a list of subconfigures to run')
parser.add_argument('--skip', type=str,
help='File containing a list of Subconfigures to skip')
parser.add_argument('subconfigures', type=str, nargs='*',
help='Subconfigures to run if no list file is given')
args, others = parser.parse_known_args(args)
subconfigures = args.subconfigures
if args.list:
subconfigures.extend(open(args.list, 'rb').read().splitlines())
if args.skip:
skips = set(open(args.skip, 'rb').read().splitlines())
subconfigures = [s for s in subconfigures if s not in skips]
if not subconfigures:
return 0
ret = 0
for subconfigure in subconfigures:
returncode = run(subconfigure)
ret = max(returncode, ret)
if ret:
break
return ret
def main(args):
if args[0] != '--prepare':
return subconfigure(args)
topsrcdir = os.path.abspath(args[1])
subdir = args[2]
# subdir can be of the form srcdir:objdir
if ':' in subdir:
srcdir, subdir = subdir.split(':', 1)
else:
srcdir = subdir
srcdir = os.path.join(topsrcdir, srcdir)
objdir = os.path.abspath(subdir)
return prepare(srcdir, objdir, args[3], args[4:])
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))