Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2025-03-04 07:40:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update pymake with --keep-going and other fixes

Browse Source
This commit is contained in:
Benjamin Smedberg 2009-03-31 17:10:23 -04:00
parent 9068ed2a35
commit b5f84b2efb
8 changed files with 338 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build/pymake/.hg_archival.txt
View File

@ -1,2 +1,2 @@
repo: f5ab154deef2ffa97f1b2139589ae4a1962090a4
node: c07f9fe9fdaa85888314efa7001cd4fd768904c4
node: 50d8e87e8af3fb8147c33f169a8d5abe8e06ffa4

37
build/pymake/pymake/command.py
View File

@ -6,7 +6,7 @@ structure, environment, and working directory. Typically they will all share a p
except when a submake specifies -j1 when the parent make is building in parallel.
"""
import os, subprocess, sys, logging, time, traceback
import os, subprocess, sys, logging, time, traceback, re
from optparse import OptionParser
import data, parserdata, process, util
@ -15,6 +15,7 @@ import data, parserdata, process, util
makepypath = os.path.normpath(os.path.join(os.path.dirname(__file__), '../make.py'))
_simpleopts = re.compile(r'^[a-zA-Z]+\s')
def parsemakeflags(env):
"""
Parse MAKEFLAGS from the environment into a sequence of command-line arguments.
@ -26,7 +27,7 @@ def parsemakeflags(env):
if makeflags == '':
return []
if makeflags[0] not in ('-', ' '):
if _simpleopts.match(makeflags):
makeflags = '-' + makeflags
opts = []
@ -89,6 +90,9 @@ def main(args, env, cwd, cb):
op.add_option('-d',
action="store_true",
dest="verbose", default=False)
op.add_option('-k', '--keep-going',
action="store_true",
dest="keepgoing", default=False)
op.add_option('--debug-log',
dest="debuglog", default=None)
op.add_option('-C', '--directory',
@ -113,6 +117,9 @@ def main(args, env, cwd, cb):
shortflags = []
longflags = []
if options.keepgoing:
shortflags.append('k');
loglevel = logging.WARNING
if options.verbose:
loglevel = logging.DEBUG
@ -128,7 +135,7 @@ def main(args, env, cwd, cb):
else:
workdir = os.path.join(cwd, options.directory)
shortflags.append('j%i' % (options.jobcount,))
longflags.append('-j%i' % (options.jobcount,))
makeflags = ''.join(shortflags) + ' ' + ' '.join(longflags)
@ -150,23 +157,24 @@ def main(args, env, cwd, cb):
overrides, targets = parserdata.parsecommandlineargs(arguments)
def makecb(error, didanything, makefile, realtargets, tstack, i, firsterror):
if error is not None:
print error
if firsterror is None:
firsterror = error
def makecb(error, didanything, makefile, realtargets, tstack, i):
assert error in (True, False)
if error:
context.defer(cb, 2)
return
if i == len(realtargets):
if options.printdir:
print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
sys.stdout.flush()
context.defer(cb, firsterror and 2 or 0)
context.defer(cb, 0)
else:
deferredmake = process.makedeferrable(makecb, makefile=makefile,
realtargets=realtargets, tstack=tstack, i=i+1, firsterror=firsterror)
realtargets=realtargets, tstack=tstack, i=i+1)
makefile.gettarget(realtargets[i]).make(makefile, tstack, [], cb=deferredmake)
makefile.gettarget(realtargets[i]).make(makefile, tstack, cb=deferredmake)
def remakecb(remade, restarts, makefile):
@ -176,7 +184,8 @@ def main(args, env, cwd, cb):
makefile = data.Makefile(restarts=restarts, make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
makeflags=makeflags, makelevel=makelevel, workdir=workdir,
context=context, env=env,
targets=targets)
targets=targets,
keepgoing=options.keepgoing)
try:
overrides.execute(makefile)
@ -206,8 +215,8 @@ def main(args, env, cwd, cb):
tstack = ['<command-line>']
deferredmake = process.makedeferrable(makecb, makefile=makefile,
realtargets=realtargets, tstack=tstack, i=1, firsterror=None)
makefile.gettarget(realtargets[0]).make(makefile, tstack, [], cb=deferredmake)
realtargets=realtargets, tstack=tstack, i=1)
makefile.gettarget(realtargets[0]).make(makefile, tstack, cb=deferredmake)
context.defer(remakecb, True, 0, None)

447
build/pymake/pymake/data.py
View File

@ -2,7 +2,7 @@
A representation of makefile data structures.
"""
import logging, re, os
import logging, re, os, sys
import parserdata, parser, functions, process, util, builtins
from cStringIO import StringIO
@ -312,14 +312,6 @@ class Variables(object):
assert source in (self.SOURCE_OVERRIDE, self.SOURCE_MAKEFILE, self.SOURCE_AUTOMATIC)
assert isinstance(value, str)
def expand():
try:
d = parser.Data.fromstring(value, parserdata.Location("Expansion of variable '%s'" % (name,), 1, 0))
valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
return valueexp, None
except parser.SyntaxError, e:
return None, e
if name not in self._map:
self._map[name] = self.FLAVOR_APPEND, source, value, None
return
@ -475,131 +467,263 @@ class Pattern(object):
return self._backre.sub(r'\\\1', self.data[0]) + '%' + self.data[1]
MAKESTATE_NONE = 0
MAKESTATE_FINISHED = 1
MAKESTATE_WORKING = 2
class RemakeTargetSerially(object):
__slots__ = ('target', 'makefile', 'indent', 'rlist')
def __init__(self, target, makefile, indent, rlist):
self.target = target
self.makefile = makefile
self.indent = indent
self.rlist = rlist
self.commandscb(False)
def resolvecb(self, error, didanything):
assert error in (True, False)
if didanything:
self.target.didanything = True
if error:
self.target.error = True
self.makefile.error = True
if not self.makefile.keepgoing:
self.target.notifydone(self.makefile)
return
else:
# don't run the commands!
del self.rlist[0]
self.commandscb(error=False)
else:
self.rlist.pop(0).runcommands(self.indent, self.commandscb)
def commandscb(self, error):
assert error in (True, False)
if error:
self.target.error = True
self.makefile.error = True
if self.target.error and not self.makefile.keepgoing:
self.target.notifydone(self.makefile)
return
if not len(self.rlist):
self.target.notifydone(self.makefile)
else:
self.rlist[0].resolvedeps(True, self.resolvecb)
class RemakeTargetParallel(object):
__slots__ = ('target', 'makefile', 'indent', 'rlist', 'rulesremaining', 'currunning')
def __init__(self, target, makefile, indent, rlist):
self.target = target
self.makefile = makefile
self.indent = indent
self.rlist = rlist
self.rulesremaining = len(rlist)
self.currunning = False
for r in rlist:
makefile.context.defer(self.doresolve, r)
def doresolve(self, r):
if self.makefile.error and not self.makefile.keepgoing:
r.error = True
self.resolvecb(True, False)
else:
r.resolvedeps(False, self.resolvecb)
def resolvecb(self, error, didanything):
assert error in (True, False)
if error:
self.target.error = True
if didanything:
self.target.didanything = True
self.rulesremaining -= 1
# commandscb takes care of the details if we're currently building
# something
if self.currunning:
return
self.runnext()
def runnext(self):
assert not self.currunning
if self.makefile.error and not self.makefile.keepgoing:
self.rlist = []
else:
while len(self.rlist) and self.rlist[0].error:
del self.rlist[0]
if not len(self.rlist):
if not self.rulesremaining:
self.target.notifydone(self.makefile)
return
if self.rlist[0].depsremaining != 0:
return
self.currunning = True
self.rlist.pop(0).runcommands(self.indent, self.commandscb)
def commandscb(self, error):
assert error in (True, False)
if error:
self.target.error = True
self.makefile.error = True
assert self.currunning
self.currunning = False
self.runnext()
class RemakeRuleContext(object):
__slots__ = ('rule', 'deps', 'depsremaining', 'error', 'didanything', 'running')
def __init__(self, rule, deps):
def __init__(self, target, makefile, rule, deps,
targetstack, avoidremakeloop):
self.target = target
self.makefile = makefile
self.rule = rule
self.deps = deps
self.targetstack = targetstack
self.avoidremakeloop = avoidremakeloop
self.running = False
self.error = False
self.depsremaining = len(deps) + 1
def resolvedeps(self, target, makefile, targetstack, rulestack, serial, cb):
if serial:
self._resolvedepsserial(target, makefile, targetstack, rulestack, cb)
else:
self._resolvedepsparallel(target, makefile, targetstack, rulestack, cb)
def _resolvedepsserial(self, target, makefile, targetstack, rulestack, cb):
resolvelist = list(self.deps)
def resolvedeps(self, serial, cb):
self.resolvecb = cb
self.didanything = False
if serial:
self._resolvedepsserial()
else:
self._resolvedepsparallel()
def depfinished(error, didanything):
if error is not None:
cb(error=error, didanything=None)
def _depfinishedserial(self, error, didanything):
assert error in (True, False)
if didanything:
self.didanything = True
if error:
self.error = True
if not self.makefile.keepgoing:
self.resolvecb(error=True, didanything=self.didanything)
return
if len(self.resolvelist):
self.makefile.context.defer(self.resolvelist.pop(0).make,
self.makefile, self.targetstack, self._depfinishedserial)
else:
self.resolvecb(error=self.error, didanything=self.didanything)
if didanything:
self.didanything = True
if len(resolvelist):
makefile.context.defer(resolvelist.pop(0).make, makefile, targetstack, rulestack, depfinished)
else:
cb(error=None, didanything=self.didanything)
def _resolvedepsserial(self):
self.resolvelist = list(self.deps)
self._depfinishedserial(False, False)
depfinished(None, False)
def _startdepparallel(self, d):
if self.makefile.error:
depfinished(True, False)
else:
d.make(self.makefile, self.targetstack, self._depfinishedparallel)
def _depfinishedparallel(self, error, didanything):
assert error in (True, False)
if error:
print "<%s>: Found error" % self.target.target
self.error = True
if didanything:
self.didanything = True
def _resolvedepsparallel(self, target, makefile, targetstack, rulestack, cb):
self.depsremaining -= 1
if self.depsremaining == 0:
cb(error=None, didanything=False)
self.resolvecb(error=self.error, didanything=self.didanything)
def _resolvedepsparallel(self):
self.depsremaining -= 1
if self.depsremaining == 0:
self.resolvecb(error=self.error, didanything=self.didanything)
return
self.error = None
self.didanything = False
def startdep(d):
if self.error is not None:
depfinished(None, False)
else:
d.make(makefile, targetstack, rulestack, depfinished)
def depfinished(error, didanything):
if error is not None:
if self.error is None:
self.error = error
elif didanything:
self.didanything = True
self.depsremaining -= 1
if self.depsremaining == 0:
cb(error=self.error, didanything=self.didanything)
for d in self.deps:
makefile.context.defer(startdep, d)
self.makefile.context.defer(self._startdepparallel, d)
def runcommands(self, target, makefile, avoidremakeloop, indent, cb):
assert not self.running
self.running = True
if self.rule is None or not len(self.rule.commands):
if target.mtime is None:
target._beingremade()
else:
for d in self.deps:
if mtimeislater(d.mtime, target.mtime):
target._beingremade()
break
cb(error=None)
def _commandcb(self, error):
assert error in (True, False)
if error:
self.runcb(error=True)
return
def commandcb(error):
if error is not None:
cb(error=error)
return
if len(self.commands):
self.commands.pop(0)(self._commandcb)
else:
self.runcb(error=False)
if len(commands):
commands.pop(0)(commandcb)
def runcommands(self, indent, cb):
assert not self.running
self.running = True
self.runcb = cb
if self.rule is None or not len(self.rule.commands):
if self.target.mtime is None:
self.target.beingremade()
else:
cb(error=None)
for d in self.deps:
if mtimeislater(d.mtime, self.target.mtime):
self.target.beingremade()
break
cb(error=False)
return
remake = False
if target.mtime is None:
if self.target.mtime is None:
remake = True
_log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, target.target, self.rule.loc)
_log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, self.target.target, self.rule.loc)
if not remake:
if self.rule.doublecolon:
if len(self.deps) == 0:
if avoidremakeloop:
_log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, target.target, self.rule.loc)
if self.avoidremakeloop:
_log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, self.target.target, self.rule.loc)
else:
_log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, target.target, self.rule.loc)
_log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, self.target.target, self.rule.loc)
remake = True
if not remake:
for d in self.deps:
if mtimeislater(d.mtime, target.mtime):
_log.info("%sRemaking %s using rule at %s because %s is newer.", indent, target.target, self.rule.loc, d.target)
if mtimeislater(d.mtime, self.target.mtime):
_log.info("%sRemaking %s using rule at %s because %s is newer.", indent, self.target.target, self.rule.loc, d.target)
remake = True
break
if remake:
target._beingremade()
target._didanything = True
self.target.beingremade()
self.target.didanything = True
try:
commands = [c for c in self.rule.getcommands(target, makefile)]
self.commands = [c for c in self.rule.getcommands(self.target, self.makefile)]
except util.MakeError, e:
cb(error=e)
print e
sys.stdout.flush()
cb(error=True)
return
commandcb(None)
self._commandcb(False)
else:
cb(error=None)
cb(error=False)
MAKESTATE_NONE = 0
MAKESTATE_FINISHED = 1
MAKESTATE_WORKING = 2
class Target(object):
"""
@ -843,7 +967,7 @@ class Target(object):
self.vpathtarget = self.target
self.mtime = None
def _beingremade(self):
def beingremade(self):
"""
When we remake ourself, we need to reset our mtime and vpathtarget.
@ -853,37 +977,38 @@ class Target(object):
self.mtime = None
self.vpathtarget = self.target
def _notifydone(self, makefile):
assert self._state == MAKESTATE_WORKING
def notifydone(self, makefile):
assert self._state == MAKESTATE_WORKING, "State was %s" % self._state
self._state = MAKESTATE_FINISHED
for cb in self._callbacks:
makefile.context.defer(cb, error=self._makeerror, didanything=self._didanything)
makefile.context.defer(cb, error=self.error, didanything=self.didanything)
del self._callbacks
def make(self, makefile, targetstack, rulestack, cb, avoidremakeloop=False):
def make(self, makefile, targetstack, cb, avoidremakeloop=False):
"""
If we are out of date, asynchronously make ourself. This is a multi-stage process, mostly handled
by enclosed functions:
by the helper objects RemakeTargetSerially, RemakeTargetParallel,
RemakeRuleContext. These helper objects should keep us from developing
any cyclical dependencies.
* resolve dependencies (synchronous)
* gather a list of rules to execute and related dependencies (synchronous)
* for each rule (rulestart)
** remake dependencies (asynchronous, toplevel, callback to start each dependency is `depstart`,
callback when each is finished is `depfinished``
** build list of commands to execute (synchronous, in `runcommands`)
** execute each command (asynchronous, runcommands.commandcb)
* asynchronously notify rulefinished when each rule is complete
* for each rule (in parallel)
** remake dependencies (asynchronous)
** build list of commands to execute (synchronous)
** execute each command (asynchronous)
* asynchronously notify when all rules are complete
@param cb A callback function to notify when remaking is finished. It is called
thusly: callback(error=exception/None, didanything=True/False/None)
thusly: callback(error=True/False, didanything=True/False)
If there is no asynchronous activity to perform, the callback may be called directly.
"""
serial = makefile.context.jcount == 1
if self._state == MAKESTATE_FINISHED:
cb(error=self._makeerror, didanything=self._didanything)
cb(error=self.error, didanything=self.didanything)
return
if self._state == MAKESTATE_WORKING:
@ -895,25 +1020,26 @@ class Target(object):
self._state = MAKESTATE_WORKING
self._callbacks = [cb]
self._makeerror = None
self._didanything = False
self.error = False
self.didanything = False
indent = getindent(targetstack)
try:
self.resolvedeps(makefile, targetstack, rulestack, False)
self.resolvedeps(makefile, targetstack, [], False)
except util.MakeError, e:
self._makeerror = e
self._notifydone(makefile)
print e
self.error = True
self.notifydone(makefile)
return
assert self.vpathtarget is not None, "Target was never resolved!"
if not len(self.rules):
self._notifydone(makefile)
self.notifydone(makefile)
return
if self.isdoublecolon():
rulelist = [RemakeRuleContext(r, [makefile.gettarget(p) for p in r.prerequisites]) for r in self.rules]
rulelist = [RemakeRuleContext(self, makefile, r, [makefile.gettarget(p) for p in r.prerequisites], targetstack, avoidremakeloop) for r in self.rules]
else:
alldeps = []
@ -929,85 +1055,14 @@ class Target(object):
else:
alldeps.extend(rdeps)
rulelist = [RemakeRuleContext(commandrule, alldeps)]
rulelist = [RemakeRuleContext(self, makefile, commandrule, alldeps, targetstack, avoidremakeloop)]
targetstack = targetstack + [self.target]
if serial:
def resolvecb(error, didanything):
if error is not None:
self._makeerror = error
self._notifydone(makefile)
return
if didanything:
self._didanything = True
rulelist.pop(0).runcommands(self, makefile, avoidremakeloop, indent, commandscb)
def commandscb(error):
if error is not None:
self._makeerror = error
self._notifydone(makefile)
return
if not len(rulelist):
self._notifydone(makefile)
return
rulelist[0].resolvedeps(self, makefile, targetstack, rulestack, serial, resolvecb)
commandscb(None)
RemakeTargetSerially(self, makefile, indent, rulelist)
else:
def doresolve(r):
if self._makeerror is not None:
resolvecb(None, False)
else:
r.resolvedeps(self, makefile, targetstack, rulestack, serial, resolvecb)
def resolvecb(error, didanything):
if error is not None:
if self._makeerror is None:
self._makeerror = error
elif didanything:
self._didanything = didanything
if self._makeerror is not None:
r = rulelist.pop()
assert not r.running
if not len(rulelist):
self._notifydone(makefile)
return
rtop = rulelist[0]
if rtop.running or rtop.depsremaining != 0:
return
rtop.runcommands(self, makefile, avoidremakeloop, indent, commandscb)
def commandscb(error):
if error is not None:
if self._makeerror is None:
self._makeerror = error
r = rulelist.pop(0)
assert r.running
if not len(rulelist):
self._notifydone(makefile)
return
if self._makeerror is not None:
return
rtop = rulelist[0]
if rtop.running or rtop.depsremaining != 0:
return
rtop.runcommands(self, makefile, avoidremakeloop, indent, commandscb)
for r in rulelist:
makefile.context.defer(doresolve, r)
RemakeTargetParallel(self, makefile, indent, rulelist)
def dirpart(p):
d, s, f = util.strrpartition(p, '/')
@ -1084,9 +1139,10 @@ class _CommandWrapper(object):
def _cb(self, res):
if res != 0 and not self.ignoreErrors:
self.usercb(error=DataError("command '%s' failed, return code %s" % (self.cline, res), self.loc))
print "%s: command '%s' failed, return code %i" % (self.loc, self.cline, res)
self.usercb(error=True)
else:
self.usercb(error=None)
self.usercb(error=False)
def __call__(self, cb):
self.usercb = cb
@ -1218,7 +1274,7 @@ class Makefile(object):
state data.
"""
def __init__(self, workdir=None, env=None, restarts=0, make=None, makeflags=None, makelevel=0, context=None, targets=()):
def __init__(self, workdir=None, env=None, restarts=0, make=None, makeflags=None, makelevel=0, context=None, targets=(), keepgoing=False):
self.defaulttarget = None
if env is None:
@ -1232,6 +1288,7 @@ class Makefile(object):
self.exportedvars = {}
self.overrides = []
self._targets = {}
self.keepgoing = keepgoing
self._patternvariables = [] # of (pattern, variables)
self.implicitrules = []
self.parsingfinished = False
@ -1349,6 +1406,8 @@ class Makefile(object):
if len(np.rules):
self.context = process.getcontext(1)
self.error = False
def include(self, path, required=True, loc=None):
"""
Include the makefile at `path`.
@ -1412,28 +1471,30 @@ class Makefile(object):
if serial:
remakelist = [self.gettarget(f) for f in self.included]
def remakecb(error, didanything):
if error is not None:
print "Error remaking makefiles (ignored): ", error
assert error in (True, False)
if error:
print "Error remaking makefiles (ignored)"
if len(remakelist):
t = remakelist.pop(0)
t.make(self, [], [], avoidremakeloop=True, cb=remakecb)
t.make(self, [], avoidremakeloop=True, cb=remakecb)
else:
remakedone()
remakelist.pop(0).make(self, [], [], avoidremakeloop=True, cb=remakecb)
remakelist.pop(0).make(self, [], avoidremakeloop=True, cb=remakecb)
else:
o = util.makeobject(('remakesremaining',), remakesremaining=len(self.included))
def remakecb(error, didanything):
if error is not None:
print "Error remaking makefiles (ignored): ", error
assert error in (True, False)
if error:
print "Error remaking makefiles (ignored)"
o.remakesremaining -= 1
if o.remakesremaining == 0:
remakedone()
for t, mtime in mlist:
t.make(self, [], [], avoidremakeloop=True, cb=remakecb)
t.make(self, [], avoidremakeloop=True, cb=remakecb)
flagescape = re.compile(r'([\s\\])')

16
build/pymake/tests/keep-going-doublecolon.mk Normal file
View File

@ -0,0 +1,16 @@
#T commandline: ['-k']
#T returncode: 2
#T grep-for: "TEST-PASS"
all:: t1
@echo TEST-FAIL "(t1)"
all:: t2
@echo TEST-PASS
t1:
@false
t2:
touch $@

11
build/pymake/tests/keep-going-parallel.mk Normal file
View File

@ -0,0 +1,11 @@
#T commandline: ['-k', '-j2']
#T returncode: 2
#T grep-for: "TEST-PASS"
all: t1 slow1 slow2 slow3 t2
t2:
@echo TEST-PASS
slow%:
sleep 1

14
build/pymake/tests/keep-going.mk Normal file
View File

@ -0,0 +1,14 @@
#T commandline: ['-k']
#T returncode: 2
#T grep-for: "TEST-PASS"
all: t2 t3
t1:
@false
t2: t1
@echo TEST-FAIL
t3:
@echo TEST-PASS

6
build/pymake/tests/makeflags.mk Normal file
View File

@ -0,0 +1,6 @@
#T environment: {'MAKEFLAGS': 'OVAR=oval'}
all:
test "$(OVAR)" = "oval"
@echo TEST-PASS

15
build/pymake/tests/runtests.py
View File

@ -12,6 +12,7 @@ The test file may contain lines at the beginning to alter the default behavior.
#T returncode: 2
#T returncode-on: {'win32': 2}
#T environment: {'VAR': 'VALUE}
#T grep-for: "text"
"""
from subprocess import Popen, PIPE, STDOUT
@ -63,6 +64,7 @@ for makefile in makefiles:
cline += ['__WIN32__=1']
returncode = 0
grepfor = None
env = dict(os.environ)
@ -83,6 +85,8 @@ for makefile in makefiles:
elif key == 'environment':
for k, v in data.iteritems():
env[k] = v
elif key == 'grep-for':
grepfor = data
else:
print >>sys.stderr, "Unexpected #T key: %s" % key
sys.exit(1)
@ -98,10 +102,17 @@ for makefile in makefiles:
print "FAIL"
print stdout
elif returncode == 0:
if stdout.find('TEST-PASS') != -1:
if stdout.find(grepfor or 'TEST-PASS') != -1:
print "PASS"
else:
print "FAIL (no passing output)"
print "FAIL (no expected output)"
print stdout
# check that test produced the expected output while failing
elif grepfor:
if stdout.find(grepfor) != -1:
print "PASS"
else:
print "FAIL (no expected output)"
print stdout
else:
print "EXPECTED-FAIL"