mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-25 13:51:41 +00:00
Update pymake for better memory usage and faster execution.
This commit is contained in:
Benjamin Smedberg
parent
c361d99f20
commit
ba4b0a1e00
@ -1,2 +1,2 @@
|
||||
repo: f5ab154deef2ffa97f1b2139589ae4a1962090a4
|
||||
node: 50d8e87e8af3fb8147c33f169a8d5abe8e06ffa4
|
||||
node: ab32ac2a4e6842787fef44f43101c03ff515a3a3
|
||||
|
||||
@ -9,6 +9,9 @@ A drop-in or mostly drop-in replacement for GNU make.
|
||||
import sys, os
|
||||
import pymake.command, pymake.process
|
||||
|
||||
import gc
|
||||
gc.disable()
|
||||
|
||||
pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), cb=sys.exit)
|
||||
pymake.process.ParallelContext.spin()
|
||||
assert False, "Not reached"
|
||||
|
||||
@ -72,6 +72,79 @@ DEALINGS IN THE SOFTWARE."""
|
||||
|
||||
_log = logging.getLogger('pymake.execution')
|
||||
|
||||
class _MakeContext(object):
|
||||
def __init__(self, makeflags, makelevel, workdir, context, env, targets, options, overrides, cb):
|
||||
self.makeflags = makeflags
|
||||
self.makelevel = makelevel
|
||||
|
||||
self.workdir = workdir
|
||||
self.context = context
|
||||
self.env = env
|
||||
self.targets = targets
|
||||
self.options = options
|
||||
self.overrides = overrides
|
||||
self.cb = cb
|
||||
|
||||
self.restarts = 0
|
||||
|
||||
self.remakecb(True)
|
||||
|
||||
def remakecb(self, remade):
|
||||
if remade:
|
||||
if self.restarts > 0:
|
||||
_log.info("make.py[%i]: Restarting makefile parsing", self.makelevel)
|
||||
|
||||
self.makefile = data.Makefile(restarts=self.restarts,
|
||||
make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
|
||||
makeflags=self.makeflags, workdir=self.workdir,
|
||||
context=self.context, env=self.env, makelevel=self.makelevel,
|
||||
targets=self.targets, keepgoing=self.options.keepgoing)
|
||||
|
||||
self.restarts += 1
|
||||
|
||||
try:
|
||||
self.overrides.execute(self.makefile)
|
||||
for f in self.options.makefiles:
|
||||
self.makefile.include(f)
|
||||
self.makefile.finishparsing()
|
||||
self.makefile.remakemakefiles(self.remakecb)
|
||||
except util.MakeError, e:
|
||||
print e
|
||||
self.context.defer(self.cb, 2)
|
||||
|
||||
return
|
||||
|
||||
if len(self.targets) == 0:
|
||||
if self.makefile.defaulttarget is None:
|
||||
print "No target specified and no default target found."
|
||||
self.context.defer(self.cb, 2)
|
||||
return
|
||||
|
||||
_log.info("Making default target %s", self.makefile.defaulttarget)
|
||||
self.realtargets = [self.makefile.defaulttarget]
|
||||
self.tstack = ['<default-target>']
|
||||
else:
|
||||
self.realtargets = self.targets
|
||||
self.tstack = ['<command-line>']
|
||||
|
||||
self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, cb=self.makecb)
|
||||
|
||||
def makecb(self, error, didanything):
|
||||
assert error in (True, False)
|
||||
|
||||
if error:
|
||||
self.context.defer(self.cb, 2)
|
||||
return
|
||||
|
||||
if not len(self.realtargets):
|
||||
if self.options.printdir:
|
||||
print "make.py[%i]: Leaving directory '%s'" % (self.makelevel, self.workdir)
|
||||
sys.stdout.flush()
|
||||
|
||||
self.context.defer(self.cb, 0)
|
||||
else:
|
||||
self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, self.makecb)
|
||||
|
||||
def main(args, env, cwd, cb):
|
||||
"""
|
||||
Start a single makefile execution, given a command line, working directory, and environment.
|
||||
@ -101,12 +174,16 @@ def main(args, env, cwd, cb):
|
||||
dest="printversion", default=False)
|
||||
op.add_option('-j', '--jobs', type="int",
|
||||
dest="jobcount", default=1)
|
||||
op.add_option('-w', '--print-directory', action="store_true",
|
||||
dest="printdir")
|
||||
op.add_option('--no-print-directory', action="store_false",
|
||||
dest="printdir", default=True)
|
||||
|
||||
options, arguments1 = op.parse_args(parsemakeflags(env))
|
||||
options, arguments2 = op.parse_args(args, values=options)
|
||||
|
||||
op.destroy()
|
||||
|
||||
arguments = arguments1 + arguments2
|
||||
|
||||
if options.printversion:
|
||||
@ -157,69 +234,7 @@ def main(args, env, cwd, cb):
|
||||
|
||||
overrides, targets = parserdata.parsecommandlineargs(arguments)
|
||||
|
||||
def makecb(error, didanything, makefile, realtargets, tstack, i):
|
||||
assert error in (True, False)
|
||||
|
||||
if error:
|
||||
context.defer(cb, 2)
|
||||
return
|
||||
|
||||
if i == len(realtargets):
|
||||
if options.printdir:
|
||||
print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
|
||||
sys.stdout.flush()
|
||||
|
||||
context.defer(cb, 0)
|
||||
else:
|
||||
deferredmake = process.makedeferrable(makecb, makefile=makefile,
|
||||
realtargets=realtargets, tstack=tstack, i=i+1)
|
||||
|
||||
makefile.gettarget(realtargets[i]).make(makefile, tstack, cb=deferredmake)
|
||||
|
||||
|
||||
def remakecb(remade, restarts, makefile):
|
||||
if remade:
|
||||
if restarts > 0:
|
||||
_log.info("make.py[%i]: Restarting makefile parsing", makelevel)
|
||||
makefile = data.Makefile(restarts=restarts, make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
|
||||
makeflags=makeflags, makelevel=makelevel, workdir=workdir,
|
||||
context=context, env=env,
|
||||
targets=targets,
|
||||
keepgoing=options.keepgoing)
|
||||
|
||||
try:
|
||||
overrides.execute(makefile)
|
||||
for f in options.makefiles:
|
||||
makefile.include(f)
|
||||
makefile.finishparsing()
|
||||
makefile.remakemakefiles(process.makedeferrable(remakecb, restarts=restarts + 1, makefile=makefile))
|
||||
|
||||
except util.MakeError, e:
|
||||
print e
|
||||
context.defer(cb, 2)
|
||||
return
|
||||
|
||||
return
|
||||
|
||||
if len(targets) == 0:
|
||||
if makefile.defaulttarget is None:
|
||||
print "No target specified and no default target found."
|
||||
context.defer(cb, 2)
|
||||
return
|
||||
|
||||
_log.info("Making default target %s", makefile.defaulttarget)
|
||||
realtargets = [makefile.defaulttarget]
|
||||
tstack = ['<default-target>']
|
||||
else:
|
||||
realtargets = targets
|
||||
tstack = ['<command-line>']
|
||||
|
||||
deferredmake = process.makedeferrable(makecb, makefile=makefile,
|
||||
realtargets=realtargets, tstack=tstack, i=1)
|
||||
makefile.gettarget(realtargets[0]).make(makefile, tstack, cb=deferredmake)
|
||||
|
||||
context.defer(remakecb, True, 0, None)
|
||||
|
||||
_MakeContext(makeflags, makelevel, workdir, context, env, targets, options, overrides, cb)
|
||||
except (util.MakeError), e:
|
||||
print e
|
||||
if options.printdir:
|
||||
|
||||
@ -1268,6 +1268,30 @@ class PatternRule(object):
|
||||
def prerequisitesforstem(self, dir, stem):
|
||||
return [p.resolve(dir, stem) for p in self.prerequisites]
|
||||
|
||||
class _RemakeContext(object):
|
||||
def __init__(self, makefile, remakelist, mtimelist, cb):
|
||||
self.makefile = makefile
|
||||
self.remakelist = remakelist
|
||||
self.mtimelist = mtimelist # list of (target, mtime)
|
||||
self.cb = cb
|
||||
|
||||
self.remakecb(error=False, didanything=False)
|
||||
|
||||
def remakecb(self, error, didanything):
|
||||
assert error in (True, False)
|
||||
|
||||
if error:
|
||||
print "Error remaking makefiles (ignored)"
|
||||
|
||||
if len(self.remakelist):
|
||||
self.remakelist.pop(0).make(self.makefile, [], avoidremakeloop=True, cb=self.remakecb)
|
||||
else:
|
||||
for t, oldmtime in self.mtimelist:
|
||||
if t.mtime != oldmtime:
|
||||
self.cb(remade=True)
|
||||
return
|
||||
self.cb(remade=False)
|
||||
|
||||
class Makefile(object):
|
||||
"""
|
||||
The top-level data structure for makefile execution. It holds Targets, implicit rules, and other
|
||||
@ -1448,17 +1472,6 @@ class Makefile(object):
|
||||
return withoutdups(vp)
|
||||
|
||||
def remakemakefiles(self, cb):
|
||||
reparse = False
|
||||
|
||||
serial = self.context.jcount == 1
|
||||
|
||||
def remakedone():
|
||||
for t, oldmtime in mlist:
|
||||
if t.mtime != oldmtime:
|
||||
cb(remade=True)
|
||||
return
|
||||
cb(remade=False)
|
||||
|
||||
mlist = []
|
||||
for f in self.included:
|
||||
t = self.gettarget(f)
|
||||
@ -1468,33 +1481,7 @@ class Makefile(object):
|
||||
|
||||
mlist.append((t, oldmtime))
|
||||
|
||||
if serial:
|
||||
remakelist = [self.gettarget(f) for f in self.included]
|
||||
def remakecb(error, didanything):
|
||||
assert error in (True, False)
|
||||
if error:
|
||||
print "Error remaking makefiles (ignored)"
|
||||
|
||||
if len(remakelist):
|
||||
t = remakelist.pop(0)
|
||||
t.make(self, [], avoidremakeloop=True, cb=remakecb)
|
||||
else:
|
||||
remakedone()
|
||||
|
||||
remakelist.pop(0).make(self, [], avoidremakeloop=True, cb=remakecb)
|
||||
else:
|
||||
o = util.makeobject(('remakesremaining',), remakesremaining=len(self.included))
|
||||
def remakecb(error, didanything):
|
||||
assert error in (True, False)
|
||||
if error:
|
||||
print "Error remaking makefiles (ignored)"
|
||||
|
||||
o.remakesremaining -= 1
|
||||
if o.remakesremaining == 0:
|
||||
remakedone()
|
||||
|
||||
for t, mtime in mlist:
|
||||
t.make(self, [], avoidremakeloop=True, cb=remakecb)
|
||||
_RemakeContext(self, [self.gettarget(f) for f in self.included], mlist, cb)
|
||||
|
||||
flagescape = re.compile(r'([\s\\])')
|
||||
|
||||
|
||||
@ -451,7 +451,22 @@ _conditionkeywordstokenlist = TokenList.get(_conditiontokens)
|
||||
|
||||
_varsettokens = (':=', '+=', '?=', '=')
|
||||
|
||||
_parsecache = {} # realpath -> (mtime, Statements)
|
||||
def _parsefile(pathname):
|
||||
fd = open(pathname, "rU")
|
||||
stmts = parsestream(fd, pathname)
|
||||
stmts.mtime = os.fstat(fd.fileno()).st_mtime
|
||||
fd.close()
|
||||
return stmts
|
||||
|
||||
def _checktime(path, stmts):
|
||||
mtime = os.path.getmtime(path)
|
||||
if mtime != stmts.mtime:
|
||||
_log.debug("Re-parsing makefile '%s': mtimes differ", path)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
_parsecache = util.MostUsedCache(15, _parsefile, _checktime)
|
||||
|
||||
def parsefile(pathname):
|
||||
"""
|
||||
@ -460,23 +475,7 @@ def parsefile(pathname):
|
||||
"""
|
||||
|
||||
pathname = os.path.realpath(pathname)
|
||||
|
||||
mtime = os.path.getmtime(pathname)
|
||||
|
||||
if pathname in _parsecache:
|
||||
oldmtime, stmts = _parsecache[pathname]
|
||||
|
||||
if mtime == oldmtime:
|
||||
_log.debug("Using '%s' from the parser cache.", pathname)
|
||||
return stmts
|
||||
|
||||
_log.debug("Not using '%s' from the parser cache, mtimes don't match: was %s, now %s", pathname, oldmtime, mtime)
|
||||
|
||||
fd = open(pathname, "rU")
|
||||
stmts = parsestream(fd, pathname)
|
||||
fd.close()
|
||||
_parsecache[pathname] = mtime, stmts
|
||||
return stmts
|
||||
return _parsecache.get(pathname)
|
||||
|
||||
def parsestream(fd, filename):
|
||||
"""
|
||||
|
||||
@ -470,8 +470,11 @@ class EmptyDirective(Statement):
|
||||
def dump(self, fd, indent):
|
||||
print >>fd, "%sEmptyDirective: %s" % (indent, self.exp)
|
||||
|
||||
class _EvalContext(object):
|
||||
__slots__ = ('currule',)
|
||||
|
||||
class StatementList(list):
|
||||
__slots__ = ()
|
||||
__slots__ = ('mtime',)
|
||||
|
||||
def append(self, statement):
|
||||
assert isinstance(statement, Statement)
|
||||
@ -479,7 +482,7 @@ class StatementList(list):
|
||||
|
||||
def execute(self, makefile, context=None):
|
||||
if context is None:
|
||||
context = util.makeobject('currule')
|
||||
context = _EvalContext()
|
||||
|
||||
for s in self:
|
||||
s.execute(makefile, context)
|
||||
|
||||
@ -1,14 +1,5 @@
|
||||
import os
|
||||
|
||||
def makeobject(proplist, **kwargs):
|
||||
class P(object):
|
||||
__slots__ = proplist
|
||||
|
||||
p = P()
|
||||
for k, v in kwargs.iteritems():
|
||||
setattr(p, k, v)
|
||||
return p
|
||||
|
||||
class MakeError(Exception):
|
||||
def __init__(self, message, loc=None):
|
||||
self.message = message
|
||||
@ -87,3 +78,62 @@ except ImportError:
|
||||
if i:
|
||||
return True
|
||||
return False
|
||||
|
||||
class _MostUsedItem(object):
|
||||
__slots__ = ('key', 'o', 'count')
|
||||
|
||||
def __init__(self, key):
|
||||
self.key = key
|
||||
self.o = None
|
||||
self.count = 1
|
||||
|
||||
def __repr__(self):
|
||||
return "MostUsedItem(key=%r, count=%i, o=%r)" % (self.key, self.count, self.o)
|
||||
|
||||
class MostUsedCache(object):
|
||||
def __init__(self, capacity, creationfunc, verifyfunc):
|
||||
self.capacity = capacity
|
||||
self.cfunc = creationfunc
|
||||
self.vfunc = verifyfunc
|
||||
|
||||
self.d = {}
|
||||
self.active = [] # lazily sorted!
|
||||
|
||||
def setactive(self, item):
|
||||
if item in self.active:
|
||||
return
|
||||
|
||||
if len(self.active) == self.capacity:
|
||||
self.active.sort(key=lambda i: i.count)
|
||||
old = self.active.pop(0)
|
||||
old.o = None
|
||||
# print "Evicting %s" % old.key
|
||||
|
||||
self.active.append(item)
|
||||
|
||||
def get(self, key):
|
||||
item = self.d.get(key, None)
|
||||
if item is None:
|
||||
item = _MostUsedItem(key)
|
||||
self.d[key] = item
|
||||
else:
|
||||
item.count += 1
|
||||
|
||||
if item.o is not None and self.vfunc(key, item.o):
|
||||
return item.o
|
||||
|
||||
item.o = self.cfunc(key)
|
||||
self.setactive(item)
|
||||
return item.o
|
||||
|
||||
def verify(self):
|
||||
for k, v in self.d.iteritems():
|
||||
if v.o:
|
||||
assert v in self.active
|
||||
else:
|
||||
assert v not in self.active
|
||||
|
||||
def debugitems(self):
|
||||
l = [i.key for i in self.active]
|
||||
l.sort()
|
||||
return l
|
||||
|
||||
@ -41,5 +41,38 @@ class GetPatSubstTest(unittest.TestCase):
|
||||
for word in words))
|
||||
self.assertEqual(a, e, 'Pattern(%r).subst(%r, %r)' % (s, r, d))
|
||||
|
||||
class LRUTest(unittest.TestCase):
|
||||
# getkey, expected, funccount, debugitems
|
||||
expected = (
|
||||
(0, '', 1, (0,)),
|
||||
(0, '', 2, (0,)),
|
||||
(1, ' ', 3, (1, 0)),
|
||||
(1, ' ', 3, (1, 0)),
|
||||
(0, '', 4, (0, 1)),
|
||||
(2, ' ', 5, (2, 0, 1)),
|
||||
(1, ' ', 5, (1, 2, 0)),
|
||||
(3, ' ', 6, (3, 1, 2)),
|
||||
)
|
||||
|
||||
def spaceFunc(self, l):
|
||||
self.funccount += 1
|
||||
return ''.ljust(l)
|
||||
|
||||
def runTest(self):
|
||||
self.funccount = 0
|
||||
c = pymake.util.LRUCache(3, self.spaceFunc, lambda k, v: k % 2)
|
||||
self.assertEqual(tuple(c.debugitems()), ())
|
||||
|
||||
for i in xrange(0, len(self.expected)):
|
||||
k, e, fc, di = self.expected[i]
|
||||
|
||||
v = c.get(k)
|
||||
self.assertEqual(v, e)
|
||||
self.assertEqual(self.funccount, fc,
|
||||
"funccount, iteration %i, got %i expected %i" % (i, self.funccount, fc))
|
||||
goti = tuple(c.debugitems())
|
||||
self.assertEqual(goti, di,
|
||||
"debugitems, iteration %i, got %r expected %r" % (i, goti, di))
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user