From cc75858eb7c3d014aefd9fa397b6368f9279f6fd Mon Sep 17 00:00:00 2001 From: Jesse Ruderman Date: Fri, 25 Jun 2010 14:47:19 -0700 Subject: [PATCH] Bug 570287 - New stack fixer that uses breakpad symbol files. r=ted --- build/Makefile.in | 3 + build/automation.py.in | 35 ++++--- build/mobile/remoteautomation.py | 2 +- testing/mochitest/Makefile.in | 3 +- tools/rb/fix_stack_using_bpsyms.py | 160 +++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+), 15 deletions(-) create mode 100755 tools/rb/fix_stack_using_bpsyms.py diff --git a/build/Makefile.in b/build/Makefile.in index 4e1c4fda02af..d2db7cec9f9c 100644 --- a/build/Makefile.in +++ b/build/Makefile.in @@ -111,6 +111,9 @@ libs:: bloaturls.txt libs:: bloatcycle.html $(INSTALL) $< $(DIST)/bin/res +libs:: $(topsrcdir)/tools/rb/fix_stack_using_bpsyms.py + $(INSTALL) $< $(DIST)/bin + ifeq ($(OS_ARCH),Darwin) libs:: $(topsrcdir)/tools/rb/fix-macosx-stack.pl $(INSTALL) $< $(DIST)/bin diff --git a/build/automation.py.in b/build/automation.py.in index fa601460bf8f..067bb982b57a 100644 --- a/build/automation.py.in +++ b/build/automation.py.in @@ -649,35 +649,44 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t self.log.info("Can't trigger Breakpad, just killing process") proc.kill() - def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo): + def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath): """ Look for timeout or crashes and return the status after the process terminates """ stackFixerProcess = None - stackFixerModule = None + stackFixerFunction = None didTimeout = False if proc.stdout is None: self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection") else: logsource = proc.stdout - if self.IS_DEBUG_BUILD and self.IS_LINUX: - # Run logsource through fix-linux-stack.pl + + if self.IS_DEBUG_BUILD and (self.IS_MAC or self.IS_LINUX) and symbolsPath and os.path.exists(symbolsPath): + # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files) + # This method is preferred for Tinderbox builds, since native symbols may have been stripped. + sys.path.insert(0, utilityPath) + import fix_stack_using_bpsyms as stackFixerModule + stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath) + del sys.path[0] + elif self.IS_DEBUG_BUILD and self.IS_MAC and False: + # Run each line through a function in fix_macosx_stack.py (uses atos) + sys.path.insert(0, utilityPath) + import fix_macosx_stack as stackFixerModule + stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line) + del sys.path[0] + elif self.IS_DEBUG_BUILD and self.IS_LINUX: + # Run logsource through fix-linux-stack.pl (uses addr2line) + # This method is preferred for developer machines, so we don't have to run "make buildsymbols". stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")], stdin=logsource, stdout=subprocess.PIPE) logsource = stackFixerProcess.stdout - if self.IS_DEBUG_BUILD and self.IS_MAC and False: - # Import fix_macosx_stack.py from utilityPath - sys.path.insert(0, utilityPath) - import fix_macosx_stack as stackFixerModule - del sys.path[0] - (line, didTimeout) = self.readWithTimeout(logsource, timeout) hitMaxTime = False while line != "" and not didTimeout: if "TEST-START" in line and "|" in line: self.lastTestSeen = line.split("|")[1].strip() - if stackFixerModule: - line = stackFixerModule.fixSymbols(line) + if stackFixerFunction: + line = stackFixerFunction(line) self.log.info(line.rstrip()) (line, didTimeout) = self.readWithTimeout(logsource, timeout) if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime): @@ -814,7 +823,7 @@ user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless t stderr = subprocess.STDOUT) self.log.info("INFO | automation.py | Application pid: %d", proc.pid) - status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo) + status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath) self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime)) # Do a final check for zombie child processes. diff --git a/build/mobile/remoteautomation.py b/build/mobile/remoteautomation.py index bbb1612c2192..f5503ebb9ffe 100644 --- a/build/mobile/remoteautomation.py +++ b/build/mobile/remoteautomation.py @@ -66,7 +66,7 @@ class RemoteAutomation(Automation): def setProduct(self, product): self._product = product - def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo): + def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsDir): # maxTime is used to override the default timeout, we should honor that status = proc.wait(timeout = maxTime) diff --git a/testing/mochitest/Makefile.in b/testing/mochitest/Makefile.in index e2017013ecc5..a7cc259639ae 100644 --- a/testing/mochitest/Makefile.in +++ b/testing/mochitest/Makefile.in @@ -125,13 +125,14 @@ GARBAGE += runtests.py libs:: $(_SERV_FILES) $(INSTALL) $^ $(_DEST_DIR) -# Binaries that don't get packaged with the build, +# Binaries and scripts that don't get packaged with the build, # but that we need for the test harness TEST_HARNESS_BINS := \ xpcshell$(BIN_SUFFIX) \ ssltunnel$(BIN_SUFFIX) \ certutil$(BIN_SUFFIX) \ pk12util$(BIN_SUFFIX) \ + fix_stack_using_bpsyms.py \ $(NULL) ifeq ($(OS_ARCH),WINNT) diff --git a/tools/rb/fix_stack_using_bpsyms.py b/tools/rb/fix_stack_using_bpsyms.py new file mode 100755 index 000000000000..dd2dc6d61751 --- /dev/null +++ b/tools/rb/fix_stack_using_bpsyms.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is fix_stack_using_bpsyms.py. +# +# The Initial Developer of the Original Code is the Mozilla Foundation. +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Jesse Ruderman +# L. David Baron +# Ted Mielczarek +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +from __future__ import with_statement + +import sys +import os +import re +import bisect + +def prettyFileName(name): + if name.startswith("../") or name.startswith("..\\"): + # dom_quickstubs.cpp and many .h files show up with relative paths that are useless + # and/or don't correspond to the layout of the source tree. + return os.path.basename(name) + ":" + elif name.startswith("hg:"): + (junk, repo, path, rev) = name.split(":") + # We could construct an hgweb URL with /file/ or /annotate/, like this: + # return "http://%s/annotate/%s/%s#l" % (repo, rev, path) + return path + ":" + return name + ":" + +class readSymbolFile: + def __init__(self, fn): + addrs = [] # list of addresses, which will be sorted once we're done initializing + funcs = {} # hash: address --> (function name + possible file/line) + files = {} # hash: filenum (string) --> prettified filename ready to have a line number appended + with open(fn) as f: + for line in f: + line = line.rstrip() + # http://code.google.com/p/google-breakpad/wiki/SymbolFiles + if line.startswith("FUNC "): + # FUNC
+ (junk, rva, size, ss, name) = line.split(None, 4) + rva = int(rva,16) + funcs[rva] = name + addrs.append(rva) + lastFuncName = name + elif line.startswith("PUBLIC "): + # PUBLIC
+ (junk, rva, ss, name) = line.split(None, 3) + rva = int(rva,16) + funcs[rva] = name + addrs.append(rva) + elif line.startswith("FILE "): + # FILE + (junk, filenum, name) = line.split(None, 2) + files[filenum] = prettyFileName(name) + elif line[0] in "0123456789abcdef": + # This is one of the "line records" corresponding to the last FUNC record + #
+ (rva, size, line, filenum) = line.split(None) + rva = int(rva,16) + file = files[filenum] + name = lastFuncName + " [" + file + line + "]" + funcs[rva] = name + addrs.append(rva) + # skip everything else + #print "Loaded %d functions from symbol file %s" % (len(funcs), os.path.basename(fn)) + self.addrs = sorted(addrs) + self.funcs = funcs + def addrToSymbol(self, address): + i = bisect.bisect(self.addrs, address) - 1 + if i > 0: + #offset = address - self.addrs[i] + return self.funcs[self.addrs[i]] + else: + return "" + + +def guessSymbolFile(fn, symbolsDir): + """Guess a symbol file based on an object file's basename, ignoring the path and UUID.""" + fn = os.path.basename(fn) + d1 = os.path.join(symbolsDir, fn) + if not os.path.exists(d1): + return None + uuids = os.listdir(d1) + if len(uuids) == 0: + raise Exception("Missing symbol file for " + fn) + if len(uuids) > 1: + raise Exception("Ambiguous symbol file for " + fn) + return os.path.join(d1, uuids[0], fn + ".sym") + +parsedSymbolFiles = {} +def addressToSymbol(file, address, symbolsDir): + p = None + if not file in parsedSymbolFiles: + symfile = guessSymbolFile(file, symbolsDir) + if symfile: + p = readSymbolFile(symfile) + else: + p = None + parsedSymbolFiles[file] = p + else: + p = parsedSymbolFiles[file] + if p: + return p.addrToSymbol(address) + else: + return "" + +line_re = re.compile("^(.*) ?\[([^ ]*) \+(0x[0-9A-F]{1,8})\](.*)$") +balance_tree_re = re.compile("^([ \|0-9-]*)") + +def fixSymbols(line, symbolsDir): + result = line_re.match(line) + if result is not None: + # before allows preservation of balance trees + # after allows preservation of counts + (before, file, address, after) = result.groups() + address = int(address, 16) + # throw away the bad symbol, but keep balance tree structure + before = balance_tree_re.match(before).groups()[0] + symbol = addressToSymbol(file, address, symbolsDir) + if not symbol: + symbol = "%s + 0x%x" % (os.path.basename(file), address) + return before + symbol + after + "\n" + else: + return line + +if __name__ == "__main__": + symbolsDir = sys.argv[1] + for line in iter(sys.stdin.readline, ''): + print fixSymbols(line, symbolsDir),