Bug 840094 - Change how nsZipArchive logging works. r=taras,r=gps

Now log in a single file given by the MOZ_JAR_LOG_FILE environment variable.
Log entries contain the URI of the Zip archive, followed by the path in the
archive.
* * *
Bug 840094 - Fixup for debug builds failure because of nsZipArchive::CloseArchive being called several times
This commit is contained in:
Mike Hommey 2013-02-19 11:02:12 +01:00
parent 5c0bbd0b29
commit 026abf6d83
9 changed files with 219 additions and 47 deletions

View File

@ -20,7 +20,7 @@ from automationutils import getDebuggerInfo, addCommonOptions
PORT = 8888
PROFILE_DIRECTORY = os.path.abspath(os.path.join(SCRIPT_DIR, "./pgoprofile"))
MOZ_JAR_LOG_DIR = os.path.abspath(os.getenv("JARLOG_DIR"))
MOZ_JAR_LOG_FILE = os.path.abspath(os.getenv("JARLOG_FILE"))
os.chdir(SCRIPT_DIR)
class EasyServer(SocketServer.TCPServer):
@ -47,7 +47,7 @@ if __name__ == '__main__':
automation.initializeProfile(PROFILE_DIRECTORY)
browserEnv = automation.environment()
browserEnv["XPCOM_DEBUG_BREAK"] = "warn"
browserEnv["MOZ_JAR_LOG_DIR"] = MOZ_JAR_LOG_DIR
browserEnv["MOZ_JAR_LOG_FILE"] = MOZ_JAR_LOG_FILE
url = "http://localhost:%d/index.html" % PORT
appPath = os.path.join(SCRIPT_DIR, automation.DEFAULT_APP)

View File

@ -203,7 +203,8 @@ endif
profiledbuild::
$(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1
$(MAKE) -C $(PGO_OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT=
MOZ_PGO_INSTRUMENTED=1 OBJDIR=${PGO_OBJDIR} JARLOG_DIR=${PGO_OBJDIR}/jarlog/en-US $(PROFILE_GEN_SCRIPT)
rm -f ${PGO_OBJDIR}/jarlog/en-US.log
MOZ_PGO_INSTRUMENTED=1 OBJDIR=${PGO_OBJDIR} JARLOG_FILE=${PGO_OBJDIR}/jarlog/en-US.log $(PROFILE_GEN_SCRIPT)
$(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild
$(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1

View File

@ -55,6 +55,9 @@
# endif
#endif /* XP_UNIX */
#ifdef XP_WIN
#include "private/pprio.h" // To get PR_ImportFile
#endif
using namespace mozilla;
@ -70,6 +73,71 @@ static uint32_t HashName(const char* aName, uint16_t nameLen);
static nsresult ResolveSymlink(const char *path);
#endif
class ZipArchiveLogger {
public:
void Write(const nsACString &zip, const char *entry) const {
if (!fd) {
char *env = PR_GetEnv("MOZ_JAR_LOG_FILE");
if (!env)
return;
nsCOMPtr<nsIFile> logFile;
nsresult rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(logFile));
if (NS_FAILED(rv))
return;
// Create the log file and its parent directory (in case it doesn't exist)
logFile->Create(nsIFile::NORMAL_FILE_TYPE, 0644);
PRFileDesc* file;
#ifdef XP_WIN
// PR_APPEND is racy on Windows, so open a handle ourselves with flags that
// will work, and use PR_ImportFile to make it a PRFileDesc.
// This can go away when bug 840435 is fixed.
nsAutoString path;
logFile->GetPath(path);
if (path.IsEmpty())
return;
HANDLE handle = CreateFileW(path.get(), FILE_APPEND_DATA, FILE_SHARE_WRITE,
NULL, OPEN_ALWAYS, 0, NULL);
if (handle == INVALID_HANDLE_VALUE)
return;
file = PR_ImportFile((PROsfd)handle);
if (!file)
return;
#else
rv = logFile->OpenNSPRFileDesc(PR_WRONLY|PR_CREATE_FILE|PR_APPEND, 0644, &file);
if (NS_FAILED(rv))
return;
#endif
fd = file;
}
nsCString buf(zip);
buf.Append(" ");
buf.Append(entry);
buf.Append('\n');
PR_Write(fd, buf.get(), buf.Length());
}
void AddRef() {
MOZ_ASSERT(refCnt >= 0);
++refCnt;
}
void Release() {
MOZ_ASSERT(refCnt > 0);
if ((0 == --refCnt) && fd) {
PR_Close(fd);
fd = NULL;
}
}
private:
int refCnt;
mutable PRFileDesc *fd;
};
static ZipArchiveLogger zipLog;
//***********************************************************
// For every inflation the following allocations are done:
// malloc(1 * 9520)
@ -189,27 +257,9 @@ nsresult nsZipArchive::OpenArchive(nsZipHandle *aZipHandle)
//-- get table of contents for archive
nsresult rv = BuildFileList();
char *env = PR_GetEnv("MOZ_JAR_LOG_DIR");
if (env && NS_SUCCEEDED(rv) && aZipHandle->mFile) {
nsCOMPtr<nsIFile> logFile;
nsresult rv2 = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(logFile));
if (!NS_SUCCEEDED(rv2))
return rv;
// Create a directory for the log (in case it doesn't exist)
logFile->Create(nsIFile::DIRECTORY_TYPE, 0700);
nsAutoString name;
nsCOMPtr<nsIFile> file = aZipHandle->mFile.GetBaseFile();
file->GetLeafName(name);
name.Append(NS_LITERAL_STRING(".log"));
logFile->Append(name);
PRFileDesc* fd;
rv2 = logFile->OpenNSPRFileDesc(PR_WRONLY|PR_CREATE_FILE|PR_APPEND, 0644, &fd);
if (NS_SUCCEEDED(rv2))
mLog = fd;
if (NS_SUCCEEDED(rv)) {
if (aZipHandle->mFile)
aZipHandle->mFile.GetURIString(mURI);
}
return rv;
}
@ -300,13 +350,8 @@ MOZ_WIN_MEM_TRY_BEGIN
if ((len == item->nameLength) &&
(!memcmp(aEntryName, item->Name(), len))) {
if (mLog) {
// Successful GetItem() is a good indicator that the file is about to be read
char *tmp = PL_strdup(aEntryName);
tmp[len]='\n';
PR_Write(mLog, tmp, len+1);
PL_strfree(tmp);
}
// Successful GetItem() is a good indicator that the file is about to be read
zipLog.Write(mURI, aEntryName);
return item; //-- found it
}
item = item->next;
@ -742,6 +787,8 @@ nsZipArchive::nsZipArchive()
: mRefCnt(0)
, mBuiltSynthetics(false)
{
zipLog.AddRef();
MOZ_COUNT_CTOR(nsZipArchive);
// initialize the table to NULL
@ -756,6 +803,8 @@ nsZipArchive::~nsZipArchive()
CloseArchive();
MOZ_COUNT_DTOR(nsZipArchive);
zipLog.Release();
}

View File

@ -217,9 +217,8 @@ private:
// file handle
nsRefPtr<nsZipHandle> mFd;
// logging handle
mozilla::AutoFDClose mLog;
// file URI, for logging
nsCString mURI;
private:
//--- private methods ---

View File

@ -349,7 +349,7 @@ def parse_manifest(root, path, fileobj=None):
if not fileobj:
fileobj = open(path)
linenum = 0
for line in fileobj.readlines():
for line in fileobj:
linenum += 1
with errors.context(path, linenum):
e = parse_manifest_line(base, line)

View File

@ -11,6 +11,8 @@ from zipfile import (
ZIP_DEFLATED,
)
from collections import OrderedDict
from urlparse import urlparse, ParseResult
import mozpack.path
JAR_STORED = ZIP_STORED
JAR_DEFLATED = ZIP_DEFLATED
@ -279,6 +281,12 @@ class JarFileReader(object):
'''
return self.read().splitlines(True)
def __iter__(self):
'''
Iterator, to support the "for line in fileobj" constructs.
'''
return iter(self.readlines())
def seek(self, pos, whence=os.SEEK_SET):
'''
Change the current position in the uncompressed data. Subsequent reads
@ -732,3 +740,60 @@ class Deflater(object):
if self.compressed:
return self._deflated.getvalue()
return self._data.getvalue()
class JarLog(dict):
'''
Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE.
The jar log is then available as a dict with the jar path as key (see
canonicalize for more details on the key value), and the corresponding
access log as a list value. Only the first access to a given member of
a jar is stored.
'''
def __init__(self, file=None, fileobj=None):
if not fileobj:
fileobj = open(file, 'r')
urlmap = {}
for line in fileobj:
url, path = line.strip().split(None, 1)
if not url or not path:
continue
if url not in urlmap:
urlmap[url] = JarLog.canonicalize(url)
jar = urlmap[url]
entry = self.setdefault(jar, [])
if path not in entry:
entry.append(path)
@staticmethod
def canonicalize(url):
'''
The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method
returns a unique value corresponding to such urls.
- file:///{path} becomes {path}
- jar:file:///{path}!/{subpath} becomes ({path}, {subpath})
- jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes
({path}, {subpath}, {subpath2})
'''
if not isinstance(url, ParseResult):
# Assume that if it doesn't start with jar: or file:, it's a path.
if not url.startswith(('jar:', 'file:')):
url = 'file:///' + os.path.abspath(url)
url = urlparse(url)
assert url.scheme
assert url.scheme in ('jar', 'file')
if url.scheme == 'jar':
path = JarLog.canonicalize(url.path)
if isinstance(path, tuple):
return path[:-1] + tuple(path[-1].split('!/', 1))
return tuple(path.split('!/', 1))
if url.scheme == 'file':
assert os.path.isabs(url.path)
path = url.path
# On Windows, url.path will be /drive:/path ; on Unix systems,
# /path. As we want drive:/path instead of /drive:/path on Windows,
# remove the leading /.
if os.path.isabs(path[1:]):
path = path[1:]
path = os.path.realpath(path)
return mozpack.path.normsep(os.path.normcase(path))

View File

@ -9,11 +9,16 @@ from mozpack.mozjar import (
JarReader,
JarWriter,
Deflater,
OrderedDict,
JarLog,
)
from collections import OrderedDict
from mozpack.test.test_files import MockDest
import unittest
import mozunit
from cStringIO import StringIO
from urllib import pathname2url
import mozpack.path
import os
class TestJarStruct(unittest.TestCase):
@ -255,5 +260,56 @@ class TestPreload(unittest.TestCase):
self.assertEqual(files[2].filename, 'foo')
class TestJarLog(unittest.TestCase):
def test_jarlog(self):
base = 'file:' + pathname2url(os.path.abspath(os.curdir))
s = StringIO('\n'.join([
base + '/bar/baz.jar first',
base + '/bar/baz.jar second',
base + '/bar/baz.jar third',
base + '/bar/baz.jar second',
base + '/bar/baz.jar second',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar first',
'jar:' + base + '/qux.zip!/omni.ja other/stuff',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar third',
'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff',
'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' +
' deeply/nested/stuff',
]))
log = JarLog(fileobj=s)
canonicalize = lambda p: \
mozpack.path.normsep(os.path.normcase(os.path.realpath(p)))
baz_jar = canonicalize('bar/baz.jar')
qux_zip = canonicalize('qux.zip')
self.assertEqual(set(log.keys()), set([
baz_jar,
(qux_zip, 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'),
]))
self.assertEqual(log[baz_jar], [
'first',
'second',
'third',
])
self.assertEqual(log[(qux_zip, 'omni.ja')], [
'stuff',
'other/stuff',
])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')],
['nested/stuff'])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip',
'omni.ja')], ['deeply/nested/stuff'])
# The above tests also indirectly check the value returned by
# JarLog.canonicalize for various jar: and file: urls, but
# JarLog.canonicalize also supports plain paths.
self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')),
baz_jar)
self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar)
if __name__ == '__main__':
mozunit.main()

View File

@ -98,7 +98,7 @@ endif # LIBXUL_SDK
_ABS_DIST = $(call core_abspath,$(DIST))
JARLOG_DIR = $(call core_abspath,$(DEPTH)/jarlog/)
JARLOG_DIR_AB_CD = $(JARLOG_DIR)/$(AB_CD)
JARLOG_FILE_AB_CD = $(JARLOG_DIR)/$(AB_CD).log
TAR_CREATE_FLAGS := --exclude=.mkdir.done $(TAR_CREATE_FLAGS)
CREATE_FINAL_TAR = $(TAR) -c --owner=0 --group=0 --numeric-owner \
@ -591,7 +591,7 @@ stage-package: $(MOZ_PKG_MANIFEST)
$(addprefix --removals ,$(MOZ_PKG_REMOVALS)) \
$(if $(filter-out 0,$(MOZ_PKG_FATAL_WARNINGS)),,--ignore-errors) \
$(if $(MOZ_PACKAGER_MINIFY),--minify) \
$(if $(JARLOG_DIR),--jarlogs $(JARLOG_DIR_AB_CD)) \
$(if $(JARLOG_DIR),$(addprefix --jarlog ,$(wildcard $(JARLOG_FILE_AB_CD)))) \
$(if $(OPTIMIZEJARS),--optimizejars) \
$(addprefix --unify ,$(UNIFY_DIST)) \
$(MOZ_PKG_MANIFEST) $(DIST) $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR) \

View File

@ -230,8 +230,8 @@ def main():
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--jarlogs', default='', help='Base directory where ' +
'to find jar content access logs')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
@ -331,13 +331,15 @@ def main():
libname)))
# Setup preloading
if args.jarlogs:
jarlogs = FileFinder(args.jarlogs)
for p, log in jarlogs:
if p.endswith('.log'):
p = p[:-4]
if copier.contains(p) and isinstance(copier[p], Jarrer):
copier[p].preload([l.strip() for l in log.open().readlines()])
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():