gecko-dev/startupcache/StartupCache.cpp
2019-05-21 17:32:45 +00:00

635 lines
19 KiB
C++

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "prio.h"
#include "PLDHashTable.h"
#include "mozilla/IOInterposer.h"
#include "mozilla/MemoryReporting.h"
#include "mozilla/scache/StartupCache.h"
#include "nsAutoPtr.h"
#include "nsClassHashtable.h"
#include "nsComponentManagerUtils.h"
#include "nsCRT.h"
#include "nsDirectoryServiceUtils.h"
#include "nsIClassInfo.h"
#include "nsIFile.h"
#include "nsIObserver.h"
#include "nsIObserverService.h"
#include "nsIOutputStream.h"
#include "nsIStorageStream.h"
#include "nsIStreamBufferAccess.h"
#include "nsIStringStream.h"
#include "nsISupports.h"
#include "nsITimer.h"
#include "nsIZipWriter.h"
#include "nsIZipReader.h"
#include "nsZipArchive.h"
#include "mozilla/Omnijar.h"
#include "prenv.h"
#include "mozilla/Telemetry.h"
#include "nsThreadUtils.h"
#include "nsXULAppAPI.h"
#include "nsIProtocolHandler.h"
#include "GeckoProfiler.h"
#ifdef IS_BIG_ENDIAN
# define SC_ENDIAN "big"
#else
# define SC_ENDIAN "little"
#endif
#if PR_BYTES_PER_WORD == 4
# define SC_WORDSIZE "4"
#else
# define SC_WORDSIZE "8"
#endif
namespace mozilla {
namespace scache {
MOZ_DEFINE_MALLOC_SIZE_OF(StartupCacheMallocSizeOf)
NS_IMETHODIMP
StartupCache::CollectReports(nsIHandleReportCallback* aHandleReport,
nsISupports* aData, bool aAnonymize) {
MOZ_COLLECT_REPORT(
"explicit/startup-cache/mapping", KIND_NONHEAP, UNITS_BYTES,
SizeOfMapping(),
"Memory used to hold the mapping of the startup cache from file. "
"This memory is likely to be swapped out shortly after start-up.");
MOZ_COLLECT_REPORT("explicit/startup-cache/data", KIND_HEAP, UNITS_BYTES,
HeapSizeOfIncludingThis(StartupCacheMallocSizeOf),
"Memory used by the startup cache for things other than "
"the file mapping.");
return NS_OK;
}
#define STARTUP_CACHE_NAME "startupCache." SC_WORDSIZE "." SC_ENDIAN
StartupCache* StartupCache::GetSingleton() {
if (!gStartupCache) {
if (!XRE_IsParentProcess()) {
return nullptr;
}
#ifdef MOZ_DISABLE_STARTUPCACHE
return nullptr;
#else
StartupCache::InitSingleton();
#endif
}
return StartupCache::gStartupCache;
}
void StartupCache::DeleteSingleton() { StartupCache::gStartupCache = nullptr; }
nsresult StartupCache::InitSingleton() {
nsresult rv;
StartupCache::gStartupCache = new StartupCache();
rv = StartupCache::gStartupCache->Init();
if (NS_FAILED(rv)) {
StartupCache::gStartupCache = nullptr;
}
return rv;
}
StaticRefPtr<StartupCache> StartupCache::gStartupCache;
bool StartupCache::gShutdownInitiated;
bool StartupCache::gIgnoreDiskCache;
NS_IMPL_ISUPPORTS(StartupCache, nsIMemoryReporter)
StartupCache::StartupCache()
: mArchive(nullptr), mStartupWriteInitiated(false), mWriteThread(nullptr) {}
StartupCache::~StartupCache() {
if (mTimer) {
mTimer->Cancel();
}
// Generally, the in-memory table should be empty here,
// but an early shutdown means either mTimer didn't run
// or the write thread is still running.
WaitOnWriteThread();
// If we shutdown quickly timer wont have fired. Instead of writing
// it on the main thread and block the shutdown we simply wont update
// the startup cache. Always do this if the file doesn't exist since
// we use it part of the package step.
if (!mArchive) {
WriteToDisk();
}
UnregisterWeakMemoryReporter(this);
}
nsresult StartupCache::Init() {
// workaround for bug 653936
nsCOMPtr<nsIProtocolHandler> jarInitializer(
do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "jar"));
nsresult rv;
// This allows to override the startup cache filename
// which is useful from xpcshell, when there is no ProfLDS directory to keep
// cache in.
char* env = PR_GetEnv("MOZ_STARTUP_CACHE");
if (env && *env) {
rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false,
getter_AddRefs(mFile));
} else {
nsCOMPtr<nsIFile> file;
rv = NS_GetSpecialDirectory("ProfLDS", getter_AddRefs(file));
if (NS_FAILED(rv)) {
// return silently, this will fail in mochitests's xpcshell process.
return rv;
}
rv = file->AppendNative(NS_LITERAL_CSTRING("startupCache"));
NS_ENSURE_SUCCESS(rv, rv);
// Try to create the directory if it's not there yet
rv = file->Create(nsIFile::DIRECTORY_TYPE, 0777);
if (NS_FAILED(rv) && rv != NS_ERROR_FILE_ALREADY_EXISTS) return rv;
rv = file->AppendNative(NS_LITERAL_CSTRING(STARTUP_CACHE_NAME));
NS_ENSURE_SUCCESS(rv, rv);
mFile = file;
}
NS_ENSURE_TRUE(mFile, NS_ERROR_UNEXPECTED);
mObserverService = do_GetService("@mozilla.org/observer-service;1");
if (!mObserverService) {
NS_WARNING("Could not get observerService.");
return NS_ERROR_UNEXPECTED;
}
mListener = new StartupCacheListener();
rv = mObserverService->AddObserver(mListener, NS_XPCOM_SHUTDOWN_OBSERVER_ID,
false);
NS_ENSURE_SUCCESS(rv, rv);
rv = mObserverService->AddObserver(mListener, "startupcache-invalidate",
false);
NS_ENSURE_SUCCESS(rv, rv);
rv = LoadArchive();
// Sometimes we don't have a cache yet, that's ok.
// If it's corrupted, just remove it and start over.
if (gIgnoreDiskCache || (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND)) {
NS_WARNING("Failed to load startupcache file correctly, removing!");
InvalidateCache();
}
RegisterWeakMemoryReporter(this);
return NS_OK;
}
/**
* LoadArchive can be called from the main thread or while reloading cache on
* write thread.
*/
nsresult StartupCache::LoadArchive() {
if (gIgnoreDiskCache) return NS_ERROR_FAILURE;
mArchive = new nsZipArchive();
nsresult rv = mArchive->OpenArchive(mFile);
if (NS_FAILED(rv)) {
mArchive = nullptr;
}
return rv;
}
namespace {
nsresult GetBufferFromZipArchive(nsZipArchive* zip, bool doCRC, const char* id,
UniquePtr<char[]>* outbuf, uint32_t* length) {
if (!zip) return NS_ERROR_NOT_AVAILABLE;
nsZipItemPtr<char> zipItem(zip, id, doCRC);
if (!zipItem) return NS_ERROR_NOT_AVAILABLE;
*outbuf = zipItem.Forget();
*length = zipItem.Length();
return NS_OK;
}
} /* anonymous namespace */
// NOTE: this will not find a new entry until it has been written to disk!
// Consumer should take ownership of the resulting buffer.
nsresult StartupCache::GetBuffer(const char* id, UniquePtr<char[]>* outbuf,
uint32_t* length) {
AUTO_PROFILER_LABEL("StartupCache::GetBuffer", OTHER);
NS_ASSERTION(NS_IsMainThread(),
"Startup cache only available on main thread");
WaitOnWriteThread();
if (!mStartupWriteInitiated) {
CacheEntry* entry;
nsDependentCString idStr(id);
mTable.Get(idStr, &entry);
if (entry) {
*outbuf = MakeUnique<char[]>(entry->size);
memcpy(outbuf->get(), entry->data.get(), entry->size);
*length = entry->size;
Telemetry::AccumulateCategorical(
Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitMemory);
return NS_OK;
}
}
nsresult rv = GetBufferFromZipArchive(mArchive, true, id, outbuf, length);
if (NS_SUCCEEDED(rv)) {
Telemetry::AccumulateCategorical(
Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitDisk);
return rv;
}
Telemetry::AccumulateCategorical(
Telemetry::LABELS_STARTUP_CACHE_REQUESTS::Miss);
RefPtr<nsZipArchive> omnijar =
mozilla::Omnijar::GetReader(mozilla::Omnijar::APP);
// no need to checksum omnijarred entries
rv = GetBufferFromZipArchive(omnijar, false, id, outbuf, length);
if (NS_SUCCEEDED(rv)) return rv;
omnijar = mozilla::Omnijar::GetReader(mozilla::Omnijar::GRE);
// no need to checksum omnijarred entries
return GetBufferFromZipArchive(omnijar, false, id, outbuf, length);
}
// Makes a copy of the buffer, client retains ownership of inbuf.
nsresult StartupCache::PutBuffer(const char* id, UniquePtr<char[]>&& inbuf,
uint32_t len) {
NS_ASSERTION(NS_IsMainThread(),
"Startup cache only available on main thread");
WaitOnWriteThread();
if (StartupCache::gShutdownInitiated) {
return NS_ERROR_NOT_AVAILABLE;
}
nsDependentCString idStr(id);
// Cache it for now, we'll write all together later.
auto entry = mTable.LookupForAdd(idStr);
if (entry) {
NS_WARNING("Existing entry in StartupCache.");
// Double-caching is undesirable but not an error.
return NS_OK;
}
#ifdef DEBUG
if (mArchive) {
nsZipItem* zipItem = mArchive->GetItem(id);
NS_ASSERTION(zipItem == nullptr, "Existing entry in disk StartupCache.");
}
#endif
entry.OrInsert(
[&inbuf, &len]() { return new CacheEntry(std::move(inbuf), len); });
mPendingWrites.AppendElement(idStr);
return ResetStartupWriteTimer();
}
size_t StartupCache::SizeOfMapping() {
return mArchive ? mArchive->SizeOfMapping() : 0;
}
size_t StartupCache::HeapSizeOfIncludingThis(
mozilla::MallocSizeOf aMallocSizeOf) const {
// This function could measure more members, but they haven't been found by
// DMD to be significant. They can be added later if necessary.
size_t n = aMallocSizeOf(this);
n += mTable.ShallowSizeOfExcludingThis(aMallocSizeOf);
for (auto iter = mTable.ConstIter(); !iter.Done(); iter.Next()) {
n += iter.Data()->SizeOfIncludingThis(aMallocSizeOf);
}
n += mPendingWrites.ShallowSizeOfExcludingThis(aMallocSizeOf);
return n;
}
struct CacheWriteHolder {
nsCOMPtr<nsIZipWriter> writer;
nsCOMPtr<nsIStringInputStream> stream;
PRTime time;
};
static void CacheCloseHelper(const nsACString& key, const CacheEntry* data,
const CacheWriteHolder* holder) {
MOZ_ASSERT(data); // assert key was found in mTable.
nsresult rv;
nsIStringInputStream* stream = holder->stream;
nsIZipWriter* writer = holder->writer;
stream->ShareData(data->data.get(), data->size);
#ifdef DEBUG
bool hasEntry;
rv = writer->HasEntry(key, &hasEntry);
NS_ASSERTION(NS_SUCCEEDED(rv) && hasEntry == false,
"Existing entry in disk StartupCache.");
#endif
rv = writer->AddEntryStream(key, holder->time, true, stream, false);
if (NS_FAILED(rv)) {
NS_WARNING("cache entry deleted but not written to disk.");
}
}
/**
* WriteToDisk writes the cache out to disk. Callers of WriteToDisk need to call
* WaitOnWriteThread to make sure there isn't a write happening on another
* thread
*/
void StartupCache::WriteToDisk() {
nsresult rv;
mStartupWriteInitiated = true;
if (mTable.Count() == 0) return;
nsCOMPtr<nsIZipWriter> zipW = do_CreateInstance("@mozilla.org/zipwriter;1");
if (!zipW) return;
rv = zipW->Open(mFile, PR_RDWR | PR_CREATE_FILE);
if (NS_FAILED(rv)) {
NS_WARNING("could not open zipfile for write");
return;
}
// If we didn't have an mArchive member, that means that we failed to
// open the startup cache for reading. Therefore, we need to record
// the time of creation in a zipfile comment; this has been useful for
// Telemetry statistics.
PRTime now = PR_Now();
if (!mArchive) {
nsCString comment;
comment.Assign((char*)&now, sizeof(now));
zipW->SetComment(comment);
}
nsCOMPtr<nsIStringInputStream> stream =
do_CreateInstance("@mozilla.org/io/string-input-stream;1", &rv);
if (NS_FAILED(rv)) {
NS_WARNING("Couldn't create string input stream.");
return;
}
CacheWriteHolder holder;
holder.stream = stream;
holder.writer = zipW;
holder.time = now;
for (auto& key : mPendingWrites) {
CacheCloseHelper(key, mTable.Get(key), &holder);
}
mPendingWrites.Clear();
mTable.Clear();
// Close the archive so Windows doesn't choke.
mArchive = nullptr;
zipW->Close();
// We succesfully wrote the archive to disk; mark the disk file as trusted
gIgnoreDiskCache = false;
// Our reader's view of the archive is outdated now, reload it.
LoadArchive();
}
void StartupCache::InvalidateCache(bool memoryOnly) {
if (memoryOnly) {
// The memoryOnly option is just for testing purposes. We want to ensure
// that we're nuking the in-memory form but that we preserve everything
// on disk.
WriteToDisk();
return;
}
WaitOnWriteThread();
mPendingWrites.Clear();
mTable.Clear();
mArchive = nullptr;
nsresult rv = mFile->Remove(false);
if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST &&
rv != NS_ERROR_FILE_NOT_FOUND) {
gIgnoreDiskCache = true;
return;
}
gIgnoreDiskCache = false;
LoadArchive();
}
void StartupCache::IgnoreDiskCache() {
gIgnoreDiskCache = true;
if (gStartupCache) gStartupCache->InvalidateCache();
}
/*
* WaitOnWriteThread() is called from a main thread to wait for the worker
* thread to finish. However since the same code is used in the worker thread
* and main thread, the worker thread can also call WaitOnWriteThread() which is
* a no-op.
*/
void StartupCache::WaitOnWriteThread() {
NS_ASSERTION(NS_IsMainThread(),
"Startup cache should only wait for io thread on main thread");
if (!mWriteThread || mWriteThread == PR_GetCurrentThread()) return;
PR_JoinThread(mWriteThread);
mWriteThread = nullptr;
}
void StartupCache::ThreadedWrite(void* aClosure) {
AUTO_PROFILER_REGISTER_THREAD("StartupCache");
NS_SetCurrentThreadName("StartupCache");
mozilla::IOInterposer::RegisterCurrentThread();
/*
* It is safe to use the pointer passed in aClosure to reference the
* StartupCache object because the thread's lifetime is tightly coupled to
* the lifetime of the StartupCache object; this thread is joined in the
* StartupCache destructor, guaranteeing that this function runs if and only
* if the StartupCache object is valid.
*/
StartupCache* startupCacheObj = static_cast<StartupCache*>(aClosure);
startupCacheObj->WriteToDisk();
mozilla::IOInterposer::UnregisterCurrentThread();
}
/*
* The write-thread is spawned on a timeout(which is reset with every write).
* This can avoid a slow shutdown. After writing out the cache, the zipreader is
* reloaded on the worker thread.
*/
void StartupCache::WriteTimeout(nsITimer* aTimer, void* aClosure) {
/*
* It is safe to use the pointer passed in aClosure to reference the
* StartupCache object because the timer's lifetime is tightly coupled to
* the lifetime of the StartupCache object; this timer is canceled in the
* StartupCache destructor, guaranteeing that this function runs if and only
* if the StartupCache object is valid.
*/
StartupCache* startupCacheObj = static_cast<StartupCache*>(aClosure);
startupCacheObj->mWriteThread = PR_CreateThread(
PR_USER_THREAD, StartupCache::ThreadedWrite, startupCacheObj,
PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD, 0);
}
// We don't want to refcount StartupCache, so we'll just
// hold a ref to this and pass it to observerService instead.
NS_IMPL_ISUPPORTS(StartupCacheListener, nsIObserver)
nsresult StartupCacheListener::Observe(nsISupports* subject, const char* topic,
const char16_t* data) {
StartupCache* sc = StartupCache::GetSingleton();
if (!sc) return NS_OK;
if (strcmp(topic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
// Do not leave the thread running past xpcom shutdown
sc->WaitOnWriteThread();
StartupCache::gShutdownInitiated = true;
} else if (strcmp(topic, "startupcache-invalidate") == 0) {
sc->InvalidateCache(data && nsCRT::strcmp(data, u"memoryOnly") == 0);
}
return NS_OK;
}
nsresult StartupCache::GetDebugObjectOutputStream(
nsIObjectOutputStream* aStream, nsIObjectOutputStream** aOutStream) {
NS_ENSURE_ARG_POINTER(aStream);
#ifdef DEBUG
auto* stream = new StartupCacheDebugOutputStream(aStream, &mWriteObjectMap);
NS_ADDREF(*aOutStream = stream);
#else
NS_ADDREF(*aOutStream = aStream);
#endif
return NS_OK;
}
nsresult StartupCache::ResetStartupWriteTimer() {
mStartupWriteInitiated = false;
nsresult rv = NS_OK;
if (!mTimer)
mTimer = NS_NewTimer();
else
rv = mTimer->Cancel();
NS_ENSURE_SUCCESS(rv, rv);
// Wait for 10 seconds, then write out the cache.
mTimer->InitWithNamedFuncCallback(StartupCache::WriteTimeout, this, 60000,
nsITimer::TYPE_ONE_SHOT,
"StartupCache::WriteTimeout");
return NS_OK;
}
bool StartupCache::StartupWriteComplete() {
WaitOnWriteThread();
return mStartupWriteInitiated && mTable.Count() == 0;
}
// StartupCacheDebugOutputStream implementation
#ifdef DEBUG
NS_IMPL_ISUPPORTS(StartupCacheDebugOutputStream, nsIObjectOutputStream,
nsIBinaryOutputStream, nsIOutputStream)
bool StartupCacheDebugOutputStream::CheckReferences(nsISupports* aObject) {
nsresult rv;
nsCOMPtr<nsIClassInfo> classInfo = do_QueryInterface(aObject);
if (!classInfo) {
NS_ERROR("aObject must implement nsIClassInfo");
return false;
}
uint32_t flags;
rv = classInfo->GetFlags(&flags);
NS_ENSURE_SUCCESS(rv, false);
if (flags & nsIClassInfo::SINGLETON) return true;
nsISupportsHashKey* key = mObjectMap->GetEntry(aObject);
if (key) {
NS_ERROR(
"non-singleton aObject is referenced multiple times in this"
"serialization, we don't support that.");
return false;
}
mObjectMap->PutEntry(aObject);
return true;
}
// nsIObjectOutputStream implementation
nsresult StartupCacheDebugOutputStream::WriteObject(nsISupports* aObject,
bool aIsStrongRef) {
nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
NS_ASSERTION(rootObject.get() == aObject,
"bad call to WriteObject -- call WriteCompoundObject!");
bool check = CheckReferences(aObject);
NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
return mBinaryStream->WriteObject(aObject, aIsStrongRef);
}
nsresult StartupCacheDebugOutputStream::WriteSingleRefObject(
nsISupports* aObject) {
nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
NS_ASSERTION(rootObject.get() == aObject,
"bad call to WriteSingleRefObject -- call WriteCompoundObject!");
bool check = CheckReferences(aObject);
NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
return mBinaryStream->WriteSingleRefObject(aObject);
}
nsresult StartupCacheDebugOutputStream::WriteCompoundObject(
nsISupports* aObject, const nsIID& aIID, bool aIsStrongRef) {
nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
nsCOMPtr<nsISupports> roundtrip;
rootObject->QueryInterface(aIID, getter_AddRefs(roundtrip));
NS_ASSERTION(roundtrip.get() == aObject,
"bad aggregation or multiple inheritance detected by call to "
"WriteCompoundObject!");
bool check = CheckReferences(aObject);
NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
return mBinaryStream->WriteCompoundObject(aObject, aIID, aIsStrongRef);
}
nsresult StartupCacheDebugOutputStream::WriteID(nsID const& aID) {
return mBinaryStream->WriteID(aID);
}
char* StartupCacheDebugOutputStream::GetBuffer(uint32_t aLength,
uint32_t aAlignMask) {
return mBinaryStream->GetBuffer(aLength, aAlignMask);
}
void StartupCacheDebugOutputStream::PutBuffer(char* aBuffer, uint32_t aLength) {
mBinaryStream->PutBuffer(aBuffer, aLength);
}
#endif // DEBUG
} // namespace scache
} // namespace mozilla