Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-09 19:35:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 923016 - Implementation of cache index, r=honzab

Browse Source
This commit is contained in:
Michal Novotny 2014-02-18 16:05:07 +01:00
parent 746849fb45
commit a5234f73a3
16 changed files with 5237 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
netwerk/cache2/CacheFile.cpp
View File

@ -8,6 +8,7 @@
#include "CacheFileChunk.h"
#include "CacheFileInputStream.h"
#include "CacheFileOutputStream.h"
#include "CacheIndex.h"
#include "nsThreadUtils.h"
#include "mozilla/DebugOnly.h"
#include <algorithm>
@ -135,6 +136,12 @@ public:
return NS_ERROR_UNEXPECTED;
}
NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult)
{
MOZ_CRASH("DoomFileHelper::OnFileRenamed should not be called!");
return NS_ERROR_UNEXPECTED;
}
private:
virtual ~DoomFileHelper()
{
@ -226,9 +233,36 @@ CacheFile::Init(const nsACString &aKey,
mReady = true;
mDataSize = mMetadata->Offset();
}
else
else {
flags = CacheFileIOManager::CREATE;
if (!mKeyIsHash) {
// Have a look into index and change to CREATE_NEW when we are sure
// that the entry does not exist.
CacheIndex::EntryStatus status;
rv = CacheIndex::HasEntry(mKey, &status);
if (status == CacheIndex::DOES_NOT_EXIST) {
LOG(("CacheFile::Init() - Forcing CREATE_NEW flag since we don't have"
" this entry according to index"));
flags = CacheFileIOManager::CREATE_NEW;
// make sure we can use this entry immediately
mMetadata = new CacheFileMetadata(mKey);
mReady = true;
mDataSize = mMetadata->Offset();
// Notify callback now and don't store it in mListener, no further
// operation can change the result.
nsRefPtr<NotifyCacheFileListenerEvent> ev;
ev = new NotifyCacheFileListenerEvent(aCallback, NS_OK, true);
rv = NS_DispatchToCurrentThread(ev);
NS_ENSURE_SUCCESS(rv, rv);
aCallback = nullptr;
}
}
}
if (aPriority)
flags |= CacheFileIOManager::PRIORITY;
if (aKeyIsHash)
@ -474,10 +508,12 @@ CacheFile::OnFileOpened(CacheFileHandle *aHandle, nsresult aResult)
mHandle = aHandle;
if (mMetadata) {
InitIndexEntry();
// The entry was initialized as createNew, don't try to read metadata.
mMetadata->SetHandle(mHandle);
// Write all cached chunks, otherwise thay may stay unwritten.
// Write all cached chunks, otherwise they may stay unwritten.
mCachedChunks.Enumerate(&CacheFile::WriteAllCachedChunks, this);
return NS_OK;
@ -542,6 +578,8 @@ CacheFile::OnMetadataRead(nsresult aResult)
isNew = true;
mMetadata->MarkDirty();
}
InitIndexEntry();
}
nsCOMPtr<CacheFileListener> listener;
@ -609,6 +647,13 @@ CacheFile::OnEOFSet(CacheFileHandle *aHandle, nsresult aResult)
return NS_ERROR_UNEXPECTED;
}
nsresult
CacheFile::OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult)
{
MOZ_CRASH("CacheFile::OnFileRenamed should not be called!");
return NS_ERROR_UNEXPECTED;
}
nsresult
CacheFile::OpenInputStream(nsIInputStream **_retval)
{
@ -786,6 +831,10 @@ CacheFile::SetExpirationTime(uint32_t aExpirationTime)
NS_ENSURE_TRUE(mMetadata, NS_ERROR_UNEXPECTED);
PostWriteTimer();
if (mHandle && !mHandle->IsDoomed())
CacheFileIOManager::UpdateIndexEntry(mHandle, nullptr, &aExpirationTime);
return mMetadata->SetExpirationTime(aExpirationTime);
}
@ -828,6 +877,10 @@ CacheFile::SetFrecency(uint32_t aFrecency)
NS_ENSURE_TRUE(mMetadata, NS_ERROR_UNEXPECTED);
PostWriteTimer();
if (mHandle && !mHandle->IsDoomed())
CacheFileIOManager::UpdateIndexEntry(mHandle, &aFrecency, nullptr);
return mMetadata->SetFrecency(aFrecency);
}
@ -1477,5 +1530,33 @@ CacheFile::PadChunkWithZeroes(uint32_t aChunkIdx)
return NS_OK;
}
nsresult
CacheFile::InitIndexEntry()
{
MOZ_ASSERT(mHandle);
if (mHandle->IsDoomed())
return NS_OK;
nsresult rv;
rv = CacheFileIOManager::InitIndexEntry(mHandle,
mMetadata->AppId(),
mMetadata->IsAnonymous(),
mMetadata->IsInBrowser());
NS_ENSURE_SUCCESS(rv, rv);
uint32_t expTime;
mMetadata->GetExpirationTime(&expTime);
uint32_t frecency;
mMetadata->GetFrecency(&frecency);
rv = CacheFileIOManager::UpdateIndexEntry(mHandle, &frecency, &expTime);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
} // net
} // mozilla

3
netwerk/cache2/CacheFile.h
View File

@ -71,6 +71,7 @@ public:
NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult);
NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnMetadataRead(nsresult aResult);
NS_IMETHOD OnMetadataWritten(nsresult aResult);
@ -155,6 +156,8 @@ private:
nsresult PadChunkWithZeroes(uint32_t aChunkIdx);
nsresult InitIndexEntry();
mozilla::Mutex mLock;
bool mOpeningFile;
bool mReady;

16
netwerk/cache2/CacheFileChunk.cpp
View File

@ -182,7 +182,7 @@ CacheFileChunk::InitNew(CacheFileChunkListener *aCallback)
nsresult
CacheFileChunk::Read(CacheFileHandle *aHandle, uint32_t aLen,
CacheHashUtils::Hash16_t aHash,
CacheHash::Hash16_t aHash,
CacheFileChunkListener *aCallback)
{
mFile->AssertOwnsLock();
@ -342,7 +342,7 @@ CacheFileChunk::Index()
return mIndex;
}
CacheHashUtils::Hash16_t
CacheHash::Hash16_t
CacheFileChunk::Hash()
{
mFile->AssertOwnsLock();
@ -351,7 +351,7 @@ CacheFileChunk::Hash()
MOZ_ASSERT(!mListener);
MOZ_ASSERT(IsReady());
return CacheHashUtils::Hash16(BufForReading(), mDataSize);
return CacheHash::Hash16(BufForReading(), mDataSize);
}
uint32_t
@ -517,8 +517,7 @@ CacheFileChunk::OnDataRead(CacheFileHandle *aHandle, char *aBuf,
MOZ_ASSERT(mListener);
if (NS_SUCCEEDED(aResult)) {
CacheHashUtils::Hash16_t hash = CacheHashUtils::Hash16(mRWBuf,
mRWBufSize);
CacheHash::Hash16_t hash = CacheHash::Hash16(mRWBuf, mRWBufSize);
if (hash != mReadHash) {
LOG(("CacheFileChunk::OnDataRead() - Hash mismatch! Hash of the data is"
" %hx, hash in metadata is %hx. [this=%p, idx=%d]",
@ -590,6 +589,13 @@ CacheFileChunk::OnEOFSet(CacheFileHandle *aHandle, nsresult aResult)
return NS_ERROR_UNEXPECTED;
}
nsresult
CacheFileChunk::OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult)
{
MOZ_CRASH("CacheFileChunk::OnFileRenamed should not be called!");
return NS_ERROR_UNEXPECTED;
}
bool
CacheFileChunk::IsReady()
{

19
netwerk/cache2/CacheFileChunk.h
View File

@ -73,18 +73,18 @@ public:
void InitNew(CacheFileChunkListener *aCallback);
nsresult Read(CacheFileHandle *aHandle, uint32_t aLen,
CacheHashUtils::Hash16_t aHash,
CacheHash::Hash16_t aHash,
CacheFileChunkListener *aCallback);
nsresult Write(CacheFileHandle *aHandle, CacheFileChunkListener *aCallback);
void WaitForUpdate(CacheFileChunkListener *aCallback);
nsresult CancelWait(CacheFileChunkListener *aCallback);
nsresult NotifyUpdateListeners();
uint32_t Index();
CacheHashUtils::Hash16_t Hash();
uint32_t DataSize();
void UpdateDataSize(uint32_t aOffset, uint32_t aLen,
bool aEOF);
uint32_t Index();
CacheHash::Hash16_t Hash();
uint32_t DataSize();
void UpdateDataSize(uint32_t aOffset, uint32_t aLen,
bool aEOF);
NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
@ -92,6 +92,7 @@ public:
NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult);
NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult);
bool IsReady();
bool IsDirty();
@ -125,9 +126,9 @@ private:
char *mBuf;
uint32_t mBufSize;
char *mRWBuf;
uint32_t mRWBufSize;
CacheHashUtils::Hash16_t mReadHash;
char *mRWBuf;
uint32_t mRWBufSize;
CacheHash::Hash16_t mReadHash;
nsRefPtr<CacheFile> mFile; // is null if chunk is cached to
// prevent reference cycles

785
netwerk/cache2/CacheFileIOManager.cpp
View File

File diff suppressed because it is too large Load Diff

73
netwerk/cache2/CacheFileIOManager.h
View File

@ -16,11 +16,21 @@
#include "nsTHashtable.h"
#include "prio.h"
//#define DEBUG_HANDLES 1
class nsIFile;
namespace mozilla {
namespace net {
#ifdef DEBUG_HANDLES
class CacheFileHandlesEntry;
#endif
const char kEntriesDir[] = "entries";
const char kDoomedDir[] = "doomed";
class CacheFileHandle : public nsISupports
{
public:
@ -28,13 +38,17 @@ public:
bool DispatchRelease();
CacheFileHandle(const SHA1Sum::Hash *aHash, bool aPriority);
CacheFileHandle(const nsACString &aKey, bool aPriority);
CacheFileHandle(const CacheFileHandle &aOther);
void Log();
bool IsDoomed() { return mIsDoomed; }
const SHA1Sum::Hash *Hash() { return mHash; }
int64_t FileSize() { return mFileSize; }
uint32_t FileSizeInK();
bool IsPriority() { return mPriority; }
bool FileExists() { return mFileExists; }
bool IsClosed() { return mClosed; }
bool IsSpecialFile() { return !mHash; }
nsCString & Key() { return mKey; }
private:
@ -71,6 +85,10 @@ public:
void ClearAll();
uint32_t HandleCount();
#ifdef DEBUG_HANDLES
void Log(CacheFileHandlesEntry *entry);
#endif
class HandleHashKey : public PLDHashEntryHdr
{
public:
@ -160,6 +178,7 @@ public:
nsresult aResult) = 0;
NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) = 0;
NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) = 0;
NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) = 0;
};
NS_DEFINE_STATIC_IID_ACCESSOR(CacheFileIOListener, CACHEFILEIOLISTENER_IID)
@ -172,11 +191,12 @@ public:
NS_DECL_NSITIMERCALLBACK
enum {
OPEN = 0U,
CREATE = 1U,
CREATE_NEW = 2U,
PRIORITY = 4U,
NOHASH = 8U
OPEN = 0U,
CREATE = 1U,
CREATE_NEW = 2U,
PRIORITY = 4U,
NOHASH = 8U,
SPECIAL_FILE = 16U
};
CacheFileIOManager();
@ -186,6 +206,7 @@ public:
static nsresult OnProfile();
static already_AddRefed<nsIEventTarget> IOTarget();
static already_AddRefed<CacheIOThread> IOThread();
static bool IsOnIOThread();
static bool IsOnIOThreadOrCeased();
static bool IsShutdown();
@ -215,6 +236,18 @@ public:
static nsresult TruncateSeekSetEOF(CacheFileHandle *aHandle,
int64_t aTruncatePos, int64_t aEOFPos,
CacheFileIOListener *aCallback);
static nsresult RenameFile(CacheFileHandle *aHandle,
const nsACString &aNewName,
CacheFileIOListener *aCallback);
static nsresult InitIndexEntry(CacheFileHandle *aHandle,
uint32_t aAppId,
bool aAnonymous,
bool aInBrowser);
static nsresult UpdateIndexEntry(CacheFileHandle *aHandle,
const uint32_t *aFrecency,
const uint32_t *aExpirationTime);
static nsresult UpdateIndexEntry();
enum EEnumerateMode {
ENTRIES,
@ -239,6 +272,8 @@ private:
friend class DoomFileByKeyEvent;
friend class ReleaseNSPRHandleEvent;
friend class TruncateSeekSetEOFEvent;
friend class RenameFileEvent;
friend class CacheIndex;
friend class MetadataWriteScheduleEvent;
virtual ~CacheFileIOManager();
@ -249,6 +284,9 @@ private:
nsresult OpenFileInternal(const SHA1Sum::Hash *aHash,
uint32_t aFlags,
CacheFileHandle **_retval);
nsresult OpenSpecialFileInternal(const nsACString &aKey,
uint32_t aFlags,
CacheFileHandle **_retval);
nsresult CloseHandleInternal(CacheFileHandle *aHandle);
nsresult ReadInternal(CacheFileHandle *aHandle, int64_t aOffset,
char *aBuf, int32_t aCount);
@ -259,10 +297,14 @@ private:
nsresult ReleaseNSPRHandleInternal(CacheFileHandle *aHandle);
nsresult TruncateSeekSetEOFInternal(CacheFileHandle *aHandle,
int64_t aTruncatePos, int64_t aEOFPos);
nsresult RenameFileInternal(CacheFileHandle *aHandle,
const nsACString &aNewName);
nsresult CreateFile(CacheFileHandle *aHandle);
static void GetHashStr(const SHA1Sum::Hash *aHash, nsACString &_retval);
static void HashToStr(const SHA1Sum::Hash *aHash, nsACString &_retval);
static nsresult StrToHash(const nsACString &aHash, SHA1Sum::Hash *_retval);
nsresult GetFile(const SHA1Sum::Hash *aHash, nsIFile **_retval);
nsresult GetSpecialFile(const nsACString &aKey, nsIFile **_retval);
nsresult GetDoomedFile(nsIFile **_retval);
nsresult CheckAndCreateDir(nsIFile *aFile, const char *aDir);
nsresult CreateCacheTree();
@ -273,15 +315,16 @@ private:
nsresult UnscheduleMetadataWriteInternal(CacheFile * aFile);
nsresult ShutdownMetadataWriteSchedulingInternal();
static CacheFileIOManager *gInstance;
bool mShuttingDown;
nsRefPtr<CacheIOThread> mIOThread;
nsCOMPtr<nsIFile> mCacheDirectory;
bool mTreeCreated;
CacheFileHandles mHandles;
nsTArray<CacheFileHandle *> mHandlesByLastUsed;
nsTArray<nsRefPtr<CacheFile> > mScheduledMetadataWrites;
nsCOMPtr<nsITimer> mMetadataWritesTimer;
static CacheFileIOManager *gInstance;
bool mShuttingDown;
nsRefPtr<CacheIOThread> mIOThread;
nsCOMPtr<nsIFile> mCacheDirectory;
bool mTreeCreated;
CacheFileHandles mHandles;
nsTArray<CacheFileHandle *> mHandlesByLastUsed;
nsTArray<nsRefPtr<CacheFileHandle> > mSpecialHandles;
nsTArray<nsRefPtr<CacheFile> > mScheduledMetadataWrites;
nsCOMPtr<nsITimer> mMetadataWritesTimer;
};
} // net

209
netwerk/cache2/CacheFileMetadata.cpp
View File

@ -6,10 +6,14 @@
#include "CacheFileMetadata.h"
#include "CacheFileIOManager.h"
#include "nsICacheEntry.h"
#include "CacheHashUtils.h"
#include "CacheFileChunk.h"
#include "nsILoadContextInfo.h"
#include "../cache/nsCacheUtils.h"
#include "nsIFile.h"
#include "mozilla/Telemetry.h"
#include "mozilla/DebugOnly.h"
#include "prnetdb.h"
@ -19,8 +23,6 @@ namespace net {
#define kMinMetadataRead 1024 // TODO find optimal value from telemetry
#define kAlignSize 4096
#define NO_EXPIRATION_TIME 0xFFFFFFFF
NS_IMPL_ISUPPORTS1(CacheFileMetadata, CacheFileIOListener)
CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString &aKey, bool aKeyIsHash)
@ -35,14 +37,22 @@ CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString
, mWriteBuf(nullptr)
, mElementsSize(0)
, mIsDirty(false)
, mAnonymous(false)
, mInBrowser(false)
, mAppId(nsILoadContextInfo::NO_APP_ID)
{
LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p, handle=%p, key=%s]",
this, aHandle, PromiseFlatCString(aKey).get()));
MOZ_COUNT_CTOR(CacheFileMetadata);
memset(&mMetaHdr, 0, sizeof(CacheFileMetadataHeader));
mMetaHdr.mExpirationTime = NO_EXPIRATION_TIME;
mMetaHdr.mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;
mKey = aKey;
if (!aKeyIsHash) {
DebugOnly<nsresult> rv;
rv = ParseKey(aKey);
MOZ_ASSERT(NS_SUCCEEDED(rv));
}
}
CacheFileMetadata::~CacheFileMetadata()
@ -79,16 +89,45 @@ CacheFileMetadata::CacheFileMetadata(const nsACString &aKey)
, mWriteBuf(nullptr)
, mElementsSize(0)
, mIsDirty(true)
, mAnonymous(false)
, mInBrowser(false)
, mAppId(nsILoadContextInfo::NO_APP_ID)
{
LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p, key=%s]",
this, PromiseFlatCString(aKey).get()));
MOZ_COUNT_CTOR(CacheFileMetadata);
memset(&mMetaHdr, 0, sizeof(CacheFileMetadataHeader));
mMetaHdr.mExpirationTime = NO_EXPIRATION_TIME;
mMetaHdr.mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;
mMetaHdr.mFetchCount++;
mKey = aKey;
mMetaHdr.mKeySize = mKey.Length();
DebugOnly<nsresult> rv;
rv = ParseKey(aKey);
MOZ_ASSERT(NS_SUCCEEDED(rv));
}
CacheFileMetadata::CacheFileMetadata()
: mHandle(nullptr)
, mKeyIsHash(false)
, mHashArray(nullptr)
, mHashArraySize(0)
, mHashCount(0)
, mOffset(0)
, mBuf(nullptr)
, mBufSize(0)
, mWriteBuf(nullptr)
, mElementsSize(0)
, mIsDirty(false)
, mAnonymous(false)
, mInBrowser(false)
, mAppId(nsILoadContextInfo::NO_APP_ID)
{
LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p]", this));
MOZ_COUNT_CTOR(CacheFileMetadata);
memset(&mMetaHdr, 0, sizeof(CacheFileMetadataHeader));
}
void
@ -221,13 +260,13 @@ CacheFileMetadata::WriteMetadata(uint32_t aOffset,
mIsDirty = false;
mWriteBuf = static_cast<char *>(moz_xmalloc(sizeof(uint32_t) +
mHashCount * sizeof(CacheHashUtils::Hash16_t) +
mHashCount * sizeof(CacheHash::Hash16_t) +
sizeof(CacheFileMetadataHeader) + mKey.Length() + 1 +
mElementsSize + sizeof(uint32_t)));
char *p = mWriteBuf + sizeof(uint32_t);
memcpy(p, mHashArray, mHashCount * sizeof(CacheHashUtils::Hash16_t));
p += mHashCount * sizeof(CacheHashUtils::Hash16_t);
memcpy(p, mHashArray, mHashCount * sizeof(CacheHash::Hash16_t));
p += mHashCount * sizeof(CacheHash::Hash16_t);
memcpy(p, &mMetaHdr, sizeof(CacheFileMetadataHeader));
p += sizeof(CacheFileMetadataHeader);
memcpy(p, mKey.get(), mKey.Length());
@ -237,9 +276,9 @@ CacheFileMetadata::WriteMetadata(uint32_t aOffset,
memcpy(p, mBuf, mElementsSize);
p += mElementsSize;
CacheHashUtils::Hash32_t hash;
hash = CacheHashUtils::Hash(mWriteBuf + sizeof(uint32_t),
p - mWriteBuf - sizeof(uint32_t));
CacheHash::Hash32_t hash;
hash = CacheHash::Hash(mWriteBuf + sizeof(uint32_t),
p - mWriteBuf - sizeof(uint32_t));
*reinterpret_cast<uint32_t *>(mWriteBuf) = PR_htonl(hash);
*reinterpret_cast<uint32_t *>(p) = PR_htonl(aOffset);
@ -280,6 +319,72 @@ CacheFileMetadata::WriteMetadata(uint32_t aOffset,
return NS_OK;
}
nsresult
CacheFileMetadata::SyncReadMetadata(nsIFile *aFile)
{
LOG(("CacheFileMetadata::SyncReadMetadata() [this=%p]", this));
MOZ_ASSERT(!mListener);
MOZ_ASSERT(!mHandle);
MOZ_ASSERT(!mHashArray);
MOZ_ASSERT(!mBuf);
MOZ_ASSERT(!mWriteBuf);
MOZ_ASSERT(mKey.IsEmpty());
nsresult rv;
int64_t fileSize;
rv = aFile->GetFileSize(&fileSize);
NS_ENSURE_SUCCESS(rv, rv);
PRFileDesc *fd;
rv = aFile->OpenNSPRFileDesc(PR_RDONLY, 0600, &fd);
NS_ENSURE_SUCCESS(rv, rv);
int64_t offset = PR_Seek64(fd, fileSize - sizeof(uint32_t), PR_SEEK_SET);
if (offset == -1) {
PR_Close(fd);
return NS_ERROR_FAILURE;
}
uint32_t metaOffset;
int32_t bytesRead = PR_Read(fd, &metaOffset, sizeof(uint32_t));
if (bytesRead != sizeof(uint32_t)) {
PR_Close(fd);
return NS_ERROR_FAILURE;
}
metaOffset = PR_ntohl(metaOffset);
if (metaOffset > fileSize) {
PR_Close(fd);
return NS_ERROR_FAILURE;
}
mBufSize = fileSize - metaOffset;
mBuf = static_cast<char *>(moz_xmalloc(mBufSize));
DoMemoryReport(MemoryUsage());
offset = PR_Seek64(fd, metaOffset, PR_SEEK_SET);
if (offset == -1) {
PR_Close(fd);
return NS_ERROR_FAILURE;
}
bytesRead = PR_Read(fd, mBuf, mBufSize);
PR_Close(fd);
if (bytesRead != static_cast<int32_t>(mBufSize)) {
return NS_ERROR_FAILURE;
}
mKeyIsHash = true;
rv = ParseMetadata(metaOffset, 0);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
const char *
CacheFileMetadata::GetElement(const char *aKey)
{
@ -361,7 +466,7 @@ CacheFileMetadata::SetElement(const char *aKey, const char *aValue)
return NS_OK;
}
CacheHashUtils::Hash16_t
CacheHash::Hash16_t
CacheFileMetadata::GetHash(uint32_t aIndex)
{
MOZ_ASSERT(aIndex < mHashCount);
@ -369,7 +474,7 @@ CacheFileMetadata::GetHash(uint32_t aIndex)
}
nsresult
CacheFileMetadata::SetHash(uint32_t aIndex, CacheHashUtils::Hash16_t aHash)
CacheFileMetadata::SetHash(uint32_t aIndex, CacheHash::Hash16_t aHash)
{
LOG(("CacheFileMetadata::SetHash() [this=%p, idx=%d, hash=%x]",
this, aIndex, aHash));
@ -381,13 +486,13 @@ CacheFileMetadata::SetHash(uint32_t aIndex, CacheHashUtils::Hash16_t aHash)
if (aIndex > mHashCount) {
return NS_ERROR_INVALID_ARG;
} else if (aIndex == mHashCount) {
if ((aIndex + 1) * sizeof(CacheHashUtils::Hash16_t) > mHashArraySize) {
if ((aIndex + 1) * sizeof(CacheHash::Hash16_t) > mHashArraySize) {
// reallocate hash array buffer
if (mHashArraySize == 0)
mHashArraySize = 32 * sizeof(CacheHashUtils::Hash16_t);
mHashArraySize = 32 * sizeof(CacheHash::Hash16_t);
else
mHashArraySize *= 2;
mHashArray = static_cast<CacheHashUtils::Hash16_t *>(
mHashArray = static_cast<CacheHash::Hash16_t *>(
moz_xrealloc(mHashArray, mHashArraySize));
}
@ -646,6 +751,13 @@ CacheFileMetadata::OnEOFSet(CacheFileHandle *aHandle, nsresult aResult)
return NS_ERROR_UNEXPECTED;
}
nsresult
CacheFileMetadata::OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult)
{
MOZ_CRASH("CacheFileMetadata::OnFileRenamed should not be called!");
return NS_ERROR_UNEXPECTED;
}
void
CacheFileMetadata::InitEmptyMetadata()
{
@ -656,7 +768,7 @@ CacheFileMetadata::InitEmptyMetadata()
}
mOffset = 0;
mMetaHdr.mFetchCount = 1;
mMetaHdr.mExpirationTime = NO_EXPIRATION_TIME;
mMetaHdr.mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;
mMetaHdr.mKeySize = mKey.Length();
DoMemoryReport(MemoryUsage());
@ -675,7 +787,7 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset)
uint32_t hashCount = aMetaOffset / kChunkSize;
if (aMetaOffset % kChunkSize)
hashCount++;
uint32_t hashesLen = hashCount * sizeof(CacheHashUtils::Hash16_t);
uint32_t hashesLen = hashCount * sizeof(CacheHash::Hash16_t);
uint32_t hdrOffset = hashesOffset + hashesLen;
uint32_t keyOffset = hdrOffset + sizeof(CacheFileMetadataHeader);
@ -714,6 +826,10 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset)
if (mKeyIsHash) {
// get the original key
origKey.Assign(mBuf + keyOffset, keySize);
rv = ParseKey(origKey);
if (NS_FAILED(rv))
return rv;
}
else {
if (keySize != mKey.Length()) {
@ -730,9 +846,8 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset)
}
// check metadata hash (data from hashesOffset to metaposOffset)
CacheHashUtils::Hash32_t hash;
hash = CacheHashUtils::Hash(mBuf + hashesOffset,
metaposOffset - hashesOffset);
CacheHash::Hash32_t hash;
hash = CacheHash::Hash(mBuf + hashesOffset, metaposOffset - hashesOffset);
if (hash != PR_ntohl(*(reinterpret_cast<uint32_t *>(mBuf + aBufOffset)))) {
LOG(("CacheFileMetadata::ParseMetadata() - Metadata hash mismatch! Hash of "
@ -749,7 +864,7 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset)
mHashArraySize = hashesLen;
mHashCount = hashCount;
if (mHashArraySize) {
mHashArray = static_cast<CacheHashUtils::Hash16_t *>(
mHashArray = static_cast<CacheHash::Hash16_t *>(
moz_xmalloc(mHashArraySize));
memcpy(mHashArray, mBuf + hashesOffset, mHashArraySize);
}
@ -813,5 +928,57 @@ CacheFileMetadata::EnsureBuffer(uint32_t aSize)
DoMemoryReport(MemoryUsage());
}
nsresult
CacheFileMetadata::ParseKey(const nsACString &aKey)
{
if (aKey.Length() < 4) {
return NS_ERROR_FAILURE;
}
if (aKey[1] == '-') {
mAnonymous = false;
}
else if (aKey[1] == 'A') {
mAnonymous = true;
}
else {
return NS_ERROR_FAILURE;
}
if (aKey[2] != ':') {
return NS_ERROR_FAILURE;
}
int32_t appIdEndIdx = aKey.FindChar(':', 3);
if (appIdEndIdx == kNotFound) {
return NS_ERROR_FAILURE;
}
if (aKey[appIdEndIdx - 1] == 'B') {
mInBrowser = true;
appIdEndIdx--;
} else {
mInBrowser = false;
}
if (appIdEndIdx < 3) {
return NS_ERROR_FAILURE;
}
if (appIdEndIdx == 3) {
mAppId = nsILoadContextInfo::NO_APP_ID;
}
else {
nsAutoCString appIdStr(Substring(aKey, 3, appIdEndIdx - 3));
nsresult rv;
int64_t appId64 = appIdStr.ToInteger64(&rv);
if (NS_FAILED(rv) || appId64 > PR_UINT32_MAX)
return NS_ERROR_FAILURE;
mAppId = appId64;
}
return NS_OK;
}
} // net
} // mozilla

18
netwerk/cache2/CacheFileMetadata.h
View File

@ -65,6 +65,7 @@ public:
const nsACString &aKey,
bool aKeyIsHash);
CacheFileMetadata(const nsACString &aKey);
CacheFileMetadata();
void SetHandle(CacheFileHandle *aHandle);
@ -74,13 +75,17 @@ public:
nsresult ReadMetadata(CacheFileMetadataListener *aListener);
nsresult WriteMetadata(uint32_t aOffset,
CacheFileMetadataListener *aListener);
nsresult SyncReadMetadata(nsIFile *aFile);
bool IsAnonymous() { return mAnonymous; }
bool IsInBrowser() { return mInBrowser; }
uint32_t AppId() { return mAppId; }
const char * GetElement(const char *aKey);
nsresult SetElement(const char *aKey, const char *aValue);
CacheHashUtils::Hash16_t GetHash(uint32_t aIndex);
nsresult SetHash(uint32_t aIndex,
CacheHashUtils::Hash16_t aHash);
CacheHash::Hash16_t GetHash(uint32_t aIndex);
nsresult SetHash(uint32_t aIndex, CacheHash::Hash16_t aHash);
nsresult SetExpirationTime(uint32_t aExpirationTime);
nsresult GetExpirationTime(uint32_t *_retval);
@ -103,6 +108,7 @@ public:
NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult);
NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult);
private:
virtual ~CacheFileMetadata();
@ -111,11 +117,12 @@ private:
nsresult ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset);
nsresult CheckElements(const char *aBuf, uint32_t aSize);
void EnsureBuffer(uint32_t aSize);
nsresult ParseKey(const nsACString &aKey);
nsRefPtr<CacheFileHandle> mHandle;
nsCString mKey;
bool mKeyIsHash;
CacheHashUtils::Hash16_t *mHashArray;
CacheHash::Hash16_t *mHashArray;
uint32_t mHashArraySize;
uint32_t mHashCount;
int64_t mOffset;
@ -126,6 +133,9 @@ private:
CacheFileMetadataHeader mMetaHdr;
uint32_t mElementsSize;
bool mIsDirty;
bool mAnonymous;
bool mInBrowser;
uint32_t mAppId;
nsCOMPtr<CacheFileMetadataListener> mListener;
};

117
netwerk/cache2/CacheHashUtils.cpp
View File

@ -10,7 +10,7 @@ namespace mozilla {
namespace net {
/**
* CacheHashUtils::Hash(const char * key, uint32_t initval)
* CacheHash::Hash(const char * key, uint32_t initval)
*
* See http://burtleburtle.net/bob/hash/evahash.html for more information
* about this hash function.
@ -31,11 +31,11 @@ static inline void hashmix(uint32_t& a, uint32_t& b, uint32_t& c)
c -= a; c -= b; c ^= (b>>15);
}
CacheHashUtils::Hash32_t
CacheHashUtils::Hash(const char *aData, uint32_t aSize, uint32_t aInitval)
CacheHash::Hash32_t
CacheHash::Hash(const char *aData, uint32_t aSize, uint32_t aInitval)
{
const uint8_t *k = reinterpret_cast<const uint8_t*>(aData);
uint32_t a, b, c, len/*, length*/;
uint32_t a, b, c, len;
// length = PL_strlen(key);
/* Set up the internal state */
@ -75,13 +75,118 @@ CacheHashUtils::Hash(const char *aData, uint32_t aSize, uint32_t aInitval)
return c;
}
CacheHashUtils::Hash16_t
CacheHashUtils::Hash16(const char *aData, uint32_t aSize, uint32_t aInitval)
CacheHash::Hash16_t
CacheHash::Hash16(const char *aData, uint32_t aSize, uint32_t aInitval)
{
Hash32_t hash = Hash(aData, aSize, aInitval);
return (hash & 0xFFFF);
}
NS_IMPL_ISUPPORTS0(CacheHash)
CacheHash::CacheHash(uint32_t aInitval)
: mA(0x9e3779b9)
, mB(0x9e3779b9)
, mC(aInitval)
, mPos(0)
, mBuf(0)
, mBufPos(0)
, mLength(0)
, mFinalized(false)
{}
void
CacheHash::Feed(uint32_t aVal, uint8_t aLen)
{
switch (mPos) {
case 0:
mA += aVal;
mPos ++;
break;
case 1:
mB += aVal;
mPos ++;
break;
case 2:
mPos = 0;
if (aLen == 4) {
mC += aVal;
hashmix(mA, mB, mC);
}
else {
mC += aVal << 8;
}
}
mLength += aLen;
}
void
CacheHash::Update(const char *aData, uint32_t aLen)
{
const uint8_t *data = reinterpret_cast<const uint8_t*>(aData);
MOZ_ASSERT(!mFinalized);
if (mBufPos) {
while (mBufPos != 4 && aLen) {
mBuf += uint32_t(*data) << 8*mBufPos;
data++;
mBufPos++;
aLen--;
}
if (mBufPos == 4) {
mBufPos = 0;
Feed(mBuf);
mBuf = 0;
}
}
if (!aLen)
return;
while (aLen >= 4) {
Feed(data[0] + (uint32_t(data[1]) << 8) + (uint32_t(data[2]) << 16) +
(uint32_t(data[3]) << 24));
data += 4;
aLen -= 4;
}
switch (aLen) {
case 3: mBuf += data[2] << 16;
case 2: mBuf += data[1] << 8;
case 1: mBuf += data[0];
}
mBufPos = aLen;
}
CacheHash::Hash32_t
CacheHash::GetHash()
{
if (!mFinalized)
{
if (mBufPos) {
Feed(mBuf, mBufPos);
}
mC += mLength;
hashmix(mA, mB, mC);
mFinalized = true;
}
return mC;
}
CacheHash::Hash16_t
CacheHash::GetHash16()
{
Hash32_t hash = GetHash();
return (hash & 0xFFFF);
}
} // net
} // mozilla

23
netwerk/cache2/CacheHashUtils.h
View File

@ -5,6 +5,7 @@
#ifndef CacheHashUtils__h__
#define CacheHashUtils__h__
#include "nsISupports.h"
#include "mozilla/Types.h"
#include "prnetdb.h"
#include "nsPrintfCString.h"
@ -22,15 +23,35 @@
namespace mozilla {
namespace net {
class CacheHashUtils
class CacheHash : public nsISupports
{
public:
NS_DECL_THREADSAFE_ISUPPORTS
typedef uint16_t Hash16_t;
typedef uint32_t Hash32_t;
static Hash32_t Hash(const char* aData, uint32_t aSize, uint32_t aInitval=0);
static Hash16_t Hash16(const char* aData, uint32_t aSize,
uint32_t aInitval=0);
CacheHash(uint32_t aInitval=0);
void Update(const char *aData, uint32_t aLen);
Hash32_t GetHash();
Hash16_t GetHash16();
private:
virtual ~CacheHash() {}
void Feed(uint32_t aVal, uint8_t aLen = 4);
uint32_t mA, mB, mC;
uint8_t mPos;
uint32_t mBuf;
uint8_t mBufPos;
uint32_t mLength;
bool mFinalized;
};

1
netwerk/cache2/CacheIOThread.h
View File

@ -35,6 +35,7 @@ public:
MANAGEMENT,
CLOSE,
EVICT,
BUILD_OR_UPDATE_INDEX,
LAST_LEVEL
};

3178
netwerk/cache2/CacheIndex.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

853
netwerk/cache2/CacheIndex.h Normal file
View File

@ -0,0 +1,853 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef CacheIndex__h__
#define CacheIndex__h__
#include "CacheLog.h"
#include "CacheFileIOManager.h"
#include "nsIRunnable.h"
#include "CacheHashUtils.h"
#include "nsICacheEntry.h"
#include "nsILoadContextInfo.h"
#include "nsTHashtable.h"
#include "mozilla/SHA1.h"
#include "mozilla/Mutex.h"
#include "mozilla/Endian.h"
#include "mozilla/TimeStamp.h"
class nsIFile;
class nsIDirectoryEnumerator;
class nsITimer;
#ifdef DEBUG
#define DEBUG_STATS 1
#endif
namespace mozilla {
namespace net {
class CacheFileMetadata;
typedef struct {
// Version of the index. The index must be ignored and deleted when the file
// on disk was written with a newer version.
uint32_t mVersion;
// Timestamp of time when the last successful write of the index started.
// During update process we use this timestamp for a quick validation of entry
// files. If last modified time of the file is lower than this timestamp, we
// skip parsing of such file since the information in index should be up to
// date.
uint32_t mTimeStamp;
// We set this flag as soon as possible after parsing index during startup
// and clean it after we write journal to disk during shutdown. We ignore the
// journal and start update process whenever this flag is set during index
// parsing.
uint32_t mIsDirty;
} CacheIndexHeader;
struct CacheIndexRecord {
SHA1Sum::Hash mHash;
uint32_t mFrecency;
uint32_t mExpirationTime;
uint32_t mAppId;
/*
* 1000 0000 0000 0000 0000 0000 0000 0000 : initialized
* 0100 0000 0000 0000 0000 0000 0000 0000 : anonymous
* 0010 0000 0000 0000 0000 0000 0000 0000 : inBrowser
* 0001 0000 0000 0000 0000 0000 0000 0000 : removed
* 0000 1000 0000 0000 0000 0000 0000 0000 : dirty
* 0000 0100 0000 0000 0000 0000 0000 0000 : fresh
* 0000 0011 0000 0000 0000 0000 0000 0000 : reserved
* 0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB)
*/
uint32_t mFlags;
CacheIndexRecord()
: mFrecency(0)
, mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME)
, mAppId(nsILoadContextInfo::NO_APP_ID)
, mFlags(0)
{}
};
class CacheIndexEntry : public PLDHashEntryHdr
{
public:
typedef const SHA1Sum::Hash& KeyType;
typedef const SHA1Sum::Hash* KeyTypePointer;
CacheIndexEntry(KeyTypePointer aKey)
{
MOZ_COUNT_CTOR(CacheIndexEntry);
mRec = new CacheIndexRecord();
LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec));
memcpy(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash));
}
CacheIndexEntry(const CacheIndexEntry& aOther)
{
NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!");
}
~CacheIndexEntry()
{
MOZ_COUNT_DTOR(CacheIndexEntry);
LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]",
mRec));
delete mRec;
}
// KeyEquals(): does this entry match this key?
bool KeyEquals(KeyTypePointer aKey) const
{
return memcmp(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0;
}
// KeyToPointer(): Convert KeyType to KeyTypePointer
static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }
// HashKey(): calculate the hash number
static PLDHashNumber HashKey(KeyTypePointer aKey)
{
return (reinterpret_cast<const uint32_t *>(aKey))[0];
}
// ALLOW_MEMMOVE can we move this class with memmove(), or do we have
// to use the copy constructor?
enum { ALLOW_MEMMOVE = true };
bool operator==(const CacheIndexEntry& aOther) const
{
return KeyEquals(&aOther.mRec->mHash);
}
CacheIndexEntry& operator=(const CacheIndexEntry& aOther)
{
MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash,
sizeof(SHA1Sum::Hash)) == 0);
mRec->mFrecency = aOther.mRec->mFrecency;
mRec->mExpirationTime = aOther.mRec->mExpirationTime;
mRec->mAppId = aOther.mRec->mAppId;
mRec->mFlags = aOther.mRec->mFlags;
return *this;
}
void InitNew()
{
mRec->mFrecency = 0;
mRec->mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;
mRec->mAppId = nsILoadContextInfo::NO_APP_ID;
mRec->mFlags = 0;
}
void Init(uint32_t aAppId, bool aAnonymous, bool aInBrowser)
{
MOZ_ASSERT(mRec->mFrecency == 0);
MOZ_ASSERT(mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME);
MOZ_ASSERT(mRec->mAppId == nsILoadContextInfo::NO_APP_ID);
// When we init the entry it must be fresh and may be dirty
MOZ_ASSERT((mRec->mFlags & ~kDirtyMask) == kFreshMask);
mRec->mAppId = aAppId;
mRec->mFlags |= kInitializedMask;
if (aAnonymous) {
mRec->mFlags |= kAnonymousMask;
}
if (aInBrowser) {
mRec->mFlags |= kInBrowserMask;
}
}
const SHA1Sum::Hash * Hash() { return &mRec->mHash; }
bool IsInitialized() { return !!(mRec->mFlags & kInitializedMask); }
uint32_t AppId() { return mRec->mAppId; }
bool Anonymous() { return !!(mRec->mFlags & kAnonymousMask); }
bool InBrowser() { return !!(mRec->mFlags & kInBrowserMask); }
bool IsRemoved() { return !!(mRec->mFlags & kRemovedMask); }
void MarkRemoved() { mRec->mFlags |= kRemovedMask; }
bool IsDirty() { return !!(mRec->mFlags & kDirtyMask); }
void MarkDirty() { mRec->mFlags |= kDirtyMask; }
void ClearDirty() { mRec->mFlags &= ~kDirtyMask; }
bool IsFresh() { return !!(mRec->mFlags & kFreshMask); }
void MarkFresh() { mRec->mFlags |= kFreshMask; }
void SetFrecency(uint32_t aFrecency) { mRec->mFrecency = aFrecency; }
uint32_t GetFrecency() { return mRec->mFrecency; }
void SetExpirationTime(uint32_t aExpirationTime)
{
mRec->mExpirationTime = aExpirationTime;
}
uint32_t GetExpirationTime() { return mRec->mExpirationTime; }
// Sets filesize in kilobytes.
void SetFileSize(uint32_t aFileSize)
{
if (aFileSize > kFileSizeMask) {
LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, "
"truncating to %u", kFileSizeMask));
aFileSize = kFileSizeMask;
}
mRec->mFlags &= ~kFileSizeMask;
mRec->mFlags |= aFileSize;
}
// Returns filesize in kilobytes.
uint32_t GetFileSize() { return mRec->mFlags & kFileSizeMask; }
bool IsFileEmpty() { return GetFileSize() == 0; }
void WriteToBuf(void *aBuf)
{
CacheIndexRecord *dst = reinterpret_cast<CacheIndexRecord *>(aBuf);
// Copy the whole record to the buffer.
memcpy(aBuf, mRec, sizeof(CacheIndexRecord));
// Dirty and fresh flags should never go to disk, since they make sense only
// during current session.
dst->mFlags &= ~kDirtyMask;
dst->mFlags &= ~kFreshMask;
#if defined(IS_LITTLE_ENDIAN)
// Data in the buffer are in machine byte order and we want them in network
// byte order.
NetworkEndian::writeUint32(&dst->mFrecency, dst->mFrecency);
NetworkEndian::writeUint32(&dst->mExpirationTime, dst->mExpirationTime);
NetworkEndian::writeUint32(&dst->mAppId, dst->mAppId);
NetworkEndian::writeUint32(&dst->mFlags, dst->mFlags);
#endif
}
void ReadFromBuf(void *aBuf)
{
CacheIndexRecord *src= reinterpret_cast<CacheIndexRecord *>(aBuf);
MOZ_ASSERT(memcmp(&mRec->mHash, &src->mHash,
sizeof(SHA1Sum::Hash)) == 0);
mRec->mFrecency = NetworkEndian::readUint32(&src->mFrecency);
mRec->mExpirationTime = NetworkEndian::readUint32(&src->mExpirationTime);
mRec->mAppId = NetworkEndian::readUint32(&src->mAppId);
mRec->mFlags = NetworkEndian::readUint32(&src->mFlags);
}
void Log() {
LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u,"
" initialized=%u, removed=%u, dirty=%u, anonymous=%u, inBrowser=%u, "
"appId=%u, frecency=%u, expirationTime=%u, size=%u]",
this, LOGSHA1(mRec->mHash), IsFresh(), IsInitialized(), IsRemoved(),
IsDirty(), Anonymous(), InBrowser(), AppId(), GetFrecency(),
GetExpirationTime(), GetFileSize()));
}
private:
friend class CacheIndex;
friend class CacheIndexEntryAutoManage;
static const uint32_t kInitializedMask = 0x80000000;
static const uint32_t kAnonymousMask = 0x40000000;
static const uint32_t kInBrowserMask = 0x20000000;
// This flag is set when the entry was removed. We need to keep this
// information in memory until we write the index file.
static const uint32_t kRemovedMask = 0x10000000;
// This flag is set when the information in memory is not in sync with the
// information in index file on disk.
static const uint32_t kDirtyMask = 0x08000000;
// This flag is set when the information about the entry is fresh, i.e.
// we've created or opened this entry during this session, or we've seen
// this entry during update or build process.
static const uint32_t kFreshMask = 0x04000000;
static const uint32_t kReservedMask = 0x03000000;
// FileSize in kilobytes
static const uint32_t kFileSizeMask = 0x00FFFFFF;
CacheIndexRecord *mRec;
};
class CacheIndexStats
{
public:
CacheIndexStats()
: mCount(0)
, mNotInitialized(0)
, mRemoved(0)
, mDirty(0)
, mFresh(0)
, mEmpty(0)
, mSize(0)
#ifdef DEBUG
, mStateLogged(false)
, mDisableLogging(false)
#endif
{
}
bool operator==(const CacheIndexStats& aOther) const
{
return
#ifdef DEBUG
aOther.mStateLogged == mStateLogged &&
#endif
aOther.mCount == mCount &&
aOther.mNotInitialized == mNotInitialized &&
aOther.mRemoved == mRemoved &&
aOther.mDirty == mDirty &&
aOther.mFresh == mFresh &&
aOther.mEmpty == mEmpty &&
aOther.mSize == mSize;
}
#ifdef DEBUG
void DisableLogging() {
mDisableLogging = true;
}
#endif
void Log() {
LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, "
"dirty=%u, fresh=%u, empty=%u, size=%lld]", mCount, mNotInitialized,
mRemoved, mDirty, mFresh, mEmpty, mSize));
}
#ifdef DEBUG
bool StateLogged() {
return mStateLogged;
}
#endif
uint32_t Count() {
MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!");
return mCount;
}
uint32_t Dirty() {
MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!");
return mDirty;
}
uint32_t Fresh() {
MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!");
return mFresh;
}
uint32_t ActiveEntriesCount() {
MOZ_ASSERT(!mStateLogged, "CacheIndexStats::ActiveEntriesCount() - state "
"logged!");
return mCount - mRemoved - mNotInitialized - mEmpty;
}
int64_t Size() {
MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!");
return mSize;
}
void BeforeChange(CacheIndexEntry *aEntry) {
#ifdef DEBUG_STATS
if (!mDisableLogging) {
LOG(("CacheIndexStats::BeforeChange()"));
Log();
}
#endif
MOZ_ASSERT(!mStateLogged, "CacheIndexStats::BeforeChange() - state "
"logged!");
#ifdef DEBUG
mStateLogged = true;
#endif
if (aEntry) {
MOZ_ASSERT(mCount);
mCount--;
if (aEntry->IsDirty()) {
MOZ_ASSERT(mDirty);
mDirty--;
}
if (aEntry->IsFresh()) {
MOZ_ASSERT(mFresh);
mFresh--;
}
if (aEntry->IsRemoved()) {
MOZ_ASSERT(mRemoved);
mRemoved--;
} else {
if (!aEntry->IsInitialized()) {
MOZ_ASSERT(mNotInitialized);
mNotInitialized--;
} else {
if (aEntry->IsFileEmpty()) {
MOZ_ASSERT(mEmpty);
mEmpty--;
} else {
MOZ_ASSERT(mSize);
mSize -= aEntry->GetFileSize();
}
}
}
}
}
void AfterChange(CacheIndexEntry *aEntry) {
MOZ_ASSERT(mStateLogged, "CacheIndexStats::AfterChange() - state not "
"logged!");
#ifdef DEBUG
mStateLogged = false;
#endif
if (aEntry) {
++mCount;
if (aEntry->IsDirty()) {
mDirty++;
}
if (aEntry->IsFresh()) {
mFresh++;
}
if (aEntry->IsRemoved()) {
mRemoved++;
} else {
if (!aEntry->IsInitialized()) {
mNotInitialized++;
} else {
if (aEntry->IsFileEmpty()) {
mEmpty++;
} else {
mSize += aEntry->GetFileSize();
}
}
}
}
#ifdef DEBUG_STATS
if (!mDisableLogging) {
LOG(("CacheIndexStats::AfterChange()"));
Log();
}
#endif
}
private:
uint32_t mCount;
uint32_t mNotInitialized;
uint32_t mRemoved;
uint32_t mDirty;
uint32_t mFresh;
uint32_t mEmpty;
int64_t mSize;
#ifdef DEBUG
// We completely remove the data about an entry from the stats in
// BeforeChange() and set this flag to true. The entry is then modified,
// deleted or created and the data is again put into the stats and this flag
// set to false. Statistics must not be read during this time since the
// information is not correct.
bool mStateLogged;
// Disables logging in this instance of CacheIndexStats
bool mDisableLogging;
#endif
};
class CacheIndex : public CacheFileIOListener
, public nsIRunnable
{
public:
NS_DECL_THREADSAFE_ISUPPORTS
NS_DECL_NSIRUNNABLE
CacheIndex();
static nsresult Init(nsIFile *aCacheDirectory);
static nsresult PreShutdown();
static nsresult Shutdown();
// Following methods can be called only on IO thread.
// Add entry to the index. The entry shouldn't be present in index. This
// method is called whenever a new handle for a new entry file is created. The
// newly created entry is not initialized and it must be either initialized
// with InitEntry() or removed with RemoveEntry().
static nsresult AddEntry(const SHA1Sum::Hash *aHash);
// Inform index about an existing entry that should be present in index. This
// method is called whenever a new handle for an existing entry file is
// created. Like in case of AddEntry(), either InitEntry() or RemoveEntry()
// must be called on the entry, since the entry is not initizlized if the
// index is outdated.
static nsresult EnsureEntryExists(const SHA1Sum::Hash *aHash);
// Initialize the entry. It MUST be present in index. Call to AddEntry() or
// EnsureEntryExists() must precede the call to this method.
static nsresult InitEntry(const SHA1Sum::Hash *aHash,
uint32_t aAppId,
bool aAnonymous,
bool aInBrowser);
// Remove entry from index. The entry should be present in index.
static nsresult RemoveEntry(const SHA1Sum::Hash *aHash);
// Update some information in entry. The entry MUST be present in index and
// MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to
// InitEntry() must precede the call to this method.
// Pass nullptr if the value didn't change.
static nsresult UpdateEntry(const SHA1Sum::Hash *aHash,
const uint32_t *aFrecency,
const uint32_t *aExpirationTime,
const uint32_t *aSize);
enum EntryStatus {
EXISTS = 0,
DOES_NOT_EXIST = 1,
DO_NOT_KNOW = 2
};
// Returns status of the entry in index for the given key. It can be called
// on any thread.
static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval);
private:
friend class CacheIndexEntryAutoManage;
friend class CacheIndexAutoLock;
friend class CacheIndexAutoUnlock;
virtual ~CacheIndex();
NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
nsresult aResult);
NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult);
NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult);
void Lock();
void Unlock();
void AssertOwnsLock();
nsresult InitInternal(nsIFile *aCacheDirectory);
void PreShutdownInternal();
// This method returns false when index is not initialized or is shut down.
bool IsIndexUsable();
// This method checks whether the entry has the same values of appId,
// isAnonymous and isInBrowser. We don't expect to find a collision since
// these values are part of the key that we hash and we use a strong hash
// function.
static bool IsCollision(CacheIndexEntry *aEntry,
uint32_t aAppId,
bool aAnonymous,
bool aInBrowser);
// Checks whether any of the information about the entry has changed.
static bool HasEntryChanged(CacheIndexEntry *aEntry,
const uint32_t *aFrecency,
const uint32_t *aExpirationTime,
const uint32_t *aSize);
// Merge all pending operations from mPendingUpdates into mIndex.
void ProcessPendingOperations();
static PLDHashOperator UpdateEntryInIndex(CacheIndexEntry *aEntry,
void* aClosure);
// Following methods perform writing of the index file.
//
// The index is written periodically, but not earlier than once in
// kMinDumpInterval and there must be at least kMinUnwrittenChanges
// differences between index on disk and in memory. Index is always first
// written to a temporary file and the old index file is replaced when the
// writing process succeeds.
//
// Starts writing of index when both limits (minimal delay between writes and
// minimum number of changes in index) were exceeded.
bool WriteIndexToDiskIfNeeded();
// Starts writing of index file.
void WriteIndexToDisk();
// Serializes part of mIndex hashtable to the write buffer a writes the buffer
// to the file.
void WriteRecords();
// Finalizes writing process.
void FinishWrite(bool aSucceeded);
static PLDHashOperator CopyRecordsToRWBuf(CacheIndexEntry *aEntry,
void* aClosure);
static PLDHashOperator ApplyIndexChanges(CacheIndexEntry *aEntry,
void* aClosure);
// Following methods perform writing of the journal during shutdown. All these
// methods must be called only during shutdown since they write/delete files
// directly on the main thread instead of using CacheFileIOManager that does
// it asynchronously on IO thread. Journal contains only entries that are
// dirty, i.e. changes that are not present in the index file on the disk.
// When the log is written successfully, the dirty flag in index file is
// cleared.
nsresult GetFile(const nsACString &aName, nsIFile **_retval);
nsresult RemoveFile(const nsACString &aName);
void RemoveIndexFromDisk();
// Writes journal to the disk and clears dirty flag in index header.
nsresult WriteLogToDisk();
static PLDHashOperator WriteEntryToLog(CacheIndexEntry *aEntry,
void* aClosure);
// Following methods perform reading of the index from the disk.
//
// Index is read at startup just after initializing the CacheIndex. There are
// 3 files used when manipulating with index: index file, journal file and
// a temporary file. All files contain the hash of the data, so we can check
// whether the content is valid and complete. Index file contains also a dirty
// flag in the index header which is unset on a clean shutdown. During opening
// and reading of the files we determine the status of the whole index from
// the states of the separate files. Following table shows all possible
// combinations:
//
// index, journal, tmpfile
// M * * - index is missing -> BUILD
// I * * - index is invalid -> BUILD
// D * * - index is dirty -> UPDATE
// C M * - index is dirty -> UPDATE
// C I * - unexpected state -> UPDATE
// C V E - unexpected state -> UPDATE
// C V M - index is up to date -> READY
//
// where the letters mean:
// * - any state
// E - file exists
// M - file is missing
// I - data is invalid (parsing failed or hash didn't match)
// D - dirty (data in index file is correct, but dirty flag is set)
// C - clean (index file is clean)
// V - valid (data in journal file is correct)
//
// Note: We accept the data from journal only when the index is up to date as
// a whole (i.e. C,V,M state).
//
// We rename the journal file to the temporary file as soon as possible after
// initial test to ensure that we start update process on the next startup if
// FF crashes during parsing of the index.
//
// Initiates reading index from disk.
void ReadIndexFromDisk();
// Starts reading data from index file.
void StartReadingIndex();
// Parses data read from index file.
void ParseRecords();
// Starts reading data from journal file.
void StartReadingJournal();
// Parses data read from journal file.
void ParseJournal();
// Merges entries from journal into mIndex.
void MergeJournal();
// In debug build this method checks that we have no fresh entry in mIndex
// after we finish reading index and before we process pending operations.
void EnsureNoFreshEntry();
// In debug build this method is called after processing pending operations
// to make sure mIndexStats contains correct information.
void EnsureCorrectStats();
static PLDHashOperator SumIndexStats(CacheIndexEntry *aEntry, void* aClosure);
// Finalizes reading process.
void FinishRead(bool aSucceeded);
static PLDHashOperator ProcessJournalEntry(CacheIndexEntry *aEntry,
void* aClosure);
// Following methods perform updating and building of the index.
// Timer callback that starts update or build process.
static void DelayedBuildUpdate(nsITimer *aTimer, void *aClosure);
// Posts timer event that start update or build process.
nsresult ScheduleBuildUpdateTimer(uint32_t aDelay);
nsresult SetupDirectoryEnumerator();
void InitEntryFromDiskData(CacheIndexEntry *aEntry,
CacheFileMetadata *aMetaData,
int64_t aFileSize);
// Starts build process or fires a timer when it is too early after startup.
void StartBuildingIndex();
// Iterates through all files in entries directory that we didn't create/open
// during this session, parses them and adds the entries to the index.
void BuildIndex();
// Finalizes build process.
void FinishBuild(bool aSucceeded);
bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState = false);
// Starts update process or fires a timer when it is too early after startup.
void StartUpdatingIndex();
// Iterates through all files in entries directory that we didn't create/open
// during this session and theirs last modified time is newer than timestamp
// in the index header. Parses the files and adds the entries to the index.
void UpdateIndex();
// Finalizes update process.
void FinishUpdate(bool aSucceeded);
static PLDHashOperator RemoveNonFreshEntries(CacheIndexEntry *aEntry,
void* aClosure);
enum EState {
// Initial state in which the index is not usable
// Possible transitions:
// -> READING
INITIAL = 0,
// Index is being read from the disk.
// Possible transitions:
// -> INITIAL - We failed to dispatch a read event.
// -> BUILDING - No or corrupted index file was found.
// -> UPDATING - No or corrupted journal file was found.
// - Dirty flag was set in index header.
// -> READY - Index was read successfully or was interrupted by
// pre-shutdown.
// -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
READING = 1,
// Index is being written to the disk.
// Possible transitions:
// -> READY - Writing of index finished or was interrupted by
// pre-shutdown..
// -> UPDATING - Writing of index finished, but index was found outdated
// during writing.
// -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
WRITING = 2,
// Index is being build.
// Possible transitions:
// -> READY - Building of index finished or was interrupted by
// pre-shutdown.
// -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
BUILDING = 3,
// Index is being updated.
// Possible transitions:
// -> READY - Updating of index finished or was interrupted by
// pre-shutdown.
// -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
UPDATING = 4,
// Index is ready.
// Possible transitions:
// -> UPDATING - Index was found outdated.
// -> SHUTDOWN - Index is shutting down.
READY = 5,
// Index is shutting down.
SHUTDOWN = 6
};
#ifdef PR_LOGGING
static char const * StateString(EState aState);
#endif
void ChangeState(EState aNewState);
// Allocates and releases buffer used for reading and writing index.
void AllocBuffer();
void ReleaseBuffer();
// Methods used by CacheIndexEntryAutoManage to keep the arrays up to date.
void InsertRecordToFrecencyArray(CacheIndexRecord *aRecord);
void InsertRecordToExpirationArray(CacheIndexRecord *aRecord);
void RemoveRecordFromFrecencyArray(CacheIndexRecord *aRecord);
void RemoveRecordFromExpirationArray(CacheIndexRecord *aRecord);
static CacheIndex *gInstance;
nsCOMPtr<nsIFile> mCacheDirectory;
mozilla::Mutex mLock;
EState mState;
// Timestamp of time when the index was initialized. We use it to delay
// initial update or build of index.
TimeStamp mStartTime;
// Set to true in PreShutdown(), it is checked on variaous places to prevent
// starting any process (write, update, etc.) during shutdown.
bool mShuttingDown;
// When set to true, update process should start as soon as possible. This
// flag is set whenever we find some inconsistency which would be fixed by
// update process. The flag is checked always when switching to READY state.
// To make sure we start the update process as soon as possible, methods that
// set this flag should also call StartUpdatingIndexIfNeeded() to cover the
// case when we are currently in READY state.
bool mIndexNeedsUpdate;
// Whether the index file on disk exists and is valid.
bool mIndexOnDiskIsValid;
// When something goes wrong during updating or building process, we don't
// mark index clean (and also don't write journal) to ensure that update or
// build will be initiated on the next start.
bool mDontMarkIndexClean;
// Timestamp value from index file. It is used during update process to skip
// entries that were last modified before this timestamp.
uint32_t mIndexTimeStamp;
// Timestamp of last time the index was dumped to disk.
// NOTE: The index might not be necessarily dumped at this time. The value
// is used to schedule next dump of the index.
TimeStamp mLastDumpTime;
// Timer of delayed update/build.
nsCOMPtr<nsITimer> mTimer;
// Helper members used when reading/writing index from/to disk.
// Contains number of entries that should be skipped:
// - in hashtable when writing index because they were already written
// - in index file when reading index because they were already read
uint32_t mSkipEntries;
// Number of entries that should be written to disk. This is number of entries
// in hashtable that are initialized and are not marked as removed when writing
// begins.
uint32_t mProcessEntries;
char *mRWBuf;
uint32_t mRWBufSize;
uint32_t mRWBufPos;
nsRefPtr<CacheHash> mRWHash;
// When reading index from disk, we open index, journal and tmpindex files at
// the same time. This value tell us how many times CacheIndex::OnFileOpened()
// will be called and identifies the handle.
uint32_t mReadOpenCount;
// Reading of index failed completely if true.
bool mReadFailed;
// Reading of journal succeeded if true.
bool mJournalReadSuccessfully;
// Handle used for writing and reading index file.
nsRefPtr<CacheFileHandle> mIndexHandle;
// Handle used for reading journal file.
nsRefPtr<CacheFileHandle> mJournalHandle;
// Directory enumerator used when building and updating index.
nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator;
// Main index hashtable.
nsTHashtable<CacheIndexEntry> mIndex;
// We cannot add, remove or change any entry in mIndex in states READING and
// WRITING. We track all changes in mPendingUpdates during these states.
nsTHashtable<CacheIndexEntry> mPendingUpdates;
// Contains information statistics for mIndex + mPendingUpdates.
CacheIndexStats mIndexStats;
// When reading journal, we must first parse the whole file and apply the
// changes iff the journal was read successfully. mTmpJournal is used to store
// entries from the journal file. We throw away all these entries if parsing
// of the journal fails or the hash does not match.
nsTHashtable<CacheIndexEntry> mTmpJournal;
// Arrays that keep entry records ordered by eviction preference. When looking
// for an entry to evict, we first try to find an expired entry. If there is
// no expired entry, we take the entry with lowest valid frecency. Zero
// frecency is an initial value and such entries are stored at the end of the
// array. Uninitialized entries and entries marked as deleted are not present
// in these arrays.
nsTArray<CacheIndexRecord *> mFrecencyArray;
nsTArray<CacheIndexRecord *> mExpirationArray;
};
} // net
} // mozilla
#endif

1
netwerk/cache2/CacheStorageService.cpp
View File

@ -1324,6 +1324,7 @@ private:
NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult) { return NS_OK; }
NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) { return NS_OK; }
NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) { return NS_OK; }
nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
};

1
netwerk/cache2/moz.build
View File

@ -39,6 +39,7 @@ SOURCES += [
'CacheFileIOManager.cpp',
'CacheFileMetadata.cpp',
'CacheFileOutputStream.cpp',
'CacheIndex.cpp',
'CacheLog.cpp',
'CacheStorage.cpp',
'CacheStorageService.cpp',

3
netwerk/cache2/nsICacheEntry.idl
View File

@ -198,6 +198,9 @@ interface nsICacheEntry : nsISupports
*/
boolean hasWriteAccess(in boolean aWriteAllowed);
const unsigned long NO_EXPIRATION_TIME = 0xFFFFFFFF;
// *************** GET RID OF THESE ??? ***************
void setDataSize(in unsigned long size);
attribute nsCacheStoragePolicy storagePolicy;