Landing DISKCACHE1_BRANCH (part Deux) which enables dynamic eviction and efficient cache miss detection for the disk cache (bug 72506) r=beard, sr=darin.

This commit is contained in:
gordon%netscape.com 2001-04-04 03:30:45 +00:00
parent 58c9f11eb3
commit 4a0c37aac2
17 changed files with 478 additions and 270 deletions

View File

@ -31,6 +31,8 @@ interface nsISimpleEnumerator;
interface nsICacheListener;
interface nsITransport;
interface nsIFile;
interface nsICacheMetaDataVisitor;
[scriptable, uuid(49c1a11d-f5d2-4f09-8262-551e64908ada)]
interface nsICacheEntryDescriptor : nsICacheEntryInfo
@ -116,6 +118,20 @@ interface nsICacheEntryDescriptor : nsICacheEntryInfo
string getMetaDataElement(in string key);
void setMetaDataElement(in string key, in string value);
nsISimpleEnumerator getMetaDataEnumerator(); /* todo */
/**
* Visitor will be called with key/value pair for each meta data element.
*/
void visitMetaData(in nsICacheMetaDataVisitor visitor);
};
[scriptable, uuid(22f9a49c-3cf8-4c23-8006-54efb11ac562)]
interface nsICacheMetaDataVisitor : nsISupports
{
/**
* Called for each key/value pair in the meta data for a cache entry
*/
boolean visitMetaDataElement(in string key,
in string value);
};

View File

@ -26,32 +26,20 @@
#include "nsCache.h"
#include "nsReadableUtils.h"
/**
* Cache Service Utility Functions
*/
#if 0
// time conversion utils from nsCachedNetData.cpp
// Convert PRTime to unix-style time_t, i.e. seconds since the epoch
PRUint32
ConvertPRTimeToSeconds(PRTime time64)
{
double fpTime;
LL_L2D(fpTime, time64);
return (PRUint32)(fpTime * 1e-6 + 0.5);
}
#if defined(PR_LOGGING)
PRLogModuleInfo * gCacheLog = nsnull;
// Convert unix-style time_t, i.e. seconds since the epoch, to PRTime
PRTime
ConvertSecondsToPRTime(PRUint32 seconds)
void
CacheLogInit()
{
PRInt64 t64;
LL_I2L(t64, seconds);
PRInt64 mil;
LL_I2L(mil, 1000000);
LL_MUL(t64, t64, mil);
return t64;
if (gCacheLog) return;
gCacheLog = PR_NewLogModule("cache");
NS_ASSERTION(gCacheLog, "\n### failed to allocate cache log.\n");
}
#endif

View File

@ -33,13 +33,23 @@
#include "nsAReadableString.h"
#include "prtime.h"
#include "nsError.h"
#include "prlog.h"
#if 0
// Convert PRTime to unix-style time_t, i.e. seconds since the epoch
PRUint32 ConvertPRTimeToSeconds(PRTime time64);
// Convert unix-style time_t, i.e. seconds since the epoch, to PRTime
PRTime ConvertSecondsToPRTime(PRUint32 seconds);
// PR_LOG args = "format string", arg, arg, ...
#if defined(PR_LOGGING)
extern PRLogModuleInfo * gCacheLog;
void CacheLogInit();
#define CACHE_LOG_INIT() CacheLogInit()
#define CACHE_LOG_ALWAYS(args) PR_LOG(gCacheLog, PR_LOG_ALWAYS, args)
#define CACHE_LOG_ERROR(args) PR_LOG(gCacheLog, PR_LOG_ERROR, args)
#define CACHE_LOG_WARNING(args) PR_LOG(gCacheLog, PR_LOG_WARNING, args)
#define CACHE_LOG_DEBUG(args) PR_LOG(gCacheLog, PR_LOG_DEBUG, args)
#else
#define CACHE_LOG_INIT() {}
#define CACHE_LOG_ALWAYS(args) {}
#define CACHE_LOG_ERROR(args) {}
#define CACHE_LOG_WARNING(args) {}
#define CACHE_LOG_DEBUG(args) {}
#endif

View File

@ -115,6 +115,18 @@ nsCacheEntry::SetMetaDataElement( const nsAReadableCString& key,
}
nsresult
nsCacheEntry::VisitMetaDataElements( nsICacheMetaDataVisitor * visitor)
{
NS_ENSURE_ARG_POINTER(visitor);
if (mMetaData)
mMetaData->VisitElements(visitor);
return NS_OK;
}
nsresult
nsCacheEntry::FlattenMetaData(char ** data, PRUint32 * size)
{
@ -535,7 +547,6 @@ nsCacheEntryHashTable::MoveEntry(PLDHashTable * /* table */,
const PLDHashEntryHdr *from,
PLDHashEntryHdr *to)
{
to->keyHash = from->keyHash;
((nsCacheEntryHashTableEntry *)to)->cacheEntry =
((nsCacheEntryHashTableEntry *)from)->cacheEntry;
}
@ -545,7 +556,6 @@ void PR_CALLBACK
nsCacheEntryHashTable::ClearEntry(PLDHashTable * /* table */,
PLDHashEntryHdr * hashEntry)
{
((nsCacheEntryHashTableEntry *)hashEntry)->keyHash = 0;
((nsCacheEntryHashTableEntry *)hashEntry)->cacheEntry = 0;
}

View File

@ -95,8 +95,10 @@ public:
nsresult SetMetaDataElement( const nsAReadableCString& key,
const nsAReadableCString& value);
nsresult FlattenMetaData(char ** data, PRUint32 * size);
nsresult UnflattenMetaData(char * data, PRUint32 size);
nsresult VisitMetaDataElements( nsICacheMetaDataVisitor * visitor);
nsresult FlattenMetaData( char ** data, PRUint32 * size);
nsresult UnflattenMetaData( char * data, PRUint32 size);
PRUint32 MetaDataSize() { return mMetaSize;}
@ -122,8 +124,13 @@ public:
eStreamDataMask = 0x00001000,
eActiveMask = 0x00002000,
eInitializedMask = 0x00004000,
eValidMask = 0x00008000
eValidMask = 0x00008000,
eBindingMask = 0x00010000
};
void MarkBinding() { mFlags |= eBindingMask; }
void ClearBinding() { mFlags &= ~eBindingMask; }
PRBool IsBinding() { return (mFlags & eBindingMask) != 0; }
void MarkEntryDirty() { mFlags |= eEntryDirtyMask; }
void MarkEntryClean() { mFlags &= ~eEntryDirtyMask; }

View File

@ -395,12 +395,12 @@ nsCacheEntryDescriptor::SetMetaDataElement(const char *key, const char *value)
NS_IMETHODIMP
nsCacheEntryDescriptor::GetMetaDataEnumerator(nsISimpleEnumerator ** result)
nsCacheEntryDescriptor::VisitMetaData(nsICacheMetaDataVisitor * visitor)
{
NS_ENSURE_ARG_POINTER(result);
NS_ENSURE_ARG_POINTER(visitor);
if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE;
return NS_ERROR_NOT_IMPLEMENTED;
return mCacheEntry->VisitMetaDataElements(visitor);
}

View File

@ -23,6 +23,7 @@
#include "nsCacheMetaData.h"
#include "nsString.h"
#include "nsICacheEntryDescriptor.h"
/*
@ -167,7 +168,7 @@ nsCacheMetaData::FlattenMetaData(char ** data, PRUint32 * size)
*data = new char[*size];
if (*data == nsnull) return NS_ERROR_OUT_OF_MEMORY;
char* state = *data;
PL_DHashTableEnumerate(&table, AccumulateElements, &state);
PL_DHashTableEnumerate(&table, AccumulateElement, &state);
}
return NS_OK;
@ -194,6 +195,15 @@ nsCacheMetaData::UnflattenMetaData(char * data, PRUint32 size)
return rv;
}
nsresult
nsCacheMetaData::VisitElements(nsICacheMetaDataVisitor * visitor)
{
(void) PL_DHashTableEnumerate(&table, VisitElement, visitor);
return NS_OK;
}
/*
* hash table operation callback functions
*/
@ -231,7 +241,6 @@ nsCacheMetaData::MoveEntry(PLDHashTable * /* table */,
const PLDHashEntryHdr *from,
PLDHashEntryHdr *to)
{
to->keyHash = from->keyHash;
((nsCacheMetaDataHashTableEntry *)to)->key =
((nsCacheMetaDataHashTableEntry *)from)->key;
((nsCacheMetaDataHashTableEntry *)to)->value =
@ -243,7 +252,6 @@ void PR_CALLBACK
nsCacheMetaData::ClearEntry(PLDHashTable * /* table */,
PLDHashEntryHdr * hashEntry)
{
((nsCacheMetaDataHashTableEntry *)hashEntry)->keyHash = 0;
((nsCacheMetaDataHashTableEntry *)hashEntry)->key = 0;
((nsCacheMetaDataHashTableEntry *)hashEntry)->value = 0;
}
@ -252,7 +260,7 @@ nsCacheMetaData::ClearEntry(PLDHashTable * /* table */,
void PR_CALLBACK
nsCacheMetaData::Finalize(PLDHashTable * table)
{
(void) PL_DHashTableEnumerate(table, FreeElements, nsnull);
(void) PL_DHashTableEnumerate(table, FreeElement, nsnull);
}
@ -272,7 +280,7 @@ nsCacheMetaData::CalculateSize(PLDHashTable *table,
}
PLDHashOperator PR_CALLBACK
nsCacheMetaData::AccumulateElements(PLDHashTable *table,
nsCacheMetaData::AccumulateElement(PLDHashTable *table,
PLDHashEntryHdr *hdr,
PRUint32 number,
void *arg)
@ -289,7 +297,7 @@ nsCacheMetaData::AccumulateElements(PLDHashTable *table,
}
PLDHashOperator PR_CALLBACK
nsCacheMetaData::FreeElements(PLDHashTable *table,
nsCacheMetaData::FreeElement(PLDHashTable *table,
PLDHashEntryHdr *hdr,
PRUint32 number,
void *arg)
@ -299,3 +307,21 @@ nsCacheMetaData::FreeElements(PLDHashTable *table,
delete entry->value;
return PL_DHASH_NEXT;
}
PLDHashOperator PR_CALLBACK
nsCacheMetaData::VisitElement(PLDHashTable *table,
PLDHashEntryHdr *hdr,
PRUint32 number,
void *arg)
{
nsCacheMetaDataHashTableEntry *entry = (nsCacheMetaDataHashTableEntry *)hdr;
nsICacheMetaDataVisitor *visitor = (nsICacheMetaDataVisitor *)arg;
const char * key = entry->key ? entry->key->get() : nsnull;
const char * value = entry->value ? entry->value->get() : nsnull;
PRBool keepGoing;
nsresult rv = visitor->VisitMetaDataElement(key, value, &keepGoing);
return NS_SUCCEEDED(rv) && keepGoing ? PL_DHASH_NEXT : PL_DHASH_STOP;
}

View File

@ -31,6 +31,8 @@
#include "nsString.h"
// #include "nsAReadableString.h"
class nsICacheMetaDataVisitor;
typedef struct {
nsCString * key;
nsCString * value;
@ -65,6 +67,8 @@ public:
nsresult UnflattenMetaData(char * data, PRUint32 size);
nsresult VisitElements(nsICacheMetaDataVisitor * visitor);
private:
// PLDHashTable operation callbacks
static const void * PR_CALLBACK GetKey( PLDHashTable *table, PLDHashEntryHdr *entry);
@ -90,16 +94,21 @@ private:
void *arg);
static
PLDHashOperator PR_CALLBACK AccumulateElements(PLDHashTable *table,
PLDHashOperator PR_CALLBACK AccumulateElement(PLDHashTable *table,
PLDHashEntryHdr *hdr,
PRUint32 number,
void *arg);
static
PLDHashOperator PR_CALLBACK FreeElements(PLDHashTable *table,
PLDHashOperator PR_CALLBACK FreeElement(PLDHashTable *table,
PLDHashEntryHdr *hdr,
PRUint32 number,
void *arg);
static
PLDHashOperator PR_CALLBACK VisitElement(PLDHashTable *table,
PLDHashEntryHdr *hdr,
PRUint32 number,
void *arg);
// member variables
static PLDHashTableOps ops;

View File

@ -201,6 +201,8 @@ nsCacheService::Init()
if (mCacheServiceLock)
return NS_ERROR_ALREADY_INITIALIZED;
CACHE_LOG_INIT();
mCacheServiceLock = PR_NewLock();
if (mCacheServiceLock == nsnull)
return NS_ERROR_OUT_OF_MEMORY;
@ -275,6 +277,9 @@ nsCacheService::Shutdown()
printf("### beging nsCacheService::Shutdown()\n");
#endif
#if defined(PR_LOGGING)
LogCacheStatistics();
#endif
// Clear entries
ClearDoomList();
ClearActiveEntries();
@ -369,19 +374,31 @@ NS_IMETHODIMP nsCacheService::VisitEntries(nsICacheVisitor *visitor)
NS_IMETHODIMP nsCacheService::EvictEntries(nsCacheStoragePolicy storagePolicy)
{
nsresult rv = NS_ERROR_NOT_IMPLEMENTED;
nsresult rv;
nsAutoLock lock(mCacheServiceLock);
if (storagePolicy == nsICache::STORE_ON_DISK) {
// XXX what should we do about error handling?
if (storagePolicy == nsICache::STORE_ANYWHERE || storagePolicy == nsICache::STORE_ON_DISK) {
if (mEnableDiskDevice) {
if (!mDiskDevice) {
rv = CreateDiskDevice();
if (NS_FAILED(rv)) return rv;
}
rv = mDiskDevice->EvictEntries(nsnull);
if (NS_FAILED(rv)) return rv;
}
}
if (storagePolicy == nsICache::STORE_ANYWHERE || storagePolicy == nsICache::STORE_IN_MEMORY) {
if (mEnableMemoryDevice) {
rv = mMemoryDevice->EvictEntries(nsnull);
if (NS_FAILED(rv)) return rv;
}
}
return rv;
return NS_OK;
}
@ -715,7 +732,9 @@ nsCacheService::EnsureEntryHasDevice(nsCacheEntry * entry)
if (device == nsnull) return nsnull;
entry->MarkBinding(); // XXX
nsresult rv = device->BindEntry(entry);
entry->ClearBinding(); // XXX
if (NS_FAILED(rv)) return nsnull;
entry->SetCacheDevice(device);
@ -755,10 +774,11 @@ nsCacheService::DoomEntry_Locked(nsCacheEntry * entry)
{
if (this == nsnull) return NS_ERROR_NOT_AVAILABLE;
if (entry->IsDoomed()) return NS_OK;
nsresult rv = NS_OK;
entry->MarkDoomed();
NS_ASSERTION(!entry->IsBinding(), "Dooming entry while binding device.");
nsCacheDevice * device = entry->CacheDevice();
if (device) device->DoomEntry(entry);
@ -844,9 +864,7 @@ nsCacheService::CloseDescriptor(nsCacheEntryDescriptor * descriptor)
nsresult rv = NS_OK;
if (!entry->IsValid()) {
if (PR_CLIST_IS_EMPTY(&entry->mRequestQ)) {
rv = ProcessPendingRequests(entry);
}
rv = ProcessPendingRequests(entry);
}
if (!stillActive) {
@ -1035,3 +1053,22 @@ NS_IMETHODIMP nsCacheService::Observe(nsISupports *aSubject, const PRUnichar *aT
return Shutdown();
}
#if defined(PR_LOGGING)
void
nsCacheService::LogCacheStatistics()
{
PRUint32 hitPercentage = (PRUint32)((((double)mCacheHits) /
((double)(mCacheHits + mCacheMisses))) * 100);
CACHE_LOG_ALWAYS(("\nCache Service Statistics:\n\n"));
CACHE_LOG_ALWAYS((" TotalEntries = %d\n", mTotalEntries));
CACHE_LOG_ALWAYS((" Cache Hits = %d\n", mCacheHits));
CACHE_LOG_ALWAYS((" Cache Misses = %d\n", mCacheMisses));
CACHE_LOG_ALWAYS((" Cache Hit %% = %d%%\n", hitPercentage));
CACHE_LOG_ALWAYS((" Max Key Length = %d\n", mMaxKeyLength));
CACHE_LOG_ALWAYS((" Max Meta Size = %d\n", mMaxMetaSize));
CACHE_LOG_ALWAYS((" Max Data Size = %d\n", mMaxDataSize));
CACHE_LOG_ALWAYS(("\n"));
CACHE_LOG_ALWAYS((" Deactivate Failures = %d\n", mDeactivateFailures));
CACHE_LOG_ALWAYS((" Deactivated Unbound Entries = %d\n", mDeactivatedUnboundEntries));
}
#endif

View File

@ -141,7 +141,9 @@ private:
PLDHashEntryHdr * hdr,
PRUint32 number,
void * arg);
#if defined(PR_LOGGING)
void LogCacheStatistics();
#endif
/**
* Data Members
*/
@ -165,6 +167,7 @@ private:
PRCList mDoomedEntries;
// stats
PRUint32 mTotalEntries;
PRUint32 mCacheHits;
PRUint32 mCacheMisses;

View File

@ -48,7 +48,7 @@ public:
{
}
PRUint32 HashNumber()
PRUint32 HashNumber() const
{
return mHashNumber;
}
@ -58,7 +58,7 @@ public:
mHashNumber = hashNumber;
}
PRUint32 EvictionRank()
PRUint32 EvictionRank() const
{
return mEvictionRank;
}
@ -68,7 +68,7 @@ public:
mEvictionRank = rank;
}
PRUint32 LocationSelector()
PRUint32 LocationSelector() const
{
return (PRUint32)(mLocation & eLocationSelectorMask) >> 22;
}
@ -79,7 +79,7 @@ public:
mLocation |= (selector & eLocationSelectorMask) << 22;
}
PRUint32 BlockCount()
PRUint32 BlockCount() const
{
return (PRUint32)((mLocation & eExtraBlocksMask) >> 20) + 1;
}
@ -92,7 +92,7 @@ public:
mLocation |= (count & eExtraBlocksMask) << 20;
}
PRUint32 BlockNumber()
PRUint32 BlockNumber() const
{
return (mLocation & eBlockNumberMask);
}
@ -103,7 +103,7 @@ public:
mLocation |= blockNumber & eBlockNumberMask;
}
PRUint16 FileGeneration()
PRUint16 FileGeneration() const
{
return (mLocation & eFileGenerationMask);
}

View File

@ -43,6 +43,8 @@
#include "nsIObserverService.h"
#include "nsIPref.h"
#include "nsQuickSort.h"
static nsresult ensureCacheDirectory(nsIFile * cacheDirectory);
static const char DISK_CACHE_DEVICE_ID[] = { "disk" };
@ -99,7 +101,7 @@ NS_IMETHODIMP nsDiskCacheObserver::Observe(nsISupports *aSubject, const PRUnicha
PRInt32 cacheCapacity;
rv = prefs->GetIntPref(CACHE_DISK_CAPACITY_PREF, &cacheCapacity);
if (NS_SUCCEEDED(rv))
mDevice->setCacheCapacity(cacheCapacity);
mDevice->setCacheCapacity(cacheCapacity * 1024);
}
} else if (NS_LITERAL_STRING("profile-do-change").Equals(aTopic)) {
// XXX need to regenerate the cache directory. hopefully the
@ -595,14 +597,6 @@ private:
nsresult* mErrorPtr;
};
#if 0
inline const nsQueryElementAt
do_QueryElementAt( nsICollection* aCollection, PRUint32 aIndex, nsresult* aErrorPtr = 0 )
{
return nsQueryElementAt(aCollection, aIndex, aErrorPtr);
}
#endif
// XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
static nsCOMPtr<nsIFileTransportService> gFileTransportService;
@ -731,18 +725,16 @@ nsDiskCacheDevice::FindEntry(nsCString * key)
nsresult
nsDiskCacheDevice::DeactivateEntry(nsCacheEntry * entry)
{
if (!entry->IsDoomed()) {
nsDiskCacheEntry * diskEntry = mBoundEntries.GetEntry(entry->Key()->get());
NS_ASSERTION(diskEntry, "DeactivateEntry called for an entry we don't have!");
if (!diskEntry)
return NS_ERROR_INVALID_POINTER;
// commit any changes about this entry to disk.
updateDiskCacheEntry(diskEntry);
nsDiskCacheEntry* diskEntry = ensureDiskCacheEntry(entry);
if (mBoundEntries.GetEntry(diskEntry->getHashNumber()) == diskEntry) {
// XXX eventually, as a performance enhancement, keep entries around for a while before deleting them.
// XXX right now, to prove correctness, destroy the entries eagerly.
mBoundEntries.RemoveEntry(diskEntry);
}
if (!entry->IsDoomed()) {
// commit any changes about this entry to disk.
updateDiskCacheEntry(diskEntry);
// XXX if this entry collided with other concurrently bound entries, then its
// generation count will be non-zero. The other entries that came before it
@ -755,10 +747,8 @@ nsDiskCacheDevice::DeactivateEntry(nsCacheEntry * entry)
delete entry;
} else {
// obliterate all knowledge of this entry on disk.
nsDiskCacheEntry* diskEntry = ensureDiskCacheEntry(entry);
NS_ASSERTION(diskEntry, "nsDiskCacheDevice::DeactivateEntry");
deleteDiskCacheEntry(diskEntry);
// XXX if this entry resides on a list, then there must have been a collision
// during the entry's lifetime. use this deactivation as a trigger to scavenge
// generation numbers, and reset the live entry's generation to zero.
@ -786,21 +776,32 @@ nsDiskCacheDevice::BindEntry(nsCacheEntry * newEntry)
// XXX check for cache collision. if an entry exists on disk that has the same
// hash code as this newly bound entry, AND there is already a bound entry for
// that key, we need to ask the Cache service to doom that entry, since two
// that key, we need to ask the cache service to doom that entry, since two
// simultaneous entries that have the same hash code aren't allowed until
// some sort of chaining mechanism is implemented.
nsDiskCacheEntry* oldDiskEntry = mBoundEntries.GetEntry(newEntry->Key()->get());
if (oldDiskEntry) {
// set the generation count on the newly bound entry,
// so that files created will be unique and won't conflict
// with the doomed entries that are still active.
if (oldDiskEntry) {
// XXX Hacky liveness test, remove when we've figured this all out.
if (oldDiskEntry->getRefCount() > 1) {
// set the generation count on the newly bound entry,
// so that files created will be unique and won't conflict
// with the doomed entries that are still active.
newDiskEntry->setGeneration(oldDiskEntry->getGeneration() + 1);
PR_APPEND_LINK(newDiskEntry, oldDiskEntry);
// XXX Whom do we tell about this impending doom?
nsCacheEntry* oldEntry = oldDiskEntry->getCacheEntry();
// XXX Yes Virginia, a doomed entry can be bound.
// NS_ASSERTION(!oldEntry->IsDoomed(), "a bound entry is doomed!");
if (!oldEntry->IsDoomed())
nsCacheService::GlobalInstance()->DoomEntry_Locked(oldEntry);
else
mBoundEntries.RemoveEntry(oldDiskEntry);
} else {
// XXX somehow we didn't hear about the entry going away. Ask gordon.
NS_NOTREACHED("bound disk cache entry with no corresponding cache entry.");
mBoundEntries.RemoveEntry(oldDiskEntry);
}
// XXX Whom do we tell about this impending doom?
nsCacheService::GlobalInstance()->DoomEntry_Locked(oldDiskEntry->getCacheEntry());
}
rv = mBoundEntries.AddEntry(newDiskEntry);
@ -884,12 +885,10 @@ nsDiskCacheDevice::OnDataSizeChange(nsCacheEntry * entry, PRInt32 deltaSize)
{
PRUint32 newCacheSize = (mCacheMap->DataSize() += deltaSize);
#if 0
if (newCacheSize > mCacheCapacity) {
// XXX go toss out some disk cache entries.
evictDiskCacheEntries();
}
#endif
return NS_OK;
}
@ -910,30 +909,29 @@ nsDiskCacheDevice::Visit(nsICacheVisitor * visitor)
return NS_OK;
}
nsresult
nsDiskCacheDevice::EvictEntries(const char * clientID)
{
nsCOMPtr<nsISupportsArray> entries;
nsresult rv = scanDiskCacheEntries(getter_AddRefs(entries));
if (NS_FAILED(rv)) return rv;
nsresult rv;
PRUint32 prefixLength = (clientID ? nsCRT::strlen(clientID) : 0);
PRUint32 newDataSize = mCacheMap->DataSize();
PRUint32 newEntryCount = mCacheMap->EntryCount();
// XXX make sure meta data is up to date.
rv = updateDiskCacheEntries();
if (NS_FAILED(rv)) return rv;
PRUint32 count;
entries->Count(&count);
for (PRUint32 i = 0; i < count; ++i) {
nsCOMPtr<nsICacheEntryInfo> info = do_QueryElementAt(entries, i, &rv);
if (NS_SUCCEEDED(rv)) {
nsDiskCacheEntryInfo* entryInfo = (nsDiskCacheEntryInfo*) info.get();
const char* key = entryInfo->Key();
// if filtering by clientID, make sure key prefix and clientID match.
if (clientID && nsCRT::strncmp(clientID, key, prefixLength) != 0)
continue;
for (PRUint32 i = 1; i < nsDiskCacheMap::kBucketsPerTable; ++i) {
nsDiskCacheRecord* bucket = mCacheMap->GetBucket(i);
for (PRUint32 j = 0; j < nsDiskCacheMap::kRecordsPerBucket; ++j) {
nsDiskCacheRecord* record = bucket++;
if (record->HashNumber() == 0)
break;
// if the entry is currently in use, then doom it rather than evicting right here.
nsDiskCacheEntry* diskEntry = mBoundEntries.GetEntry(key);
nsDiskCacheEntry* diskEntry = mBoundEntries.GetEntry(record->HashNumber());
if (diskEntry) {
nsCacheService::GlobalInstance()->DoomEntry_Locked(diskEntry->getCacheEntry());
continue;
@ -941,30 +939,62 @@ nsDiskCacheDevice::EvictEntries(const char * clientID)
// delete the metadata file.
nsCOMPtr<nsIFile> metaFile;
rv = getFileForKey(key, PR_TRUE, 0, getter_AddRefs(metaFile));
rv = getFileForHashNumber(record->HashNumber(), PR_TRUE, 0, getter_AddRefs(metaFile));
if (NS_SUCCEEDED(rv)) {
if (clientID) {
// if filtering by clientID, make sure key prefix and clientID match.
// if anything fails, assume they don't by continuing with the loop.
nsCOMPtr<nsIInputStream> input;
rv = openInputStream(metaFile, getter_AddRefs(input));
if (NS_FAILED(rv)) continue;
// read the metadata file.
MetaDataFile metaDataFile;
rv = metaDataFile.Read(input);
input->Close();
if (NS_FAILED(rv)) continue;
if (nsCRT::strncmp(clientID, metaDataFile.mKey, prefixLength) != 0)
continue;
}
rv = metaFile->Delete(PR_FALSE);
}
PRUint32 dataSize = 0;
// delete the data file
nsCOMPtr<nsIFile> dataFile;
rv = getFileForKey(key, PR_FALSE, 0, getter_AddRefs(dataFile));
rv = getFileForHashNumber(record->HashNumber(), PR_FALSE, 0, getter_AddRefs(dataFile));
if (NS_SUCCEEDED(rv)) {
PRInt64 fileSize;
rv = dataFile->GetFileSize(&fileSize);
if (NS_SUCCEEDED(rv))
LL_L2I(dataSize, fileSize);
rv = dataFile->Delete(PR_FALSE);
}
// update the cache size.
PRUint32 dataSize;
info->GetDataSize(&dataSize);
newDataSize -= dataSize;
// remove from cache map.
nsDiskCacheRecord* deletedRecord = mCacheMap->GetRecord(record->HashNumber());
if (record->HashNumber() == deletedRecord->HashNumber() && record->FileGeneration() == deletedRecord->FileGeneration())
mCacheMap->DeleteRecord(deletedRecord);
// update the cache size/entry count.
if (newDataSize >= dataSize) newDataSize -= dataSize;
if (newEntryCount) --newEntryCount;
}
}
mCacheMap->DataSize() = newDataSize;
if (clientID) {
mCacheMap->DataSize() = newDataSize;
mCacheMap->EntryCount() = newEntryCount;
} else {
mCacheMap->DataSize() = 0;
mCacheMap->EntryCount() = 0;
}
return NS_OK;
}
void nsDiskCacheDevice::setPrefsObserver(nsIObserver* observer)
{
mPrefsObserver = observer;
@ -982,8 +1012,13 @@ void nsDiskCacheDevice::setCacheDirectory(nsILocalFile* cacheDirectory)
void nsDiskCacheDevice::setCacheCapacity(PRUint32 capacity)
{
// XXX start evicting entries if the new size is smaller!
mCacheCapacity = capacity;
if (mInitialized) {
// XXX start evicting entries if the new size is smaller!
// XXX need to enter cache service lock here!
if (mCacheMap->DataSize() > capacity)
evictDiskCacheEntries();
}
}
PRUint32 nsDiskCacheDevice::getCacheCapacity()
@ -1101,33 +1136,36 @@ inline PRBool isMetaDataFile(const char* name)
return (name[8] == 'm');
}
// XXX All these transports and opening/closing of files. We need a way to cache open files,
// XXX and to seek. Perhaps I should just be using ANSI FILE objects for all of the metadata
// XXX operations.
nsresult nsDiskCacheDevice::visitEntries(nsICacheVisitor * visitor)
{
nsCOMPtr<nsISimpleEnumerator> entries;
nsresult rv = mCacheDirectory->GetDirectoryEntries(getter_AddRefs(entries));
if (NS_FAILED(rv)) return rv;
nsresult rv;
// XXX make sure meta data is up to date.
rv = updateDiskCacheEntries();
if (NS_FAILED(rv)) return rv;
nsDiskCacheEntryInfo* entryInfo = new nsDiskCacheEntryInfo();
if (!entryInfo) return NS_ERROR_OUT_OF_MEMORY;
nsCOMPtr<nsICacheEntryInfo> ref(entryInfo);
for (PRBool more; NS_SUCCEEDED(entries->HasMoreElements(&more)) && more;) {
nsCOMPtr<nsISupports> next;
rv = entries->GetNext(getter_AddRefs(next));
if (NS_FAILED(rv)) break;
nsCOMPtr<nsIFile> file(do_QueryInterface(next, &rv));
if (NS_FAILED(rv)) break;
nsXPIDLCString name;
rv = file->GetLeafName(getter_Copies(name));
if (isMetaDataFile(name)) {
// this must be a metadata file.
for (PRUint32 i = 1; i < nsDiskCacheMap::kBucketsPerTable; ++i) {
nsDiskCacheRecord* bucket = mCacheMap->GetBucket(i);
for (PRUint32 j = 0; j < nsDiskCacheMap::kRecordsPerBucket; ++j) {
nsDiskCacheRecord* record = bucket++;
if (record->HashNumber() == 0)
break;
nsCOMPtr<nsIFile> file;
rv = getFileForHashNumber(record->HashNumber(), PR_TRUE, record->FileGeneration(), getter_AddRefs(file));
if (NS_FAILED(rv)) continue;
nsCOMPtr<nsIInputStream> input;
rv = openInputStream(file, getter_AddRefs(input));
if (NS_FAILED(rv)) continue;
if (NS_FAILED(rv)) {
// delete non-existent record.
mCacheMap->DeleteRecord(record);
--bucket;
continue;
}
// read the metadata file.
rv = entryInfo->Read(input);
@ -1225,48 +1263,44 @@ nsresult nsDiskCacheDevice::readDiskCacheEntry(const char * key, nsDiskCacheEntr
nsCOMPtr<nsIFile> file;
nsresult rv = getFileForKey(key, PR_TRUE, 0, getter_AddRefs(file));
if (NS_FAILED(rv)) return rv;
PRBool exists;
rv = file->Exists(&exists);
if (NS_FAILED(rv) || !exists) return NS_ERROR_NOT_AVAILABLE;
nsCacheEntry* entry = nsnull;
do {
nsCOMPtr<nsIInputStream> input;
rv = openInputStream(file, getter_AddRefs(input));
if (NS_FAILED(rv)) break;
// read the metadata file.
MetaDataFile metaDataFile;
rv = metaDataFile.Read(input);
input->Close();
if (NS_FAILED(rv)) break;
// Ensure that the keys match.
if (nsCRT::strcmp(key, metaDataFile.mKey) != 0) break;
rv = NS_NewCacheEntry(&entry, key, PR_TRUE, nsICache::STORE_ON_DISK, this);
if (NS_FAILED(rv)) return rv;
nsCOMPtr<nsIInputStream> input;
rv = openInputStream(file, getter_AddRefs(input));
if (NS_FAILED(rv)) return rv;
// initialize the entry.
entry->SetFetchCount(metaDataFile.mFetchCount);
entry->SetLastFetched(metaDataFile.mLastFetched);
entry->SetLastModified(metaDataFile.mLastModified);
entry->SetExpirationTime(metaDataFile.mExpirationTime);
entry->SetDataSize(metaDataFile.mDataSize);
// read the metadata file.
MetaDataFile metaDataFile;
rv = metaDataFile.Read(input);
input->Close();
if (NS_FAILED(rv)) return rv;
// restore the metadata.
if (metaDataFile.mMetaDataSize) {
rv = entry->UnflattenMetaData(metaDataFile.mMetaData, metaDataFile.mMetaDataSize);
if (NS_FAILED(rv)) break;
}
// celebrate!
*result = ensureDiskCacheEntry(entry);
if (!*result) break;
return NS_OK;
} while (0);
// Ensure that the keys match.
if (nsCRT::strcmp(key, metaDataFile.mKey) != 0) return NS_ERROR_NOT_AVAILABLE;
nsCacheEntry* entry = nsnull;
rv = NS_NewCacheEntry(&entry, key, PR_TRUE, nsICache::STORE_ON_DISK, this);
if (NS_FAILED(rv)) return rv;
// oh, auto_ptr<> would be nice right about now.
// initialize the entry.
entry->SetFetchCount(metaDataFile.mFetchCount);
entry->SetLastFetched(metaDataFile.mLastFetched);
entry->SetLastModified(metaDataFile.mLastModified);
entry->SetExpirationTime(metaDataFile.mExpirationTime);
entry->SetDataSize(metaDataFile.mDataSize);
// restore the metadata.
if (metaDataFile.mMetaDataSize) {
rv = entry->UnflattenMetaData(metaDataFile.mMetaData, metaDataFile.mMetaDataSize);
if (NS_FAILED(rv)) goto error;
}
// celebrate!
*result = ensureDiskCacheEntry(entry);
if (!*result) goto error;
return NS_OK;
error:
// oh, auto_ptr<> would be nice right about now.
delete entry;
return NS_ERROR_NOT_AVAILABLE;
}
@ -1294,7 +1328,9 @@ nsresult nsDiskCacheDevice::deleteDiskCacheEntry(nsDiskCacheEntry * diskEntry)
}
// remove from cache map.
mCacheMap->DeleteRecord(diskEntry->getHashNumber());
nsDiskCacheRecord* record = mCacheMap->GetRecord(diskEntry->getHashNumber());
if (record->HashNumber() == diskEntry->getHashNumber() && record->FileGeneration() == diskEntry->getGeneration())
mCacheMap->DeleteRecord(record);
return NS_OK;
}
@ -1445,53 +1481,86 @@ nsresult nsDiskCacheDevice::scanDiskCacheEntries(nsISupportsArray ** result)
return NS_OK;
}
static int compareRecords(const void* e1, const void* e2, void* /*unused*/)
{
const nsDiskCacheRecord* r1 = (const nsDiskCacheRecord*) e1;
const nsDiskCacheRecord* r2 = (const nsDiskCacheRecord*) e2;
return (r1->EvictionRank() - r2->EvictionRank());
}
nsresult nsDiskCacheDevice::evictDiskCacheEntries()
{
nsCOMPtr<nsISupportsArray> entries;
nsresult rv = scanDiskCacheEntries(getter_AddRefs(entries));
if (NS_FAILED(rv)) return rv;
nsresult rv;
if (mCacheMap->DataSize() < mCacheCapacity) return NS_OK;
// these are sorted in oldest to newest order.
PRUint32 count;
entries->Count(&count);
for (PRUint32 i = 0; i < count; ++i) {
nsCOMPtr<nsICacheEntryInfo> info = do_QueryElementAt(entries, i, &rv);
if (NS_SUCCEEDED(rv)) {
nsDiskCacheEntryInfo* entryInfo = (nsDiskCacheEntryInfo*) info.get();
const char* key = entryInfo->Key();
// XXX if this entry is currently active, then leave it alone,
// as it is likely to be modified very soon.
nsDiskCacheEntry* diskEntry = mBoundEntries.GetEntry(key);
if (diskEntry) continue;
// delete the metadata file.
nsCOMPtr<nsIFile> metaFile;
rv = getFileForKey(key, PR_TRUE, 0, getter_AddRefs(metaFile));
if (NS_SUCCEEDED(rv)) {
rv = metaFile->Delete(PR_FALSE);
}
// delete the data file
nsCOMPtr<nsIFile> dataFile;
rv = getFileForKey(key, PR_FALSE, 0, getter_AddRefs(dataFile));
if (NS_SUCCEEDED(rv)) {
rv = dataFile->Delete(PR_FALSE);
}
// update the cache size.
PRUint32 dataSize;
info->GetDataSize(&dataSize);
if ((mCacheMap->DataSize() -= dataSize) <= mCacheCapacity)
break;
// XXX make sure meta data is up to date.
rv = updateDiskCacheEntries();
if (NS_FAILED(rv)) return rv;
// 1. gather all records into an array, sorted by eviction rank. keep deleting them until we recover enough space.
PRUint32 count = 0;
nsDiskCacheRecord* sortedRecords = new nsDiskCacheRecord[mCacheMap->EntryCount()];
if (sortedRecords) {
for (PRUint32 i = 1; i < nsDiskCacheMap::kBucketsPerTable; ++i) {
nsDiskCacheRecord* bucket = mCacheMap->GetBucket(i);
for (PRUint32 j = 0; j < nsDiskCacheMap::kRecordsPerBucket; ++j) {
nsDiskCacheRecord* record = bucket++;
if (record->HashNumber() == 0)
break;
sortedRecords[count++] = *record;
}
}
NS_QuickSort((void*)sortedRecords, count, sizeof(nsDiskCacheRecord), compareRecords, nsnull);
} else {
return NS_ERROR_OUT_OF_MEMORY;
}
// these are sorted by eviction rank. lower eviction ranks are more eligible for eviction.
for (PRUint32 i = 0; i < count; ++i) {
nsDiskCacheRecord* record = &sortedRecords[i];
// XXX if this entry is currently active, then leave it alone,
// as it is likely to be modified very soon.
nsDiskCacheEntry* diskEntry = mBoundEntries.GetEntry(record->HashNumber());
if (diskEntry) continue;
// delete the metadata file.
nsCOMPtr<nsIFile> metaFile;
rv = getFileForHashNumber(record->HashNumber(), PR_TRUE, 0, getter_AddRefs(metaFile));
if (NS_SUCCEEDED(rv)) {
rv = metaFile->Delete(PR_FALSE);
}
PRUint32 dataSize = 0;
// delete the data file
nsCOMPtr<nsIFile> dataFile;
rv = getFileForHashNumber(record->HashNumber(), PR_FALSE, 0, getter_AddRefs(dataFile));
if (NS_SUCCEEDED(rv)) {
PRInt64 fileSize;
rv = dataFile->GetFileSize(&fileSize);
if (NS_SUCCEEDED(rv))
LL_L2I(dataSize, fileSize);
rv = dataFile->Delete(PR_FALSE);
}
// remove from cache map.
nsDiskCacheRecord* deletedRecord = mCacheMap->GetRecord(record->HashNumber());
if (record->HashNumber() == deletedRecord->HashNumber() && record->FileGeneration() == deletedRecord->FileGeneration())
mCacheMap->DeleteRecord(deletedRecord);
// update the cache size.
if ((mCacheMap->DataSize() -= dataSize) <= mCacheCapacity)
break;
}
delete[] sortedRecords;
return NS_OK;
}
nsresult nsDiskCacheDevice::readCacheMap()
{
nsCOMPtr<nsIFile> file;
@ -1538,6 +1607,8 @@ nsresult nsDiskCacheDevice::updateCacheMap(nsDiskCacheEntry * diskEntry)
if (record->HashNumber() != 0) {
// eviction of eldest entry in this bucket.
evictDiskCacheRecord(record);
} else {
mCacheMap->EntryCount() += 1;
}
// newly bound record. fill in the blanks.
record->SetHashNumber(diskEntry->getHashNumber());

View File

@ -62,11 +62,6 @@ public:
{
return mTransports[mode - 1];
}
nsCOMPtr<nsITransport>& getMetaTransport(nsCacheAccessMode mode)
{
return mMetaTransports[mode - 1];
}
#endif
nsCacheEntry* getCacheEntry()
@ -89,12 +84,16 @@ public:
return mHashNumber;
}
nsrefcnt getRefCount()
{
return mRefCnt;
}
static PLDHashNumber Hash(const char* key);
private:
#ifdef MOZ_NEW_CACHE_REUSE_TRANSPORTS
nsCOMPtr<nsITransport> mTransports[3];
nsCOMPtr<nsITransport> mMetaTransports[3];
#endif
nsCacheEntry* mCacheEntry;
PRUint32 mGeneration;

View File

@ -50,35 +50,36 @@ nsDiskCacheRecord* nsDiskCacheMap::GetRecord(PRUint32 hashNumber)
return oldestRecord;
}
void nsDiskCacheMap::DeleteRecord(PRUint32 hashNumber)
void nsDiskCacheMap::DeleteRecord(nsDiskCacheRecord* deletedRecord)
{
PRUint32 hashNumber = deletedRecord->HashNumber();
nsDiskCacheBucket& bucket = mBuckets[(hashNumber & (kBucketsPerTable - 1))];
for (int r = 0; r < kRecordsPerBucket; ++r) {
nsDiskCacheRecord* record = &bucket.mRecords[r];
if (record->HashNumber() == hashNumber) {
nsDiskCacheRecord* deletedRecord = record;
nsDiskCacheRecord* lastRecord = nsnull;
// XXX use binary search to find the end, much quicker.
// find the last record, to fill in the deleted record.
for (int j = r + 1; j < kRecordsPerBucket; ++j) {
record = &bucket.mRecords[j];
if (record->HashNumber() == 0) {
lastRecord = record - 1;
break;
}
}
// copy the last record, to the newly deleted record.
if (lastRecord && deletedRecord != lastRecord) {
*deletedRecord = *lastRecord;
deletedRecord = lastRecord;
}
// mark record as free.
deletedRecord->SetHashNumber(0);
NS_ASSERTION(deletedRecord >= &bucket.mRecords[0] &&
deletedRecord < &bucket.mRecords[kRecordsPerBucket],
"invalid record to delete.");
nsDiskCacheRecord* limit = &bucket.mRecords[kRecordsPerBucket];
nsDiskCacheRecord* lastRecord = nsnull;
// XXX use binary search to find the end, much quicker.
// find the last record, to fill in the deleted record.
for (nsDiskCacheRecord* record = deletedRecord + 1; record < limit; ++record) {
if (record->HashNumber() == 0) {
lastRecord = record - 1;
break;
}
}
// copy the last record, to the newly deleted record.
if (lastRecord && deletedRecord != lastRecord) {
*deletedRecord = *lastRecord;
deletedRecord = lastRecord;
}
// mark record as free.
deletedRecord->SetHashNumber(0);
// reduce the number of entries.
mHeader.mEntryCount--;
}
nsresult nsDiskCacheMap::Read(nsIInputStream* input)
{
nsresult rv;

View File

@ -64,20 +64,21 @@ public:
~nsDiskCacheMap();
PRUint32& DataSize() { return mHeader.mDataSize; }
PRUint32& EntryCount() { return mHeader.mEntryCount; }
nsDiskCacheRecord* GetRecord(PRUint32 hashNumber);
void DeleteRecord(PRUint32 hashNumber);
nsresult Read(nsIInputStream* input);
nsresult Write(nsIOutputStream* output);
void DeleteRecord(nsDiskCacheRecord* record);
enum {
kRecordsPerBucket = 256,
kBucketsPerTable = (1 << 5) // must be a power of 2!
};
nsDiskCacheRecord* GetBucket(PRUint32 index) { return mBuckets[index].mRecords; }
nsresult Read(nsIInputStream* input);
nsresult Write(nsIOutputStream* output);
private:
struct nsDiskCacheBucket {
nsDiskCacheRecord mRecords[kRecordsPerBucket];

View File

@ -18,7 +18,8 @@
* Rights Reserved.
*
* Contributor(s):
* Gordon Sheridan, 22-February-2001
* Gordon Sheridan, <gordon@netscape.com>
* Patrick C. Beard <beard@netscape.com>
*/
#include "nsMemoryCacheDevice.h"
@ -155,6 +156,10 @@ nsMemoryCacheDevice::DeactivateEntry(nsCacheEntry * entry)
#if debug
// XXX verify we've removed it from mMemCacheEntries & eviction list
#endif
// update statistics
mTotalSize -= entry->Size();
--mEntryCount;
delete entry;
return NS_OK;
}
@ -177,15 +182,17 @@ nsMemoryCacheDevice::BindEntry(nsCacheEntry * entry)
{
NS_ASSERTION(PR_CLIST_IS_EMPTY(entry),"entry is already on a list!");
// append entry to the eviction list
PR_APPEND_LINK(entry, &mEvictionList);
if (!entry->IsDoomed()) {
// append entry to the eviction list
PR_APPEND_LINK(entry, &mEvictionList);
// add entry to hashtable of mem cache entries
nsresult rv = mMemCacheEntries.AddEntry(entry);
if (NS_FAILED(rv)) {
PR_REMOVE_AND_INIT_LINK(entry);
return rv;
}
// add entry to hashtable of mem cache entries
nsresult rv = mMemCacheEntries.AddEntry(entry);
if (NS_FAILED(rv)) {
PR_REMOVE_AND_INIT_LINK(entry);
return rv;
}
}
// add size of entry to memory totals
++mEntryCount;
@ -206,10 +213,6 @@ nsMemoryCacheDevice::DoomEntry(nsCacheEntry * entry)
// remove entry from our eviction list
PR_REMOVE_AND_INIT_LINK(entry);
// adjust our totals
mTotalSize -= entry->Size();
mInactiveSize -= entry->Size();
}
@ -276,6 +279,25 @@ nsMemoryCacheDevice::AdjustMemoryLimits(PRUint32 softLimit, PRUint32 hardLimit
}
void
nsMemoryCacheDevice::EvictEntry(nsCacheEntry * entry)
{
// remove entry from our hashtable
mMemCacheEntries.RemoveEntry(entry);
// remove entry from the eviction list
PR_REMOVE_AND_INIT_LINK(entry);
// update statistics
PRUint32 memoryRecovered = entry->Size();
mTotalSize -= memoryRecovered;
mInactiveSize -= memoryRecovered;
--mEntryCount;
delete entry;
}
void
nsMemoryCacheDevice::EvictEntriesIfNecessary(void)
{
@ -293,20 +315,8 @@ nsMemoryCacheDevice::EvictEntriesIfNecessary(void)
continue;
}
// remove entry from our hashtable
mMemCacheEntries.RemoveEntry(entry);
// remove entry from the eviction list
next = (nsCacheEntry *)PR_NEXT_LINK(entry);
PR_REMOVE_AND_INIT_LINK(entry);
// update statistics
PRUint32 memoryRecovered = entry->Size();
mTotalSize -= memoryRecovered;
mInactiveSize -= memoryRecovered;
--mEntryCount;
delete entry;
EvictEntry(entry);
entry = next;
if ((mTotalSize < mHardLimit) && (mInactiveSize < mSoftLimit))
@ -353,7 +363,26 @@ nsMemoryCacheDevice::Visit(nsICacheVisitor * visitor)
nsresult
nsMemoryCacheDevice::EvictEntries(const char * clientID)
{
return NS_ERROR_NOT_IMPLEMENTED;
nsCacheEntry * entry;
PRUint32 prefixLength = nsCRT::strlen(clientID);
PRCList * elem = PR_LIST_HEAD(&mEvictionList);
while (elem != &mEvictionList) {
entry = (nsCacheEntry *)elem;
elem = PR_NEXT_LINK(elem);
const char * key = entry->Key()->get();
if (clientID && nsCRT::strncmp(clientID, key, prefixLength) != 0)
continue;
if (entry->IsInUse()) {
nsresult rv = nsCacheService::GlobalInstance()->DoomEntry_Locked(entry);
if (NS_FAILED(rv)) return rv;
} else {
EvictEntry(entry);
}
}
return NS_OK;
}

View File

@ -67,8 +67,9 @@ public:
private:
friend class nsMemoryCacheDeviceInfo;
void AdjustMemoryLimits(PRUint32 softLimit, PRUint32 hardLimit);
void EvictEntriesIfNecessary(void);
void AdjustMemoryLimits( PRUint32 softLimit, PRUint32 hardLimit);
void EvictEntry( nsCacheEntry * entry );
void EvictEntriesIfNecessary();
nsCacheEntryHashTable mMemCacheEntries;
PRCList mEvictionList;