Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2025-02-27 21:00:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

bug 814010: diskcache: skip buffer when writing to file. r=michal.novotny

Browse Source
This commit is contained in:
Alfred Kayser 2013-01-06 14:51:12 +01:00
parent a5972c7b9b
commit 4a4870b372
2 changed files with 103 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

356
netwerk/cache/nsDiskCacheStreams.cpp vendored
View File

@ -16,6 +16,10 @@
#include "mozilla/TimeStamp.h"
// we pick 16k as the max buffer size because that is the threshold above which
// we are unable to store the data in the cache block files
// see nsDiskCacheMap.[cpp,h]
#define kMaxBufferSize (16 * 1024)
// Assumptions:
// - cache descriptors live for life of streams
@ -290,22 +294,13 @@ nsDiskCacheOutputStream::IsNonBlocking(bool * nonBlocking)
*****************************************************************************/
NS_IMPL_THREADSAFE_ISUPPORTS0(nsDiskCacheStreamIO)
// we pick 16k as the max buffer size because that is the threshold above which
// we are unable to store the data in the cache block files
// see nsDiskCacheMap.[cpp,h]
#define kMaxBufferSize (16 * 1024)
nsDiskCacheStreamIO::nsDiskCacheStreamIO(nsDiskCacheBinding * binding)
: mBinding(binding)
, mOutStream(nullptr)
, mInStreamCount(0)
, mFD(nullptr)
, mStreamPos(0)
, mStreamEnd(0)
, mBufPos(0)
, mBufEnd(0)
, mBufSize(0)
, mBufDirty(false)
, mBuffer(nullptr)
{
mDevice = (nsDiskCacheDevice *)mBinding->mCacheEntry->CacheDevice();
@ -372,7 +367,7 @@ nsDiskCacheStreamIO::GetInputStream(uint32_t offset, nsIInputStream ** inputStre
} else if (!mBuffer) {
// read block file for data
rv = ReadCacheBlocks();
rv = ReadCacheBlocks(mStreamEnd);
if (NS_FAILED(rv)) return rv;
}
// else, mBuffer already contains all of the data (left over from a
@ -402,18 +397,10 @@ nsDiskCacheStreamIO::GetOutputStream(uint32_t offset, nsIOutputStream ** outputS
NS_ASSERTION(mInStreamCount == 0, "we already have input streams open");
if (mOutStream || mInStreamCount) return NS_ERROR_NOT_AVAILABLE;
// mBuffer lazily allocated, but might exist if a previous stream already
// created one.
mBufPos = 0;
mStreamPos = 0;
mStreamEnd = mBinding->mCacheEntry->DataSize();
nsresult rv;
if (offset) {
rv = Seek(PR_SEEK_SET, offset);
if (NS_FAILED(rv)) return rv;
}
rv = SetEOF();
// Inits file or buffer and truncate at the desired offset
nsresult rv = SeekAndTruncate(offset);
if (NS_FAILED(rv)) return rv;
// create a new output stream
@ -439,8 +426,6 @@ nsDiskCacheStreamIO::CloseOutputStream(nsDiskCacheOutputStream * outputStream)
{
nsCacheServiceAutoLock lock(LOCK_TELEM(NSDISKCACHESTREAMIO_CLOSEOUTPUTSTREAM)); // grab service lock
nsresult rv;
if (outputStream != mOutStream) {
NS_WARNING("mismatched output streams");
return NS_ERROR_UNEXPECTED;
@ -448,13 +433,12 @@ nsDiskCacheStreamIO::CloseOutputStream(nsDiskCacheOutputStream * outputStream)
// output stream is closing
if (!mBinding) { // if we're severed, just clear member variables
NS_ASSERTION(!mBufDirty, "oops");
mOutStream = nullptr;
outputStream->ReleaseStreamIO();
return NS_ERROR_NOT_AVAILABLE;
}
rv = Flush();
nsresult rv = Flush();
if (NS_FAILED(rv))
NS_WARNING("Flush() failed");
@ -470,11 +454,10 @@ nsDiskCacheStreamIO::Flush()
CACHE_LOG_DEBUG(("CACHE: Flush [%x doomed=%u]\n",
mBinding->mRecord.HashNumber(), mBinding->mDoomed));
if (!mBufDirty) {
if (mFD) {
(void) PR_Close(mFD);
mFD = nullptr;
}
// When writing to a file, just close the file
if (mFD) {
(void) PR_Close(mFD);
mFD = nullptr;
return NS_OK;
}
@ -487,8 +470,6 @@ nsDiskCacheStreamIO::Flush()
if (mStreamEnd <= kMaxBufferSize) {
// store data (if any) in cache block files
mBufDirty = false;
// delete existing storage
nsDiskCacheRecord * record = &mBinding->mRecord;
if (record->DataLocationInitialized()) {
@ -497,16 +478,12 @@ nsDiskCacheStreamIO::Flush()
NS_WARNING("cacheMap->DeleteStorage() failed.");
return rv;
}
if (mFD) {
PR_Close(mFD);
mFD = nullptr;
}
}
// flush buffer to block files
written = true;
if (mStreamEnd > 0) {
rv = cacheMap->WriteDataCacheBlocks(mBinding, mBuffer, mBufEnd);
rv = cacheMap->WriteDataCacheBlocks(mBinding, mBuffer, mStreamEnd);
if (NS_FAILED(rv)) {
NS_WARNING("WriteDataCacheBlocks() failed.");
written = false;
@ -515,7 +492,7 @@ nsDiskCacheStreamIO::Flush()
}
if (!written) {
// make sure we save as separate file
// failed to store in cacheblocks, save as separate file
rv = FlushBufferToFile(); // initializes DataFileLocation() if necessary
if (mFD) {
@ -532,13 +509,6 @@ nsDiskCacheStreamIO::Flush()
// close mFD first if possible before returning if FlushBufferToFile
// failed
NS_ENSURE_SUCCESS(rv, rv);
// since the data location is on disk as a single file, the only value
// in keeping mBuffer around is to avoid an extra malloc the next time
// we need to write to this file. reading will use a file descriptor.
// therefore, it's probably not worth optimizing for the subsequent
// write, so we unconditionally delete mBuffer here.
DeleteBuffer();
}
// XXX do we need this here? WriteDataCacheBlocks() calls UpdateRecord()
@ -565,8 +535,8 @@ nsDiskCacheStreamIO::Write( const char * buffer,
uint32_t count,
uint32_t * bytesWritten)
{
nsresult rv = NS_OK;
nsCacheServiceAutoLock lock(LOCK_TELEM(NSDISKCACHESTREAMIO_WRITE)); // grab service lock
// grab service lock
nsCacheServiceAutoLock lock(LOCK_TELEM(NSDISKCACHESTREAMIO_WRITE));
if (!mBinding) return NS_ERROR_NOT_AVAILABLE;
if (mInStreamCount) {
@ -577,62 +547,53 @@ nsDiskCacheStreamIO::Write( const char * buffer,
}
NS_ASSERTION(count, "Write called with count of zero");
NS_ASSERTION(mBufPos <= mBufEnd, "streamIO buffer corrupted");
uint32_t bytesLeft = count;
bool flushed = false;
while (bytesLeft) {
if (mBufPos == mBufSize) {
if (mBufSize < kMaxBufferSize) {
// Not writing to file, and it will fit in the cachedatablocks?
if (!mFD && (mStreamEnd + count <= kMaxBufferSize)) {
// We have more data than the current buffer size?
if ((mStreamEnd + count > mBufSize) && (mBufSize < kMaxBufferSize)) {
// Try to increase buffer to the max size, no problem if not
// succesful, we just use what is available.
char *newbuf = (char *) realloc(mBuffer, kMaxBufferSize);
if (newbuf) {
// Use the new larger buffer
mBuffer = newbuf;
mBufSize = kMaxBufferSize;
char *buffer = mBuffer;
mBuffer = (char *) realloc(mBuffer, mBufSize);
if (!mBuffer) {
free(buffer);
mBufSize = 0;
break;
}
} else {
nsresult rv = FlushBufferToFile();
if (NS_FAILED(rv)) break;
flushed = true;
}
}
uint32_t chunkSize = bytesLeft;
if (chunkSize > (mBufSize - mBufPos))
chunkSize = mBufSize - mBufPos;
memcpy(mBuffer + mBufPos, buffer, chunkSize);
mBufDirty = true;
mBufPos += chunkSize;
bytesLeft -= chunkSize;
buffer += chunkSize;
if (mBufEnd < mBufPos)
mBufEnd = mBufPos;
}
if (bytesLeft) {
*bytesWritten = 0;
return NS_ERROR_FAILURE;
}
*bytesWritten = count;
// update mStreamPos, mStreamEnd
mStreamPos += count;
if (mStreamEnd < mStreamPos) {
mStreamEnd = mStreamPos;
NS_ASSERTION(mBinding->mCacheEntry->DataSize() == mStreamEnd, "bad stream");
// If we have flushed to a file, update the file size
if (flushed && mFD) {
UpdateFileSize();
// Store in the buffer but only if it fits
if ((count > 0) && (mStreamEnd + count <= mBufSize)) {
memcpy(mBuffer + mStreamEnd, buffer, count);
mStreamEnd += count;
*bytesWritten = count;
return NS_OK;
}
}
return rv;
// There are more bytes than fit in the buffer/cacheblocks, switch to file
if (!mFD) {
// Opens a cache file and write the buffer to it
nsresult rv = FlushBufferToFile();
if (NS_FAILED(rv)) {
*bytesWritten = 0;
return rv;
}
}
// Write directly to the file
if (PR_Write(mFD, buffer, count) != (int32_t)count) {
NS_WARNING("failed to write all data");
*bytesWritten = 0;
return NS_ERROR_UNEXPECTED; // NS_ErrorAccordingToNSPR()
}
mStreamEnd += count;
*bytesWritten = count;
UpdateFileSize();
NS_ASSERTION(mBinding->mCacheEntry->DataSize() == mStreamEnd, "bad stream");
return NS_OK;
}
@ -691,10 +652,11 @@ nsDiskCacheStreamIO::OpenCacheFile(int flags, PRFileDesc ** fd)
nsresult
nsDiskCacheStreamIO::ReadCacheBlocks()
nsDiskCacheStreamIO::ReadCacheBlocks(uint32_t bufferSize)
{
NS_ASSERTION(mStreamEnd == mBinding->mCacheEntry->DataSize(), "bad stream");
NS_ASSERTION(mStreamEnd <= kMaxBufferSize, "data too large for buffer");
NS_ASSERTION(bufferSize <= kMaxBufferSize, "bufferSize too large for buffer");
NS_ASSERTION(mStreamEnd <= bufferSize, "data too large for buffer");
nsDiskCacheRecord * record = &mBinding->mRecord;
if (!record->DataLocationInitialized()) return NS_OK;
@ -703,23 +665,16 @@ nsDiskCacheStreamIO::ReadCacheBlocks()
if (!mBuffer) {
// allocate buffer
mBuffer = (char *) malloc(mStreamEnd);
mBuffer = (char *) malloc(bufferSize);
if (!mBuffer) {
return NS_ERROR_OUT_OF_MEMORY;
}
mBufSize = mStreamEnd;
mBufSize = bufferSize;
}
// read data stored in cache block files
nsDiskCacheMap *map = mDevice->CacheMap(); // get map reference
nsresult rv = map->ReadDataCacheBlocks(mBinding, mBuffer, mStreamEnd);
if (NS_FAILED(rv)) return rv;
// update streamIO variables
mBufPos = 0;
mBufEnd = mStreamEnd;
return NS_OK;
return map->ReadDataCacheBlocks(mBinding, mBuffer, mStreamEnd);
}
@ -747,18 +702,15 @@ nsDiskCacheStreamIO::FlushBufferToFile()
mozilla::fallocate(mFD, NS_MIN<int64_t>(dataSize, kPreallocateLimit));
}
// write buffer
int32_t bytesWritten = PR_Write(mFD, mBuffer, mBufEnd);
if (uint32_t(bytesWritten) != mBufEnd) {
// write buffer to the file
if (PR_Write(mFD, mBuffer, mStreamEnd) != (int32_t)mStreamEnd) {
NS_WARNING("failed to flush all data");
return NS_ERROR_UNEXPECTED; // NS_ErrorAccordingToNSPR()
}
mBufDirty = false;
// reset buffer
mBufPos = 0;
mBufEnd = 0;
// buffer is no longer valid
DeleteBuffer();
return NS_OK;
}
@ -767,11 +719,8 @@ void
nsDiskCacheStreamIO::DeleteBuffer()
{
if (mBuffer) {
NS_ASSERTION(!mBufDirty, "deleting dirty buffer");
free(mBuffer);
mBuffer = nullptr;
mBufPos = 0;
mBufEnd = 0;
mBufSize = 0;
}
}
@ -787,157 +736,50 @@ nsDiskCacheStreamIO::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf)
return usage;
}
// NOTE: called with service lock held
nsresult
nsDiskCacheStreamIO::Seek(int32_t whence, int32_t offset)
nsDiskCacheStreamIO::SeekAndTruncate(uint32_t offset)
{
int32_t newPos;
if (!mBinding) return NS_ERROR_NOT_AVAILABLE;
if (uint32_t(offset) > mStreamEnd) return NS_ERROR_FAILURE;
if (mBinding->mRecord.DataLocationInitialized()) {
if (mBinding->mRecord.DataFile() == 0) {
if (!mFD) {
// we need an mFD, we better open it now
nsresult rv = OpenCacheFile(PR_RDWR | PR_CREATE_FILE, &mFD);
if (NS_FAILED(rv)) return rv;
}
}
}
if (mFD) {
// do we have data in the buffer that needs to be flushed?
if (mBufDirty) {
// XXX optimization: are we just moving within the current buffer?
nsresult rv = FlushBufferToFile();
// Set the current end to the desired offset
mStreamEnd = offset;
// Currently stored in file?
if (mBinding->mRecord.DataLocationInitialized() &&
(mBinding->mRecord.DataFile() == 0)) {
if (!mFD) {
// we need an mFD, we better open it now
nsresult rv = OpenCacheFile(PR_RDWR | PR_CREATE_FILE, &mFD);
if (NS_FAILED(rv)) return rv;
}
newPos = PR_Seek(mFD, offset, (PRSeekWhence)whence);
if (newPos == -1)
return NS_ErrorAccordingToNSPR();
mStreamPos = (uint32_t) newPos;
mBufPos = 0;
mBufEnd = 0;
return NS_OK;
}
// else, seek in mBuffer
switch(whence) {
case PR_SEEK_SET:
newPos = offset;
break;
case PR_SEEK_CUR: // relative from current posistion
newPos = offset + (uint32_t)mStreamPos;
break;
case PR_SEEK_END: // relative from end
newPos = offset + (uint32_t)mBufEnd;
break;
default:
return NS_ERROR_INVALID_ARG;
}
// read data into mBuffer if not read yet.
if (mStreamEnd && !mBufEnd) {
if (newPos > 0) {
nsresult rv = ReadCacheBlocks();
if (NS_FAILED(rv)) return rv;
if (offset) {
if (PR_Seek(mFD, offset, PR_SEEK_SET) == -1)
return NS_ErrorAccordingToNSPR();
}
}
// stream buffer sanity checks
NS_ASSERTION(mBufEnd <= kMaxBufferSize, "bad stream");
NS_ASSERTION(mBufPos <= mBufEnd, "bad stream");
NS_ASSERTION(mStreamPos == mBufPos, "bad stream");
NS_ASSERTION(mStreamEnd == mBufEnd, "bad stream");
if ((newPos < 0) || (uint32_t(newPos) > mBufEnd)) {
NS_WARNING("seek offset out of range");
return NS_ERROR_INVALID_ARG;
}
mStreamPos = newPos;
mBufPos = newPos;
return NS_OK;
}
// called only from nsDiskCacheOutputStream::Tell
nsresult
nsDiskCacheStreamIO::Tell(uint32_t * result)
{
NS_ENSURE_ARG_POINTER(result);
*result = mStreamPos;
return NS_OK;
}
// NOTE: called with service lock held
nsresult
nsDiskCacheStreamIO::SetEOF()
{
nsresult rv;
bool needToCloseFD = false;
NS_ASSERTION(mStreamPos <= mStreamEnd, "bad stream");
if (!mBinding) return NS_ERROR_NOT_AVAILABLE;
if (mBinding->mRecord.DataLocationInitialized()) {
if (mBinding->mRecord.DataFile() == 0) {
if (!mFD) {
// we need an mFD, we better open it now
rv = OpenCacheFile(PR_RDWR | PR_CREATE_FILE, &mFD);
if (NS_FAILED(rv)) return rv;
needToCloseFD = true;
}
} else {
// data in cache block files
if ((mStreamPos != 0) && (mStreamPos != mBufPos)) {
// only read data if there will be some left after truncation
rv = ReadCacheBlocks();
if (NS_FAILED(rv)) return rv;
}
// We need to make sure we reflect this change in Flush().
// In particular, if mStreamPos is 0 and we never write to
// the buffer, we want the storage to be deleted.
mBufDirty = true;
}
}
if (mFD) {
rv = nsDiskCache::Truncate(mFD, mStreamPos);
#ifdef DEBUG
uint32_t oldSizeK = (mStreamEnd + 0x03FF) >> 10;
NS_ASSERTION(mBinding->mRecord.DataFileSize() == oldSizeK, "bad disk cache entry size");
} else {
// data stored in buffer.
NS_ASSERTION(mStreamEnd <= kMaxBufferSize, "buffer truncation inadequate");
NS_ASSERTION(mBufPos == mStreamPos, "bad stream");
NS_ASSERTION(mBuffer ? mBufEnd == mStreamEnd : true, "bad stream");
#endif
}
NS_ASSERTION(mStreamEnd == mBinding->mCacheEntry->DataSize(), "cache entry not updated");
// we expect nsCacheEntryDescriptor::TransportWrapper::OpenOutputStream()
// to eventually update the cache entry
mStreamEnd = mStreamPos;
mBufEnd = mBufPos;
if (mFD) {
nsDiskCache::Truncate(mFD, offset);
UpdateFileSize();
if (needToCloseFD) {
// When we starting at zero again, close file and start with buffer.
// If offset is non-zero (and within buffer) an option would be
// to read the file into the buffer, but chance is high that it is
// rewritten to the file anyway.
if (offset == 0) {
// close file descriptor
(void) PR_Close(mFD);
mFD = nullptr;
}
return NS_OK;
}
// read data into mBuffer if not read yet.
if (offset && !mBuffer) {
nsresult rv = ReadCacheBlocks(kMaxBufferSize);
if (NS_FAILED(rv)) return rv;
}
return NS_OK;
// stream buffer sanity check
NS_ASSERTION(mStreamEnd <= kMaxBufferSize, "bad stream");
return NS_OK;
}

16
netwerk/cache/nsDiskCacheStreams.h vendored
View File

@ -37,10 +37,6 @@ public:
uint32_t count,
uint32_t * bytesWritten);
nsresult Seek(int32_t whence, int32_t offset);
nsresult Tell(uint32_t * position);
nsresult SetEOF();
nsresult ClearBinding();
void IncrementInputStreamCount() { PR_ATOMIC_INCREMENT(&mInStreamCount); }
@ -55,16 +51,17 @@ public:
// GCC 2.95.2 requires this to be defined, although we never call it.
// and OS/2 requires that it not be private
nsDiskCacheStreamIO() { NS_NOTREACHED("oops"); }
private:
private:
void Close();
nsresult OpenCacheFile(int flags, PRFileDesc ** fd);
nsresult ReadCacheBlocks();
nsresult ReadCacheBlocks(uint32_t bufferSize);
nsresult FlushBufferToFile();
void UpdateFileSize();
void DeleteBuffer();
nsresult Flush();
nsresult SeekAndTruncate(uint32_t offset);
nsDiskCacheBinding * mBinding; // not an owning reference
nsDiskCacheDevice * mDevice;
@ -72,14 +69,9 @@ private:
int32_t mInStreamCount;
PRFileDesc * mFD;
uint32_t mStreamPos; // for Output Streams
uint32_t mStreamEnd;
uint32_t mBufPos; // current mark in buffer
uint32_t mBufEnd; // current end of data in buffer
uint32_t mStreamEnd; // current size of data
uint32_t mBufSize; // current end of buffer
bool mBufDirty;
char * mBuffer;
};
#endif // _nsDiskCacheStreams_h_