Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-16 06:45:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 481347: Split uploads into multiple chunks to get around server upload limit and curtail memory usage

Browse Source
This commit is contained in:
Dan Mills 2009-07-22 16:21:33 -07:00
parent e041769fd8
commit ddefe825fc
2 changed files with 30 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
services/sync/modules/constants.js
View File

@ -40,7 +40,7 @@ const EXPORTED_SYMBOLS = ["WEAVE_VERSION", "STORAGE_VERSION",
'MODE_CREATE', 'MODE_APPEND', 'MODE_TRUNCATE',
'PERMS_FILE', 'PERMS_PASSFILE', 'PERMS_DIRECTORY',
'ONE_BYTE', 'ONE_KILOBYTE', 'ONE_MEGABYTE',
'CONNECTION_TIMEOUT', 'KEEP_DELTAS',
'CONNECTION_TIMEOUT', 'MAX_UPLOAD_RECORDS',
'WEAVE_STATUS_OK', 'WEAVE_STATUS_FAILED',
'WEAVE_STATUS_PARTIAL', 'SERVER_LOW_QUOTA',
'SERVER_DOWNTIME', 'SERVER_UNREACHABLE',
@ -79,7 +79,10 @@ const ONE_MEGABYTE = 1024 * ONE_KILOBYTE;
const CONNECTION_TIMEOUT = 30000;
const KEEP_DELTAS = 25;
// How many records to upload in a single POST
// If there are more, multiple POST calls will be made.
// Record size limit is currently 10K, so 100 is a bit over 1MB
const MAX_UPLOAD_RECORDS = 100;
// Top-level statuses:
const WEAVE_STATUS_OK = "Sync succeeded.";

37
services/sync/modules/engines.js
View File

@ -459,14 +459,28 @@ SyncEngine.prototype = {
// don't cache the outgoing items, we won't need them later
this._store.cache.enabled = false;
let count = 0;
for (let id in this._tracker.changedIDs) {
let out = this._createRecord(id);
this._log.trace("Outgoing:\n" + out);
// skip getting siblings of already processed and deleted records
if (!out.deleted && !(out.id in meta))
this._store.createMetaRecords(out.id, meta);
out.encrypt(ID.get("WeaveCryptoID"));
up.pushData(JSON.parse(out.serialize())); // FIXME: inefficient
if ((++count % MAX_UPLOAD_RECORDS) == 0) {
// partial upload
this._log.info("Uploading " + (count - MAX_UPLOAD_RECORDS) + " - " +
count + " out of " + outnum + " records");
up.post();
if (up.data.modified > this.lastSync)
this.lastSync = up.data.modified;
up.clearRecords();
}
Sync.sleep(0);
}
@ -474,22 +488,21 @@ SyncEngine.prototype = {
// now add short depth-and-index-only records, except the ones we're
// sending as full records
let count = 0;
let metaCount = 0;
for each (let obj in meta) {
if (!(obj.id in this._tracker.changedIDs)) {
up.pushData(obj);
count++;
}
if (!(obj.id in this._tracker.changedIDs)) {
up.pushData(obj);
metaCount++;
}
}
this._log.info("Uploading " + outnum + " records + " + count + " index/depth records)");
// do the upload
// final upload
this._log.info("Uploading " +
(count >= MAX_UPLOAD_RECORDS? "last batch of " : "") +
count + " records, and " + metaCount + " index/depth records");
up.post();
// save last modified date
let mod = up.data.modified;
if (mod > this.lastSync)
this.lastSync = mod;
if (up.data.modified > this.lastSync)
this.lastSync = up.data.modified;
}
this._tracker.clearChangedIDs();
},