mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-13 21:35:39 +00:00
Bug 1186955 - Telemetry should discard pings that are too big. r=gfritzsche
This commit is contained in:
parent
467aa72e3e
commit
e6f236aeff
@ -4707,6 +4707,48 @@
|
||||
"n_buckets": 20,
|
||||
"description": "Time (ms) it takes for checking if the pending pings are over-quota"
|
||||
},
|
||||
"TELEMETRY_PING_SIZE_EXCEEDED_SEND": {
|
||||
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||
"expires_in_version": "never",
|
||||
"kind": "count",
|
||||
"description": "Number of Telemetry pings discarded before sending because they exceeded the maximum size"
|
||||
},
|
||||
"TELEMETRY_PING_SIZE_EXCEEDED_PENDING": {
|
||||
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||
"expires_in_version": "never",
|
||||
"kind": "count",
|
||||
"description": "Number of Telemetry pending pings discarded because they exceeded the maximum size"
|
||||
},
|
||||
"TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED": {
|
||||
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||
"expires_in_version": "never",
|
||||
"kind": "count",
|
||||
"description": "Number of archived Telemetry pings discarded because they exceeded the maximum size"
|
||||
},
|
||||
"TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB": {
|
||||
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||
"expires_in_version": "never",
|
||||
"kind": "linear",
|
||||
"high": "30",
|
||||
"n_buckets": 29,
|
||||
"description": "The size (MB) of the Telemetry pending pings exceeding the maximum file size"
|
||||
},
|
||||
"TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB": {
|
||||
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||
"expires_in_version": "never",
|
||||
"kind": "linear",
|
||||
"high": "30",
|
||||
"n_buckets": 29,
|
||||
"description": "The size (MB) of the Telemetry archived, compressed, pings exceeding the maximum file size"
|
||||
},
|
||||
"TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB": {
|
||||
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||
"expires_in_version": "never",
|
||||
"kind": "linear",
|
||||
"high": "30",
|
||||
"n_buckets": 29,
|
||||
"description": "The size (MB) of the ping data submitted to Telemetry exceeding the maximum size"
|
||||
},
|
||||
"TELEMETRY_DISCARDED_CONTENT_PINGS_COUNT": {
|
||||
"alert_emails": ["perf-telemetry-alerts@mozilla.com"],
|
||||
"expires_in_version": "never",
|
||||
|
@ -928,6 +928,19 @@ let TelemetrySendImpl = {
|
||||
let utf8Payload = converter.ConvertFromUnicode(JSON.stringify(networkPayload));
|
||||
utf8Payload += converter.Finish();
|
||||
Telemetry.getHistogramById("TELEMETRY_STRINGIFY").add(new Date() - startTime);
|
||||
|
||||
// Check the size and drop pings which are too big.
|
||||
const pingSizeBytes = utf8Payload.length;
|
||||
if (pingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) {
|
||||
this._log.error("_doPing - submitted ping exceeds the size limit, size: " + pingSizeBytes);
|
||||
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND").add();
|
||||
Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB")
|
||||
.add(Math.floor(pingSizeBytes / 1024 / 1024));
|
||||
// We don't need to call |request.abort()| as it was not sent yet.
|
||||
this._pendingPingRequests.delete(id);
|
||||
return TelemetryStorage.removePendingPing(id);
|
||||
}
|
||||
|
||||
let payloadStream = Cc["@mozilla.org/io/string-input-stream;1"]
|
||||
.createInstance(Ci.nsIStringInputStream);
|
||||
startTime = new Date();
|
||||
|
@ -59,6 +59,9 @@ const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB
|
||||
// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes).
|
||||
const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB
|
||||
|
||||
// The maximum size a pending/archived ping can take on disk.
|
||||
const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB
|
||||
|
||||
// This special value is submitted when the archive is outside of the quota.
|
||||
const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;
|
||||
|
||||
@ -135,6 +138,12 @@ this.TelemetryStorage = {
|
||||
return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");
|
||||
},
|
||||
|
||||
/**
|
||||
* The maximum size a ping can have, in bytes.
|
||||
*/
|
||||
get MAXIMUM_PING_SIZE() {
|
||||
return PING_FILE_MAXIMUM_SIZE_BYTES;
|
||||
},
|
||||
/**
|
||||
* Shutdown & block on any outstanding async activity in this module.
|
||||
*
|
||||
@ -644,13 +653,27 @@ let TelemetryStorageImpl = {
|
||||
const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type);
|
||||
const pathCompressed = path + "lz4";
|
||||
|
||||
// Purge pings which are too big.
|
||||
let checkSize = function*(path) {
|
||||
const fileSize = (yield OS.File.stat(path)).size;
|
||||
if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||
Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")
|
||||
.add(Math.floor(fileSize / 1024 / 1024));
|
||||
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();
|
||||
yield OS.File.remove(path, {ignoreAbsent: true});
|
||||
throw new Error("loadArchivedPing - exceeded the maximum ping size: " + fileSize);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
// Try to load a compressed version of the archived ping first.
|
||||
this._log.trace("loadArchivedPing - loading ping from: " + pathCompressed);
|
||||
yield* checkSize(pathCompressed);
|
||||
return yield this.loadPingFile(pathCompressed, /*compressed*/ true);
|
||||
} catch (ex if ex.becauseNoSuchFile) {
|
||||
// If that fails, look for the uncompressed version.
|
||||
this._log.trace("loadArchivedPing - compressed ping not found, loading: " + path);
|
||||
yield* checkSize(path);
|
||||
return yield this.loadPingFile(path, /*compressed*/ false);
|
||||
}
|
||||
}),
|
||||
@ -671,6 +694,8 @@ let TelemetryStorageImpl = {
|
||||
this._log.trace("_removeArchivedPing - removing ping from: " + path);
|
||||
yield OS.File.remove(path, {ignoreAbsent: true});
|
||||
yield OS.File.remove(pathCompressed, {ignoreAbsent: true});
|
||||
// Remove the ping from the cache.
|
||||
this._archivedPings.delete(id);
|
||||
}),
|
||||
|
||||
/**
|
||||
@ -813,6 +838,19 @@ let TelemetryStorageImpl = {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Enforce a maximum file size limit on archived pings.
|
||||
if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||
this._log.error("_enforceArchiveQuota - removing file exceeding size limit, size: " + fileSize);
|
||||
// We just remove the ping from the disk, we don't bother removing it from pingList
|
||||
// since it won't contribute to the quota.
|
||||
yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type)
|
||||
.catch(e => this._log.error("_enforceArchiveQuota - failed to remove archived ping" + ping.id));
|
||||
Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")
|
||||
.add(Math.floor(fileSize / 1024 / 1024));
|
||||
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();
|
||||
continue;
|
||||
}
|
||||
|
||||
archiveSizeInBytes += fileSize;
|
||||
|
||||
if (archiveSizeInBytes < SAFE_QUOTA) {
|
||||
@ -857,9 +895,6 @@ let TelemetryStorageImpl = {
|
||||
// This list is guaranteed to be in order, so remove the pings at its
|
||||
// beginning (oldest).
|
||||
yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type);
|
||||
|
||||
// Remove outdated pings from the cache.
|
||||
this._archivedPings.delete(ping.id);
|
||||
}
|
||||
|
||||
const endTimeStamp = Policy.now().getTime();
|
||||
@ -1191,15 +1226,36 @@ let TelemetryStorageImpl = {
|
||||
});
|
||||
},
|
||||
|
||||
loadPendingPing: function(id) {
|
||||
loadPendingPing: Task.async(function*(id) {
|
||||
this._log.trace("loadPendingPing - id: " + id);
|
||||
let info = this._pendingPings.get(id);
|
||||
if (!info) {
|
||||
this._log.trace("loadPendingPing - unknown id " + id);
|
||||
return Promise.reject(new Error("TelemetryStorage.loadPendingPing - no ping with id " + id));
|
||||
throw new Error("TelemetryStorage.loadPendingPing - no ping with id " + id);
|
||||
}
|
||||
|
||||
return this.loadPingFile(info.path, false).catch(e => {
|
||||
// Try to get the dimension of the ping. If that fails, update the histograms.
|
||||
let fileSize = 0;
|
||||
try {
|
||||
fileSize = (yield OS.File.stat(info.path)).size;
|
||||
} catch (e if e instanceof OS.File.Error && e.becauseNoSuchFile) {
|
||||
// Fall through and let |loadPingFile| report the error.
|
||||
}
|
||||
|
||||
// Purge pings which are too big.
|
||||
if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||
yield this.removePendingPing(id);
|
||||
Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")
|
||||
.add(Math.floor(fileSize / 1024 / 1024));
|
||||
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();
|
||||
throw new Error("loadPendingPing - exceeded the maximum ping size: " + fileSize);
|
||||
}
|
||||
|
||||
// Try to load the ping file. Update the related histograms on failure.
|
||||
let ping;
|
||||
try {
|
||||
ping = yield this.loadPingFile(info.path, false);
|
||||
} catch(e) {
|
||||
// If we failed to load the ping, check what happened and update the histogram.
|
||||
// Then propagate the rejection.
|
||||
if (e instanceof PingReadError) {
|
||||
@ -1207,10 +1263,11 @@ let TelemetryStorageImpl = {
|
||||
} else if (e instanceof PingParseError) {
|
||||
Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").add();
|
||||
}
|
||||
throw e;
|
||||
};
|
||||
|
||||
return Promise.reject(e);
|
||||
});
|
||||
},
|
||||
return ping;
|
||||
}),
|
||||
|
||||
removePendingPing: function(id) {
|
||||
let info = this._pendingPings.get(id);
|
||||
@ -1281,6 +1338,21 @@ let TelemetryStorageImpl = {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Enforce a maximum file size limit on pending pings.
|
||||
if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||
this._log.error("_scanPendingPings - removing file exceeding size limit " + file.path);
|
||||
try {
|
||||
yield OS.File.remove(file.path);
|
||||
} catch (ex) {
|
||||
this._log.error("_scanPendingPings - failed to remove file " + file.path, ex);
|
||||
} finally {
|
||||
Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")
|
||||
.add(Math.floor(info.size / 1024 / 1024));
|
||||
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let id = OS.Path.basename(file.path);
|
||||
if (!UUID_REGEX.test(id)) {
|
||||
this._log.trace("_scanPendingPings - filename is not a UUID: " + id);
|
||||
|
Loading…
Reference in New Issue
Block a user