Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-16 23:05:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1162538 - Add the probes for archived ping count and evicted ping count. r=gfritzsche

Browse Source
This commit is contained in:
Alessio Placitelli 2015-05-27 02:48:00 +02:00
parent 9c0da8f5d7
commit 44d11931b3
2 changed files with 137 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
toolkit/components/telemetry/Histograms.json
View File

@ -4431,6 +4431,84 @@
"extended_statistics_ok": true,
"description": "Number of histograms with total count low errors"
},
"TELEMETRY_ARCHIVE_DIRECTORIES_COUNT": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "linear",
"high": "13",
"n_buckets": 12,
"description": "Number of directories in the archive at scan"
},
"TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "linear",
"high": "13",
"n_buckets": 12,
"description": "The age of the oldest Telemetry archive directory in months"
},
"TELEMETRY_ARCHIVE_SCAN_PING_COUNT": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "exponential",
"high": "100000",
"n_buckets": 100,
"description": "Number of Telemetry pings in the archive at scan"
},
"TELEMETRY_ARCHIVE_SESSION_PING_COUNT": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "count",
"description": "Number of Telemetry pings added to the archive during the session"
},
"TELEMETRY_ARCHIVE_SIZE_MB": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "linear",
"high": "300",
"n_buckets": 60,
"description": "The size of the Telemetry archive (MB)"
},
"TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "exponential",
"high": "100000",
"n_buckets": 100,
"description": "Number of Telemetry pings evicted from the archive during cleanup, because they were over the quota"
},
"TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "linear",
"high": "13",
"n_buckets": 12,
"description": "Number of Telemetry directories evicted from the archive during cleanup, because they were too old"
},
"TELEMETRY_ARCHIVE_EVICTING_DIRS_MS": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "exponential",
"high": "300000",
"n_buckets": 20,
"description": "Time (ms) it takes for evicting old directories"
},
"TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "exponential",
"high": "300000",
"n_buckets": 20,
"description": "Time (ms) it takes for checking if the archive is over-quota"
},
"TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS": {
"alert_emails": ["telemetry-client-dev@mozilla.com"],
"expires_in_version": "never",
"kind": "exponential",
"high": "300000",
"n_buckets": 20,
"description": "Time (ms) it takes for evicting over-quota pings"
},
"TELEMETRY_DISCARDED_CONTENT_PINGS_COUNT": {
"alert_emails": ["perf-telemetry-alerts@mozilla.com"],
"expires_in_version": "never",

75
toolkit/components/telemetry/TelemetryStorage.jsm
View File

@ -27,6 +27,7 @@ const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetryStorage::";
const Telemetry = Services.telemetry;
const Utils = TelemetryUtils;
// Compute the path of the pings archive on the first use.
const DATAREPORTING_DIR = "datareporting";
@ -59,6 +60,9 @@ const MAX_ARCHIVED_PINGS_RETENTION_MS = 180 * 24 * 60 * 60 * 1000; // 180 days
// Maximum space the archive can take on disk (in Bytes).
const ARCHIVE_QUOTA_BYTES = 120 * 1024 * 1024; // 120 MB
// This special value is submitted when the archive is outside of the quota.
const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;
// The number of outstanding saved pings that we have issued loading
// requests for.
let pingsLoaded = 0;
@ -513,6 +517,8 @@ let TelemetryStorageImpl = {
timestampCreated: creationDate.getTime(),
type: ping.type,
});
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").add();
}),
/**
@ -587,12 +593,15 @@ let TelemetryStorageImpl = {
_purgeOldPings: Task.async(function*() {
this._log.trace("_purgeOldPings");
const now = Policy.now().getTime();
const nowDate = Policy.now();
const now = nowDate.getTime();
let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);
let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir);
// Keep track of the newest removed month to update the cache, if needed.
let newestRemovedMonth = null;
let newestRemovedMonthTimestamp = null;
let evictedDirsCount = 0;
let maxDirAgeInMonths = 0;
// Walk through the monthly subdirs of the form <YYYY-MM>/
for (let dir of subdirs) {
@ -613,18 +622,20 @@ let TelemetryStorageImpl = {
}
// If this archive directory is older than 180 days, remove it.
if (!TelemetryUtils.areTimesClose(archiveDate.getTime(), now,
MAX_ARCHIVED_PINGS_RETENTION_MS)) {
if ((now - archiveDate.getTime()) > MAX_ARCHIVED_PINGS_RETENTION_MS) {
try {
yield OS.File.removeDir(dir.path);
evictedDirsCount++;
// Update the newest removed month.
if (archiveDate > newestRemovedMonth) {
newestRemovedMonth = archiveDate;
}
newestRemovedMonthTimestamp = Math.max(archiveDate, newestRemovedMonthTimestamp);
} catch (ex) {
this._log.error("_purgeOldPings - Unable to remove " + dir.path, ex);
}
} else {
// We're not removing this directory, so record the age for the oldest directory.
const dirAgeInMonths = Utils.getElapsedTimeInMonths(archiveDate, nowDate);
maxDirAgeInMonths = Math.max(dirAgeInMonths, maxDirAgeInMonths);
}
}
@ -633,17 +644,27 @@ let TelemetryStorageImpl = {
// Refresh the cache: we could still skip this, but it's cheap enough to keep it
// to avoid introducing task dependencies.
if (newestRemovedMonth) {
if (newestRemovedMonthTimestamp) {
// Scan the archive cache for pings older than the newest directory pruned above.
for (let [id, info] of this._archivedPings) {
const timestampCreated = new Date(info.timestampCreated);
if (timestampCreated.getTime() > newestRemovedMonth.getTime()) {
if (timestampCreated.getTime() > newestRemovedMonthTimestamp) {
continue;
}
// Remove outdated pings from the cache.
this._archivedPings.delete(id);
}
}
const endTimestamp = Policy.now().getTime();
// Save the time it takes to evict old directories and the eviction count.
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS")
.add(evictedDirsCount);
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_DIRS_MS")
.add(Math.ceil(endTimestamp - now));
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE")
.add(maxDirAgeInMonths);
}),
/**
@ -652,6 +673,7 @@ let TelemetryStorageImpl = {
*/
_enforceArchiveQuota: Task.async(function*() {
this._log.trace("_enforceArchiveQuota");
const now = Policy.now().getTime();
// Build an ordered list, from newer to older, of archived pings.
let pingList = [for (p of this._archivedPings) {
@ -698,8 +720,19 @@ let TelemetryStorageImpl = {
}
}
// Check if we're using too much space. If not, bail out.
// Save the time it takes to check if the archive is over-quota.
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS")
.add(Math.round(Policy.now().getTime() - now));
let submitProbes = (sizeInMB, evictedPings, elapsedMs) => {
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").add(sizeInMB);
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").add(evictedPings);
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS").add(elapsedMs);
};
// Check if we're using too much space. If not, submit the archive size and bail out.
if (archiveSizeInBytes < Policy.getArchiveQuota()) {
submitProbes(Math.round(archiveSizeInBytes / 1024 / 1024), 0, 0);
return;
}
@ -722,6 +755,10 @@ let TelemetryStorageImpl = {
// Remove outdated pings from the cache.
this._archivedPings.delete(ping.id);
}
const endTimestamp = Policy.now().getTime();
submitProbes(ARCHIVE_SIZE_PROBE_SPECIAL_VALUE, pingsToPurge.length,
Math.ceil(endTimestamp - now));
}),
_cleanArchive: Task.async(function*() {
@ -782,20 +819,24 @@ let TelemetryStorageImpl = {
_scanArchive: Task.async(function*() {
this._log.trace("_scanArchive");
let submitProbes = (pingCount, dirCount) => {
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT")
.add(pingCount);
Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT")
.add(dirCount);
};
if (!(yield OS.File.exists(gPingsArchivePath))) {
submitProbes(0, 0);
return new Map();
}
let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);
let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir);
let subdirs =
(yield dirIterator.nextBatch()).filter(e => e.isDir).filter(e => isValidArchiveDir(e.name));
// Walk through the monthly subdirs of the form <YYYY-MM>/
for (let dir of subdirs) {
if (!isValidArchiveDir(dir.name)) {
this._log.warn("_scanArchive - skipping invalidly named subdirectory " + dir.path);
continue;
}
this._log.trace("_scanArchive - checking in subdir: " + dir.path);
let pingIterator = new OS.File.DirectoryIterator(dir.path);
let pings = (yield pingIterator.nextBatch()).filter(e => !e.isDir);
@ -830,6 +871,8 @@ let TelemetryStorageImpl = {
// Mark the archive as scanned, so we no longer hit the disk.
this._scannedArchiveDirectory = true;
// Update the ping and directories count histograms.
submitProbes(this._archivedPings.size, subdirs.length);
return this._archivedPings;
}),