Bug 875562 - Part 2: Create CrashManager API for managing crash data; r=ted, Yoric

The tree doesn't have a robust and reusable API for interfacing with
crash data. This patch is the start of a new API.

In this patch, the CrashManager type is introduced. It has APIs for
retrieving the lists of files related to crash dumps. In subsequent
patches, I will convert existing code in the tree that does similar
things to the new API. I will also build the events/timeline API onto
this type.

I made CrashManager generic because I hate, hate, hate singletons and
global variables. Allowing it to be instantiated multiple times with
different options (instead of say binding a global instance to ProfD)
makes the testing story much, much nicer. That is reason enough, IMO. In
a subsequent patch, I'll add an XPCOM service that instantiates the
"global" instance of CrashManager with the appropriate options.

It was tempting to add this code into the existing CrashReports.jsm.
However, this file does not import cleanly in xpcshell tests and I
didn't want to bloat scope to include fixing that file... yet.
CrashReports.jsm is using synchronous I/O. So, depending on how
adventerous I feel, I may replace consumers of CrashReports.jsm with the
new CrashManager.jsm, remove CrashReports.jsm, and eliminate another
source of synchronous I/O in the tree.

--HG--
extra : rebase_source : 379fa6a78b53bc0dea0c7c64e8b1bdcf85b61a7c

toolkit/components/crashes/CrashManager.jsm

@@ -0,0 +1,157 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/osfile.jsm", this)
+Cu.import("resource://gre/modules/Task.jsm", this);
+
+this.EXPORTED_SYMBOLS = [
+  "CrashManager",
+];
+
+/**
+ * A gateway to crash-related data.
+ *
+ * This type is generic and can be instantiated any number of times.
+ * However, most applications will typically only have one instance
+ * instantiated and that instance will point to profile and user appdata
+ * directories.
+ *
+ * Instances are created by passing an object with properties.
+ * Recognized properties are:
+ *
+ *   pendingDumpsDir (string) (required)
+ *     Where dump files that haven't been uploaded are located.
+ *
+ *   submittedDumpsDir (string) (required)
+ *     Where records of uploaded dumps are located.
+ */
+this.CrashManager = function (options) {
+  for (let k of ["pendingDumpsDir", "submittedDumpsDir"]) {
+    if (!(k in options)) {
+      throw new Error("Required key not present in options: " + k);
+    }
+  }
+
+  for (let [k, v] of Iterator(options)) {
+    switch (k) {
+      case "pendingDumpsDir":
+        this._pendingDumpsDir = v;
+        break;
+
+      case "submittedDumpsDir":
+        this._submittedDumpsDir = v;
+        break;
+
+      default:
+        throw new Error("Unknown property in options: " + k);
+    }
+  }
+};
+
+this.CrashManager.prototype = Object.freeze({
+  DUMP_REGEX: /^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.dmp$/i,
+  SUBMITTED_REGEX: /^bp-(?:hr-)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.txt$/i,
+
+  /**
+   * Obtain a list of all dumps pending upload.
+   *
+   * The returned value is a promise that resolves to an array of objects
+   * on success. Each element in the array has the following properties:
+   *
+   *   id (string)
+   *      The ID of the crash (a UUID).
+   *
+   *   path (string)
+   *      The filename of the crash (<UUID.dmp>)
+   *
+   *   date (Date)
+   *      When this dump was created
+   *
+   * The returned arry is sorted by the modified time of the file backing
+   * the entry, oldest to newest.
+   *
+   * @return Promise<Array>
+   */
+  pendingDumps: function () {
+    return this._getDirectoryEntries(this._pendingDumpsDir, this.DUMP_REGEX);
+  },
+
+  /**
+   * Obtain a list of all dump files corresponding to submitted crashes.
+   *
+   * The returned value is a promise that resolves to an Array of
+   * objects. Each object has the following properties:
+   *
+   *   path (string)
+   *     The path of the file this entry comes from.
+   *
+   *   id (string)
+   *     The crash UUID.
+   *
+   *   date (Date)
+   *     The (estimated) date this crash was submitted.
+   *
+   * The returned array is sorted by the modified time of the file backing
+   * the entry, oldest to newest.
+   *
+   * @return Promise<Array>
+   */
+  submittedDumps: function () {
+    return this._getDirectoryEntries(this._submittedDumpsDir,
+                                     this.SUBMITTED_REGEX);
+  },
+
+  /**
+   * Helper to obtain all directory entries in a path that match a regexp.
+   *
+   * The resolved promise is an array of objects with the properties:
+   *
+   *   path -- String filename
+   *   id -- regexp.match()[1] (likely the crash ID)
+   *   date -- Date mtime of the file
+   */
+  _getDirectoryEntries: function (path, re) {
+    return Task.spawn(function* () {
+      try {
+        yield OS.File.stat(path);
+      } catch (ex if ex instanceof OS.File.Error && ex.becauseNoSuchFile) {
+          return [];
+      }
+
+      let it = new OS.File.DirectoryIterator(path);
+      let entries = [];
+
+      try {
+        yield it.forEach((entry, index, it) => {
+          if (entry.isDir) {
+            return;
+          }
+
+          let match = re.exec(entry.name);
+          if (!match) {
+            return;
+          }
+
+          return OS.File.stat(entry.path).then((info) => {
+            entries.push({
+              path: entry.path,
+              id: match[1],
+              date: info.lastModificationDate,
+            });
+          });
+        });
+      } finally {
+        it.close();
+      }
+
+      entries.sort((a, b) => { return a.date - b.date; });
+
+      return entries;
+    }.bind(this));
+  },
+});

toolkit/components/crashes/moz.build

@@ -0,0 +1,11 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXTRA_JS_MODULES += [
+    'CrashManager.jsm',
+]
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell/xpcshell.ini']

toolkit/components/crashes/tests/xpcshell/test_crash_manager.js

@@ -0,0 +1,173 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/CrashManager.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/osfile.jsm", this);
+
+let DUMMY_DIR_COUNT = 0;
+
+function getManager() {
+  function mkdir(f) {
+    if (f.exists()) {
+      return;
+    }
+
+    dump("Creating directory: " + f.path + "\n");
+    f.create(Ci.nsIFile.DIRECTORY_TYPE, dirMode);
+  }
+
+  const dirMode = OS.Constants.libc.S_IRWXU;
+  let baseFile = do_get_tempdir();
+  let pendingD = baseFile.clone();
+  let submittedD = baseFile.clone();
+  pendingD.append("dummy-dir-" + DUMMY_DIR_COUNT++);
+  submittedD.append("dummy-dir-" + DUMMY_DIR_COUNT++);
+  mkdir(pendingD);
+  mkdir(submittedD);
+
+  let m = new CrashManager({
+    pendingDumpsDir: pendingD.path,
+    submittedDumpsDir: submittedD.path,
+  });
+
+  m.create_dummy_dump = function (submitted=false, date=new Date(), hr=false) {
+    let uuid = Cc["@mozilla.org/uuid-generator;1"]
+                 .getService(Ci.nsIUUIDGenerator)
+                 .generateUUID()
+                 .toString();
+    uuid = uuid.substring(1, uuid.length - 1);
+
+    let file;
+    let mode;
+    if (submitted) {
+      file = submittedD.clone();
+      if (hr) {
+        file.append("bp-hr-" + uuid + ".txt");
+      } else {
+        file.append("bp-" + uuid + ".txt");
+      }
+      mode = OS.Constants.libc.S_IRUSR | OS.Constants.libc.S_IWUSR |
+             OS.Constants.libc.S_IRGRP | OS.Constants.libc.S_IROTH;
+    } else {
+      file = pendingD.clone();
+      file.append(uuid + ".dmp");
+      mode = OS.Constants.libc.S_IRUSR | OS.Constants.libc.S_IWUSR;
+    }
+
+    file.create(file.NORMAL_FILE_TYPE, mode);
+    file.lastModifiedTime = date.getTime();
+    dump("Created fake crash: " + file.path + "\n");
+
+    return uuid;
+  };
+
+  m.create_ignored_dump_file = function (filename, submitted=false) {
+    let file;
+    if (submitted) {
+      file = submittedD.clone();
+    } else {
+      file = pendingD.clone();
+    }
+
+    file.append(filename);
+    file.create(file.NORMAL_FILE_TYPE,
+                OS.Constants.libc.S_IRUSR | OS.Constants.libc.S_IWUSR);
+    dump("Created ignored dump file: " + file.path + "\n");
+  };
+
+  return m;
+}
+
+function run_test() {
+  run_next_test();
+}
+
+add_task(function* test_constructor_ok() {
+  let m = new CrashManager({
+    pendingDumpsDir: "/foo",
+    submittedDumpsDir: "/bar",
+  });
+  Assert.ok(m);
+});
+
+add_task(function* test_constructor_invalid() {
+  Assert.throws(() => {
+    new CrashManager({foo: true});
+  });
+});
+
+add_task(function* test_get_manager() {
+  let m = getManager();
+  Assert.ok(m);
+
+  m.create_dummy_dump(true);
+  m.create_dummy_dump(false);
+});
+
+add_task(function* test_pending_dumps() {
+  let m = getManager();
+  let now = Date.now();
+  let ids = [];
+  const COUNT = 5;
+
+  for (let i = 0; i < COUNT; i++) {
+    ids.push(m.create_dummy_dump(false, new Date(now - i * 86400000)));
+  }
+  m.create_ignored_dump_file("ignored", false);
+
+  let entries = yield m.pendingDumps();
+  Assert.equal(entries.length, COUNT, "proper number detected.");
+
+  for (let entry of entries) {
+    Assert.equal(typeof(entry), "object", "entry is an object");
+    Assert.ok("id" in entry, "id in entry");
+    Assert.ok("path" in entry, "path in entry");
+    Assert.ok("date" in entry, "date in entry");
+    Assert.notEqual(ids.indexOf(entry.id), -1, "ID is known");
+  }
+
+  for (let i = 0; i < COUNT; i++) {
+    Assert.equal(entries[i].id, ids[COUNT-i-1], "Entries sorted by mtime");
+  }
+});
+
+add_task(function* test_submitted_dumps() {
+  let m = getManager();
+  let COUNT = 5;
+
+  for (let i = 0; i < COUNT; i++) {
+    m.create_dummy_dump(true);
+  }
+  m.create_ignored_dump_file("ignored", true);
+
+  let entries = yield m.submittedDumps();
+  Assert.equal(entries.length, COUNT, "proper number detected.");
+
+  let hrID = m.create_dummy_dump(true, new Date(), true);
+  entries = yield m.submittedDumps();
+  Assert.equal(entries.length, COUNT + 1, "hr- in filename detected.");
+
+  let gotIDs = new Set([e.id for (e of entries)]);
+  Assert.ok(gotIDs.has(hrID));
+});
+
+add_task(function* test_submitted_and_pending() {
+  let m = getManager();
+  let pendingIDs = [];
+  let submittedIDs = [];
+
+  pendingIDs.push(m.create_dummy_dump(false));
+  pendingIDs.push(m.create_dummy_dump(false));
+  submittedIDs.push(m.create_dummy_dump(true));
+
+  let submitted = yield m.submittedDumps();
+  let pending = yield m.pendingDumps();
+
+  Assert.equal(submitted.length, submittedIDs.length);
+  Assert.equal(pending.length, pendingIDs.length);
+});

toolkit/components/crashes/tests/xpcshell/xpcshell.ini

@@ -0,0 +1,5 @@
+[DEFAULT]
+head =
+tail =
+
+[test_crash_manager.js]

toolkit/components/moz.build

@@ -48,6 +48,9 @@ PARALLEL_DIRS += [
     'workerlz4',
 ]
 
+if CONFIG['MOZ_CRASHREPORTER']:
+    PARALLEL_DIRS += ['crashes']
+
 if CONFIG['MOZ_SOCIAL']:
     PARALLEL_DIRS += ['social']