Bug 1363428 - Add reftest-specific endpoints to Marionette, r=ato

This adds commands to start a reftest session, run a test, and end the
session. It as assumed that after you start a reftest session you will
just run reftests until you end the session. When starting a session
the user provides a string indicating when screenshots should be
taken, and an object mapping urls to a count of the number of times
that url is expected to be used in the session, to help with
caching. Running the tests takes a url to a test, an expected status,
a timeout, and a nested list of possible references, in which each
entry at a specific level is combined by OR and nested references are
combined by AND.

The implementation is heavilly inspired by the existing reftest
harness, starting a minimal window with no tabs, and loading the urls
directly in there. In order to get a screenshot in the e10s case we
have to pass the DRAW_VIEW and USE_WIDGET_LAYERS flags when taking the
screenshot.

For performance we heavily cache canvases; for references that will be
repeated we cache the full canvas with image, and we also cache a
single canvas to use for all other screenshots to avoid the overhead
of repeatedly creating a new canvas element.

MozReview-Commit-ID: JOFvtmH7tg

layout/tools/reftest/reftest-content.js

@@ -447,6 +447,7 @@ function WaitForTestEnd(contentRootElement, inPrintMode, spellCheckedElements) {
     var state = STATE_WAITING_TO_FIRE_INVALIDATE_EVENT;
 
     function AfterPaintListener(event) {
+        dump("AfterPaintListener\n");
         LogInfo("AfterPaintListener in " + event.target.document.location.href);
         if (event.target.document != currentDoc) {
             // ignore paint events for subframes or old documents in the window.

testing/marionette/driver.js

@@ -34,6 +34,7 @@ Cu.import("chrome://marionette/content/legacyaction.js");
 Cu.import("chrome://marionette/content/logging.js");
 Cu.import("chrome://marionette/content/modal.js");
 Cu.import("chrome://marionette/content/proxy.js");
+Cu.import("chrome://marionette/content/reftest.js");
 Cu.import("chrome://marionette/content/session.js");
 Cu.import("chrome://marionette/content/wait.js");
 
@@ -3247,6 +3248,65 @@ GeckoDriver.prototype.localizeProperty = function (cmd, resp) {
   resp.body.value = l10n.localizeProperty(urls, id);
 }
 
+/**
+ * Initialize the reftest mode
+ */
+GeckoDriver.prototype.setupReftest = function* (cmd, resp) {
+  if (this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:setup with a reftest session already active");
+  }
+
+  if (this.context !== Context.CHROME) {
+    throw new UnsupportedOperationError("Must set chrome context before running reftests");
+  }
+
+  let {urlCount = {}, screenshot = "unexpected"} = cmd.parameters;
+  if (!["always", "fail", "unexpected"].includes(screenshot)) {
+    throw new InvalidArgumentError("Value of `screenshot` should be 'always', 'fail' or 'unexpected'");
+  }
+
+  this._reftest = new reftest.Runner(this);
+
+  yield this._reftest.setup(urlCount, screenshot);
+};
+
+
+/**
+ * Run a reftest
+ */
+GeckoDriver.prototype.runReftest = function* (cmd, resp) {
+  let {test, references, expected, timeout} = cmd.parameters;
+
+  if (!this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:run before reftest:start");
+  }
+
+  assert.string(test);
+  assert.string(expected);
+  assert.array(references);
+
+  let result = yield this._reftest.run(test, references, expected, timeout);
+
+  resp.body.value = result;
+};
+
+/**
+ * End a reftest run
+ *
+ * Closes the reftest window (without changing the current window handle),
+ * and removes cached canvases.
+ */
+GeckoDriver.prototype.teardownReftest = function* (cmd, resp) {
+  if (!this._reftest) {
+    throw new UnsupportedOperationError("Called reftest:teardown before reftest:start");
+  }
+
+  this._reftest.abort();
+
+  this._reftest = null;
+};
+
+
 GeckoDriver.prototype.commands = {
   "getMarionetteID": GeckoDriver.prototype.getMarionetteID,
   "sayHello": GeckoDriver.prototype.sayHello,
@@ -3333,6 +3393,10 @@ GeckoDriver.prototype.commands = {
 
   "addon:install": GeckoDriver.prototype.installAddon,
   "addon:uninstall": GeckoDriver.prototype.uninstallAddon,
+
+  "reftest:setup": GeckoDriver.prototype.setupReftest,
+  "reftest:run": GeckoDriver.prototype.runReftest,
+  "reftest:teardown": GeckoDriver.prototype.teardownReftest,
 };
 
 function copy (obj) {

testing/marionette/jar.mn

@@ -34,6 +34,8 @@ marionette.jar:
   content/transport.js (transport.js)
   content/packets.js (packets.js)
   content/stream-utils.js (stream-utils.js)
+  content/reftest.js (reftest.js)
+  content/reftest.xul (reftest.xul)
 #ifdef ENABLE_TESTS
   content/test.xul (chrome/test.xul)
   content/test2.xul (chrome/test2.xul)

testing/marionette/listener.js

@@ -488,6 +488,7 @@ var deleteAllCookiesFn = dispatch(deleteAllCookies);
 var executeFn = dispatch(execute);
 var executeInSandboxFn = dispatch(executeInSandbox);
 var sendKeysToElementFn = dispatch(sendKeysToElement);
+var reftestWaitFn = dispatch(reftestWait);
 
 /**
  * Start all message listeners
@@ -535,6 +536,7 @@ function startListeners() {
   addMessageListenerId("Marionette:getCookies", getCookiesFn);
   addMessageListenerId("Marionette:deleteAllCookies", deleteAllCookiesFn);
   addMessageListenerId("Marionette:deleteCookie", deleteCookieFn);
+  addMessageListenerId("Marionette:reftestWait", reftestWaitFn);
 }
 
 /**
@@ -1080,14 +1082,15 @@ function waitForPageLoaded(msg) {
  * driver (in chrome space).
  */
 function get(msg) {
-  let {command_id, pageTimeout, url} = msg.json;
-  let loadEventExpected = true;
+  let {command_id, pageTimeout, url, loadEventExpected=null} = msg.json;
 
   try {
     if (typeof url == "string") {
       try {
         let requestedURL = new URL(url).toString();
+        if (loadEventExpected === null) {
           loadEventExpected = navigate.isLoadEventExpected(requestedURL);
+        }
       } catch (e) {
         sendError(new InvalidArgumentError("Malformed URL: " + e.message), command_id);
         return;
@@ -1735,5 +1738,116 @@ function takeScreenshot(format, opts = {}) {
   }
 }
 
+function flushRendering() {
+  let content = curContainer.frame;
+  let anyPendingPaintsGeneratedInDescendants = false;
+
+  let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor)
+    .getInterface(Ci.nsIDOMWindowUtils);
+
+  function flushWindow(win) {
+    let utils = win.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+    let afterPaintWasPending = utils.isMozAfterPaintPending;
+
+    let root = win.document.documentElement;
+    if (root) {
+      try {
+        // Flush pending restyles and reflows for this window
+        root.getBoundingClientRect();
+      } catch (e) {
+        logger.warning(`flushWindow failed: ${e}`);
+      }
+    }
+
+    if (!afterPaintWasPending && utils.isMozAfterPaintPending) {
+      anyPendingPaintsGeneratedInDescendants = true;
+    }
+
+    for (let i = 0; i < win.frames.length; ++i) {
+      flushWindow(win.frames[i]);
+    }
+  }
+  flushWindow(content);
+
+  if (anyPendingPaintsGeneratedInDescendants &&
+      !windowUtils.isMozAfterPaintPending) {
+    logger.error("Internal error: descendant frame generated a MozAfterPaint event, but the root document doesn't have one!");
+  }
+
+  logger.debug(`flushRendering ${windowUtils.isMozAfterPaintPending}`);
+  return windowUtils.isMozAfterPaintPending;
+}
+
+function* reftestWait(url, remote) {
+  let win = curContainer.frame;
+  let document = curContainer.frame.document;
+
+  let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+
+
+  let reftestWait = false;
+
+  if (document.location.href !== url || document.readyState != "complete") {
+    logger.debug(`Waiting for page load of ${url}`);
+    yield new Promise(resolve => {
+      let maybeResolve = (event) => {
+        if (event.target === curContainer.frame.document &&
+            event.target.location.href === url) {
+          win = curContainer.frame;
+          document = curContainer.frame.document;
+          reftestWait = document.documentElement.classList.contains("reftest-wait");
+          removeEventListener("load", maybeResolve, {once: true});
+          win.setTimeout(resolve, 0);
+        }
+      };
+      addEventListener("load", maybeResolve, true);
+    });
+  } else {
+    // Ensure that the event loop has spun at least once since load,
+    // so that setTimeout(fn, 0) in the load event has run
+    reftestWait = document.documentElement.classList.contains("reftest-wait");
+    yield new Promise(resolve => win.setTimeout(resolve, 0));
+  };
+
+  let root = document.documentElement;
+  if (reftestWait) {
+    // Check again in case reftest-wait was removed since the load event
+    if (root.classList.contains("reftest-wait")) {
+      logger.debug("Waiting for reftest-wait removal");
+      yield new Promise(resolve => {
+        let observer = new win.MutationObserver(() => {
+          if (!root.classList.contains("reftest-wait")) {
+            observer.disconnect();
+            logger.debug("reftest-wait removed");
+            win.setTimeout(resolve, 0);
+          }
+        });
+        observer.observe(root, {attributes: true});
+      });
+    }
+
+    logger.debug("Waiting for rendering");
+
+    yield new Promise(resolve => {
+      let maybeResolve = () => {
+        if (flushRendering()) {
+          win.addEventListener("MozAfterPaint", maybeResolve, {once: true});
+        } else {
+          win.setTimeout(resolve, 0);
+        }
+      };
+      maybeResolve();
+    });
+  } else {
+    flushRendering();
+  }
+
+  if (remote) {
+    windowUtils.updateLayerTree();
+  }
+}
+
 // Call register self when we get loaded
 registerSelf();

testing/marionette/reftest.js

@@ -0,0 +1,353 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+
+Cu.import("chrome://marionette/content/assert.js");
+Cu.import("chrome://marionette/content/capture.js");
+
+this.EXPORTED_SYMBOLS = ["reftest"];
+
+const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul";
+const PREF_E10S = "browser.tabs.remote.autostart";
+
+const logger = Log.repository.getLogger("Marionette");
+
+const SCREENSHOT_MODE = {
+  unexpected: 0,
+  fail: 1,
+  always: 2
+};
+
+const STATUS = {
+  PASS: "PASS",
+  FAIL: "FAIL",
+  ERROR: "ERROR",
+  TIMEOUT: "TIMEOUT",
+};
+
+/**
+ * Implements an fast runner for web-platform-tests format reftests
+ * c.f. http://web-platform-tests.org/writing-tests/reftests.html
+ */
+let reftest = {};
+
+reftest.Runner = class {
+  constructor(driver) {
+    this.driver = driver;
+    this.canvasCache = new Map([[null, []]]);
+    this.windowUtils = null;
+    this.lastUrl = null;
+    this.remote = Preferences.get(PREF_E10S);
+  }
+
+  /**
+   * Setup the required environment for running reftests.
+   *
+   * This will open a non-browser window in which the tests will
+   * be loaded, and set up various caches for the reftest run.
+   *
+   * @param {Object.<Number>} urlCount
+   *     Object holding a map of URL: number of times the URL
+   *     will be opened during the reftest run, where that's
+   *     greater than 1.
+   * @param {string} screenshotMode
+   *     String enum representing when screenshots should be taken
+   */
+  *setup(urlCount, screenshotMode) {
+    this.parentWindow =  assert.window(this.driver.getCurrentWindow());
+
+    this.screenshotMode = SCREENSHOT_MODE[screenshotMode] ||
+        SCREENSHOT_MODE["unexpected"];
+
+    this.urlCount = Object.keys(urlCount || {})
+        .reduce((map, key) => map.set(key, urlCount[key]), new Map());
+
+    yield this.ensureWindow();
+  };
+
+  *ensureWindow() {
+    if (this.reftestWin && !this.reftestWin.closed) {
+      return this.reftestWin;
+    }
+
+    let reftestWin = yield this.openWindow();
+
+    let found = this.driver.findWindow([reftestWin], () => true);
+    yield this.driver.setWindowHandle(found, true);
+
+    this.windowUtils = reftestWin.QueryInterface(Ci.nsIInterfaceRequestor)
+      .getInterface(Ci.nsIDOMWindowUtils);
+    this.reftestWin = reftestWin;
+    return reftestWin;
+  }
+
+  *openWindow() {
+    let reftestWin;
+    yield new Promise(resolve => {
+      reftestWin = this.parentWindow.openDialog("chrome://marionette/content/reftest.xul",
+                                                "reftest",
+                                                "chrome,dialog,height=600,width=600,all",
+                                                () => resolve());
+    });
+
+    let browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
+    browser.permanentKey = {};
+    browser.setAttribute("id", "browser");
+    browser.setAttribute("anonid", "initialBrowser");
+    browser.setAttribute("type", "content");
+    browser.setAttribute("primary", "true");
+
+    if (this.remote) {
+      browser.setAttribute("remote", "true");
+      browser.setAttribute("remoteType", "web");
+    }
+    // Make sure the browser element is exactly 600x600, no matter
+    // what size our window is
+    const window_style = `padding: 0px; margin: 0px; border:none;
+min-width: 600px; min-height: 600px; max-width: 600px; max-height: 600px`;
+    browser.setAttribute("style", window_style);
+
+    let doc = reftestWin.document.documentElement;
+    while (doc.firstChild) {
+      doc.firstChild.remove();
+    }
+    doc.appendChild(browser);
+    reftestWin.gBrowser = browser;
+
+    return reftestWin;
+  }
+
+  abort() {
+    this.driver.close();
+    this.reftestWin = null;
+  }
+
+  /**
+   * Run a specific reftest.
+   *
+   * The assumed semantics are those of web-platform-tests where
+   * references form a tree and each test must meet all the conditions
+   * to reach one leaf node of the tree in order for the overall test
+   * to pass.
+   *
+   * @param {string} testUrl
+   *     URL of the test itself.
+   * @param {Array.<Array>} references
+   *     Array representing a tree of references to try. Each item in
+   *     the array represents a single reference node and has the form
+   *     [referenceUrl, references, relation], where referenceUrl is a
+   *     string to the url, relation is either "==" or "!=" depending on
+   *     the type of reftest, and references is another array containing
+   *     items of the same form, representing further comparisons treated
+   *     as AND with the current item. Sibling entries are treated as OR.
+   *     For example with testUrl of T:
+   *       references = [[A, [[B, [], ==]], ==]]
+   *       Must have T == A AND A == B to pass
+   *
+   *       references = [[A, [], ==], [B, [], !=]
+   *       Must have T == A OR T != B
+   *
+   *       references = [[A, [[B, [], ==], [C, [], ==]], ==], [D, [], ]]
+   *       Must have (T == A AND A == B) OR (T == A AND A == C) OR (T == D)
+   * @param {string} expected
+   *     Expected test outcome (e.g. PASS, FAIL).
+   * @param {number} timeout
+   *     Test timeout in ms
+   *
+   * @return {Object}
+   *     Result object with fields status, message and extra.
+   */
+  *run(testUrl, references, expected, timeout) {
+
+    let timeoutHandle;
+
+    let timeoutPromise = new Promise(resolve => {
+      timeoutHandle = this.parentWindow.setTimeout(() => {
+        resolve({status: STATUS.TIMEOUT, message: null, extra: {}});
+      }, timeout);
+    });
+
+    let testRunner = Task.spawn(function*() {
+      let result;
+      try {
+        result = yield this.runTest(testUrl, references, expected, timeout);
+      } catch (e) {
+        result = {status: STATUS.ERROR, message: e.stack, extra: {}};
+      }
+      return result;
+    }.bind(this));
+
+    let result = yield Promise.race([testRunner, timeoutPromise]);
+    this.parentWindow.clearTimeout(timeoutHandle);
+    if(result.status === STATUS.TIMEOUT) {
+      this.abort();
+    }
+
+    return result;
+  }
+
+  *runTest(testUrl, references, expected, timeout) {
+
+    let win = yield this.ensureWindow();
+
+    win.innerWidth = 600;
+    win.innerHeight = 600;
+
+    let message = "";
+
+    let screenshotData = [];
+
+    let stack = [];
+    for (let i = references.length-1; i >= 0; i--) {
+      let item = references[i];
+      stack.push([testUrl, item[0], item[1], item[2]]);
+    }
+
+    let status = STATUS.FAIL;
+
+    while (stack.length) {
+      let [lhsUrl, rhsUrl, references, relation] = stack.pop();
+      message += `Testing ${lhsUrl} ${relation} ${rhsUrl}\n`;
+
+      let comparison = yield this.compareUrls(win, lhsUrl, rhsUrl, relation, timeout);
+
+      function recordScreenshot() {
+        let toBase64 =  screenshot => screenshot.canvas.toDataURL().split(",")[1];
+        screenshotData.push([{url: lhsUrl, screenshot: toBase64(comparison.lhs)},
+                             relation,
+                             {url:rhsUrl, screenshot: toBase64(comparison.rhs)}]);
+      }
+
+      if (this.screenshotMode === SCREENSHOT_MODE.always) {
+        recordScreenshot();
+      }
+
+      if (comparison.passed) {
+        if (references.length) {
+          for (let i = references.length - 1; i >= 0; i--) {
+            let item = references[i];
+            stack.push([testUrl, item[0], item[1], item[2]]);
+          }
+        } else {
+          // Reached a leaf node so all of one reference chain passed
+          status = STATUS.PASS;
+          if (this.screenshotMode <= SCREENSHOT_MODE.fail && expected != status) {
+            recordScreenshot();
+          }
+          break;
+        }
+      } else if (!stack.length) {
+        // If we don't have any alternatives to try then this will be the last iteration,
+        // so save the failing screenshots if required
+        if (this.screenshotMode === SCREENSHOT_MODE.fail ||
+            (this.screenshotMode === SCREENSHOT_MODE.unexpected && expected != status)) {
+          recordScreenshot();
+        }
+      }
+
+      // Return any reusable canvases to the pool
+      let canvasPool = this.canvasCache.get(null);
+      [comparison.lhs, comparison.rhs].map(screenshot => {
+        if (screenshot.reuseCanvas) {
+          canvasPool.push(screenshot.canvas);
+        }
+      });
+      logger.debug(`Canvas pool is of length ${canvasPool.length}`);
+    }
+
+    let result = {status, message, extra: {}};
+    if (screenshotData.length) {
+      // For now the tbpl formatter only accepts one screenshot, so just return the
+      // last one we took.
+      result.extra.reftest_screenshots = screenshotData[screenshotData.length - 1];
+    }
+
+    return result;
+  };
+
+  *compareUrls(win, lhsUrl, rhsUrl, relation, timeout) {
+    logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`);
+
+    // Take the reference screenshot first so that if we pause
+    // we see the test rendering
+    let rhs = yield this.screenshot(win, rhsUrl, timeout);
+    let lhs = yield this.screenshot(win, lhsUrl, timeout);
+
+    let maxDifferences = {};
+
+    let differences = this.windowUtils.compareCanvases(lhs.canvas, rhs.canvas, {});
+
+    let passed;
+    switch (relation) {
+    case "==":
+      passed = differences === 0;
+      break;
+    case "!=":
+      passed = differences !== 0;
+      break;
+    default:
+      throw new InvalidArgumentError("Reftest operator should be '==' or '!='");
+    }
+
+    return {lhs, rhs, passed};
+  }
+
+  *screenshot(win, url, timeout) {
+    let canvas = null;
+    let remainingCount = this.urlCount.get(url) || 1;
+    let cache = remainingCount > 1;
+    logger.debug(`screenshot ${url} remainingCount: ${remainingCount} cache: ${cache}`);
+    let reuseCanvas = false;
+    if (this.canvasCache.has(url)) {
+      logger.debug(`screenshot ${url} taken from cache`);
+      canvas = this.canvasCache.get(url);
+      if (!cache) {
+        this.canvasCache.delete(url);
+      }
+    } else {
+      let canvases = this.canvasCache.get(null);
+      if (canvases.length) {
+        canvas = canvases.pop();
+      } else {
+        canvas = null;
+      }
+      reuseCanvas = !cache;
+
+      let ctxInterface = win.CanvasRenderingContext2D;
+      let flags = ctxInterface.DRAWWINDOW_DRAW_CARET |
+          ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS |
+          ctxInterface.DRAWWINDOW_DRAW_VIEW;
+
+      logger.debug(`Starting load of ${url}`);
+      if (this.lastUrl === url) {
+        logger.debug(`Refreshing page`);
+        yield this.driver.listener.refresh({commandId: this.driver.listener.activeMessageId,
+                                            pageTimeout: timeout});
+      } else {
+        yield this.driver.listener.get({commandId: this.driver.listener.activeMessageId,
+                                        url: url,
+                                        pageTimeout: timeout,
+                                        loadEventExpected: false});
+        this.lastUrl = url;
+      }
+
+      this.driver.curBrowser.contentBrowser.focus();
+      yield this.driver.listener.reftestWait(url, this.remote);
+
+      canvas = capture.canvas(win, 0, 0, win.innerWidth, win.innerHeight, {canvas, flags});
+    }
+    if (cache) {
+      this.canvasCache.set(url, canvas);
+    };
+    this.urlCount.set(url, remainingCount - 1);
+    return {canvas, reuseCanvas};
+  }
+};

testing/marionette/reftest.xul

@@ -0,0 +1 @@
+<window id="reftest" xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" onload="window.arguments[0]()"></window>

testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py

@@ -184,7 +184,9 @@ class FirefoxBrowser(Browser):
                                       "dom.disable_open_during_load": False,
                                       "network.dns.localDomains": ",".join(hostnames),
                                       "network.proxy.type": 0,
-                                      "places.history.enabled": False})
+                                      "places.history.enabled": False,
+                                      "dom.send_after_paint_to_content": True,
+                                      "layout.interruptible-reflow.enabled": False})
         if self.e10s:
             self.profile.set_preferences({"browser.tabs.remote.autostart": True})