diff --git a/layout/tools/reftest/reftest-content.js b/layout/tools/reftest/reftest-content.js index 7016d6df13b7..ff0de8c6ca0d 100644 --- a/layout/tools/reftest/reftest-content.js +++ b/layout/tools/reftest/reftest-content.js @@ -447,6 +447,7 @@ function WaitForTestEnd(contentRootElement, inPrintMode, spellCheckedElements) { var state = STATE_WAITING_TO_FIRE_INVALIDATE_EVENT; function AfterPaintListener(event) { + dump("AfterPaintListener\n"); LogInfo("AfterPaintListener in " + event.target.document.location.href); if (event.target.document != currentDoc) { // ignore paint events for subframes or old documents in the window. diff --git a/testing/marionette/driver.js b/testing/marionette/driver.js index e62f35bc6cba..57f0a99c428c 100644 --- a/testing/marionette/driver.js +++ b/testing/marionette/driver.js @@ -34,6 +34,7 @@ Cu.import("chrome://marionette/content/legacyaction.js"); Cu.import("chrome://marionette/content/logging.js"); Cu.import("chrome://marionette/content/modal.js"); Cu.import("chrome://marionette/content/proxy.js"); +Cu.import("chrome://marionette/content/reftest.js"); Cu.import("chrome://marionette/content/session.js"); Cu.import("chrome://marionette/content/wait.js"); @@ -3247,6 +3248,65 @@ GeckoDriver.prototype.localizeProperty = function (cmd, resp) { resp.body.value = l10n.localizeProperty(urls, id); } +/** + * Initialize the reftest mode + */ +GeckoDriver.prototype.setupReftest = function* (cmd, resp) { + if (this._reftest) { + throw new UnsupportedOperationError("Called reftest:setup with a reftest session already active"); + } + + if (this.context !== Context.CHROME) { + throw new UnsupportedOperationError("Must set chrome context before running reftests"); + } + + let {urlCount = {}, screenshot = "unexpected"} = cmd.parameters; + if (!["always", "fail", "unexpected"].includes(screenshot)) { + throw new InvalidArgumentError("Value of `screenshot` should be 'always', 'fail' or 'unexpected'"); + } + + this._reftest = new reftest.Runner(this); + + yield this._reftest.setup(urlCount, screenshot); +}; + + +/** + * Run a reftest + */ +GeckoDriver.prototype.runReftest = function* (cmd, resp) { + let {test, references, expected, timeout} = cmd.parameters; + + if (!this._reftest) { + throw new UnsupportedOperationError("Called reftest:run before reftest:start"); + } + + assert.string(test); + assert.string(expected); + assert.array(references); + + let result = yield this._reftest.run(test, references, expected, timeout); + + resp.body.value = result; +}; + +/** + * End a reftest run + * + * Closes the reftest window (without changing the current window handle), + * and removes cached canvases. + */ +GeckoDriver.prototype.teardownReftest = function* (cmd, resp) { + if (!this._reftest) { + throw new UnsupportedOperationError("Called reftest:teardown before reftest:start"); + } + + this._reftest.abort(); + + this._reftest = null; +}; + + GeckoDriver.prototype.commands = { "getMarionetteID": GeckoDriver.prototype.getMarionetteID, "sayHello": GeckoDriver.prototype.sayHello, @@ -3333,6 +3393,10 @@ GeckoDriver.prototype.commands = { "addon:install": GeckoDriver.prototype.installAddon, "addon:uninstall": GeckoDriver.prototype.uninstallAddon, + + "reftest:setup": GeckoDriver.prototype.setupReftest, + "reftest:run": GeckoDriver.prototype.runReftest, + "reftest:teardown": GeckoDriver.prototype.teardownReftest, }; function copy (obj) { diff --git a/testing/marionette/jar.mn b/testing/marionette/jar.mn index ac4f98a3d724..b79fa0899072 100644 --- a/testing/marionette/jar.mn +++ b/testing/marionette/jar.mn @@ -34,6 +34,8 @@ marionette.jar: content/transport.js (transport.js) content/packets.js (packets.js) content/stream-utils.js (stream-utils.js) + content/reftest.js (reftest.js) + content/reftest.xul (reftest.xul) #ifdef ENABLE_TESTS content/test.xul (chrome/test.xul) content/test2.xul (chrome/test2.xul) diff --git a/testing/marionette/listener.js b/testing/marionette/listener.js index b76d488edde6..1db34427a4db 100644 --- a/testing/marionette/listener.js +++ b/testing/marionette/listener.js @@ -488,6 +488,7 @@ var deleteAllCookiesFn = dispatch(deleteAllCookies); var executeFn = dispatch(execute); var executeInSandboxFn = dispatch(executeInSandbox); var sendKeysToElementFn = dispatch(sendKeysToElement); +var reftestWaitFn = dispatch(reftestWait); /** * Start all message listeners @@ -535,6 +536,7 @@ function startListeners() { addMessageListenerId("Marionette:getCookies", getCookiesFn); addMessageListenerId("Marionette:deleteAllCookies", deleteAllCookiesFn); addMessageListenerId("Marionette:deleteCookie", deleteCookieFn); + addMessageListenerId("Marionette:reftestWait", reftestWaitFn); } /** @@ -1080,14 +1082,15 @@ function waitForPageLoaded(msg) { * driver (in chrome space). */ function get(msg) { - let {command_id, pageTimeout, url} = msg.json; - let loadEventExpected = true; + let {command_id, pageTimeout, url, loadEventExpected=null} = msg.json; try { if (typeof url == "string") { try { let requestedURL = new URL(url).toString(); - loadEventExpected = navigate.isLoadEventExpected(requestedURL); + if (loadEventExpected === null) { + loadEventExpected = navigate.isLoadEventExpected(requestedURL); + } } catch (e) { sendError(new InvalidArgumentError("Malformed URL: " + e.message), command_id); return; @@ -1735,5 +1738,116 @@ function takeScreenshot(format, opts = {}) { } } +function flushRendering() { + let content = curContainer.frame; + let anyPendingPaintsGeneratedInDescendants = false; + + let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindowUtils); + + function flushWindow(win) { + let utils = win.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindowUtils); + let afterPaintWasPending = utils.isMozAfterPaintPending; + + let root = win.document.documentElement; + if (root) { + try { + // Flush pending restyles and reflows for this window + root.getBoundingClientRect(); + } catch (e) { + logger.warning(`flushWindow failed: ${e}`); + } + } + + if (!afterPaintWasPending && utils.isMozAfterPaintPending) { + anyPendingPaintsGeneratedInDescendants = true; + } + + for (let i = 0; i < win.frames.length; ++i) { + flushWindow(win.frames[i]); + } + } + flushWindow(content); + + if (anyPendingPaintsGeneratedInDescendants && + !windowUtils.isMozAfterPaintPending) { + logger.error("Internal error: descendant frame generated a MozAfterPaint event, but the root document doesn't have one!"); + } + + logger.debug(`flushRendering ${windowUtils.isMozAfterPaintPending}`); + return windowUtils.isMozAfterPaintPending; +} + +function* reftestWait(url, remote) { + let win = curContainer.frame; + let document = curContainer.frame.document; + + let windowUtils = content.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindowUtils); + + + let reftestWait = false; + + if (document.location.href !== url || document.readyState != "complete") { + logger.debug(`Waiting for page load of ${url}`); + yield new Promise(resolve => { + let maybeResolve = (event) => { + if (event.target === curContainer.frame.document && + event.target.location.href === url) { + win = curContainer.frame; + document = curContainer.frame.document; + reftestWait = document.documentElement.classList.contains("reftest-wait"); + removeEventListener("load", maybeResolve, {once: true}); + win.setTimeout(resolve, 0); + } + }; + addEventListener("load", maybeResolve, true); + }); + } else { + // Ensure that the event loop has spun at least once since load, + // so that setTimeout(fn, 0) in the load event has run + reftestWait = document.documentElement.classList.contains("reftest-wait"); + yield new Promise(resolve => win.setTimeout(resolve, 0)); + }; + + let root = document.documentElement; + if (reftestWait) { + // Check again in case reftest-wait was removed since the load event + if (root.classList.contains("reftest-wait")) { + logger.debug("Waiting for reftest-wait removal"); + yield new Promise(resolve => { + let observer = new win.MutationObserver(() => { + if (!root.classList.contains("reftest-wait")) { + observer.disconnect(); + logger.debug("reftest-wait removed"); + win.setTimeout(resolve, 0); + } + }); + observer.observe(root, {attributes: true}); + }); + } + + logger.debug("Waiting for rendering"); + + yield new Promise(resolve => { + let maybeResolve = () => { + if (flushRendering()) { + win.addEventListener("MozAfterPaint", maybeResolve, {once: true}); + } else { + win.setTimeout(resolve, 0); + } + }; + maybeResolve(); + }); + } else { + flushRendering(); + } + + if (remote) { + windowUtils.updateLayerTree(); + } +} + // Call register self when we get loaded registerSelf(); diff --git a/testing/marionette/reftest.js b/testing/marionette/reftest.js new file mode 100644 index 000000000000..f59ae4dbebc6 --- /dev/null +++ b/testing/marionette/reftest.js @@ -0,0 +1,353 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/Log.jsm"); +Cu.import("resource://gre/modules/Preferences.jsm"); +Cu.import("resource://gre/modules/Task.jsm"); + +Cu.import("chrome://marionette/content/assert.js"); +Cu.import("chrome://marionette/content/capture.js"); + +this.EXPORTED_SYMBOLS = ["reftest"]; + +const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"; +const PREF_E10S = "browser.tabs.remote.autostart"; + +const logger = Log.repository.getLogger("Marionette"); + +const SCREENSHOT_MODE = { + unexpected: 0, + fail: 1, + always: 2 +}; + +const STATUS = { + PASS: "PASS", + FAIL: "FAIL", + ERROR: "ERROR", + TIMEOUT: "TIMEOUT", +}; + +/** + * Implements an fast runner for web-platform-tests format reftests + * c.f. http://web-platform-tests.org/writing-tests/reftests.html + */ +let reftest = {}; + +reftest.Runner = class { + constructor(driver) { + this.driver = driver; + this.canvasCache = new Map([[null, []]]); + this.windowUtils = null; + this.lastUrl = null; + this.remote = Preferences.get(PREF_E10S); + } + + /** + * Setup the required environment for running reftests. + * + * This will open a non-browser window in which the tests will + * be loaded, and set up various caches for the reftest run. + * + * @param {Object.} urlCount + * Object holding a map of URL: number of times the URL + * will be opened during the reftest run, where that's + * greater than 1. + * @param {string} screenshotMode + * String enum representing when screenshots should be taken + */ + *setup(urlCount, screenshotMode) { + this.parentWindow = assert.window(this.driver.getCurrentWindow()); + + this.screenshotMode = SCREENSHOT_MODE[screenshotMode] || + SCREENSHOT_MODE["unexpected"]; + + this.urlCount = Object.keys(urlCount || {}) + .reduce((map, key) => map.set(key, urlCount[key]), new Map()); + + yield this.ensureWindow(); + }; + + *ensureWindow() { + if (this.reftestWin && !this.reftestWin.closed) { + return this.reftestWin; + } + + let reftestWin = yield this.openWindow(); + + let found = this.driver.findWindow([reftestWin], () => true); + yield this.driver.setWindowHandle(found, true); + + this.windowUtils = reftestWin.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindowUtils); + this.reftestWin = reftestWin; + return reftestWin; + } + + *openWindow() { + let reftestWin; + yield new Promise(resolve => { + reftestWin = this.parentWindow.openDialog("chrome://marionette/content/reftest.xul", + "reftest", + "chrome,dialog,height=600,width=600,all", + () => resolve()); + }); + + let browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser"); + browser.permanentKey = {}; + browser.setAttribute("id", "browser"); + browser.setAttribute("anonid", "initialBrowser"); + browser.setAttribute("type", "content"); + browser.setAttribute("primary", "true"); + + if (this.remote) { + browser.setAttribute("remote", "true"); + browser.setAttribute("remoteType", "web"); + } + // Make sure the browser element is exactly 600x600, no matter + // what size our window is + const window_style = `padding: 0px; margin: 0px; border:none; +min-width: 600px; min-height: 600px; max-width: 600px; max-height: 600px`; + browser.setAttribute("style", window_style); + + let doc = reftestWin.document.documentElement; + while (doc.firstChild) { + doc.firstChild.remove(); + } + doc.appendChild(browser); + reftestWin.gBrowser = browser; + + return reftestWin; + } + + abort() { + this.driver.close(); + this.reftestWin = null; + } + + /** + * Run a specific reftest. + * + * The assumed semantics are those of web-platform-tests where + * references form a tree and each test must meet all the conditions + * to reach one leaf node of the tree in order for the overall test + * to pass. + * + * @param {string} testUrl + * URL of the test itself. + * @param {Array.} references + * Array representing a tree of references to try. Each item in + * the array represents a single reference node and has the form + * [referenceUrl, references, relation], where referenceUrl is a + * string to the url, relation is either "==" or "!=" depending on + * the type of reftest, and references is another array containing + * items of the same form, representing further comparisons treated + * as AND with the current item. Sibling entries are treated as OR. + * For example with testUrl of T: + * references = [[A, [[B, [], ==]], ==]] + * Must have T == A AND A == B to pass + * + * references = [[A, [], ==], [B, [], !=] + * Must have T == A OR T != B + * + * references = [[A, [[B, [], ==], [C, [], ==]], ==], [D, [], ]] + * Must have (T == A AND A == B) OR (T == A AND A == C) OR (T == D) + * @param {string} expected + * Expected test outcome (e.g. PASS, FAIL). + * @param {number} timeout + * Test timeout in ms + * + * @return {Object} + * Result object with fields status, message and extra. + */ + *run(testUrl, references, expected, timeout) { + + let timeoutHandle; + + let timeoutPromise = new Promise(resolve => { + timeoutHandle = this.parentWindow.setTimeout(() => { + resolve({status: STATUS.TIMEOUT, message: null, extra: {}}); + }, timeout); + }); + + let testRunner = Task.spawn(function*() { + let result; + try { + result = yield this.runTest(testUrl, references, expected, timeout); + } catch (e) { + result = {status: STATUS.ERROR, message: e.stack, extra: {}}; + } + return result; + }.bind(this)); + + let result = yield Promise.race([testRunner, timeoutPromise]); + this.parentWindow.clearTimeout(timeoutHandle); + if(result.status === STATUS.TIMEOUT) { + this.abort(); + } + + return result; + } + + *runTest(testUrl, references, expected, timeout) { + + let win = yield this.ensureWindow(); + + win.innerWidth = 600; + win.innerHeight = 600; + + let message = ""; + + let screenshotData = []; + + let stack = []; + for (let i = references.length-1; i >= 0; i--) { + let item = references[i]; + stack.push([testUrl, item[0], item[1], item[2]]); + } + + let status = STATUS.FAIL; + + while (stack.length) { + let [lhsUrl, rhsUrl, references, relation] = stack.pop(); + message += `Testing ${lhsUrl} ${relation} ${rhsUrl}\n`; + + let comparison = yield this.compareUrls(win, lhsUrl, rhsUrl, relation, timeout); + + function recordScreenshot() { + let toBase64 = screenshot => screenshot.canvas.toDataURL().split(",")[1]; + screenshotData.push([{url: lhsUrl, screenshot: toBase64(comparison.lhs)}, + relation, + {url:rhsUrl, screenshot: toBase64(comparison.rhs)}]); + } + + if (this.screenshotMode === SCREENSHOT_MODE.always) { + recordScreenshot(); + } + + if (comparison.passed) { + if (references.length) { + for (let i = references.length - 1; i >= 0; i--) { + let item = references[i]; + stack.push([testUrl, item[0], item[1], item[2]]); + } + } else { + // Reached a leaf node so all of one reference chain passed + status = STATUS.PASS; + if (this.screenshotMode <= SCREENSHOT_MODE.fail && expected != status) { + recordScreenshot(); + } + break; + } + } else if (!stack.length) { + // If we don't have any alternatives to try then this will be the last iteration, + // so save the failing screenshots if required + if (this.screenshotMode === SCREENSHOT_MODE.fail || + (this.screenshotMode === SCREENSHOT_MODE.unexpected && expected != status)) { + recordScreenshot(); + } + } + + // Return any reusable canvases to the pool + let canvasPool = this.canvasCache.get(null); + [comparison.lhs, comparison.rhs].map(screenshot => { + if (screenshot.reuseCanvas) { + canvasPool.push(screenshot.canvas); + } + }); + logger.debug(`Canvas pool is of length ${canvasPool.length}`); + } + + let result = {status, message, extra: {}}; + if (screenshotData.length) { + // For now the tbpl formatter only accepts one screenshot, so just return the + // last one we took. + result.extra.reftest_screenshots = screenshotData[screenshotData.length - 1]; + } + + return result; + }; + + *compareUrls(win, lhsUrl, rhsUrl, relation, timeout) { + logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`); + + // Take the reference screenshot first so that if we pause + // we see the test rendering + let rhs = yield this.screenshot(win, rhsUrl, timeout); + let lhs = yield this.screenshot(win, lhsUrl, timeout); + + let maxDifferences = {}; + + let differences = this.windowUtils.compareCanvases(lhs.canvas, rhs.canvas, {}); + + let passed; + switch (relation) { + case "==": + passed = differences === 0; + break; + case "!=": + passed = differences !== 0; + break; + default: + throw new InvalidArgumentError("Reftest operator should be '==' or '!='"); + } + + return {lhs, rhs, passed}; + } + + *screenshot(win, url, timeout) { + let canvas = null; + let remainingCount = this.urlCount.get(url) || 1; + let cache = remainingCount > 1; + logger.debug(`screenshot ${url} remainingCount: ${remainingCount} cache: ${cache}`); + let reuseCanvas = false; + if (this.canvasCache.has(url)) { + logger.debug(`screenshot ${url} taken from cache`); + canvas = this.canvasCache.get(url); + if (!cache) { + this.canvasCache.delete(url); + } + } else { + let canvases = this.canvasCache.get(null); + if (canvases.length) { + canvas = canvases.pop(); + } else { + canvas = null; + } + reuseCanvas = !cache; + + let ctxInterface = win.CanvasRenderingContext2D; + let flags = ctxInterface.DRAWWINDOW_DRAW_CARET | + ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS | + ctxInterface.DRAWWINDOW_DRAW_VIEW; + + logger.debug(`Starting load of ${url}`); + if (this.lastUrl === url) { + logger.debug(`Refreshing page`); + yield this.driver.listener.refresh({commandId: this.driver.listener.activeMessageId, + pageTimeout: timeout}); + } else { + yield this.driver.listener.get({commandId: this.driver.listener.activeMessageId, + url: url, + pageTimeout: timeout, + loadEventExpected: false}); + this.lastUrl = url; + } + + this.driver.curBrowser.contentBrowser.focus(); + yield this.driver.listener.reftestWait(url, this.remote); + + canvas = capture.canvas(win, 0, 0, win.innerWidth, win.innerHeight, {canvas, flags}); + } + if (cache) { + this.canvasCache.set(url, canvas); + }; + this.urlCount.set(url, remainingCount - 1); + return {canvas, reuseCanvas}; + } +}; diff --git a/testing/marionette/reftest.xul b/testing/marionette/reftest.xul new file mode 100644 index 000000000000..aed3a8a30b24 --- /dev/null +++ b/testing/marionette/reftest.xul @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py index bacf10c57c02..ed0180962a30 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py @@ -184,7 +184,9 @@ class FirefoxBrowser(Browser): "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(hostnames), "network.proxy.type": 0, - "places.history.enabled": False}) + "places.history.enabled": False, + "dom.send_after_paint_to_content": True, + "layout.interruptible-reflow.enabled": False}) if self.e10s: self.profile.set_preferences({"browser.tabs.remote.autostart": True})