Bug 1514856 - move image data array traversal to worker thread when calculating contrast ratio for text nodes. r=jdescottes,ochameau

MozReview-Commit-ID: K3twiMih7e9

Differential Revision: https://phabricator.services.mozilla.com/D15113

--HG--
rename : devtools/server/actors/utils/accessibility.js => devtools/server/actors/accessibility/contrast.js
extra : moz-landing-system : lando

devtools/server/actors/accessibility/accessible.js

@@ -8,7 +8,7 @@ const { Ci, Cu } = require("chrome");
 const { Actor, ActorClassWithSpec } = require("devtools/shared/protocol");
 const { accessibleSpec } = require("devtools/shared/specs/accessibility");
 
-loader.lazyRequireGetter(this, "getContrastRatioFor", "devtools/server/actors/utils/accessibility", true);
+loader.lazyRequireGetter(this, "getContrastRatioFor", "devtools/server/actors/accessibility/contrast", true);
 loader.lazyRequireGetter(this, "isDefunct", "devtools/server/actors/utils/accessibility", true);
 loader.lazyRequireGetter(this, "findCssSelector", "devtools/shared/inspector/css-logic", true);
 
@@ -380,16 +380,18 @@ const AccessibleActor = ActorClassWithSpec(accessibleSpec, {
   /**
    * Calculate the contrast ratio of the given accessible.
    */
-  _getContrastRatio() {
+  async _getContrastRatio() {
     if (!this._isValidTextLeaf(this.rawAccessible)) {
       return null;
     }
 
     const { DOMNode: rawNode } = this.rawAccessible;
-    return getContrastRatioFor(rawNode.parentNode, {
+    const contrastRatio = await getContrastRatioFor(rawNode.parentNode, {
       bounds: this.bounds,
       win: rawNode.ownerGlobal,
     });
+
+    return contrastRatio;
   },
 
   /**
@@ -399,8 +401,15 @@ const AccessibleActor = ActorClassWithSpec(accessibleSpec, {
    *         Audit results for the accessible object.
   */
   async audit() {
+    // More audit steps will be added here in the near future. In addition to colour
+    // contrast ratio we will add autits for to the missing names, invalid states, etc.
+    // (For example see bug 1518808).
+    const [ contrastRatio ] = await Promise.all([
+      this._getContrastRatio(),
+    ]);
+
     return this.isDefunct ? null : {
-      contrastRatio: this._getContrastRatio(),
+      contrastRatio,
     };
   },
 

devtools/server/actors/accessibility/contrast.js

@@ -0,0 +1,168 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+loader.lazyRequireGetter(this, "colorUtils", "devtools/shared/css/color", true);
+loader.lazyRequireGetter(this, "CssLogic", "devtools/server/actors/inspector/css-logic", true);
+loader.lazyRequireGetter(this, "getBounds", "devtools/server/actors/highlighters/utils/accessibility", true);
+loader.lazyRequireGetter(this, "getCurrentZoom", "devtools/shared/layout/utils", true);
+loader.lazyRequireGetter(this, "addPseudoClassLock", "devtools/server/actors/highlighters/utils/markup", true);
+loader.lazyRequireGetter(this, "removePseudoClassLock", "devtools/server/actors/highlighters/utils/markup", true);
+loader.lazyRequireGetter(this, "DevToolsWorker", "devtools/shared/worker/worker", true);
+
+const WORKER_URL = "resource://devtools/server/actors/accessibility/worker.js";
+const HIGHLIGHTED_PSEUDO_CLASS = ":-moz-devtools-highlighted";
+
+loader.lazyGetter(this, "worker", () => new DevToolsWorker(WORKER_URL));
+
+/**
+ * Get text style properties for a given node, if possible.
+ * @param  {DOMNode} node
+ *         DOM node for which text styling information is to be calculated.
+ * @return {Object}
+ *         Color and text size information for a given DOM node.
+ */
+function getTextProperties(node) {
+  const computedStyles = CssLogic.getComputedStyle(node);
+  if (!computedStyles) {
+    return null;
+  }
+
+  const { color, "font-size": fontSize, "font-weight": fontWeight } = computedStyles;
+  const opacity = parseFloat(computedStyles.opacity);
+
+  let { r, g, b, a } = colorUtils.colorToRGBA(color, true);
+  a = opacity * a;
+  const textRgbaColor = new colorUtils.CssColor(`rgba(${r}, ${g}, ${b}, ${a})`, true);
+  // TODO: For cases where text color is transparent, it likely comes from the color of
+  // the background that is underneath it (commonly from background-clip: text
+  // property). With some additional investigation it might be possible to calculate the
+  // color contrast where the color of the background is used as text color and the
+  // color of the ancestor's background is used as its background.
+  if (textRgbaColor.isTransparent()) {
+    return null;
+  }
+
+  const isBoldText = parseInt(fontWeight, 10) >= 600;
+  const isLargeText = Math.ceil(parseFloat(fontSize) * 72) / 96 >= (isBoldText ? 14 : 18);
+
+  return {
+    // Blend text color taking its alpha into account asuming white background.
+    color: colorUtils.blendColors([r, g, b, a]),
+    isLargeText,
+  };
+}
+
+/**
+ * Get canvas rendering context for the current target window bound by the bounds of the
+ * accessible objects.
+ * @param  {Object}  win
+ *         Current target window.
+ * @param  {Object}  bounds
+ *         Bounds for the accessible object.
+ * @param  {null|DOMNode} node
+ *         If not null, a node that corresponds to the accessible object to be used to
+ *         make its text color transparent.
+ * @return {CanvasRenderingContext2D}
+ *         Canvas rendering context for the current window.
+ */
+function getImageCtx(win, bounds, node) {
+  const doc = win.document;
+  const canvas = doc.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
+  const scale = getCurrentZoom(win);
+
+  const { left, top, width, height } = bounds;
+  canvas.width = width / scale;
+  canvas.height = height / scale;
+  const ctx = canvas.getContext("2d", { alpha: false });
+
+  // If node is passed, make its color related text properties invisible.
+  if (node) {
+    addPseudoClassLock(node, HIGHLIGHTED_PSEUDO_CLASS);
+  }
+
+  ctx.drawWindow(win, left / scale, top / scale, width / scale, height / scale, "#fff",
+                 ctx.DRAWWINDOW_USE_WIDGET_LAYERS);
+
+  // Restore all inline styling.
+  if (node) {
+    removePseudoClassLock(node, HIGHLIGHTED_PSEUDO_CLASS);
+  }
+
+  return ctx;
+}
+
+/**
+ * Calculates the contrast ratio of the referenced DOM node.
+ *
+ * @param  {DOMNode} node
+ *         The node for which we want to calculate the contrast ratio.
+ * @param  {Object}  options
+ *         - bounds   {Object}
+ *                    Bounds for the accessible object.
+ *         - win      {Object}
+ *                    Target window.
+ *
+ * @return {Object}
+ *         An object that may contain one or more of the following fields: error,
+ *         isLargeText, value, min, max values for contrast.
+*/
+async function getContrastRatioFor(node, options = {}) {
+  const props = getTextProperties(node);
+  if (!props) {
+    return {
+      error: true,
+    };
+  }
+
+  const bounds = getBounds(options.win, options.bounds);
+  const textContext = getImageCtx(options.win, bounds);
+  const backgroundContext = getImageCtx(options.win, bounds, node);
+
+  const { data: dataText } = textContext.getImageData(0, 0, bounds.width, bounds.height);
+  const { data: dataBackground } = backgroundContext.getImageData(
+    0, 0, bounds.width, bounds.height);
+
+  const rgba = await worker.performTask("getBgRGBA", {
+    dataTextBuf: dataText.buffer,
+    dataBackgroundBuf: dataBackground.buffer,
+  }, [ dataText.buffer, dataBackground.buffer ]);
+
+  if (!rgba) {
+    return {
+      error: true,
+    };
+  }
+
+  const { color, isLargeText } = props;
+  if (rgba.value) {
+    return {
+      value: colorUtils.calculateContrastRatio(rgba.value, color),
+      color,
+      backgroundColor: rgba.value,
+      isLargeText,
+    };
+  }
+
+  let min = colorUtils.calculateContrastRatio(rgba.min, color);
+  let max = colorUtils.calculateContrastRatio(rgba.max, color);
+
+  // Flip minimum and maximum contrast ratios if necessary.
+  if (min > max) {
+    [min, max] = [max, min];
+    [rgba.min, rgba.max] = [rgba.max, rgba.min];
+  }
+
+  return {
+    min,
+    max,
+    color,
+    backgroundColorMin: rgba.min,
+    backgroundColorMax: rgba.max,
+    isLargeText,
+  };
+}
+
+exports.getContrastRatioFor = getContrastRatioFor;

devtools/server/actors/accessibility/moz.build

@@ -6,7 +6,9 @@ DevToolsModules(
   'accessibility-parent.js',
   'accessibility.js',
   'accessible.js',
+  'contrast.js',
   'walker.js',
+  'worker.js',
 )
 
 with Files('**'):

devtools/server/actors/accessibility/worker.js

@@ -0,0 +1,102 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+/* eslint-env worker */
+
+/**
+ * Import `createTask` to communicate with `devtools/shared/worker`.
+ */
+importScripts("resource://gre/modules/workers/require.js");
+const { createTask } = require("resource://devtools/shared/worker/helper.js");
+
+/**
+ * @see LineGraphWidget.prototype.setDataFromTimestamps in Graphs.js
+ * @param number id
+ * @param array timestamps
+ * @param number interval
+ * @param number duration
+ */
+createTask(self, "getBgRGBA", ({ dataTextBuf, dataBackgroundBuf }) =>
+  getBgRGBA(dataTextBuf, dataBackgroundBuf));
+
+/**
+ * Calculates the luminance of a rgba tuple based on the formula given in
+ * https://www.w3.org/TR/2008/REC-WCAG20-20081211/#relativeluminancedef
+ *
+ * @param {Array} rgba An array with [r,g,b,a] values.
+ * @return {Number} The calculated luminance.
+ */
+function calculateLuminance(rgba) {
+  for (let i = 0; i < 3; i++) {
+    rgba[i] /= 255;
+    rgba[i] = (rgba[i] < 0.03928) ? (rgba[i] / 12.92) :
+                                    Math.pow(((rgba[i] + 0.055) / 1.055), 2.4);
+  }
+  return 0.2126 * rgba[0] + 0.7152 * rgba[1] + 0.0722 * rgba[2];
+}
+
+/**
+ * Get RGBA or a range of RGBAs for the background pixels under the text. If luminance is
+ * uniform, only return one value of RGBA, otherwise return values that correspond to the
+ * min and max luminances.
+ * @param  {ImageData} dataTextBuf
+ *         pixel data for the accessible object with text visible.
+ * @param  {ImageData} dataBackgroundBuf
+ *         pixel data for the accessible object with transparent text.
+ * @return {Object}
+ *         RGBA or a range of RGBAs with min and max values.
+ */
+function getBgRGBA(dataTextBuf, dataBackgroundBuf) {
+  let min = [0, 0, 0, 1];
+  let max = [255, 255, 255, 1];
+  let minLuminance = 1;
+  let maxLuminance = 0;
+  const luminances = {};
+  const dataText = new Uint8ClampedArray(dataTextBuf);
+  const dataBackground = new Uint8ClampedArray(dataBackgroundBuf);
+
+  let foundDistinctColor = false;
+  for (let i = 0; i < dataText.length; i = i + 4) {
+    const tR = dataText[i];
+    const bgR = dataBackground[i];
+    const tG = dataText[i + 1];
+    const bgG = dataBackground[i + 1];
+    const tB = dataText[i + 2];
+    const bgB = dataBackground[i + 2];
+
+    // Ignore pixels that are the same where pixels that are different between the two
+    // images are assumed to belong to the text within the node.
+    if (tR === bgR && tG === bgG && tB === bgB) {
+      continue;
+    }
+
+    foundDistinctColor = true;
+
+    const bgColor = `rgb(${bgR}, ${bgG}, ${bgB})`;
+    let luminance = luminances[bgColor];
+
+    if (!luminance) {
+      // Calculate luminance for the RGB value and store it to only measure once.
+      luminance = calculateLuminance([bgR, bgG, bgB]);
+      luminances[bgColor] = luminance;
+    }
+
+    if (minLuminance >= luminance) {
+      minLuminance = luminance;
+      min = [bgR, bgG, bgB, 1];
+    }
+
+    if (maxLuminance <= luminance) {
+      maxLuminance = luminance;
+      max = [bgR, bgG, bgB, 1];
+    }
+  }
+
+  if (!foundDistinctColor) {
+    return null;
+  }
+
+  return minLuminance === maxLuminance ? { value: max } : { min, max };
+}

devtools/server/actors/utils/accessibility.js

@@ -5,221 +5,7 @@
 "use strict";
 
 loader.lazyRequireGetter(this, "Ci", "chrome", true);
-loader.lazyRequireGetter(this, "colorUtils", "devtools/shared/css/color", true);
-loader.lazyRequireGetter(this, "CssLogic", "devtools/server/actors/inspector/css-logic", true);
-loader.lazyRequireGetter(this, "getBounds", "devtools/server/actors/highlighters/utils/accessibility", true);
-loader.lazyRequireGetter(this, "getCurrentZoom", "devtools/shared/layout/utils", true);
 loader.lazyRequireGetter(this, "Services");
-loader.lazyRequireGetter(this, "addPseudoClassLock", "devtools/server/actors/highlighters/utils/markup", true);
-loader.lazyRequireGetter(this, "removePseudoClassLock", "devtools/server/actors/highlighters/utils/markup", true);
-
-const HIGHLIGHTED_PSEUDO_CLASS = ":-moz-devtools-highlighted";
-
-/**
- * Get text style properties for a given node, if possible.
- * @param  {DOMNode} node
- *         DOM node for which text styling information is to be calculated.
- * @return {Object}
- *         Color and text size information for a given DOM node.
- */
-function getTextProperties(node) {
-  const computedStyles = CssLogic.getComputedStyle(node);
-  if (!computedStyles) {
-    return null;
-  }
-
-  const { color, "font-size": fontSize, "font-weight": fontWeight } = computedStyles;
-  const opacity = parseFloat(computedStyles.opacity);
-
-  let { r, g, b, a } = colorUtils.colorToRGBA(color, true);
-  a = opacity * a;
-  const textRgbaColor = new colorUtils.CssColor(`rgba(${r}, ${g}, ${b}, ${a})`, true);
-  // TODO: For cases where text color is transparent, it likely comes from the color of
-  // the background that is underneath it (commonly from background-clip: text
-  // property). With some additional investigation it might be possible to calculate the
-  // color contrast where the color of the background is used as text color and the
-  // color of the ancestor's background is used as its background.
-  if (textRgbaColor.isTransparent()) {
-    return null;
-  }
-
-  const isBoldText = parseInt(fontWeight, 10) >= 600;
-  const isLargeText = Math.ceil(parseFloat(fontSize) * 72) / 96 >= (isBoldText ? 14 : 18);
-
-  return {
-    // Blend text color taking its alpha into account asuming white background.
-    color: colorUtils.blendColors([r, g, b, a]),
-    isLargeText,
-  };
-}
-
-/**
- * Get canvas rendering context for the current target window bound by the bounds of the
- * accessible objects.
- * @param  {Object}  win
- *         Current target window.
- * @param  {Object}  bounds
- *         Bounds for the accessible object.
- * @param  {null|DOMNode} node
- *         If not null, a node that corresponds to the accessible object to be used to
- *         make its text color transparent.
- * @return {CanvasRenderingContext2D}
- *         Canvas rendering context for the current window.
- */
-function getImageCtx(win, bounds, node) {
-  const doc = win.document;
-  const canvas = doc.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
-  const scale = getCurrentZoom(win);
-
-  const { left, top, width, height } = bounds;
-  canvas.width = width / scale;
-  canvas.height = height / scale;
-  const ctx = canvas.getContext("2d", { alpha: false });
-
-  // If node is passed, make its color related text properties invisible.
-  if (node) {
-    addPseudoClassLock(node, HIGHLIGHTED_PSEUDO_CLASS);
-  }
-
-  ctx.drawWindow(win, left / scale, top / scale, width / scale, height / scale, "#fff",
-                 ctx.DRAWWINDOW_USE_WIDGET_LAYERS);
-
-  // Restore all inline styling.
-  if (node) {
-    removePseudoClassLock(node, HIGHLIGHTED_PSEUDO_CLASS);
-  }
-
-  return ctx;
-}
-
-/**
- * Get RGBA or a range of RGBAs for the background pixels under the text. If luminance is
- * uniform, only return one value of RGBA, otherwise return values that correspond to the
- * min and max luminances.
- * @param  {ImageData} dataText
- *         pixel data for the accessible object with text visible.
- * @param  {ImageData} dataBackground
- *         pixel data for the accessible object with transparent text.
- * @return {Object}
- *         RGBA or a range of RGBAs with min and max values.
- */
-function getBgRGBA(dataText, dataBackground) {
-  let min = [0, 0, 0, 1];
-  let max = [255, 255, 255, 1];
-  let minLuminance = 1;
-  let maxLuminance = 0;
-  const luminances = {};
-
-  let foundDistinctColor = false;
-  for (let i = 0; i < dataText.length; i = i + 4) {
-    const tR = dataText[i];
-    const bgR = dataBackground[i];
-    const tG = dataText[i + 1];
-    const bgG = dataBackground[i + 1];
-    const tB = dataText[i + 2];
-    const bgB = dataBackground[i + 2];
-
-    // Ignore pixels that are the same where pixels that are different between the two
-    // images are assumed to belong to the text within the node.
-    if (tR === bgR && tG === bgG && tB === bgB) {
-      continue;
-    }
-
-    foundDistinctColor = true;
-
-    const bgColor = `rgb(${bgR}, ${bgG}, ${bgB})`;
-    let luminance = luminances[bgColor];
-
-    if (!luminance) {
-      // Calculate luminance for the RGB value and store it to only measure once.
-      luminance = colorUtils.calculateLuminance([bgR, bgG, bgB]);
-      luminances[bgColor] = luminance;
-    }
-
-    if (minLuminance >= luminance) {
-      minLuminance = luminance;
-      min = [bgR, bgG, bgB, 1];
-    }
-
-    if (maxLuminance <= luminance) {
-      maxLuminance = luminance;
-      max = [bgR, bgG, bgB, 1];
-    }
-  }
-
-  if (!foundDistinctColor) {
-    return null;
-  }
-
-  return minLuminance === maxLuminance ? { value: max } : { min, max };
-}
-
-/**
- * Calculates the contrast ratio of the referenced DOM node.
- *
- * @param  {DOMNode} node
- *         The node for which we want to calculate the contrast ratio.
- * @param  {Object}  options
- *         - bounds   {Object}
- *                    Bounds for the accessible object.
- *         - win      {Object}
- *                    Target window.
- *
- * @return {Object}
- *         An object that may contain one or more of the following fields: error,
- *         isLargeText, value, min, max values for contrast.
-*/
-function getContrastRatioFor(node, options = {}) {
-  const props = getTextProperties(node);
-  if (!props) {
-    return {
-      error: true,
-    };
-  }
-
-  const bounds = getBounds(options.win, options.bounds);
-  const textContext = getImageCtx(options.win, bounds);
-  const backgroundContext = getImageCtx(options.win, bounds, node);
-
-  const { data: dataText } = textContext.getImageData(0, 0, bounds.width, bounds.height);
-  const { data: dataBackground } = backgroundContext.getImageData(
-    0, 0, bounds.width, bounds.height);
-
-  const rgba = getBgRGBA(dataText, dataBackground);
-  if (!rgba) {
-    return {
-      error: true,
-    };
-  }
-
-  const { color, isLargeText } = props;
-  if (rgba.value) {
-    return {
-      value: colorUtils.calculateContrastRatio(rgba.value, color),
-      color,
-      backgroundColor: rgba.value,
-      isLargeText,
-    };
-  }
-
-  let min = colorUtils.calculateContrastRatio(rgba.min, color);
-  let max = colorUtils.calculateContrastRatio(rgba.max, color);
-
-  // Flip minimum and maximum contrast ratios if necessary.
-  if (min > max) {
-    [min, max] = [max, min];
-    [rgba.min, rgba.max] = [rgba.max, rgba.min];
-  }
-
-  return {
-    min,
-    max,
-    color,
-    backgroundColorMin: rgba.min,
-    backgroundColorMax: rgba.max,
-    isLargeText,
-  };
-}
 
 /**
  * Helper function that determines if nsIAccessible object is in defunct state.
@@ -251,5 +37,4 @@ function isDefunct(accessible) {
   return defunct;
 }
 
-exports.getContrastRatioFor = getContrastRatioFor;
 exports.isDefunct = isDefunct;

devtools/shared/worker/tests/browser/browser_worker-01.js

@@ -9,6 +9,7 @@
 const WORKER_URL =
   "resource://devtools/client/shared/widgets/GraphsWorker.js";
 
+const BUFFER_SIZE = 8;
 const count = 100000;
 const WORKER_DATA = (function() {
   const timestamps = [];
@@ -25,6 +26,7 @@ add_task(async function() {
 
   await testWorker("JSM", () => ChromeUtils.import("resource://devtools/shared/worker/worker.js", {}));
   await testWorker("CommonJS", () => require("devtools/shared/worker/worker"));
+  await testTransfer();
 });
 
 async function testWorker(context, workerFactory) {
@@ -45,3 +47,20 @@ async function testWorker(context, workerFactory) {
 
   worker.destroy();
 }
+
+async function testTransfer() {
+  const { workerify } =
+    ChromeUtils.import("resource://devtools/shared/worker/worker.js", {});
+  const workerFn = workerify(({ buf }) => buf.byteLength);
+  const buf = new ArrayBuffer(BUFFER_SIZE);
+
+  is(buf.byteLength, BUFFER_SIZE, "Size of the buffer before transfer is correct.");
+
+  is((await workerFn({ buf })), 8, "Sent array buffer to worker");
+  is(buf.byteLength, 8, "Array buffer was copied, not transferred.");
+
+  is((await workerFn({ buf }, [ buf ])), 8, "Sent array buffer to worker");
+  is(buf.byteLength, 0, "Array buffer was transferred, not copied.");
+
+  workerFn.destroy();
+}

devtools/shared/worker/worker.js

@@ -60,9 +60,11 @@
    *        The name of the task to execute in the worker.
    * @param {any} data
    *        Data to be passed into the task implemented by the worker.
+   * @param {undefined|Array} transfer
+   *        Optional array of transferable objects to transfer ownership of.
    * @return {Promise}
    */
-  DevToolsWorker.prototype.performTask = function(task, data) {
+  DevToolsWorker.prototype.performTask = function(task, data, transfer) {
     if (this._destroyed) {
       return Promise.reject("Cannot call performTask on a destroyed DevToolsWorker");
     }
@@ -76,7 +78,7 @@
             ": " +
             JSON.stringify(payload, null, 2));
     }
-    worker.postMessage(payload);
+    worker.postMessage(payload, transfer);
 
     return new Promise((resolve, reject) => {
       const listener = ({ data: result }) => {
@@ -146,7 +148,8 @@
     const url = URL.createObjectURL(blob);
     const worker = new DevToolsWorker(url);
 
-    const wrapperFn = data => worker.performTask("workerifiedTask", data);
+    const wrapperFn = (data, transfer) =>
+      worker.performTask("workerifiedTask", data, transfer);
 
     wrapperFn.destroy = function() {
       URL.revokeObjectURL(url);