Bug 634139 - Add a service for finding the representative color in an image. r=MattN

--HG--
extra : rebase_source : 111c2a3e6b0abfd8b75b90afbe5e736f80ff2939

browser/installer/package-manifest.in

@@ -401,6 +401,7 @@
 @BINPATH@/components/nsPlacesExpiration.js
 @BINPATH@/components/PlacesProtocolHandler.js
 @BINPATH@/components/PlacesCategoriesStarter.js
+@BINPATH@/components/ColorAnalyzer.js
 @BINPATH@/components/PageThumbsProtocol.js
 @BINPATH@/components/nsDefaultCLH.manifest
 @BINPATH@/components/nsDefaultCLH.js

toolkit/components/places/ClusterLib.js

@@ -0,0 +1,248 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Class that can run the hierarchical clustering algorithm with the given
+ * parameters.
+ *
+ * @param distance
+ *        Function that should return the distance between two items.
+ *        Defaults to clusterlib.euclidean_distance.
+ * @param merge
+ *        Function that should take in two items and return a merged one.
+ *        Defaults to clusterlib.average_linkage.
+ * @param threshold
+ *        The maximum distance between two items for which their clusters
+ *        can be merged.
+ */
+function HierarchicalClustering(distance, merge, threshold) {
+  this.distance = distance || clusterlib.euclidean_distance;
+  this.merge = merge || clusterlib.average_linkage;
+  this.threshold = threshold == undefined ? Infinity : threshold;
+}
+
+HierarchicalClustering.prototype = {
+  /**
+   * Run the hierarchical clustering algorithm on the given items to produce
+   * a final set of clusters.  Uses the parameters set in the constructor.
+   *
+   * @param items
+   *        An array of "things" to cluster - this is the domain-specific
+   *        collection you're trying to cluster (colors, points, etc.)
+   * @param snapshotGap
+   *        How many iterations of the clustering algorithm to wait between
+   *        calling the snapshotCallback
+   * @param snapshotCallback
+   *        If provided, will be called as clusters are merged to let you view
+   *        the progress of the algorithm.  Passed the current array of
+   *        clusters, cached distances, and cached closest clusters.
+   *
+   * @return An array of merged clusters.  The represented item can be
+   *         found in the "item" property of the cluster.
+   */
+  cluster: function HC_cluster(items, snapshotGap, snapshotCallback) {
+    // array of all remaining clusters
+    let clusters = [];
+    // 2D matrix of distances between each pair of clusters, indexed by key
+    let distances = [];
+    // closest cluster key for each cluster, indexed by key
+    let neighbors = [];
+    // an array of all clusters, but indexed by key
+    let clustersByKey = [];
+
+    // set up clusters from the initial items array
+    for (let index = 0; index < items.length; index++) {
+      let cluster = {
+        // the item this cluster represents
+        item: items[index],
+        // a unique key for this cluster, stays constant unless merged itself
+        key: index,
+        // index of cluster in clusters array, can change during any merge
+        index: index,
+        // how many clusters have been merged into this one
+        size: 1
+      };
+      clusters[index] = cluster;
+      clustersByKey[index] = cluster;
+      distances[index] = [];
+      neighbors[index] = 0;
+    }
+
+    // initialize distance matrix and cached neighbors
+    for (let i = 0; i < clusters.length; i++) {
+      for (let j = 0; j <= i; j++) {
+        var dist = (i == j) ? Infinity :
+          this.distance(clusters[i].item, clusters[j].item);
+        distances[i][j] = dist;
+        distances[j][i] = dist;
+
+        if (dist < distances[i][neighbors[i]]) {
+          neighbors[i] = j;
+        }
+      }
+    }
+
+    // merge the next two closest clusters until none of them are close enough
+    let next = null, i = 0;
+    for (; next = this.closestClusters(clusters, distances, neighbors); i++) {
+      if (snapshotCallback && (i % snapshotGap) == 0) {
+        snapshotCallback(clusters);
+      }
+      this.mergeClusters(clusters, distances, neighbors, clustersByKey,
+                         clustersByKey[next[0]], clustersByKey[next[1]]);
+    }
+    return clusters;
+  },
+
+  /**
+   * Once we decide to merge two clusters in the cluster method, actually
+   * merge them.  Alters the given state of the algorithm.
+   *
+   * @param clusters
+   *        The array of all remaining clusters
+   * @param distances
+   *        Cached distances between pairs of clusters
+   * @param neighbors
+   *        Cached closest clusters
+   * @param clustersByKey
+   *        Array of all clusters, indexed by key
+   * @param cluster1
+   *        First cluster to merge
+   * @param cluster2
+   *        Second cluster to merge
+   */
+  mergeClusters: function HC_mergeClus(clusters, distances, neighbors,
+                                       clustersByKey, cluster1, cluster2) {
+    let merged = { item: this.merge(cluster1.item, cluster2.item),
+                   left: cluster1,
+                   right: cluster2,
+                   key: cluster1.key,
+                   size: cluster1.size + cluster2.size };
+
+    clusters[cluster1.index] = merged;
+    clusters.splice(cluster2.index, 1);
+    clustersByKey[cluster1.key] = merged;
+
+    // update distances with new merged cluster
+    for (let i = 0; i < clusters.length; i++) {
+      var ci = clusters[i];
+      var dist;
+      if (cluster1.key == ci.key) {
+        dist = Infinity;
+      } else if (this.merge == clusterlib.single_linkage) {
+        dist = distances[cluster1.key][ci.key];
+        if (distances[cluster1.key][ci.key] >
+            distances[cluster2.key][ci.key]) {
+          dist = distances[cluster2.key][ci.key];
+        }
+      } else if (this.merge == clusterlib.complete_linkage) {
+        dist = distances[cluster1.key][ci.key];
+        if (distances[cluster1.key][ci.key] <
+            distances[cluster2.key][ci.key]) {
+          dist = distances[cluster2.key][ci.key];
+        }
+      } else if (this.merge == clusterlib.average_linkage) {
+        dist = (distances[cluster1.key][ci.key] * cluster1.size
+              + distances[cluster2.key][ci.key] * cluster2.size)
+              / (cluster1.size + cluster2.size);
+      } else {
+        dist = this.distance(ci.item, cluster1.item);
+      }
+
+      distances[cluster1.key][ci.key] = distances[ci.key][cluster1.key]
+                                      = dist;
+    }
+
+    // update cached neighbors
+    for (let i = 0; i < clusters.length; i++) {
+      var key1 = clusters[i].key;
+      if (neighbors[key1] == cluster1.key ||
+          neighbors[key1] == cluster2.key) {
+        let minKey = key1;
+        for (let j = 0; j < clusters.length; j++) {
+          var key2 = clusters[j].key;
+          if (distances[key1][key2] < distances[key1][minKey]) {
+            minKey = key2;
+          }
+        }
+        neighbors[key1] = minKey;
+      }
+      clusters[i].index = i;
+    }
+  },
+
+  /**
+   * Given the current state of the algorithm, return the keys of the two
+   * clusters that are closest to each other so we know which ones to merge
+   * next.
+   *
+   * @param clusters
+   *        The array of all remaining clusters
+   * @param distances
+   *        Cached distances between pairs of clusters
+   * @param neighbors
+   *        Cached closest clusters
+   *
+   * @return An array of two keys of clusters to merge, or null if there are
+   *         no more clusters close enough to merge
+   */
+  closestClusters: function HC_closestClus(clusters, distances, neighbors) {
+    let minKey = 0, minDist = Infinity;
+    for (let i = 0; i < clusters.length; i++) {
+      var key = clusters[i].key;
+      if (distances[key][neighbors[key]] < minDist) {
+        minKey = key;
+        minDist = distances[key][neighbors[key]];
+      }
+    }
+    if (minDist < this.threshold) {
+      return [minKey, neighbors[minKey]];
+    }
+    return null;
+  }
+};
+
+let clusterlib = {
+  hcluster: function hcluster(items, distance, merge, threshold, snapshotGap,
+                              snapshotCallback) {
+    return (new HierarchicalClustering(distance, merge, threshold))
+           .cluster(items, snapshotGap, snapshotCallback);
+  },
+
+  single_linkage: function single_linkage(cluster1, cluster2) {
+    return cluster1;
+  },
+
+  complete_linkage: function complete_linkage(cluster1, cluster2) {
+    return cluster1;
+  },
+
+  average_linkage: function average_linkage(cluster1, cluster2) {
+    return cluster1;
+  },
+
+  euclidean_distance: function euclidean_distance(v1, v2) {
+    let total = 0;
+    for (let i = 0; i < v1.length; i++) {
+      total += Math.pow(v2[i] - v1[i], 2);
+    }
+    return Math.sqrt(total);
+  },
+
+  manhattan_distance: function manhattan_distance(v1, v2) {
+    let total = 0;
+    for (let i = 0; i < v1.length; i++) {
+      total += Math.abs(v2[i] - v1[i]);
+    }
+    return total;
+  },
+
+  max_distance: function max_distance(v1, v2) {
+    let max = 0;
+    for (let i = 0; i < v1.length; i++) {
+      max = Math.max(max, Math.abs(v2[i] - v1[i]));
+    }
+    return max;
+  }
+};

toolkit/components/places/ColorAnalyzer.js

@@ -0,0 +1,90 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const Ci = Components.interfaces;
+const Cc = Components.classes;
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+const XHTML_NS = "http://www.w3.org/1999/xhtml";
+const MAXIMUM_PIXELS = Math.pow(128, 2);
+
+function ColorAnalyzer() {
+  // a queue of callbacks for each job we give to the worker
+  this.callbacks = [];
+
+  this.hiddenWindowDoc = Cc["@mozilla.org/appshell/appShellService;1"].
+                         getService(Ci.nsIAppShellService).
+                         hiddenDOMWindow.document;
+
+  this.worker = new ChromeWorker("resource://gre/modules/ColorAnalyzer_worker.js");
+  this.worker.onmessage = this.onWorkerMessage.bind(this);
+  this.worker.onerror = this.onWorkerError.bind(this);
+}
+
+ColorAnalyzer.prototype = {
+  findRepresentativeColor: function ColorAnalyzer_frc(imageURI, callback) {
+    function cleanup() {
+      image.removeEventListener("load", loadListener);
+      image.removeEventListener("error", errorListener);
+    }
+    let image = this.hiddenWindowDoc.createElementNS(XHTML_NS, "img");
+    let loadListener = this.onImageLoad.bind(this, image, callback, cleanup);
+    let errorListener = this.onImageError.bind(this, image, callback, cleanup);
+    image.addEventListener("load", loadListener);
+    image.addEventListener("error", errorListener);
+    image.src = imageURI.spec;
+  },
+
+  onImageLoad: function ColorAnalyzer_onImageLoad(image, callback, cleanup) {
+    if (image.naturalWidth * image.naturalHeight > MAXIMUM_PIXELS) {
+      // this will probably take too long to process - fail
+      callback.onComplete(false);
+    } else {
+      let canvas = this.hiddenWindowDoc.createElementNS(XHTML_NS, "canvas");
+      canvas.width = image.naturalWidth;
+      canvas.height = image.naturalHeight;
+      let ctx = canvas.getContext("2d");
+      ctx.drawImage(image, 0, 0);
+      this.startJob(ctx.getImageData(0, 0, canvas.width, canvas.height),
+                    callback);
+    }
+    cleanup();
+  },
+
+  onImageError: function ColorAnalyzer_onImageError(image, callback, cleanup) {
+    Cu.reportError("ColorAnalyzer: image at " + image.src + " didn't load");
+    callback.onComplete(false);
+    cleanup();
+  },
+
+  startJob: function ColorAnalyzer_startJob(imageData, callback) {
+    this.callbacks.push(callback);
+    this.worker.postMessage({ imageData: imageData, maxColors: 1 });
+  },
+
+  onWorkerMessage: function ColorAnalyzer_onWorkerMessage(event) {
+    // colors can be empty on failure
+    if (event.data.colors.length < 1) {
+      this.callbacks.shift().onComplete(false);
+    } else {
+      this.callbacks.shift().onComplete(true, event.data.colors[0]);
+    }
+  },
+
+  onWorkerError: function ColorAnalyzer_onWorkerError(error) {
+    // this shouldn't happen, but just in case
+    error.preventDefault();
+    Cu.reportError("ColorAnalyzer worker: " + error.message);
+    this.callbacks.shift().onComplete(false);
+  },
+
+  classID: Components.ID("{d056186c-28a0-494e-aacc-9e433772b143}"),
+  QueryInterface: XPCOMUtils.generateQI([Ci.mozIColorAnalyzer])
+};
+
+let NSGetFactory = XPCOMUtils.generateNSGetFactory([ColorAnalyzer]);

toolkit/components/places/ColorAnalyzer_worker.js

@@ -0,0 +1,392 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+importScripts("ClusterLib.js", "ColorConversion.js");
+
+// Offsets in the ImageData pixel array to reach pixel colors
+const PIXEL_RED = 0;
+const PIXEL_GREEN = 1;
+const PIXEL_BLUE = 2;
+const PIXEL_ALPHA = 3;
+
+// Number of components in one ImageData pixel (RGBA)
+const NUM_COMPONENTS = 4;
+
+// Shift a color represented as a 24 bit integer by N bits to get a component
+const RED_SHIFT = 16;
+const GREEN_SHIFT = 8;
+
+// Only run the N most frequent unique colors through the clustering algorithm
+// Images with more than this many unique colors will have reduced accuracy.
+const MAX_COLORS_TO_MERGE = 500;
+
+// Each cluster of colors has a mean color in the Lab color space.
+// If the euclidean distance between the means of two clusters is greater
+// than or equal to this threshold, they won't be merged.
+const MERGE_THRESHOLD = 12;
+
+// The highest the distance handicap can be for large clusters
+const MAX_SIZE_HANDICAP = 5;
+// If the handicap is below this number, it is cut off to zero
+const SIZE_HANDICAP_CUTOFF = 2;
+
+// If potential background colors deviate from the mean background color by
+// this threshold or greater, finding a background color will fail
+const BACKGROUND_THRESHOLD = 10;
+
+// Alpha component of colors must be larger than this in order to make it into
+// the clustering algorithm or be considered a background color (0 - 255).
+const MIN_ALPHA = 25;
+
+// The euclidean distance in the Lab color space under which merged colors
+// are weighted lower for being similar to the background color
+const BACKGROUND_WEIGHT_THRESHOLD = 15;
+
+// The range in which color chroma differences will affect desirability.
+// Colors with chroma outside of the range take on the desirability of
+// their nearest extremes. Should be roughly 0 - 150.
+const CHROMA_WEIGHT_UPPER = 90;
+const CHROMA_WEIGHT_LOWER = 1;
+const CHROMA_WEIGHT_MIDDLE = (CHROMA_WEIGHT_UPPER + CHROMA_WEIGHT_LOWER) / 2;
+
+/**
+ * When we receive a message from the outside world, find the representative
+ * colors of the given image.  The colors will be posted back to the caller
+ * through the "colors" property on the event data object as an array of
+ * integers.  Colors of lower indices are more representative.
+ * This array can be empty if this worker can't find a color.
+ *
+ * @param event
+ *        A MessageEvent whose data should have the following properties:
+ *          imageData - A DOM ImageData instance to analyze
+ *          maxColors - The maximum number of representative colors to find,
+ *                      defaults to 1 if not provided
+ */
+onmessage = function(event) {
+  let imageData = event.data.imageData;
+  let pixels = imageData.data;
+  let width = imageData.width;
+  let height = imageData.height;
+  let maxColors = event.data.maxColors;
+  if (typeof(maxColors) != "number") {
+    maxColors = 1;
+  }
+
+  let allColors = getColors(pixels, width, height);
+
+  // Only merge top colors by frequency for speed.
+  let mergedColors = mergeColors(allColors.slice(0, MAX_COLORS_TO_MERGE),
+                                 width * height, MERGE_THRESHOLD);
+
+  let backgroundColor = getBackgroundColor(pixels, width, height);
+
+  mergedColors = mergedColors.map(function(cluster) {
+    // metadata holds a bunch of information about the color represented by
+    // this cluster
+    let metadata = cluster.item;
+
+    // the basis of color desirability is how much of the image the color is
+    // responsible for, but we'll need to weigh this number differently
+    // depending on other factors
+    metadata.desirability = metadata.ratio;
+    let weight = 1;
+
+    // if the color is close to the background color, we don't want it
+    if (backgroundColor != null) {
+      let backgroundDistance = labEuclidean(metadata.mean, backgroundColor);
+      if (backgroundDistance < BACKGROUND_WEIGHT_THRESHOLD) {
+        weight = backgroundDistance / BACKGROUND_WEIGHT_THRESHOLD;
+      }
+    }
+
+    // prefer more interesting colors, but don't knock low chroma colors
+    // completely out of the running (lower bound), and we don't really care
+    // if a color is slightly more intense than another on the higher end
+    let chroma = labChroma(metadata.mean);
+    if (chroma < CHROMA_WEIGHT_LOWER) {
+      chroma = CHROMA_WEIGHT_LOWER;
+    } else if (chroma > CHROMA_WEIGHT_UPPER) {
+      chroma = CHROMA_WEIGHT_UPPER;
+    }
+    weight *= chroma / CHROMA_WEIGHT_MIDDLE;
+
+    metadata.desirability *= weight;
+    return metadata;
+  });
+
+  // only send back the most desirable colors
+  mergedColors.sort(function(a, b) {
+    return b.desirability - a.desirability;
+  });
+  mergedColors = mergedColors.map(function(metadata) {
+    return metadata.color;
+  }).slice(0, maxColors);
+  postMessage({ colors: mergedColors });
+};
+
+/**
+ * Given the pixel data and dimensions of an image, return an array of objects
+ * associating each unique color and its frequency in the image, sorted
+ * descending by frequency.  Sufficiently transparent colors are ignored.
+ *
+ * @param pixels
+ *        Pixel data array for the image to get colors from (ImageData.data).
+ * @param width
+ *        Width of the image, in # of pixels.
+ * @param height
+ *        Height of the image, in # of pixels.
+ *
+ * @return An array of objects with color and freq properties, sorted
+ *         descending by freq
+ */
+function getColors(pixels, width, height) {
+  let colorFrequency = {};
+  for (let x = 0; x < width; x++) {
+    for (let y = 0; y < height; y++) {
+      let offset = (x * NUM_COMPONENTS) + (y * NUM_COMPONENTS * width);
+
+      if (pixels[offset + PIXEL_ALPHA] < MIN_ALPHA) {
+        continue;
+      }
+
+      let color = pixels[offset + PIXEL_RED] << RED_SHIFT
+                | pixels[offset + PIXEL_GREEN] << GREEN_SHIFT
+                | pixels[offset + PIXEL_BLUE];
+
+      if (color in colorFrequency) {
+        colorFrequency[color]++;
+      } else {
+        colorFrequency[color] = 1;
+      }
+    }
+  }
+
+  let colors = [];
+  for (var color in colorFrequency) {
+    colors.push({ color: +color, freq: colorFrequency[+color] });
+  }
+  colors.sort(descendingFreqSort);
+  return colors;
+}
+
+/**
+ * Given an array of objects from getColors, the number of pixels in the
+ * image, and a merge threshold, run the clustering algorithm on the colors
+ * and return the set of merged clusters.
+ *
+ * @param colorFrequencies
+ *        An array of objects from getColors to cluster
+ * @param numPixels
+ *        The number of pixels in the image
+ * @param threshold
+ *        The maximum distance between two clusters for which those clusters
+ *        can be merged.
+ *
+ * @return An array of merged clusters
+ *
+ * @see clusterlib.hcluster
+ * @see getColors
+ */
+function mergeColors(colorFrequencies, numPixels, threshold) {
+  let items = colorFrequencies.map(function(colorFrequency) {
+    let color = colorFrequency.color;
+    let freq = colorFrequency.freq;
+    return {
+      mean: rgb2lab(color >> RED_SHIFT, color >> GREEN_SHIFT & 0xff,
+                    color & 0xff),
+      // the canonical color of the cluster
+      // (one w/ highest freq or closest to mean)
+      color: color,
+      colors: [color],
+      highFreq: freq,
+      highRatio: freq / numPixels,
+      // the individual color w/ the highest frequency in this cluster
+      highColor: color,
+      // ratio of image taken up by colors in this cluster
+      ratio: freq / numPixels,
+      freq: freq,
+    };
+  });
+
+  let merged = clusterlib.hcluster(items, distance, merge, threshold);
+  return merged;
+}
+
+function descendingFreqSort(a, b) {
+  return b.freq - a.freq;
+}
+
+/**
+ * Given two items for a pair of clusters (as created in mergeColors above),
+ * determine the distance between them so we know if we should merge or not.
+ * Uses the euclidean distance between their mean colors in the lab color
+ * space, weighted so larger items are harder to merge.
+ *
+ * @param item1
+ *        The first item to compare
+ * @param item2
+ *        The second item to compare
+ *
+ * @return The distance between the two items
+ */
+function distance(item1, item2) {
+  // don't cluster large blocks of color unless they're really similar
+  let minRatio = Math.min(item1.ratio, item2.ratio);
+  let dist = labEuclidean(item1.mean, item2.mean);
+  let handicap = Math.min(MAX_SIZE_HANDICAP, dist * minRatio);
+  if (handicap <= SIZE_HANDICAP_CUTOFF) {
+    handicap = 0;
+  }
+  return dist + handicap;
+}
+
+/**
+ * Find the euclidean distance between two colors in the Lab color space.
+ *
+ * @param color1
+ *        The first color to compare
+ * @param color2
+ *        The second color to compare
+ *
+ * @return The euclidean distance between the two colors
+ */
+function labEuclidean(color1, color2) {
+  return Math.sqrt(
+      Math.pow(color2.lightness - color1.lightness, 2)
+    + Math.pow(color2.a - color1.a, 2)
+    + Math.pow(color2.b - color1.b, 2));
+}
+
+/**
+ * Given items from two clusters we know are appropriate for merging,
+ * merge them together into a third item such that its metadata describes both
+ * input items.  The "color" property is set to the color in the new item that
+ * is closest to its mean color.
+ *
+ * @param item1
+ *        The first item to merge
+ * @param item2
+ *        The second item to merge
+ *
+ * @return An item that represents the merging of the given items
+ */
+function merge(item1, item2) {
+  let lab1 = item1.mean;
+  let lab2 = item2.mean;
+
+  /* algorithm tweak point - weighting the mean of the cluster */
+  let num1 = item1.freq;
+  let num2 = item2.freq;
+
+  let total = num1 + num2;
+
+  let mean = {
+    lightness: (lab1.lightness * num1 + lab2.lightness * num2) / total,
+    a: (lab1.a * num1 + lab2.a * num2) / total,
+    b: (lab1.b * num1 + lab2.b * num2) / total
+  };
+
+  let colors = item1.colors.concat(item2.colors);
+
+  // get the canonical color of the new cluster
+  let color;
+  let avgFreq = colors.length / (item1.freq + item2.freq);
+  if ((item1.highFreq > item2.highFreq) && (item1.highFreq > avgFreq * 2)) {
+    color = item1.highColor;
+  } else if (item2.highFreq > avgFreq * 2) {
+    color = item2.highColor;
+  } else {
+    // if there's no stand-out color
+    let minDist = Infinity, closest = 0;
+    for (let i = 0; i < colors.length; i++) {
+      let color = colors[i];
+      let lab = rgb2lab(color >> RED_SHIFT, color >> GREEN_SHIFT & 0xff,
+                        color & 0xff);
+      let dist = labEuclidean(lab, mean);
+      if (dist < minDist) {
+        minDist = dist;
+        closest = i;
+      }
+    }
+    color = colors[closest];
+  }
+
+  const higherItem = item1.highFreq > item2.highFreq ? item1 : item2;
+
+  return {
+    mean: mean,
+    color: color,
+    highFreq: higherItem.highFreq,
+    highColor: higherItem.highColor,
+    highRatio: higherItem.highRatio,
+    ratio: item1.ratio + item2.ratio,
+    freq: item1.freq + item2.freq,
+    colors: colors,
+  };
+}
+
+/**
+ * Find the background color of the given image.
+ *
+ * @param pixels
+ *        The pixel data for the image (an array of component integers)
+ * @param width
+ *        The width of the image
+ * @param height
+ *        The height of the image
+ *
+ * @return The background color of the image as a Lab object, or null if we
+ *         can't determine the background color
+ */
+function getBackgroundColor(pixels, width, height) {
+  // we'll assume that if the four corners are roughly the same color,
+  // then that's the background color
+  let coordinates = [[0, 0], [width - 1, 0], [width - 1, height - 1],
+                     [0, height - 1]];
+
+  // find the corner colors in LAB
+  let cornerColors = [];
+  for (let i = 0; i < coordinates.length; i++) {
+    let offset = (coordinates[i][0] * NUM_COMPONENTS)
+               + (coordinates[i][1] * NUM_COMPONENTS * width);
+    if (pixels[offset + PIXEL_ALPHA] < MIN_ALPHA) {
+      // we can't make very accurate judgements below this opacity
+      continue;
+    }
+    cornerColors.push(rgb2lab(pixels[offset + PIXEL_RED],
+                              pixels[offset + PIXEL_GREEN],
+                              pixels[offset + PIXEL_BLUE]));
+  }
+
+  // we want at least two points at acceptable alpha levels
+  if (cornerColors.length <= 1) {
+    return null;
+  }
+
+  // find the average color among the corners
+  let averageColor = { lightness: 0, a: 0, b: 0 };
+  cornerColors.forEach(function(color) {
+    for (let i in color) {
+      averageColor[i] += color[i];
+    }
+  });
+  for (let i in averageColor) {
+    averageColor[i] /= cornerColors.length;
+  }
+
+  // if we have fewer points due to low alpha, they need to be closer together
+  let threshold = BACKGROUND_THRESHOLD
+                * (cornerColors.length / coordinates.length);
+
+  // if any of the corner colors deviate enough from the average, they aren't
+  // similar enough to be considered the background color
+  for (let cornerColor of cornerColors) {
+    if (labEuclidean(cornerColor, averageColor) > threshold) {
+      return null;
+    }
+  }
+  return averageColor;
+}

toolkit/components/places/ColorConversion.js

@@ -0,0 +1,64 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Given a color in the Lab space, return its chroma (colorfulness,
+ * saturation).
+ *
+ * @param lab
+ *        The lab color to get the chroma from
+ *
+ * @return A number greater than zero that measures chroma in the image
+ */
+function labChroma(lab) {
+  return Math.sqrt(Math.pow(lab.a, 2) + Math.pow(lab.b, 2));
+}
+
+/**
+ * Given the RGB components of a color as integers from 0-255, return the
+ * color in the XYZ color space.
+ *
+ * @return An object with x, y, z properties holding those components of the
+ *         color in the XYZ color space.
+ */
+function rgb2xyz(r, g, b) {
+  r /= 255;
+  g /= 255;
+  b /= 255;
+
+  // assume sRGB
+  r = r > 0.04045 ? Math.pow(((r + 0.055) / 1.055), 2.4) : (r / 12.92);
+  g = g > 0.04045 ? Math.pow(((g + 0.055) / 1.055), 2.4) : (g / 12.92);
+  b = b > 0.04045 ? Math.pow(((b + 0.055) / 1.055), 2.4) : (b / 12.92);
+
+  return {
+    x: ((r * 0.4124) + (g * 0.3576) + (b * 0.1805)) * 100,
+    y: ((r * 0.2126) + (g * 0.7152) + (b * 0.0722)) * 100,
+    z: ((r * 0.0193) + (g * 0.1192) + (b * 0.9505)) * 100
+  };
+}
+
+/**
+ * Given the RGB components of a color as integers from 0-255, return the
+ * color in the Lab color space.
+ *
+ * @return An object with lightness, a, b properties holding those components
+ *         of the color in the Lab color space.
+ */
+function rgb2lab(r, g, b) {
+  let xyz = rgb2xyz(r, g, b),
+        x = xyz.x / 95.047,
+        y = xyz.y / 100,
+        z = xyz.z / 108.883;
+
+  x = x > 0.008856 ? Math.pow(x, 1/3) : (7.787 * x) + (16 / 116);
+  y = y > 0.008856 ? Math.pow(y, 1/3) : (7.787 * y) + (16 / 116);
+  z = z > 0.008856 ? Math.pow(z, 1/3) : (7.787 * z) + (16 / 116);
+
+  return {
+    lightness: (116 * y) - 16,
+    a:  500 * (x - y),
+    b:  200 * (y - z)
+  };
+}

toolkit/components/places/Makefile.in

@@ -28,6 +28,7 @@ XPIDLSRCS += \
   mozIAsyncFavicons.idl \
   mozIAsyncLivemarks.idl \
   mozIPlacesAutoComplete.idl \
+  mozIColorAnalyzer.idl \
   nsIAnnotationService.idl \
   nsIBrowserHistory.idl \
   nsIFaviconService.idl \
@@ -83,6 +84,7 @@ EXTRA_COMPONENTS = \
   nsTaggingService.js \
   nsPlacesExpiration.js \
   PlacesCategoriesStarter.js \
+  ColorAnalyzer.js \
   $(NULL)
 
 ifdef MOZ_XUL
@@ -92,6 +94,9 @@ endif
 EXTRA_JS_MODULES = \
   PlacesDBUtils.jsm \
   BookmarkHTMLUtils.jsm \
+  ColorAnalyzer_worker.js \
+  ColorConversion.js \
+  ClusterLib.js \
   $(NULL)
 
 EXTRA_PP_JS_MODULES = \

toolkit/components/places/mozIColorAnalyzer.idl

@@ -0,0 +1,52 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+interface nsIURI;
+
+[function, scriptable, uuid(e4089e21-71b6-40af-b546-33c21b90e874)]
+interface mozIRepresentativeColorCallback : nsISupports
+{
+  /**
+   * Will be called when color analysis finishes.
+   *
+   * @param success
+   *        True if analysis was successful, false otherwise.
+   *        Analysis can fail if the image is transparent, imageURI doesn't
+   *        resolve to a valid image, or the image is too big.
+   *
+   * @param color
+   *        The representative color as an integer in RGB form.
+   *        e.g. 0xFF0102 == rgb(255,1,2)
+   *        If success is false, color is not provided.
+   */
+  void onComplete(in boolean success, [optional] in unsigned long color);
+};
+
+[scriptable, uuid(d056186c-28a0-494e-aacc-9e433772b143)]
+interface mozIColorAnalyzer : nsISupports
+{
+  /**
+   * Given an image URI, find the most representative color for that image
+   * based on the frequency of each color.  Preference is given to colors that
+   * are more interesting.  Avoids the background color if it can be
+   * discerned.  Ignores sufficiently transparent colors.
+   *
+   * This is intended to be used on favicon images.  Larger images take longer
+   * to process, especially those with a larger number of unique colors.  If
+   * imageURI points to an image that has more than 128^2 pixels, this method
+   * will fail before analyzing it for performance reasons.
+   *
+   * @param imageURI
+   *        A URI pointing to the image - ideally a data: URI, but any scheme
+   *        that will load when setting the src attribute of a DOM img element
+   *        should work.
+   * @param callback
+   *        Function to call when the representative color is found or an
+   *        error occurs.
+   */
+  void findRepresentativeColor(in nsIURI imageURI,
+                               in mozIRepresentativeColorCallback callback);
+};

toolkit/components/places/tests/browser/Makefile.in

@@ -16,12 +16,17 @@ MOCHITEST_BROWSER_FILES = \
 	browser_bug399606.js \
 	browser_bug646422.js \
 	browser_bug680727.js \
+	browser_colorAnalyzer.js \
 	browser_notfound.js \
 	browser_redirect.js \
 	browser_visituri.js \
 	browser_visituri_nohistory.js \
 	browser_visituri_privatebrowsing.js \
 	browser_settitle.js \
+	colorAnalyzer/category-discover.png \
+	colorAnalyzer/dictionaryGeneric-16.png \
+	colorAnalyzer/extensionGeneric-16.png \
+	colorAnalyzer/localeGeneric.png \
 	$(NULL)
 
 # These are files that need to be loaded via the HTTP proxy server

toolkit/components/places/tests/browser/browser_colorAnalyzer.js

@@ -0,0 +1,343 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Services.jsm");
+
+const CA = Cc["@mozilla.org/places/colorAnalyzer;1"].
+           getService(Ci.mozIColorAnalyzer);
+
+const hiddenWindowDoc = Cc["@mozilla.org/appshell/appShellService;1"].
+                        getService(Ci.nsIAppShellService).
+                        hiddenDOMWindow.document;
+
+const XHTML_NS = "http://www.w3.org/1999/xhtml";
+
+// to make async tests easier, push them into here and call nextStep
+// when a test finishes
+let tests = [];
+function generatorTest() {
+  while (tests.length > 0) {
+    tests.shift()();
+    yield;
+  }
+}
+
+/**
+ * Passes the given uri to findRepresentativeColor.
+ * If expected is null, you expect it to fail.
+ * If expected is a function, it will call that function.
+ * If expected is a color, you expect that color to be returned.
+ * Message is used in the calls to is().
+ */
+function frcTest(uri, expected, message, skipNextStep) {
+  CA.findRepresentativeColor(Services.io.newURI(uri, "", null),
+    function(success, color) {
+      if (expected == null) {
+        ok(!success, message);
+      } else if (typeof expected == "function") {
+        expected(color, message);
+      } else {
+        ok(success, "success: " + message);
+        is(color, expected, message);
+      }
+      if (!skipNextStep) {
+        nextStep();
+      }
+    }
+  );
+}
+
+/**
+ * Handy function for getting an image into findRepresentativeColor and testing it.
+ * Makes a canvas with the given dimensions, calls paintCanvasFunc with the 2d
+ * context of the canvas, sticks the generated canvas into findRepresentativeColor.
+ * See frcTest.
+ */
+function canvasTest(width, height, paintCanvasFunc, expected, message, skipNextStep) {
+  let canvas = hiddenWindowDoc.createElementNS(XHTML_NS, "canvas");
+  canvas.width = width;
+  canvas.height = height;
+  paintCanvasFunc(canvas.getContext("2d"));
+  let uri = canvas.toDataURL();
+  frcTest(uri, expected, message, skipNextStep);
+}
+
+// simple test - draw a red box in the center, make sure we get red back
+tests.push(function test_redSquare() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "red";
+    ctx.fillRect(2, 2, 12, 12);
+  }, 0xFF0000, "redSquare analysis returns red");
+});
+
+// draw a blue square in one corner, red in the other, such that blue overlaps
+// red by one pixel, making it the dominant color
+tests.push(function test_blueOverlappingRed() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "red";
+    ctx.fillRect(0, 0, 8, 8);
+    ctx.fillStyle = "blue";
+    ctx.fillRect(7, 7, 8, 8);
+  }, 0x0000FF, "blueOverlappingRed analysis returns blue");
+});
+
+// draw a red gradient next to a solid blue rectangle to ensure that a large
+// block of similar colors beats out a smaller block of one color
+tests.push(function test_redGradientBlueSolid() {
+  canvasTest(16, 16, function(ctx) {
+    let gradient = ctx.createLinearGradient(0, 0, 1, 15);
+    gradient.addColorStop(0, "#FF0000");
+    gradient.addColorStop(1, "#FF0808");
+
+    ctx.fillStyle = gradient;
+    ctx.fillRect(0, 0, 16, 16);
+    ctx.fillStyle = "blue";
+    ctx.fillRect(9, 0, 7, 16);
+  }, function(actual, message) {
+    ok(actual > 0xFF0000 && actual < 0xFF0808, message);
+  }, "redGradientBlueSolid analysis returns redish");
+});
+
+// try a transparent image, should fail
+tests.push(function test_transparent() {
+  canvasTest(16, 16, function(ctx) {
+    //do nothing!
+  }, null, "transparent analysis fails");
+});
+
+tests.push(function test_invalidURI() {
+  frcTest("data:blah,Imnotavaliddatauri", null, "invalid URI analysis fails");
+});
+
+tests.push(function test_malformedPNGURI() {
+  frcTest("data:image/png;base64,iVBORblahblahblah", null,
+          "malformed PNG URI analysis fails");
+});
+
+tests.push(function test_unresolvableURI() {
+  frcTest("http://www.example.com/blah/idontexist.png", null,
+          "unresolvable URI analysis fails");
+});
+
+// draw a small blue box on a red background to make sure the algorithm avoids
+// using the background color
+tests.push(function test_blueOnRedBackground() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "red";
+    ctx.fillRect(0, 0, 16, 16);
+    ctx.fillStyle = "blue";
+    ctx.fillRect(4, 4, 8, 8);
+  }, 0x0000FF, "blueOnRedBackground analysis returns blue");
+});
+
+// draw a slightly different color in the corners to make sure the corner colors
+// don't have to be exactly equal to be considered the background color
+tests.push(function test_variableBackground() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "white";
+    ctx.fillRect(0, 0, 16, 16);
+    ctx.fillStyle = "#FEFEFE";
+    ctx.fillRect(15, 0, 1, 1);
+    ctx.fillStyle = "#FDFDFD";
+    ctx.fillRect(15, 15, 1, 1);
+    ctx.fillStyle = "#FCFCFC";
+    ctx.fillRect(0, 15, 1, 1);
+    ctx.fillStyle = "black";
+    ctx.fillRect(4, 4, 8, 8);
+  }, 0x000000, "variableBackground analysis returns black");
+});
+
+// like the above test, but make the colors different enough that they aren't
+// considered the background color
+tests.push(function test_tooVariableBackground() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "white";
+    ctx.fillRect(0, 0, 16, 16);
+    ctx.fillStyle = "#EEDDCC";
+    ctx.fillRect(15, 0, 1, 1);
+    ctx.fillStyle = "#DDDDDD";
+    ctx.fillRect(15, 15, 1, 1);
+    ctx.fillStyle = "#CCCCCC";
+    ctx.fillRect(0, 15, 1, 1);
+    ctx.fillStyle = "black";
+    ctx.fillRect(4, 4, 8, 8);
+  }, function(actual, message) {
+    isnot(actual, 0x000000, message);
+  }, "tooVariableBackground analysis doesn't return black");
+});
+
+// draw a small black/white box over transparent background to make sure the
+// algorithm doesn't think rgb(0,0,0) == rgba(0,0,0,0)
+tests.push(function test_transparentBackgroundConflation() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "black";
+    ctx.fillRect(2, 2, 12, 12);
+    ctx.fillStyle = "white";
+    ctx.fillRect(5, 5, 6, 6);
+  }, 0x000000, "transparentBackgroundConflation analysis returns black");
+});
+
+// make sure we fall back to the background color if we have no other choice
+// (instead of failing as if there were no colors)
+tests.push(function test_backgroundFallback() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "black";
+    ctx.fillRect(0, 0, 16, 16);
+  }, 0x000000, "backgroundFallback analysis returns black");
+});
+
+// draw red rectangle next to a pink one to make sure the algorithm picks the
+// more interesting color
+tests.push(function test_interestingColorPreference() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "#FFDDDD";
+    ctx.fillRect(0, 0, 16, 16);
+    ctx.fillStyle = "red";
+    ctx.fillRect(0, 0, 3, 16);
+  }, 0xFF0000, "interestingColorPreference analysis returns red");
+});
+
+// draw high saturation but dark red next to slightly less saturated color but
+// much lighter, to make sure the algorithm doesn't pick colors that are
+// nearly black just because of high saturation (in HSL terms)
+tests.push(function test_saturationDependence() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "hsl(0, 100%, 5%)";
+    ctx.fillRect(0, 0, 16, 16);
+    ctx.fillStyle = "hsl(0, 90%, 35%)";
+    ctx.fillRect(0, 0, 8, 16);
+  }, 0xA90808, "saturationDependence analysis returns lighter red");
+});
+
+// make sure the preference for interesting colors won't stupidly pick 1 pixel
+// of red over 169 black pixels
+tests.push(function test_interestingColorPreferenceLenient() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "black";
+    ctx.fillRect(1, 1, 13, 13);
+    ctx.fillStyle = "red";
+    ctx.fillRect(3, 3, 1, 1);
+  }, 0x000000, "interestingColorPreferenceLenient analysis returns black");
+});
+
+// ...but 6 pixels of red is more reasonable
+tests.push(function test_interestingColorPreferenceNotTooLenient() {
+  canvasTest(16, 16, function(ctx) {
+    ctx.fillStyle = "black";
+    ctx.fillRect(1, 1, 13, 13);
+    ctx.fillStyle = "red";
+    ctx.fillRect(3, 3, 3, 2);
+  }, 0xFF0000, "interestingColorPreferenceNotTooLenient analysis returns red");
+});
+
+// make sure that images larger than 128x128 fail
+tests.push(function test_imageTooLarge() {
+  canvasTest(129, 129, function(ctx) {
+    ctx.fillStyle = "red";
+    ctx.fillRect(0, 0, 129, 129);
+  }, null, "imageTooLarge analysis fails");
+});
+
+
+// these next tests are for performance (and also to make sure concurrency
+// doesn't break anything)
+
+let maxColor = Math.pow(2, 24) - 1;
+
+function getRandomColor() {
+  let randomColor = (Math.ceil(Math.random() * maxColor)).toString(16);
+  return "000000".slice(0, 6 - randomColor.length) + randomColor;
+}
+
+function testFiller(color, ctx) {
+  ctx.fillStyle = "#" + color;
+  ctx.fillRect(2, 2, 12, 12);
+}
+
+tests.push(function test_perfInSeries() {
+  let t1 = new Date();
+  let numTests = 20;
+  let allCorrect = true;
+  function nextPerfTest() {
+    let color = getRandomColor();
+    canvasTest(16, 16, testFiller.bind(this, color), function(actual) {
+      if (actual != parseInt(color, 16)) {
+        allCorrect = false;
+      }
+      if (--numTests > 0) {
+        nextPerfTest();
+      } else {
+        is(allCorrect, true, "perfInSeries colors are all correct");
+        info("perfInSeries: " + ((new Date()) - t1) + "ms");
+        nextStep();
+      }
+    }, "", true);
+  }
+  nextPerfTest();
+});
+
+tests.push(function test_perfInParallel() {
+  let t1 = new Date();
+  let numTests = 20;
+  let testsDone = 0;
+  let allCorrect = true;
+  for (let i = 0; i < numTests; i++) {
+    let color = getRandomColor();
+    canvasTest(16, 16, testFiller.bind(this, color), function(actual) {
+      if (actual != parseInt(color, 16)) {
+        allCorrect = false;
+      }
+      if (++testsDone >= 20) {
+        is(allCorrect, true, "perfInParallel colors are all correct");
+        info("perfInParallel: " + ((new Date()) - t1) + "ms");
+        nextStep();
+      }
+    }, "", true);
+  }
+});
+
+tests.push(function test_perfBigImage() {
+  let t1 = 0;
+  canvasTest(128, 128, function(ctx) {
+    // make sure to use a bunch of unique colors so the clustering algorithm
+    // actually has to do work
+    for (let y = 0; y < 128; y++) {
+      for (let x = 0; x < 128; x++) {
+        ctx.fillStyle = "#" + getRandomColor();
+        ctx.fillRect(x, y, 1, 1);
+      }
+    }
+    t1 = new Date();
+  }, function(actual) {
+    info("perfBigImage: " + ((new Date()) - t1) + "ms");
+    nextStep();
+  }, "", true);
+});
+
+
+// the rest of the tests are for coverage of "real" favicons
+// exact color isn't terribly important, just make sure it's reasonable
+const filePrefix = getRootDirectory(gTestPath);
+
+tests.push(function test_categoryDiscover() {
+  frcTest(filePrefix + "category-discover.png", 0xB28D3A,
+          "category-discover analysis returns red");
+});
+
+tests.push(function test_localeGeneric() {
+  frcTest(filePrefix + "localeGeneric.png", 0x00A400,
+          "localeGeneric analysis returns orange");
+});
+
+tests.push(function test_dictionaryGeneric() {
+  frcTest(filePrefix + "dictionaryGeneric-16.png", 0x502E1E,
+          "dictionaryGeneric-16 analysis returns blue");
+});
+
+tests.push(function test_extensionGeneric() {
+  frcTest(filePrefix + "extensionGeneric-16.png", 0x53BA3F,
+          "extensionGeneric-16 analysis returns green");
+});

toolkit/components/places/toolkitplaces.manifest

@@ -18,3 +18,7 @@ component {803938d5-e26d-4453-bf46-ad4b26e41114} PlacesCategoriesStarter.js
 contract @mozilla.org/places/categoriesStarter;1 {803938d5-e26d-4453-bf46-ad4b26e41114}
 category idle-daily PlacesCategoriesStarter @mozilla.org/places/categoriesStarter;1
 category bookmark-observers PlacesCategoriesStarter @mozilla.org/places/categoriesStarter;1
+
+# ColorAnalyzer.js
+component {d056186c-28a0-494e-aacc-9e433772b143} ColorAnalyzer.js
+contract @mozilla.org/places/colorAnalyzer;1 {d056186c-28a0-494e-aacc-9e433772b143}