Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2025-03-02 22:37:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1370930 - remove DirectoryLinksProvider, r=Mardak

Browse Source 
MozReview-Commit-ID: 4YcsNvRg7Hn

--HG--
extra : rebase_source : 15518736c9cc52cf18a0540417e6a38c9bed630a
This commit is contained in:
Gijs Kruitbosch 2018-03-01 16:53:49 +00:00
parent 645028e1cf
commit 8fc45f6ac9
18 changed files with 9 additions and 1170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
browser/app/profile/firefox.js
View File

@ -1291,9 +1291,6 @@ pref("browser.newtabpage.rows", 3);
// number of columns of newtab grid
pref("browser.newtabpage.columns", 5);
// directory tiles download URL
pref("browser.newtabpage.directory.source", "https://tiles.services.mozilla.com/v3/links/fetch/%LOCALE%/%CHANNEL%");
// Activity Stream prefs that control to which page to redirect
pref("browser.newtabpage.activity-stream.prerender", true);
#ifndef RELEASE_OR_BETA

1
browser/base/content/newtab/newTab.js
View File

@ -8,7 +8,6 @@ ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
ChromeUtils.import("resource://gre/modules/Services.jsm");
ChromeUtils.import("resource://gre/modules/PageThumbs.jsm");
ChromeUtils.import("resource://gre/modules/BackgroundPageThumbs.jsm");
ChromeUtils.import("resource:///modules/DirectoryLinksProvider.jsm");
ChromeUtils.import("resource://gre/modules/NewTabUtils.jsm");
ChromeUtils.defineModuleGetter(this, "Rect",

5
browser/base/content/newtab/sites.js
View File

@ -135,7 +135,7 @@ Site.prototype = {
*/
_render: function Site_render() {
// setup display variables
let enhanced = gAllPages.enhanced && DirectoryLinksProvider.getEnhancedLink(this.link);
let enhanced = gAllPages.enhanced;
let url = this.url;
let title = enhanced && enhanced.title ? enhanced.title :
this.link.type == "history" ? this.link.baseDomain :
@ -178,8 +178,7 @@ Site.prototype = {
*/
refreshThumbnail: function Site_refreshThumbnail() {
// Only enhance tiles if that feature is turned on
let link = gAllPages.enhanced && DirectoryLinksProvider.getEnhancedLink(this.link) ||
this.link;
let link = gAllPages.enhanced || this.link;
let thumbnail = this._querySelector(".newtab-thumbnail.thumbnail");
if (link.bgColor) {

31
browser/base/content/test/newtab/head.js
View File

@ -2,7 +2,6 @@
http://creativecommons.org/publicdomain/zero/1.0/ */
const PREF_NEWTAB_ENABLED = "browser.newtabpage.enabled";
const PREF_NEWTAB_DIRECTORYSOURCE = "browser.newtabpage.directory.source";
Services.prefs.setBoolPref(PREF_NEWTAB_ENABLED, true);
@ -17,7 +16,6 @@ ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
XPCOMUtils.defineLazyModuleGetters(this, {
NewTabUtils: "resource://gre/modules/NewTabUtils.jsm",
DirectoryLinksProvider: "resource:///modules/DirectoryLinksProvider.jsm",
PlacesTestUtils: "resource://testing-common/PlacesTestUtils.jsm",
Sanitizer: "resource:///modules/Sanitizer.jsm",
});
@ -94,31 +92,12 @@ add_task(async function setupWindowSize() {
registerCleanupFunction(function() {
Services.prefs.clearUserPref(PREF_NEWTAB_ENABLED);
Services.prefs.setCharPref(PREF_NEWTAB_DIRECTORYSOURCE, gOrigDirectorySource);
return watchLinksChangeOnce();
});
function pushPrefs(...aPrefs) {
return SpecialPowers.pushPrefEnv({"set": aPrefs});
}
/**
* Resolves promise when directory links are downloaded and written to disk
*/
function watchLinksChangeOnce() {
return new Promise(resolve => {
let observer = {
onManyLinksChanged: () => {
DirectoryLinksProvider.removeObserver(observer);
resolve();
}
};
observer.onDownloadFail = observer.onManyLinksChanged;
DirectoryLinksProvider.addObserver(observer);
});
}
add_task(async function setup() {
registerCleanupFunction(function() {
return new Promise(resolve => {
@ -139,15 +118,7 @@ add_task(async function setup() {
});
});
let promiseReady = (async function() {
await watchLinksChangeOnce();
await whenPagesUpdated();
})();
// Save the original directory source (which is set globally for tests)
gOrigDirectorySource = Services.prefs.getCharPref(PREF_NEWTAB_DIRECTORYSOURCE);
Services.prefs.setCharPref(PREF_NEWTAB_DIRECTORYSOURCE, gDirectorySource);
await promiseReady;
await whenPagesUpdated();
});
/** Perform an action on a cell within the newtab page.

1
browser/base/content/test/performance/browser_startup.js
View File

@ -68,7 +68,6 @@ const startupPhases = {
"resource:///modules/BrowserUITelemetry.jsm",
"resource:///modules/BrowserUsageTelemetry.jsm",
"resource:///modules/ContentCrashHandlers.jsm",
"resource:///modules/DirectoryLinksProvider.jsm",
"resource://gre/modules/NewTabUtils.jsm",
"resource://gre/modules/PageThumbs.jsm",
"resource://gre/modules/PlacesUtils.jsm",

10
browser/components/nsBrowserGlue.js
View File

@ -95,7 +95,6 @@ XPCOMUtils.defineLazyModuleGetters(this, {
ContextualIdentityService: "resource://gre/modules/ContextualIdentityService.jsm",
CustomizableUI: "resource:///modules/CustomizableUI.jsm",
DateTimePickerHelper: "resource://gre/modules/DateTimePickerHelper.jsm",
DirectoryLinksProvider: "resource:///modules/DirectoryLinksProvider.jsm",
ExtensionsUI: "resource:///modules/ExtensionsUI.jsm",
Feeds: "resource:///modules/Feeds.jsm",
FileUtils: "resource://gre/modules/FileUtils.jsm",
@ -998,9 +997,7 @@ BrowserGlue.prototype = {
PageThumbs.init();
DirectoryLinksProvider.init();
NewTabUtils.init();
NewTabUtils.links.addProvider(DirectoryLinksProvider);
PageActions.init();
@ -1820,7 +1817,7 @@ BrowserGlue.prototype = {
// eslint-disable-next-line complexity
_migrateUI: function BG__migrateUI() {
const UI_VERSION = 62;
const UI_VERSION = 64;
const BROWSER_DOCURL = "chrome://browser/content/browser.xul";
let currentUIVersion;
@ -2316,6 +2313,11 @@ BrowserGlue.prototype = {
}
}
if (currentUIVersion < 64) {
OS.File.remove(OS.Path.join(OS.Constants.Path.profileDir,
"directoryLinks.json"), {ignoreAbsent: true});
}
// Update the migration version.
Services.prefs.setIntPref("browser.migration.version", UI_VERSION);
},

107
browser/docs/DirectoryLinksProvider.rst
View File

@ -1,107 +0,0 @@
=============================================
Directory Links Architecture and Data Formats
=============================================
Directory links are enhancements to the new tab experience that combine content
Firefox already knows about from user browsing with external content. There is 1
kind of links:
- directory links fill in additional tiles on the new tab page if there would
have been empty tiles because the user has a clean profile or cleared history
To power the above features, DirectoryLinksProvider module downloads, at most
once per 24 hours, the directory source links as JSON with enough data for
Firefox to determine what should be shown or not.
For the directory source endpoint, the default preference values point
to Mozilla key-pinned servers with encryption. No cookies are set by the servers
and Firefox enforces this by making anonymous requests.
- default directory source endpoint:
https://tiles.services.mozilla.com/v3/links/fetch/%LOCALE%/%CHANNEL%
Preferences
===========
There is one main preference that controls downloading links.
``browser.newtabpage.directory.source``
---------------------------------------
This endpoint tells Firefox where to download directory source file as a GET
request. It should return JSON of the appropriate format containing the relevant
links data. The value can be a data URI, e.g., an empty JSON object effectively
turns off remote downloading: ``data:text/plain,{}``
The preference value will have %LOCALE% and %CHANNEL% replaced by the
appropriate values for the build of Firefox, e.g.,
- directory source endpoint:
https://tiles.services.mozilla.com/v3/links/fetch/en-US/release
Data Flow
=========
When Firefox starts, it checks for a cached directory source file. If one
exists, it checks for its timestamp to determine if a new file should be
downloaded.
If a directory source file needs to be downloaded, a GET request is made then
cached and unpacked the JSON into the different types of links. Various checks
filter out invalid links, e.g., those with http-hosted images or those that
don't fit the allowed suggestions.
When a new tab page is built, DirectoryLinksProvider module provides additional
link data that is combined with history link data to determine which links can
be displayed or not.
As the new tab page is rendered, any images for tiles are downloaded if not
already cached. The default servers hosting the images are Mozilla CDN that
don't use cookies: https://tiles.cdn.mozilla.net/ and Firefox enforces that the
images come from mozilla.net or data URIs when using the default directory
source.
Source JSON Format
==================
Firefox expects links data in a JSON object with a top level "directory" key
providing an array of tile objects.
Example
-------
Below is an example directory source file::
{
"directory": [
{
"bgColor": "",
"directoryId": 498,
"enhancedImageURI": "https://tiles.cdn.mozilla.net/images/d11ba0b3095bb19d8092cd29be9cbb9e197671ea.28088.png",
"imageURI": "https://tiles.cdn.mozilla.net/images/1332a68badf11e3f7f69bf7364e79c0a7e2753bc.5316.png",
"title": "Mozilla Community",
"type": "affiliate",
"url": "http://contribute.mozilla.org/"
}
]
}
Link Object
-----------
Each link object has various values that Firefox uses to display a tile:
- ``url`` - string url for the page to be loaded when the tile is clicked. Only
https and http URLs are allowed.
- ``title`` - string that appears below the tile.
- ``type`` - string relationship of the link to Mozilla. Expected values:
affiliate.
- ``imageURI`` - string url for the tile image to show. Only https and data URIs
are allowed.
- ``enhancedImageURI`` - string url for the image to be shown before the user
hovers. Only https and data URIs are allowed.
- ``bgColor`` - string css color for additional fill background color.
- ``directoryId`` - id of the tile to be used during ping reporting

1
browser/docs/index.rst
View File

@ -7,6 +7,5 @@ This is the nascent documentation of the Firefox front-end code.
.. toctree::
:maxdepth: 1
DirectoryLinksProvider
UITelemetry
BrowserUsageTelemetry

377
browser/modules/DirectoryLinksProvider.jsm
View File

@ -1,377 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var EXPORTED_SYMBOLS = ["DirectoryLinksProvider"];
Cu.importGlobalProperties(["XMLHttpRequest"]);
ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
ChromeUtils.import("resource://gre/modules/Services.jsm");
ChromeUtils.defineModuleGetter(this, "OS",
"resource://gre/modules/osfile.jsm");
ChromeUtils.defineModuleGetter(this, "PromiseUtils",
"resource://gre/modules/PromiseUtils.jsm");
ChromeUtils.defineModuleGetter(this, "UpdateUtils",
"resource://gre/modules/UpdateUtils.jsm");
XPCOMUtils.defineLazyGetter(this, "gTextDecoder", () => {
return new TextDecoder();
});
// The filename where directory links are stored locally
const DIRECTORY_LINKS_FILE = "directoryLinks.json";
const DIRECTORY_LINKS_TYPE = "application/json";
// The preference that tells where to obtain directory links
const PREF_DIRECTORY_SOURCE = "browser.newtabpage.directory.source";
// The preference that tells if newtab is enhanced
const PREF_NEWTAB_ENHANCED = "browser.newtabpage.enhanced";
// Only allow link urls that are http(s)
const ALLOWED_LINK_SCHEMES = new Set(["http", "https"]);
// Only allow link image urls that are https or data
const ALLOWED_IMAGE_SCHEMES = new Set(["https", "data"]);
// Only allow urls to Mozilla's CDN or empty (for data URIs)
const ALLOWED_URL_BASE = new Set(["mozilla.net", ""]);
// The frecency of a directory link
const DIRECTORY_FRECENCY = 1000;
/**
* Singleton that serves as the provider of directory links.
* Directory links are a hard-coded set of links shown if a user's link
* inventory is empty.
*/
var DirectoryLinksProvider = {
__linksURL: null,
_observers: new Set(),
// links download deferred, resolved upon download completion
_downloadDeferred: null,
// download default interval is 24 hours in milliseconds
_downloadIntervalMS: 86400000,
get _observedPrefs() {
return Object.freeze({
enhanced: PREF_NEWTAB_ENHANCED,
linksURL: PREF_DIRECTORY_SOURCE,
});
},
get _linksURL() {
if (!this.__linksURL) {
try {
this.__linksURL = Services.prefs.getCharPref(this._observedPrefs.linksURL);
this.__linksURLModified = Services.prefs.prefHasUserValue(this._observedPrefs.linksURL);
} catch (e) {
Cu.reportError("Error fetching directory links url from prefs: " + e);
}
}
return this.__linksURL;
},
/**
* Gets the currently selected locale for display.
* @return the selected locale or "en-US" if none is selected
*/
get locale() {
return Services.locale.getRequestedLocale() || "en-US";
},
/**
* Set appropriate default ping behavior controlled by enhanced pref
*/
_setDefaultEnhanced: function DirectoryLinksProvider_setDefaultEnhanced() {
if (!Services.prefs.prefHasUserValue(PREF_NEWTAB_ENHANCED)) {
let enhanced = Services.prefs.getBoolPref(PREF_NEWTAB_ENHANCED);
try {
// Default to not enhanced if DNT is set to tell websites to not track
if (Services.prefs.getBoolPref("privacy.donottrackheader.enabled")) {
enhanced = false;
}
} catch (ex) {}
Services.prefs.setBoolPref(PREF_NEWTAB_ENHANCED, enhanced);
}
},
observe: function DirectoryLinksProvider_observe(aSubject, aTopic, aData) {
if (aTopic == "nsPref:changed") {
switch (aData) {
// Re-set the default in case the user clears the pref
case this._observedPrefs.enhanced:
this._setDefaultEnhanced();
break;
case this._observedPrefs.linksURL:
delete this.__linksURL;
this._fetchAndCacheLinksIfNecessary(true);
break;
}
} else if (aTopic === "intl:requested-locales-changed") {
this._fetchAndCacheLinksIfNecessary(true);
}
},
_addPrefsObserver: function DirectoryLinksProvider_addObserver() {
for (let pref in this._observedPrefs) {
let prefName = this._observedPrefs[pref];
Services.prefs.addObserver(prefName, this);
}
},
_removePrefsObserver: function DirectoryLinksProvider_removeObserver() {
for (let pref in this._observedPrefs) {
let prefName = this._observedPrefs[pref];
Services.prefs.removeObserver(prefName, this);
}
},
_fetchAndCacheLinks: function DirectoryLinksProvider_fetchAndCacheLinks(uri) {
// Replace with the same display locale used for selecting links data
uri = uri.replace("%LOCALE%", this.locale);
uri = uri.replace("%CHANNEL%", UpdateUtils.UpdateChannel);
return this._downloadJsonData(uri).then(json => {
return OS.File.writeAtomic(this._directoryFilePath, json, {tmpPath: this._directoryFilePath + ".tmp"});
});
},
/**
* Downloads a links with json content
* @param download uri
* @return promise resolved to json string, "{}" returned if status != 200
*/
_downloadJsonData: function DirectoryLinksProvider__downloadJsonData(uri) {
return new Promise((resolve, reject) => {
let xmlHttp = this._newXHR();
xmlHttp.onload = function(aResponse) {
let json = this.responseText;
if (this.status && this.status != 200) {
json = "{}";
}
resolve(json);
};
xmlHttp.onerror = function(e) {
reject("Fetching " + uri + " results in error code: " + e.target.status);
};
try {
xmlHttp.open("GET", uri);
// Override the type so XHR doesn't complain about not well-formed XML
xmlHttp.overrideMimeType(DIRECTORY_LINKS_TYPE);
// Set the appropriate request type for servers that require correct types
xmlHttp.setRequestHeader("Content-Type", DIRECTORY_LINKS_TYPE);
xmlHttp.send();
} catch (e) {
reject("Error fetching " + uri);
Cu.reportError(e);
}
});
},
/**
* Downloads directory links if needed
* @return promise resolved immediately if no download needed, or upon completion
*/
_fetchAndCacheLinksIfNecessary: function DirectoryLinksProvider_fetchAndCacheLinksIfNecessary(forceDownload = false) {
if (this._downloadDeferred) {
// fetching links already - just return the promise
return this._downloadDeferred.promise;
}
if (forceDownload || this._needsDownload) {
this._downloadDeferred = PromiseUtils.defer();
this._fetchAndCacheLinks(this._linksURL).then(() => {
// the new file was successfully downloaded and cached, so update a timestamp
this._lastDownloadMS = Date.now();
this._downloadDeferred.resolve();
this._downloadDeferred = null;
this._callObservers("onManyLinksChanged");
},
error => {
this._downloadDeferred.resolve();
this._downloadDeferred = null;
this._callObservers("onDownloadFail");
});
return this._downloadDeferred.promise;
}
// download is not needed
return Promise.resolve();
},
/**
* @return true if download is needed, false otherwise
*/
get _needsDownload() {
// fail if last download occured less then 24 hours ago
if ((Date.now() - this._lastDownloadMS) > this._downloadIntervalMS) {
return true;
}
return false;
},
/**
* Create a new XMLHttpRequest that is anonymous, i.e., doesn't send cookies
*/
_newXHR() {
return new XMLHttpRequest({mozAnon: true});
},
/**
* Reads directory links file and parses its content
* @return a promise resolved to an object with keys 'directory' and 'suggested',
* each containing a valid list of links,
* or {'directory': [], 'suggested': []} if read or parse fails.
*/
_readDirectoryLinksFile: function DirectoryLinksProvider_readDirectoryLinksFile() {
let emptyOutput = {directory: []};
return OS.File.read(this._directoryFilePath).then(binaryData => {
let output;
try {
let json = gTextDecoder.decode(binaryData);
let linksObj = JSON.parse(json);
output = {directory: linksObj.directory || []};
} catch (e) {
Cu.reportError(e);
}
return output || emptyOutput;
},
error => {
Cu.reportError(error);
return emptyOutput;
});
},
/**
* Get the enhanced link object for a link (whether history or directory)
*/
getEnhancedLink: function DirectoryLinksProvider_getEnhancedLink(link) {
// Use the provided link if it's already enhanced
return link.enhancedImageURI && link;
},
/**
* Check if a url's scheme is in a Set of allowed schemes and if the base
* domain is allowed.
* @param url to check
* @param allowed Set of allowed schemes
* @param checkBase boolean to check the base domain
*/
isURLAllowed(url, allowed, checkBase) {
// Assume no url is an allowed url
if (!url) {
return true;
}
let scheme = "", base = "";
try {
// A malformed url will not be allowed
let uri = Services.io.newURI(url);
scheme = uri.scheme;
// URIs without base domains will be allowed
base = Services.eTLD.getBaseDomain(uri);
} catch (ex) {}
// Require a scheme match and the base only if desired
return allowed.has(scheme) && (!checkBase || ALLOWED_URL_BASE.has(base));
},
/**
* Gets the current set of directory links.
* @param aCallback The function that the array of links is passed to.
*/
getLinks: function DirectoryLinksProvider_getLinks(aCallback) {
this._readDirectoryLinksFile().then(rawLinks => {
// Only check base domain for images when using the default pref
let checkBase = !this.__linksURLModified;
let validityFilter = link => {
// Make sure the link url is allowed and images too if they exist
return this.isURLAllowed(link.url, ALLOWED_LINK_SCHEMES, false) &&
(!link.imageURI ||
this.isURLAllowed(link.imageURI, ALLOWED_IMAGE_SCHEMES, checkBase)) &&
(!link.enhancedImageURI ||
this.isURLAllowed(link.enhancedImageURI, ALLOWED_IMAGE_SCHEMES, checkBase));
};
let links = rawLinks.directory.filter(validityFilter).map((link, position) => {
link.lastVisitDate = rawLinks.directory.length - position;
link.frecency = DIRECTORY_FRECENCY;
return link;
});
return links;
}).catch(ex => {
Cu.reportError(ex);
return [];
}).then(links => {
aCallback(links);
});
},
init: function DirectoryLinksProvider_init() {
this._setDefaultEnhanced();
this._addPrefsObserver();
Services.obs.addObserver(this, "intl:requested-locales-changed");
// setup directory file path and last download timestamp
this._directoryFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, DIRECTORY_LINKS_FILE);
this._lastDownloadMS = 0;
return (async () => {
// get the last modified time of the links file if it exists
let doesFileExists = await OS.File.exists(this._directoryFilePath);
if (doesFileExists) {
let fileInfo = await OS.File.stat(this._directoryFilePath);
this._lastDownloadMS = Date.parse(fileInfo.lastModificationDate);
}
// fetch directory on startup without force
await this._fetchAndCacheLinksIfNecessary();
})();
},
/**
* Return the object to its pre-init state
*/
reset: function DirectoryLinksProvider_reset() {
delete this.__linksURL;
this._removePrefsObserver();
this._removeObservers();
Services.obs.removeObserver(this, "intl:requested-locales-changed");
},
addObserver: function DirectoryLinksProvider_addObserver(aObserver) {
this._observers.add(aObserver);
},
removeObserver: function DirectoryLinksProvider_removeObserver(aObserver) {
this._observers.delete(aObserver);
},
_callObservers(methodName, ...args) {
for (let obs of this._observers) {
if (typeof(obs[methodName]) == "function") {
try {
obs[methodName](this, ...args);
} catch (err) {
Cu.reportError(err);
}
}
}
},
_removeObservers() {
this._observers.clear();
}
};

7
browser/modules/moz.build
View File

@ -31,9 +31,6 @@ with Files("test/browser/browser_urlBar_zoom.js"):
with Files("test/unit/test_AttributionCode.js"):
BUG_COMPONENT = ("Toolkit", "Telemetry")
with Files("test/unit/test_DirectoryLinksProvider.js"):
BUG_COMPONENT = ("Firefox", "New Tab Page")
with Files("test/unit/test_E10SUtils_nested_URIs.js"):
BUG_COMPONENT = ("Core", "Security: Process Sandboxing")
@ -61,9 +58,6 @@ with Files("ContentSearch.jsm"):
with Files("ContentWebRTC.jsm"):
BUG_COMPONENT = ("Firefox", "Device Permissions")
with Files("DirectoryLinksProvider.jsm"):
BUG_COMPONENT = ("Firefox", "New Tab Page")
with Files("ExtensionsUI.jsm"):
BUG_COMPONENT = ("Toolkit", "WebExtensions: General")
@ -140,7 +134,6 @@ EXTRA_JS_MODULES += [
'ContentSearch.jsm',
'ContentWebRTC.jsm',
'ContextMenu.jsm',
'DirectoryLinksProvider.jsm',
'ExtensionsUI.jsm',
'Feeds.jsm',
'FormSubmitObserver.jsm',

534
browser/modules/test/unit/test_DirectoryLinksProvider.js
View File

@ -1,534 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
"use strict";
/**
* This file tests the DirectoryLinksProvider singleton in the DirectoryLinksProvider.jsm module.
*/
var CC = Components.Constructor;
ChromeUtils.import("resource://gre/modules/Services.jsm");
ChromeUtils.import("resource:///modules/DirectoryLinksProvider.jsm");
ChromeUtils.import("resource://gre/modules/Promise.jsm");
ChromeUtils.import("resource://gre/modules/Http.jsm");
ChromeUtils.import("resource://testing-common/httpd.js");
ChromeUtils.import("resource://gre/modules/osfile.jsm");
ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
ChromeUtils.import("resource://gre/modules/PlacesUtils.jsm");
ChromeUtils.defineModuleGetter(this, "NetUtil",
"resource://gre/modules/NetUtil.jsm");
ChromeUtils.defineModuleGetter(this, "NewTabUtils",
"resource://gre/modules/NewTabUtils.jsm");
ChromeUtils.defineModuleGetter(this, "PlacesTestUtils",
"resource://testing-common/PlacesTestUtils.jsm");
do_get_profile();
const DIRECTORY_LINKS_FILE = "directoryLinks.json";
const DIRECTORY_FRECENCY = 1000;
const kURLData = {"directory": [{"url": "http://example.com", "title": "LocalSource"}]};
const kTestURL = "data:application/json," + JSON.stringify(kURLData);
// DirectoryLinksProvider preferences
const kSourceUrlPref = DirectoryLinksProvider._observedPrefs.linksURL;
const kNewtabEnhancedPref = "browser.newtabpage.enhanced";
// httpd settings
var server;
const kDefaultServerPort = 9000;
const kBaseUrl = "http://localhost:" + kDefaultServerPort;
const kExamplePath = "/exampleTest/";
const kFailPath = "/fail/";
const kExampleURL = kBaseUrl + kExamplePath;
const kFailURL = kBaseUrl + kFailPath;
// app/profile/firefox.js are not avaialble in xpcshell: hence, preset them
const origReqLocales = Services.locale.getRequestedLocales();
Services.locale.setRequestedLocales(["en-US"]);
Services.prefs.setCharPref(kSourceUrlPref, kTestURL);
Services.prefs.setBoolPref(kNewtabEnhancedPref, true);
const kHttpHandlerData = {};
kHttpHandlerData[kExamplePath] = {"directory": [{"url": "http://example.com", "title": "RemoteSource"}]};
const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
"nsIBinaryInputStream",
"setInputStream");
var gLastRequestPath;
function getHttpHandler(path) {
let code = 200;
let body = JSON.stringify(kHttpHandlerData[path]);
if (path == kFailPath) {
code = 204;
}
return function(aRequest, aResponse) {
gLastRequestPath = aRequest.path;
aResponse.setStatusLine(null, code);
aResponse.setHeader("Content-Type", "application/json");
aResponse.write(body);
};
}
function isIdentical(actual, expected) {
if (expected == null) {
Assert.equal(actual, expected);
} else if (typeof expected == "object") {
// Make sure all the keys match up
Assert.equal(Object.keys(actual).sort() + "", Object.keys(expected).sort());
// Recursively check each value individually
Object.keys(expected).forEach(key => {
isIdentical(actual[key], expected[key]);
});
} else {
Assert.equal(actual, expected);
}
}
function fetchData() {
return new Promise(resolve => {
DirectoryLinksProvider.getLinks(linkData => {
resolve(linkData);
});
});
}
function readJsonFile(jsonFile = DIRECTORY_LINKS_FILE) {
let decoder = new TextDecoder();
let directoryLinksFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, jsonFile);
return OS.File.read(directoryLinksFilePath).then(array => {
let json = decoder.decode(array);
return JSON.parse(json);
}, () => { return ""; });
}
function cleanJsonFile(jsonFile = DIRECTORY_LINKS_FILE) {
let directoryLinksFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, jsonFile);
return OS.File.remove(directoryLinksFilePath);
}
function LinksChangeObserver() {
this.deferred = Promise.defer();
this.onManyLinksChanged = () => this.deferred.resolve();
this.onDownloadFail = this.onManyLinksChanged;
}
function promiseDirectoryDownloadOnPrefChange(pref, newValue) {
let oldValue = Services.prefs.getCharPref(pref);
if (oldValue != newValue) {
// if the preference value is already equal to newValue
// the pref service will not call our observer and we
// deadlock. Hence only setup observer if values differ
let observer = new LinksChangeObserver();
DirectoryLinksProvider.addObserver(observer);
Services.prefs.setCharPref(pref, newValue);
return observer.deferred.promise.then(() => {
DirectoryLinksProvider.removeObserver(observer);
});
}
return Promise.resolve();
}
function promiseSetupDirectoryLinksProvider(options = {}) {
return (async function() {
let linksURL = options.linksURL || kTestURL;
await DirectoryLinksProvider.init();
Services.locale.setRequestedLocales([options.locale || "en-US"]);
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, linksURL);
Assert.equal(DirectoryLinksProvider._linksURL, linksURL);
DirectoryLinksProvider._lastDownloadMS = options.lastDownloadMS || 0;
})();
}
function promiseCleanDirectoryLinksProvider() {
return (async function() {
Services.locale.setRequestedLocales(["en-US"]);
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, kTestURL);
DirectoryLinksProvider._lastDownloadMS = 0;
DirectoryLinksProvider.reset();
})();
}
function run_test() {
// Set up a mock HTTP server to serve a directory page
server = new HttpServer();
server.registerPrefixHandler(kExamplePath, getHttpHandler(kExamplePath));
server.registerPrefixHandler(kFailPath, getHttpHandler(kFailPath));
server.start(kDefaultServerPort);
NewTabUtils.init();
run_next_test();
// Teardown.
registerCleanupFunction(function() {
server.stop(function() { });
DirectoryLinksProvider.reset();
Services.locale.setRequestedLocales(origReqLocales);
Services.prefs.clearUserPref(kSourceUrlPref);
Services.prefs.clearUserPref(kNewtabEnhancedPref);
});
}
function setTimeout(fun, timeout) {
let timer = Cc["@mozilla.org/timer;1"]
.createInstance(Ci.nsITimer);
var event = {
notify() {
fun();
}
};
timer.initWithCallback(event, timeout,
Ci.nsITimer.TYPE_ONE_SHOT);
return timer;
}
add_task(async function test_fetchAndCacheLinks_local() {
await DirectoryLinksProvider.init();
await cleanJsonFile();
// Trigger cache of data or chrome uri files in profD
await DirectoryLinksProvider._fetchAndCacheLinks(kTestURL);
let data = await readJsonFile();
isIdentical(data, kURLData);
});
add_task(async function test_fetchAndCacheLinks_remote() {
await DirectoryLinksProvider.init();
await cleanJsonFile();
// this must trigger directory links json download and save it to cache file
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, kExampleURL + "%LOCALE%");
Assert.equal(gLastRequestPath, kExamplePath + "en-US");
let data = await readJsonFile();
isIdentical(data, kHttpHandlerData[kExamplePath]);
});
add_task(async function test_fetchAndCacheLinks_malformedURI() {
await DirectoryLinksProvider.init();
await cleanJsonFile();
let someJunk = "some junk";
try {
await DirectoryLinksProvider._fetchAndCacheLinks(someJunk);
do_throw("Malformed URIs should fail");
} catch (e) {
Assert.equal(e, "Error fetching " + someJunk);
}
// File should be empty.
let data = await readJsonFile();
isIdentical(data, "");
});
add_task(async function test_fetchAndCacheLinks_unknownHost() {
await DirectoryLinksProvider.init();
await cleanJsonFile();
let nonExistentServer = "http://localhost:56789/";
try {
await DirectoryLinksProvider._fetchAndCacheLinks(nonExistentServer);
do_throw("BAD URIs should fail");
} catch (e) {
Assert.ok(e.startsWith("Fetching " + nonExistentServer + " results in error code: "));
}
// File should be empty.
let data = await readJsonFile();
isIdentical(data, "");
});
add_task(async function test_fetchAndCacheLinks_non200Status() {
await DirectoryLinksProvider.init();
await cleanJsonFile();
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, kFailURL);
Assert.equal(gLastRequestPath, kFailPath);
let data = await readJsonFile();
isIdentical(data, {});
});
// To test onManyLinksChanged observer, trigger a fetch
add_task(async function test_DirectoryLinksProvider__linkObservers() {
await DirectoryLinksProvider.init();
let testObserver = new LinksChangeObserver();
DirectoryLinksProvider.addObserver(testObserver);
Assert.equal(DirectoryLinksProvider._observers.size, 1);
DirectoryLinksProvider._fetchAndCacheLinksIfNecessary(true);
await testObserver.deferred.promise;
DirectoryLinksProvider._removeObservers();
Assert.equal(DirectoryLinksProvider._observers.size, 0);
await promiseCleanDirectoryLinksProvider();
});
add_task(async function test_DirectoryLinksProvider__prefObserver_url() {
await promiseSetupDirectoryLinksProvider({linksURL: kTestURL});
let links = await fetchData();
Assert.equal(links.length, 1);
let expectedData = [{url: "http://example.com", title: "LocalSource", frecency: DIRECTORY_FRECENCY, lastVisitDate: 1}];
isIdentical(links, expectedData);
// tests these 2 things:
// 1. _linksURL is properly set after the pref change
// 2. invalid source url is correctly handled
let exampleUrl = "http://localhost:56789/bad";
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, exampleUrl);
Assert.equal(DirectoryLinksProvider._linksURL, exampleUrl);
// since the download fail, the directory file must remain the same
let newLinks = await fetchData();
isIdentical(newLinks, expectedData);
// now remove the file, and re-download
await cleanJsonFile();
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, exampleUrl + " ");
// we now should see empty links
newLinks = await fetchData();
isIdentical(newLinks, []);
await promiseCleanDirectoryLinksProvider();
});
add_task(async function test_DirectoryLinksProvider_getLinks_noDirectoryData() {
let data = {
"directory": [],
};
let dataURI = "data:application/json," + JSON.stringify(data);
await promiseSetupDirectoryLinksProvider({linksURL: dataURI});
let links = await fetchData();
Assert.equal(links.length, 0);
await promiseCleanDirectoryLinksProvider();
});
add_task(async function test_DirectoryLinksProvider_getLinks_badData() {
let data = {
"en-US": {
"en-US": [{url: "http://example.com", title: "US"}],
},
};
let dataURI = "data:application/json," + JSON.stringify(data);
await promiseSetupDirectoryLinksProvider({linksURL: dataURI});
// Make sure we get nothing for incorrectly formatted data
let links = await fetchData();
Assert.equal(links.length, 0);
await promiseCleanDirectoryLinksProvider();
});
add_task(function test_DirectoryLinksProvider_needsDownload() {
// test timestamping
DirectoryLinksProvider._lastDownloadMS = 0;
Assert.ok(DirectoryLinksProvider._needsDownload);
DirectoryLinksProvider._lastDownloadMS = Date.now();
Assert.ok(!DirectoryLinksProvider._needsDownload);
DirectoryLinksProvider._lastDownloadMS = Date.now() - (60 * 60 * 24 + 1) * 1000;
Assert.ok(DirectoryLinksProvider._needsDownload);
DirectoryLinksProvider._lastDownloadMS = 0;
});
add_task(async function test_DirectoryLinksProvider_fetchAndCacheLinksIfNecessary() {
await DirectoryLinksProvider.init();
await cleanJsonFile();
// explicitly change source url to cause the download during setup
await promiseSetupDirectoryLinksProvider({linksURL: kTestURL + " "});
await DirectoryLinksProvider._fetchAndCacheLinksIfNecessary();
// inspect lastDownloadMS timestamp which should be 5 seconds less then now()
let lastDownloadMS = DirectoryLinksProvider._lastDownloadMS;
Assert.ok((Date.now() - lastDownloadMS) < 5000);
// we should have fetched a new file during setup
let data = await readJsonFile();
isIdentical(data, kURLData);
// attempt to download again - the timestamp should not change
await DirectoryLinksProvider._fetchAndCacheLinksIfNecessary();
Assert.equal(DirectoryLinksProvider._lastDownloadMS, lastDownloadMS);
// clean the file and force the download
await cleanJsonFile();
await DirectoryLinksProvider._fetchAndCacheLinksIfNecessary(true);
data = await readJsonFile();
isIdentical(data, kURLData);
// make sure that failed download does not corrupt the file, nor changes lastDownloadMS
lastDownloadMS = DirectoryLinksProvider._lastDownloadMS;
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, "http://");
await DirectoryLinksProvider._fetchAndCacheLinksIfNecessary(true);
data = await readJsonFile();
isIdentical(data, kURLData);
Assert.equal(DirectoryLinksProvider._lastDownloadMS, lastDownloadMS);
// _fetchAndCacheLinksIfNecessary must return same promise if download is in progress
let downloadPromise = DirectoryLinksProvider._fetchAndCacheLinksIfNecessary(true);
let anotherPromise = DirectoryLinksProvider._fetchAndCacheLinksIfNecessary(true);
Assert.ok(downloadPromise === anotherPromise);
await downloadPromise;
await promiseCleanDirectoryLinksProvider();
});
add_task(async function test_DirectoryLinksProvider_fetchDirectoryOnPrefChange() {
await DirectoryLinksProvider.init();
let testObserver = new LinksChangeObserver();
DirectoryLinksProvider.addObserver(testObserver);
await cleanJsonFile();
// ensure that provider does not think it needs to download
Assert.ok(!DirectoryLinksProvider._needsDownload);
// change the source URL, which should force directory download
await promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, kExampleURL);
// then wait for testObserver to fire and test that json is downloaded
await testObserver.deferred.promise;
Assert.equal(gLastRequestPath, kExamplePath);
let data = await readJsonFile();
isIdentical(data, kHttpHandlerData[kExamplePath]);
await promiseCleanDirectoryLinksProvider();
});
add_task(async function test_DirectoryLinksProvider_fetchDirectoryOnInit() {
// ensure preferences are set to defaults
await promiseSetupDirectoryLinksProvider();
// now clean to provider, so we can init it again
await promiseCleanDirectoryLinksProvider();
await cleanJsonFile();
await DirectoryLinksProvider.init();
let data = await readJsonFile();
isIdentical(data, kURLData);
await promiseCleanDirectoryLinksProvider();
});
add_task(async function test_DirectoryLinksProvider_getLinksFromCorruptedFile() {
await promiseSetupDirectoryLinksProvider();
// write bogus json to a file and attempt to fetch from it
let directoryLinksFilePath = OS.Path.join(OS.Constants.Path.profileDir, DIRECTORY_LINKS_FILE);
await OS.File.writeAtomic(directoryLinksFilePath, '{"en-US":');
let data = await fetchData();
isIdentical(data, []);
await promiseCleanDirectoryLinksProvider();
});
add_task(async function test_DirectoryLinksProvider_getAllowedLinks() {
let data = {"directory": [
{url: "ftp://example.com"},
{url: "http://example.net"},
{url: "javascript:5"},
{url: "https://example.com"},
{url: "httpJUNKjavascript:42"},
{url: "data:text/plain,hi"},
{url: "http/bork:eh"},
]};
let dataURI = "data:application/json," + JSON.stringify(data);
await promiseSetupDirectoryLinksProvider({linksURL: dataURI});
let links = await fetchData();
Assert.equal(links.length, 2);
// The only remaining url should be http and https
Assert.equal(links[0].url, data.directory[1].url);
Assert.equal(links[1].url, data.directory[3].url);
});
add_task(async function test_DirectoryLinksProvider_getAllowedImages() {
let data = {"directory": [
{url: "http://example.com", imageURI: "ftp://example.com"},
{url: "http://example.com", imageURI: "http://example.net"},
{url: "http://example.com", imageURI: "javascript:5"},
{url: "http://example.com", imageURI: "https://example.com"},
{url: "http://example.com", imageURI: "httpJUNKjavascript:42"},
{url: "http://example.com", imageURI: "data:text/plain,hi"},
{url: "http://example.com", imageURI: "http/bork:eh"},
]};
let dataURI = "data:application/json," + JSON.stringify(data);
await promiseSetupDirectoryLinksProvider({linksURL: dataURI});
let links = await fetchData();
Assert.equal(links.length, 2);
// The only remaining images should be https and data
Assert.equal(links[0].imageURI, data.directory[3].imageURI);
Assert.equal(links[1].imageURI, data.directory[5].imageURI);
});
add_task(async function test_DirectoryLinksProvider_getAllowedImages_base() {
let data = {"directory": [
{url: "http://example1.com", imageURI: "https://example.com"},
{url: "http://example2.com", imageURI: "https://tiles.cdn.mozilla.net"},
{url: "http://example3.com", imageURI: "https://tiles2.cdn.mozilla.net"},
{url: "http://example4.com", enhancedImageURI: "https://mozilla.net"},
{url: "http://example5.com", imageURI: "data:text/plain,hi"},
]};
let dataURI = "data:application/json," + JSON.stringify(data);
await promiseSetupDirectoryLinksProvider({linksURL: dataURI});
// Pretend we're using the default pref to trigger base matching
DirectoryLinksProvider.__linksURLModified = false;
let links = await fetchData();
Assert.equal(links.length, 4);
// The only remaining images should be https with mozilla.net or data URI
Assert.equal(links[0].url, data.directory[1].url);
Assert.equal(links[1].url, data.directory[2].url);
Assert.equal(links[2].url, data.directory[3].url);
Assert.equal(links[3].url, data.directory[4].url);
});
add_task(async function test_DirectoryLinksProvider_getAllowedEnhancedImages() {
let data = {"directory": [
{url: "http://example.com", enhancedImageURI: "ftp://example.com"},
{url: "http://example.com", enhancedImageURI: "http://example.net"},
{url: "http://example.com", enhancedImageURI: "javascript:5"},
{url: "http://example.com", enhancedImageURI: "https://example.com"},
{url: "http://example.com", enhancedImageURI: "httpJUNKjavascript:42"},
{url: "http://example.com", enhancedImageURI: "data:text/plain,hi"},
{url: "http://example.com", enhancedImageURI: "http/bork:eh"},
]};
let dataURI = "data:application/json," + JSON.stringify(data);
await promiseSetupDirectoryLinksProvider({linksURL: dataURI});
let links = await fetchData();
Assert.equal(links.length, 2);
// The only remaining enhancedImages should be http and https and data
Assert.equal(links[0].enhancedImageURI, data.directory[3].enhancedImageURI);
Assert.equal(links[1].enhancedImageURI, data.directory[5].enhancedImageURI);
});
add_task(function test_DirectoryLinksProvider_setDefaultEnhanced() {
function checkDefault(expected) {
Services.prefs.clearUserPref(kNewtabEnhancedPref);
Assert.equal(Services.prefs.getBoolPref(kNewtabEnhancedPref), expected);
}
// Use the default donottrack prefs (enabled = false)
Services.prefs.clearUserPref("privacy.donottrackheader.enabled");
checkDefault(true);
// Turn on DNT - no track
Services.prefs.setBoolPref("privacy.donottrackheader.enabled", true);
checkDefault(false);
// Turn off DNT header
Services.prefs.clearUserPref("privacy.donottrackheader.enabled");
checkDefault(true);
// Clean up
Services.prefs.clearUserPref("privacy.donottrackheader.value");
});
add_task(function test_DirectoryLinksProvider_anonymous() {
Assert.ok(DirectoryLinksProvider._newXHR().mozAnon);
});

1
browser/modules/test/unit/xpcshell.ini
View File

@ -5,7 +5,6 @@ skip-if = toolkit == 'android'
[test_AttributionCode.js]
skip-if = os != 'win'
[test_DirectoryLinksProvider.js]
[test_E10SUtils_nested_URIs.js]
[test_Sanitizer_interrupted.js]
[test_SitePermissions.js]

3
browser/moz.build
View File

@ -97,9 +97,6 @@ with Files("config/**"):
with Files("docs/**"):
BUG_COMPONENT = ("Toolkit", "Telemetry")
with Files("docs/DirectoryLinksProvider.rst"):
BUG_COMPONENT = ("Firefox", "New Tab Page")
with Files("fonts/**"):
BUG_COMPONENT = ("Core", "Graphics: Text")

2
layout/tools/reftest/reftest-preferences.js
View File

@ -116,8 +116,6 @@ user_pref("browser.snippets.firstrunHomepage.enabled", false);
user_pref("general.useragent.updates.enabled", false);
// And for webapp updates. Yes, it is supposed to be an integer.
user_pref("browser.webapps.checkForUpdates", 0);
// And for about:newtab content fetch.
user_pref("browser.newtabpage.directory.source", "data:application/json,{\"reftest\":1}");
// Only allow add-ons from the profile and app and allow foreign
// injection
user_pref("extensions.enabledScopes", 5);

4
testing/profiles/prefs_general.js
View File

@ -210,10 +210,6 @@ user_pref("browser.pagethumbnails.capturing_disabled", true);
// download test runs first doesn't show the popup inconsistently.
user_pref("browser.download.panel.shown", true);
// Assume the about:newtab page's intro panels have been shown to not depend on
// which test runs first and happens to open about:newtab
user_pref("browser.newtabpage.introShown", true);
// Enable webapps testing mode, which bypasses native installation.
user_pref("browser.webapps.testing", true);

2
testing/talos/talos/config.py
View File

@ -112,8 +112,6 @@ DEFAULTS = dict(
'browser.newtabpage.activity-stream.tippyTop.service.endpoint': '',
'browser.newtabpage.activity-stream.feeds.section.topstories': False,
'browser.newtabpage.activity-stream.feeds.snippets': False,
'browser.newtabpage.directory.source':
'${webserver}/directoryLinks.json',
'browser.newtabpage.introShown': True,
'browser.safebrowsing.downloads.remote.url':
'http://127.0.0.1/safebrowsing-dummy/downloads',

87
testing/talos/talos/directoryLinks.json
View File

File diff suppressed because one or more lines are too long

3
testing/talos/talos/tests/tart/addon/content/tart.js
View File

@ -434,7 +434,6 @@ Tart.prototype = {
_startTest() {
// Save prefs and states which will change during the test, to get restored when done.
var origNewtabEnabled = Services.prefs.getBoolPref("browser.newtabpage.enabled");
var origPreload = Services.prefs.getBoolPref("browser.newtab.preload");
var origDpi = Services.prefs.getCharPref("layout.css.devPixelsPerPx");
var origPinned = this._tartTab.pinned;
@ -490,7 +489,6 @@ Tart.prototype = {
var subtests = {
init: [ // This is called before each subtest, so it's safe to assume the following prefs:
function() {
Services.prefs.setBoolPref("browser.newtabpage.enabled", true);
Services.prefs.setBoolPref("browser.newtab.preload", false);
self.pinTart();
self.makeNewTabURLChangePromise("about:blank").then(next);
@ -500,7 +498,6 @@ Tart.prototype = {
restore: [
// Restore prefs which were modified during the test
function() {
Services.prefs.setBoolPref("browser.newtabpage.enabled", origNewtabEnabled);
Services.prefs.setBoolPref("browser.newtab.preload", origPreload);
Services.prefs.setCharPref("layout.css.devPixelsPerPx", origDpi);
if (origPinned) self.pinTart(); else self.unpinTart();