Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-16 14:55:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out changeset 84391bad8a10 (bug 1741588) for causing multiple xpc/mochitest failures. CLOSED TREE

Browse Source
This commit is contained in:
Sandor Molnar 2021-12-01 15:27:13 +02:00
parent 731f86bff7
commit af76619700
6 changed files with 27 additions and 829 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

261
browser/components/pagedata/PageDataService.jsm
View File

@ -13,10 +13,7 @@ const { XPCOMUtils } = ChromeUtils.import(
XPCOMUtils.defineLazyModuleGetters(this, {
BrowserWindowTracker: "resource:///modules/BrowserWindowTracker.jsm",
Services: "resource://gre/modules/Services.jsm",
E10SUtils: "resource://gre/modules/E10SUtils.jsm",
EventEmitter: "resource://gre/modules/EventEmitter.jsm",
HiddenFrame: "resource://gre/modules/HiddenFrame.jsm",
PromiseUtils: "resource://gre/modules/PromiseUtils.jsm",
});
XPCOMUtils.defineLazyGetter(this, "logConsole", function() {
@ -28,42 +25,8 @@ XPCOMUtils.defineLazyGetter(this, "logConsole", function() {
});
});
XPCOMUtils.defineLazyServiceGetters(this, {
idleService: ["@mozilla.org/widget/useridleservice;1", "nsIUserIdleService"],
});
XPCOMUtils.defineLazyPreferenceGetter(
this,
"fetchIdleTime",
"browser.pagedata.fetchIdleTime",
300
);
const ALLOWED_SCHEMES = ["http", "https", "data", "blob"];
const BACKGROUND_WIDTH = 1024;
const BACKGROUND_HEIGHT = 768;
/**
* Shifts the first element out of the set.
*
* @param {Set<T>} set
* The set containing elements.
* @returns {T | undefined} The first element in the set or undefined if
* there is nothing in the set.
*/
function shift(set) {
let iter = set.values();
let { value, done } = iter.next();
if (done) {
return undefined;
}
set.delete(value);
return value;
}
/**
* @typedef {object} PageData
* A set of discovered from a page. Other than the `data` property this is the
@ -92,55 +55,6 @@ const PageDataService = new (class PageDataService extends EventEmitter {
*/
#pageDataCache = new Map();
/**
* The number of currently running background fetches.
* @type {number}
*/
#backgroundFetches = 0;
/**
* The list of urls waiting to be loaded in the background.
* @type {Set<string>}
*/
#backgroundQueue = new Set();
/**
* Tracks whether the user is currently idle.
* @type {boolean}
*/
#userIsIdle = false;
/**
* A hidden frame used for creating browsers for background page data loading.
* @type {HiddenFrame | null}
*/
#hiddenFrame = null;
/**
* A map of hidden browsers to a resolve function that should be passed the
* actor that was created for the browser.
*
* @type {WeakMap<Browser, (actor: PageDataParent) => void>}
*/
#backgroundBrowsers = new WeakMap();
/**
* Constructs the service.
*/
constructor() {
super();
// Limits the number of background fetches that will run at once. Set to 0 to
// effectively allow an infinite number.
XPCOMUtils.defineLazyPreferenceGetter(
this,
"MAX_BACKGROUND_FETCHES",
"browser.pagedata.maxBackgroundFetches",
5,
() => this.#startBackgroundWorkers()
);
}
/**
* Initializes a new instance of the service, not called externally.
*/
@ -176,8 +90,6 @@ const PageDataService = new (class PageDataService extends EventEmitter {
}
}
}
idleService.addIdleObserver(this, fetchIdleTime);
}
/**
@ -204,22 +116,9 @@ const PageDataService = new (class PageDataService extends EventEmitter {
}
let browser = actor.browsingContext?.embedderElement;
// If we don't have a browser then it went away before we could record,
// so we don't know where the data came from.
if (!browser) {
return;
}
// Is this a load in a background browser?
let backgroundResolve = this.#backgroundBrowsers.get(browser);
if (backgroundResolve) {
backgroundResolve(actor);
return;
}
// Otherwise we only care about pages loaded in the tab browser.
if (!this.#isATabBrowser(browser)) {
if (!browser || !this.#isATabBrowser(browser)) {
return;
}
@ -267,149 +166,6 @@ const PageDataService = new (class PageDataService extends EventEmitter {
return this.#pageDataCache.get(url) ?? null;
}
/**
* Fetches page data from the given URL using a hidden window. Note that this does not populate
* the page data cache or emit the `page-data` event.
*
* @param {string} url
* The url to retrieve data for.
* @returns {Promise<PageData|null>}
* Resolves to the found pagedata or null in case of error.
*/
async fetchPageData(url) {
if (!this.#hiddenFrame) {
this.#hiddenFrame = new HiddenFrame();
}
let windowlessBrowser = await this.#hiddenFrame.get();
let doc = windowlessBrowser.document;
let browser = doc.createXULElement("browser");
browser.setAttribute("remote", "true");
browser.setAttribute("type", "content");
browser.setAttribute(
"style",
`
width: ${BACKGROUND_WIDTH}px;
min-width: ${BACKGROUND_WIDTH}px;
height: ${BACKGROUND_HEIGHT}px;
min-height: ${BACKGROUND_HEIGHT}px;
`
);
browser.setAttribute("maychangeremoteness", "true");
doc.documentElement.appendChild(browser);
try {
let { promise, resolve } = PromiseUtils.defer();
this.#backgroundBrowsers.set(browser, resolve);
let principal = Services.scriptSecurityManager.getSystemPrincipal();
let oa = E10SUtils.predictOriginAttributes({
browser,
});
let loadURIOptions = {
triggeringPrincipal: principal,
remoteType: E10SUtils.getRemoteTypeForURI(
url,
true,
false,
E10SUtils.DEFAULT_REMOTE_TYPE,
null,
oa
),
};
browser.loadURI(url, loadURIOptions);
let actor = await promise;
return await actor.collectPageData();
} finally {
this.#backgroundBrowsers.delete(browser);
browser.remove();
}
}
/**
* Handles notifications from the idle service.
*
* @param {nsISupports} subject
* The notification's subject.
* @param {string} topic
* The notification topic.
* @param {string} data
* The data associated with the notification.
*/
observe(subject, topic, data) {
switch (topic) {
case "idle":
logConsole.debug("User went idle");
this.#userIsIdle = true;
this.#startBackgroundWorkers();
break;
case "active":
logConsole.debug("User became active");
this.#userIsIdle = false;
break;
}
}
/**
* Starts as many background workers as are allowed to process the background
* queue.
*/
#startBackgroundWorkers() {
if (!this.#userIsIdle) {
return;
}
let toStart;
if (this.MAX_BACKGROUND_FETCHES) {
toStart = this.MAX_BACKGROUND_FETCHES - this.#backgroundFetches;
} else {
toStart = this.#backgroundQueue.size;
}
for (let i = 0; i < toStart; i++) {
this.#backgroundFetch();
}
}
/**
* Starts a background fetch worker which will pull urls from the queue and
* load them until the queue is empty.
*/
async #backgroundFetch() {
this.#backgroundFetches++;
let url = shift(this.#backgroundQueue);
while (url) {
try {
let pageData = await this.fetchPageData(url);
if (pageData) {
this.#pageDataCache.set(url, pageData);
this.emit("page-data", pageData);
}
} catch (e) {
logConsole.error(e);
}
// Check whether the user became active or the worker limit changed
// dynamically.
if (
!this.#userIsIdle ||
(this.MAX_BACKGROUND_FETCHES > 0 &&
this.#backgroundFetches > this.MAX_BACKGROUND_FETCHES)
) {
break;
}
url = shift(this.#backgroundQueue);
}
this.#backgroundFetches--;
}
/**
* Queues page data retrieval for a url. The page-data notification will be
* generated if data becomes available.
@ -419,10 +175,19 @@ const PageDataService = new (class PageDataService extends EventEmitter {
* @param {string} url
* The url to retrieve data for.
*/
queueFetch(url) {
this.#backgroundQueue.add(url);
async queueFetch(url) {
// Stub-implementation that generates an empty record.
let pageData = {
url,
date: Date.now(),
data: {},
};
this.#startBackgroundWorkers();
this.#pageDataCache.set(url, pageData);
// Send out a notification. The `no-page-data` notification is intended
// for test use only.
this.emit("page-data", pageData);
}
/**

1
browser/components/pagedata/tests/browser/browser.ini
View File

@ -10,4 +10,3 @@ support-files =
head.js
[browser_pagedata_basic.js]
[browser_pagedata_background.js]

48
browser/components/pagedata/tests/browser/browser_pagedata_background.js
View File

@ -1,48 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
* Background load tests for the page data service.
*/
const TEST_URL =
"data:text/html," +
encodeURIComponent(`
<html>
<head>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:site" content="@nytimes">
<meta name="twitter:creator" content="@SarahMaslinNir">
<meta name="twitter:title" content="Parade of Fans for Houston’s Funeral">
<meta name="twitter:description" content="NEWARK - The guest list and parade of limousines">
<meta name="twitter:image" content="http://graphics8.nytimes.com/images/2012/02/19/us/19whitney-span/19whitney-span-articleLarge.jpg">
</head>
<body>
</body>
</html>
`);
add_task(async function test_pagedata_no_data() {
let pageData = await PageDataService.fetchPageData(TEST_URL);
delete pageData.date;
Assert.deepEqual(
pageData,
{
url: TEST_URL,
siteName: "@nytimes",
description: "NEWARK - The guest list and parade of limousines",
image:
"http://graphics8.nytimes.com/images/2012/02/19/us/19whitney-span/19whitney-span-articleLarge.jpg",
data: {},
},
"Should have returned the right data data"
);
Assert.equal(
PageDataService.getCached(TEST_URL),
null,
"Should not have cached this data"
);
});

14
browser/components/pagedata/tests/unit/test_pagedata_basic.js
View File

@ -59,3 +59,17 @@ add_task(async function test_pageDataDiscovered_notifies() {
"Should return the same pageData from the cache as was notified."
);
});
add_task(async function test_queueFetch_notifies() {
let promise = PageDataService.once("page-data");
PageDataService.queueFetch("https://example.org");
let pageData = await promise;
Assert.equal(
pageData.url,
"https://example.org",
"Should have notified data for the expected url"
);
Assert.deepEqual(pageData.data, {}, "Should have returned no data");
});

531
browser/components/pagedata/tests/unit/test_queue.js
View File

@ -1,531 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/ */
XPCOMUtils.defineLazyModuleGetters(this, {
PageDataService: "resource:///modules/pagedata/PageDataService.jsm",
PromiseUtils: "resource://gre/modules/PromiseUtils.jsm",
});
// Test that urls are retrieved in the expected order.
add_task(async function test_queueOrder() {
Services.prefs.setIntPref("browser.pagedata.maxBackgroundFetches", 0);
// Pretend we are idle.
PageDataService.observe(null, "idle", null);
let pageDataResults = [
{
date: Date.now(),
url: "http://www.mozilla.org/1",
siteName: "Mozilla",
data: {},
},
{
date: Date.now() - 3600,
url: "http://www.google.com/2",
siteName: "Google",
data: {},
},
{
date: Date.now() + 3600,
url: "http://www.example.com/3",
image: "http://www.example.com/banner.jpg",
data: {},
},
{
date: Date.now() / 2,
url: "http://www.wikipedia.org/4",
data: {},
},
{
date: Date.now() / 3,
url: "http://www.microsoft.com/5",
data: {
[PageDataSchema.DATA_TYPE.PRODUCT]: {
name: "Windows 11",
},
},
},
];
let requests = [];
PageDataService.fetchPageData = url => {
requests.push(url);
for (let pageData of pageDataResults) {
if (pageData.url == url) {
return Promise.resolve(pageData);
}
}
return Promise.reject(new Error("Unknown url"));
};
let { promise: completePromise, resolve } = PromiseUtils.defer();
let results = [];
let listener = (_, pageData) => {
results.push(pageData);
if (results.length == pageDataResults.length) {
resolve();
}
};
PageDataService.on("page-data", listener);
for (let pageData of pageDataResults) {
PageDataService.queueFetch(pageData.url);
}
await completePromise;
PageDataService.off("page-data", listener);
Assert.deepEqual(
requests,
pageDataResults.map(pd => pd.url)
);
// Because our fetch implementation is essentially synchronous the results
// will be in a known order. This isn't guaranteed by the API though.
Assert.deepEqual(results, pageDataResults);
for (let pageData of pageDataResults) {
Assert.deepEqual(PageDataService.getCached(pageData.url), pageData);
}
delete PageDataService.fetchPageData;
});
// Tests that limiting the number of fetches works.
add_task(async function test_queueLimit() {
Services.prefs.setIntPref("browser.pagedata.maxBackgroundFetches", 3);
// Pretend we are idle.
PageDataService.observe(null, "idle", null);
let requests = [];
PageDataService.fetchPageData = url => {
let { promise, resolve, reject } = PromiseUtils.defer();
requests.push({ url, resolve, reject });
return promise;
};
let results = [];
let listener = (_, pageData) => {
results.push(pageData?.url);
};
PageDataService.on("page-data", listener);
PageDataService.queueFetch("https://www.mozilla.org/1");
PageDataService.queueFetch("https://www.mozilla.org/2");
PageDataService.queueFetch("https://www.mozilla.org/3");
PageDataService.queueFetch("https://www.mozilla.org/4");
PageDataService.queueFetch("https://www.mozilla.org/5");
PageDataService.queueFetch("https://www.mozilla.org/6");
PageDataService.queueFetch("https://www.mozilla.org/7");
PageDataService.queueFetch("https://www.mozilla.org/8");
PageDataService.queueFetch("https://www.mozilla.org/9");
PageDataService.queueFetch("https://www.mozilla.org/10");
PageDataService.queueFetch("https://www.mozilla.org/11");
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
]
);
// Completing or rejecting a request should start new ones.
requests[1].resolve({
date: 2345,
url: "https://www.mozilla.org/2",
siteName: "Test 2",
data: {},
});
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
]
);
requests[3].reject(new Error("Fail"));
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
]
);
// Increasing the limit should start more requests.
Services.prefs.setIntPref("browser.pagedata.maxBackgroundFetches", 5);
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
]
);
// Dropping the limit shouldn't start anything new.
Services.prefs.setIntPref("browser.pagedata.maxBackgroundFetches", 3);
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
]
);
// But resolving should also not start new requests.
requests[5].resolve({
date: 345334,
url: "https://www.mozilla.org/6",
siteName: "Test 6",
data: {},
});
requests[0].resolve({
date: 343446434,
url: "https://www.mozilla.org/1",
siteName: "Test 1",
data: {},
});
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
]
);
// Until a previous request completes.
requests[4].resolve(null);
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
"https://www.mozilla.org/8",
]
);
// Inifinite queue should work.
Services.prefs.setIntPref("browser.pagedata.maxBackgroundFetches", 0);
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
"https://www.mozilla.org/8",
"https://www.mozilla.org/9",
"https://www.mozilla.org/10",
"https://www.mozilla.org/11",
]
);
requests[10].resolve({
date: 345334,
url: "https://www.mozilla.org/11",
data: {},
});
requests[2].resolve({
date: 345334,
url: "https://www.mozilla.org/3",
data: {},
});
requests[7].resolve({
date: 345334,
url: "https://www.mozilla.org/8",
data: {},
});
requests[6].resolve({
date: 345334,
url: "https://www.mozilla.org/7",
data: {},
});
requests[8].resolve({
date: 345334,
url: "https://www.mozilla.org/9",
data: {},
});
requests[9].resolve({
date: 345334,
url: "https://www.mozilla.org/10",
data: {},
});
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
"https://www.mozilla.org/8",
"https://www.mozilla.org/9",
"https://www.mozilla.org/10",
"https://www.mozilla.org/11",
]
);
PageDataService.off("page-data", listener);
delete PageDataService.fetchPageData;
Assert.deepEqual(results, [
"https://www.mozilla.org/2",
"https://www.mozilla.org/6",
"https://www.mozilla.org/1",
"https://www.mozilla.org/11",
"https://www.mozilla.org/3",
"https://www.mozilla.org/8",
"https://www.mozilla.org/7",
"https://www.mozilla.org/9",
"https://www.mozilla.org/10",
]);
});
// Tests that the user idle state stops and starts fetches.
add_task(async function test_idle() {
Services.prefs.setIntPref("browser.pagedata.maxBackgroundFetches", 3);
// Pretend we are active.
PageDataService.observe(null, "active", null);
let requests = [];
PageDataService.fetchPageData = url => {
let { promise, resolve, reject } = PromiseUtils.defer();
requests.push({ url, resolve, reject });
return promise;
};
let results = [];
let listener = (_, pageData) => {
results.push(pageData?.url);
};
PageDataService.on("page-data", listener);
PageDataService.queueFetch("https://www.mozilla.org/1");
PageDataService.queueFetch("https://www.mozilla.org/2");
PageDataService.queueFetch("https://www.mozilla.org/3");
PageDataService.queueFetch("https://www.mozilla.org/4");
PageDataService.queueFetch("https://www.mozilla.org/5");
PageDataService.queueFetch("https://www.mozilla.org/6");
PageDataService.queueFetch("https://www.mozilla.org/7");
// Let a tick pass.
await Promise.resolve();
// Nothing will start when active.
Assert.deepEqual(
requests.map(r => r.url),
[]
);
// Pretend we are idle.
PageDataService.observe(null, "idle", null);
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
]
);
// Completing or rejecting a request should start new ones.
requests[1].resolve({
date: 2345,
url: "https://www.mozilla.org/2",
data: {},
});
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
]
);
// But not when active
PageDataService.observe(null, "active", null);
requests[3].resolve({
date: 2345,
url: "https://www.mozilla.org/4",
data: {},
});
requests[0].resolve({
date: 2345,
url: "https://www.mozilla.org/1",
data: {},
});
requests[2].resolve({
date: 2345,
url: "https://www.mozilla.org/3",
data: {},
});
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
]
);
// Going idle should start more workers
PageDataService.observe(null, "idle", null);
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
]
);
requests[4].resolve({
date: 2345,
url: "https://www.mozilla.org/5",
data: {},
});
requests[5].resolve({
date: 2345,
url: "https://www.mozilla.org/6",
data: {},
});
requests[6].resolve({
date: 2345,
url: "https://www.mozilla.org/7",
data: {},
});
// Let a tick pass.
await Promise.resolve();
Assert.deepEqual(
requests.map(r => r.url),
[
"https://www.mozilla.org/1",
"https://www.mozilla.org/2",
"https://www.mozilla.org/3",
"https://www.mozilla.org/4",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
]
);
PageDataService.off("page-data", listener);
delete PageDataService.fetchPageData;
Assert.deepEqual(results, [
"https://www.mozilla.org/2",
"https://www.mozilla.org/4",
"https://www.mozilla.org/1",
"https://www.mozilla.org/3",
"https://www.mozilla.org/5",
"https://www.mozilla.org/6",
"https://www.mozilla.org/7",
]);
});

1
browser/components/pagedata/tests/unit/xpcshell.ini
View File

@ -8,6 +8,5 @@ head = head.js
[test_pagedata_basic.js]
[test_pagedata_schema.js]
[test_opengraph.js]
[test_queue.js]
[test_schemaorg.js]
[test_twitter.js]