Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-17 23:35:34 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out 2 changesets (bug 1722275, bug 1722680) for snapshots related xpcshell and bc failures. CLOSED TREE

Browse Source 
Backed out changeset 62ed0dc702e1 (bug 1722275)
Backed out changeset 70d2848c396a (bug 1722680)
This commit is contained in:
Cosmin Sabou 2021-07-28 20:51:36 +03:00
parent 8dd2d16e8a
commit 7e5aea8bff
15 changed files with 90 additions and 873 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
browser/base/content/test/static/browser_all_files_referenced.js
View File

@ -270,6 +270,9 @@ var whitelist = [
},
{ file: "chrome://browser/content/screenshots/menu-fullpage.svg" },
{ file: "chrome://browser/content/screenshots/menu-visible.svg" },
// Will be resolved by bug 1722275.
{ file: "resource://app/modules/pagedata/PageDataService.jsm" },
];
if (AppConstants.NIGHTLY_BUILD && AppConstants.platform != "win") {

3
browser/components/BrowserGlue.jsm
View File

@ -52,7 +52,6 @@ XPCOMUtils.defineLazyModuleGetters(this, {
Interactions: "resource:///modules/Interactions.jsm",
Log: "resource://gre/modules/Log.jsm",
LoginBreaches: "resource:///modules/LoginBreaches.jsm",
PageDataService: "resource:///modules/pagedata/PageDataService.jsm",
NetUtil: "resource://gre/modules/NetUtil.jsm",
NewTabUtils: "resource://gre/modules/NewTabUtils.jsm",
NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
@ -1934,7 +1933,6 @@ BrowserGlue.prototype = {
BrowserUsageTelemetry.uninit();
SearchSERPTelemetry.uninit();
Interactions.uninit();
PageDataService.uninit();
PageThumbs.uninit();
NewTabUtils.uninit();
@ -2150,7 +2148,6 @@ BrowserGlue.prototype = {
SearchSERPTelemetry.init();
Interactions.init();
PageDataService.init();
ExtensionsUI.init();
let signingRequired;

235
browser/components/pagedata/PageDataChild.jsm
View File

@ -1,235 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var EXPORTED_SYMBOLS = ["PageDataChild"];
const { XPCOMUtils } = ChromeUtils.import(
"resource://gre/modules/XPCOMUtils.jsm"
);
XPCOMUtils.defineLazyModuleGetters(this, {
PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.jsm",
PageDataCollector: "resource:///modules/pagedata/PageDataCollector.jsm",
Services: "resource://gre/modules/Services.jsm",
});
XPCOMUtils.defineLazyGetter(this, "logConsole", function() {
return console.createInstance({
prefix: "PageData",
maxLogLevel: Services.prefs.getBoolPref("browser.pagedata.log", false)
? "Debug"
: "Warn",
});
});
// We defer any attempt to check for page data for a short time after a page
// loads to allow JS to operate.
XPCOMUtils.defineLazyPreferenceGetter(
this,
"READY_DELAY",
"browser.pagedata.readyDelay",
500
);
/**
* For testing purposes.
*/
class DummyPageData extends PageDataCollector {
type = "dummy";
}
/**
* Returns the list of page data collectors for a document.
*
* @param {Document} document
* The DOM document to collect data for.
* @returns {PageDataCollector[]}
*/
function getCollectors(document) {
return [new DummyPageData(document)];
}
/**
* The actor responsible for monitoring a page for page data.
*/
class PageDataChild extends JSWindowActorChild {
#isContentWindowPrivate = true;
/**
* Used to debounce notifications about a page being ready.
* @type {Timer | null}
*/
#deferTimer = null;
/**
* The current set of page data collectors for the page and their current data
* or null if data collection has not begun.
* @type {Map<PageDataCollector, Data[]> | null}
*/
#collectors = null;
/**
* Called when the actor is created for a new page.
*/
actorCreated() {
this.#isContentWindowPrivate = PrivateBrowsingUtils.isContentWindowPrivate(
this.contentWindow
);
}
/**
* Called when the page is destroyed.
*/
didDestroy() {
if (this.#deferTimer) {
this.#deferTimer.cancel();
}
}
/**
* Called when the page has signalled it is done loading. This signal is
* debounced by READY_DELAY.
*/
#deferReady() {
if (!this.#deferTimer) {
this.#deferTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
}
// If the timer was already running this re-starts it.
this.#deferTimer.initWithCallback(
() => {
this.#deferTimer = null;
this.sendAsyncMessage("PageData:DocumentReady", {
url: this.document.documentURI,
});
},
READY_DELAY,
Ci.nsITimer.TYPE_ONE_SHOT_LOW_PRIORITY
);
}
/**
* Coalesces the data from the page data collectors into a single array.
*
* @returns {Data[]}
*/
#buildData() {
if (!this.#collectors) {
return [];
}
let results = [];
for (let data of this.#collectors.values()) {
if (data !== null) {
results = results.concat(data);
}
}
return results;
}
/**
* Begins page data collection on the page.
*/
async #beginCollection() {
if (this.#collectors !== null) {
// Already collecting.
return this.#buildData();
}
logConsole.debug("Starting collection", this.document.documentURI);
// let initialCollection = true;
this.#collectors = new Map();
let pending = [];
for (let collector of getCollectors(this.document)) {
// TODO: Implement monitoring of pages for changes, e.g. for SPAs changing
// video without reloading.
//
// The commented out code below is a first attempt, that would allow
// individual collectors to provide updates. It will need fixing to
// ensure that listeners are either removed or not re-added on fresh
// page loads, as would happen currently.
//
// collector.on("data", (type, data) => {
// this.#collectors.set(collector, data);
//
// // Do nothing if intial collection is still ongoing.
// if (!initialCollection) {
// // TODO debounce this.
// this.sendAsyncMessage("PageData:Collected", {
// url: this.document.documentURI,
// data: this.#buildData(),
// });
// }
// });
pending.push(
collector.init().then(
data => {
this.#collectors.set(collector, data);
},
error => {
this.#collectors.set(collector, []);
logConsole.error(`Failed collecting page data`, error);
}
)
);
}
await Promise.all(pending);
// initialCollection = false;
return this.#buildData();
}
/**
* Called when a message is received from the parent process.
*
* @param {ReceiveMessageArgument} msg
* The received message.
*
* @returns {Promise | undefined}
* A promise for the requested data or undefined if no data was requested.
*/
receiveMessage(msg) {
if (this.#isContentWindowPrivate) {
return undefined;
}
switch (msg.name) {
case "PageData:CheckLoaded":
// The service just started in the parent. Check if this document is
// already loaded.
if (this.document.readystate == "complete") {
this.#deferReady();
}
break;
case "PageData:Collect":
return this.#beginCollection();
}
return undefined;
}
/**
* DOM event handler.
*
* @param {Event} event
* The DOM event.
*/
handleEvent(event) {
if (this.#isContentWindowPrivate) {
return;
}
switch (event.type) {
case "DOMContentLoaded":
case "pageshow":
this.#deferReady();
break;
}
}
}

58
browser/components/pagedata/PageDataCollector.jsm
View File

@ -1,58 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var EXPORTED_SYMBOLS = ["PageDataCollector"];
const { EventEmitter } = ChromeUtils.import(
"resource://gre/modules/EventEmitter.jsm"
);
/**
* Each `PageDataCollector` is responsible for finding data about a DOM
* document. When initialized it must asynchronously discover available data and
* either report what was found or an empty array if there was no relevant data
* in the page. Following this it may continue to monitor the page and report as
* the available data changes.
*/
class PageDataCollector extends EventEmitter {
/**
* Internal, should generally not need to be overriden by child classes.
*
* @param {Document} document
* The DOM Document for the page.
*/
constructor(document) {
super();
this.document = document;
}
/**
* Starts collection of data, should be overriden by child classes. The
* current state of data in the page should be asynchronously returned from
* this method.
*
* @returns {Data[]} The data found for the page which may be an empty array.
*/
async init() {
return [];
}
/**
* Signals that the page has been destroyed.
*/
destroy() {}
/**
* Should not be overriden by child classes. Call to signal that the data in
* the page changed.
*
* @param {Data[]} data
* The data found which may be an empty array to signal that no data was found.
*/
dataFound(data) {
this.emit("data", data);
}
}

69
browser/components/pagedata/PageDataParent.jsm
View File

@ -1,69 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var EXPORTED_SYMBOLS = ["PageDataParent"];
const { XPCOMUtils } = ChromeUtils.import(
"resource://gre/modules/XPCOMUtils.jsm"
);
XPCOMUtils.defineLazyModuleGetters(this, {
PageDataService: "resource:///modules/pagedata/PageDataService.jsm",
PromiseUtils: "resource://gre/modules/PromiseUtils.jsm",
});
/**
* Receives messages from PageDataChild and passes them to the PageData service.
*/
class PageDataParent extends JSWindowActorParent {
#deferredCollection = null;
/**
* Starts data collection in the child process. Returns a promise that
* resolves to the initial set of data discovered.
*
* @returns {Promise<Data[]>}
*/
collectPageData() {
if (!this.#deferredCollection) {
this.#deferredCollection = PromiseUtils.defer();
this.sendQuery("PageData:Collect").then(
this.#deferredCollection.resolve,
this.#deferredCollection.reject
);
}
return this.#deferredCollection.promise;
}
/**
* Called when the page is destroyed.
*/
didDestroy() {
this.#deferredCollection?.reject(
new Error("Page destroyed before collection completed.")
);
}
/**
* Called when a message is received from the content process.
*
* @param {ReceiveMessageArgument} msg
* The received message.
*/
receiveMessage(msg) {
switch (msg.name) {
case "PageData:DocumentReady":
PageDataService.pageLoaded(this, msg.data.url);
break;
// TODO: This is for supporting listening to dynamic changes. See
// PageDataChild.jsm for more information.
// case "PageData:Collected":
// PageDataService.pageDataDiscovered(msg.data.url, msg.data.data);
// break;
}
}
}

122
browser/components/pagedata/PageDataService.jsm
View File

@ -11,7 +11,6 @@ const { XPCOMUtils } = ChromeUtils.import(
);
XPCOMUtils.defineLazyModuleGetters(this, {
BrowserWindowTracker: "resource:///modules/BrowserWindowTracker.jsm",
Services: "resource://gre/modules/Services.jsm",
EventEmitter: "resource://gre/modules/EventEmitter.jsm",
});
@ -25,13 +24,11 @@ XPCOMUtils.defineLazyGetter(this, "logConsole", function() {
});
});
const ALLOWED_SCHEMES = ["http", "https", "data", "blob"];
/**
* @typedef {object} Data
* An individual piece of data about a page.
* @property {number} type
* The type of data, see Snapshots.DATA_TYPE.*
* @property {string} type
* The type of data.
* @property {object} data
* The data in a format specific to the type of data.
*
@ -47,83 +44,23 @@ const ALLOWED_SCHEMES = ["http", "https", "data", "blob"];
const PageDataService = new (class PageDataService extends EventEmitter {
/**
* Caches page data discovered from browsers. The key is the url of the data.
*
* TODO: Currently the cache never expires.
*
* Caches page data discovered from browsers. The key is the url of the data. Currently the cache
* never expires.
* @type {Map<string, PageData[]>}
*/
#pageDataCache = new Map();
/**
* Initializes a new instance of the service, not called externally.
* Constructs a new instance of the service, not called externally.
*/
init() {
constructor() {
super();
if (!Services.prefs.getBoolPref("browser.pagedata.enabled", false)) {
return;
}
ChromeUtils.registerWindowActor("PageData", {
parent: {
moduleURI: "resource:///actors/PageDataParent.jsm",
},
child: {
moduleURI: "resource:///actors/PageDataChild.jsm",
events: {
DOMContentLoaded: {},
pageshow: {},
},
},
});
logConsole.debug("Service started");
for (let win of BrowserWindowTracker.orderedWindows) {
if (!win.closed) {
// Ask any existing tabs to report
for (let tab of win.gBrowser.tabs) {
let parent = tab.linkedBrowser.browsingContext.currentWindowGlobal.getActor(
"PageData"
);
parent.sendAsyncMessage("PageData:CheckLoaded");
}
}
}
}
/**
* Called when the service is destroyed. This is generally on shutdown so we
* don't really need to do much cleanup.
*/
uninit() {
logConsole.debug("Service stopped");
}
/**
* Called when the content process signals that a page is ready for data
* collection.
*
* @param {PageDataParent} actor
* The parent actor for the page.
* @param {string} url
* The url of the page.
*/
async pageLoaded(actor, url) {
let uri = Services.io.newURI(url);
if (!ALLOWED_SCHEMES.includes(uri.scheme)) {
return;
}
let browser = actor.browsingContext?.embedderElement;
// If we don't have a browser then it went away before we could record,
// so we don't know where the data came from.
if (!browser || !this.#isATabBrowser(browser)) {
return;
}
let data = await actor.collectPageData();
this.pageDataDiscovered(url, data);
}
/**
@ -136,8 +73,6 @@ const PageDataService = new (class PageDataService extends EventEmitter {
* The set of data discovered.
*/
pageDataDiscovered(url, data) {
logConsole.debug("Discovered page data", url, data);
let pageData = {
url,
date: Date.now(),
@ -146,9 +81,10 @@ const PageDataService = new (class PageDataService extends EventEmitter {
this.#pageDataCache.set(url, pageData);
// Send out a notification. The `no-page-data` notification is intended
// for test use only.
this.emit(data.length ? "page-data" : "no-page-data", pageData);
// Send out a notification if there was some data found.
if (data.length) {
this.emit("page-data", pageData);
}
}
/**
@ -167,14 +103,34 @@ const PageDataService = new (class PageDataService extends EventEmitter {
}
/**
* Determines if the given browser is contained within a tab.
* Queues page data retrieval for a url.
*
* @param {DOMElement} browser
* The browser element to check.
* @returns {boolean}
* True if the browser element is contained within a tab.
* @param {string} url
* The url to retrieve data for.
* @returns {Promise<PageData>}
* Resolves to a `PageData` (which may not contain any items of data) when the page has been
* successfully checked for data. Will resolve immediately if there is cached data available.
* Rejects if there was some failure to collect data.
*/
#isATabBrowser(browser) {
return browser.ownerGlobal.gBrowser?.getTabForBrowser(browser);
async queueFetch(url) {
let cached = this.#pageDataCache.get(url);
if (cached) {
return cached;
}
let pageData = {
url,
date: Date.now(),
data: [],
};
this.#pageDataCache.set(url, pageData);
// Send out a notification if there was some data found.
if (pageData.data.length) {
this.emit("page-data", pageData);
}
return pageData;
}
})();

6
browser/components/pagedata/docs/index.md
View File

@ -29,6 +29,12 @@ a short delay and then updated when necessary. Any data is cached in memory for
When page data has been found a `page-data` event is emitted. The event's argument holds the
`PageData` structure. The `getCached` function can be used to access any cached data for a url.
Page data can also be requested for a URL that is not currently open. In this case the service will
load the page in the background to find its data. The service operates a queueing system to reduce
resource usage. As above when any new data is found the `page-data` event is emitted. The
`queueFetch` method starts this process and returns a promise that resolves to the `PageData` or
rejects in the event of failure.
## Supported Types of page data
The following types of page data are currently supported:

9
browser/components/pagedata/moz.build
View File

@ -7,18 +7,9 @@
XPCSHELL_TESTS_MANIFESTS += [
"tests/unit/xpcshell.ini",
]
BROWSER_CHROME_MANIFESTS += [
"tests/browser/browser.ini",
]
EXTRA_JS_MODULES.pagedata += [
"PageDataCollector.jsm",
"PageDataService.jsm",
]
FINAL_TARGET_FILES.actors += [
"PageDataChild.jsm",
"PageDataParent.jsm",
]
SPHINX_TREES["docs"] = "docs"

12
browser/components/pagedata/tests/browser/browser.ini
View File

@ -1,12 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
prefs =
browser.pagedata.log=true
browser.pagedata.enabled=true
support-files =
head.js
[browser_pagedata_basic.js]

64
browser/components/pagedata/tests/browser/browser_pagedata_basic.js
View File

@ -1,64 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
* Basic tests for the page data service.
*/
const TEST_URL = "https://example.com/";
const TEST_URL2 = "https://example.com/browser";
add_task(async function test_pagedata_no_data() {
let promise = PageDataService.once("no-page-data");
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
let pageData = await promise;
Assert.equal(pageData.url, TEST_URL, "Should have returned the loaded URL");
Assert.deepEqual(pageData.data, [], "Should have returned no data");
Assert.deepEqual(
PageDataService.getCached(TEST_URL),
pageData,
"Should return the same data from the cache"
);
promise = PageDataService.once("no-page-data");
BrowserTestUtils.loadURI(browser, TEST_URL2);
await BrowserTestUtils.browserLoaded(browser, false, TEST_URL2);
pageData = await promise;
Assert.equal(
pageData.url,
TEST_URL2,
"Should have returned the loaded URL"
);
Assert.deepEqual(pageData.data, [], "Should have returned no data");
Assert.deepEqual(
PageDataService.getCached(TEST_URL2),
pageData,
"Should return the same data from the cache"
);
info("Test going back still triggers collection");
promise = PageDataService.once("no-page-data");
let locationChangePromise = BrowserTestUtils.waitForLocationChange(
gBrowser,
TEST_URL
);
browser.goBack();
await locationChangePromise;
pageData = await promise;
Assert.equal(
pageData.url,
TEST_URL,
"Should have returned the URL of the previous page"
);
Assert.deepEqual(pageData.data, [], "Should have returned no data");
Assert.deepEqual(
PageDataService.getCached(TEST_URL),
pageData,
"Should return the same data from the cache"
);
});
});

7
browser/components/pagedata/tests/browser/head.js
View File

@ -1,7 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
const { PageDataService } = ChromeUtils.import(
"resource:///modules/pagedata/PageDataService.jsm"
);

48
browser/components/pagedata/tests/unit/test_pagedata_basic.js
View File

@ -13,7 +13,6 @@ const { XPCOMUtils } = ChromeUtils.import(
XPCOMUtils.defineLazyModuleGetters(this, {
Services: "resource://gre/modules/Services.jsm",
PageDataService: "resource:///modules/pagedata/PageDataService.jsm",
Snapshots: "resource:///modules/Snapshots.jsm",
});
add_task(async function notifies() {
@ -25,39 +24,38 @@ add_task(async function notifies() {
"Should be no cached data."
);
let listener = () => {
Assert.ok(false, "Should not notify for no data.");
};
PageDataService.on("page-data", listener);
let pageData = await PageDataService.queueFetch(url);
Assert.equal(pageData.url, "https://www.mozilla.org/");
Assert.equal(pageData.data.length, 0);
pageData = PageDataService.getCached(url);
Assert.equal(pageData.url, "https://www.mozilla.org/");
Assert.equal(pageData.data.length, 0);
PageDataService.off("page-data", listener);
let promise = PageDataService.once("page-data");
PageDataService.pageDataDiscovered(url, [
{
type: Snapshots.DATA_TYPE.PRODUCT,
type: "product",
data: {
price: 276,
},
},
]);
let pageData = await promise;
Assert.equal(
pageData.url,
"https://www.mozilla.org/",
"Should have notified data for the expected url"
);
Assert.deepEqual(
pageData.data,
[
{
type: Snapshots.DATA_TYPE.PRODUCT,
data: {
price: 276,
},
},
],
"Should have returned the correct product data"
);
pageData = await promise;
Assert.equal(pageData.url, "https://www.mozilla.org/");
Assert.equal(pageData.data.length, 1);
Assert.equal(pageData.data[0].type, "product");
Assert.deepEqual(
PageDataService.getCached(url),
pageData,
"Should return the same pageData from the cache as was notified."
);
Assert.equal(PageDataService.getCached(url), pageData);
Assert.equal(await PageDataService.queueFetch(url), pageData);
});

172
browser/components/places/Snapshots.jsm
View File

@ -15,7 +15,6 @@ const VERSION_PREF = "browser.places.snapshots.version";
XPCOMUtils.defineLazyModuleGetters(this, {
PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
Services: "resource://gre/modules/Services.jsm",
PageDataService: "resource:///modules/pagedata/PageDataService.jsm",
});
/**
@ -93,8 +92,6 @@ XPCOMUtils.defineLazyPreferenceGetter(
* The document type of the snapshot.
* @property {boolean} userPersisted
* True if the user created or persisted the snapshot in some way.
* @property {Map<type, data>} pageData
* Collection of PageData by type. See PageDataService.jsm
*/
/**
@ -107,81 +104,10 @@ XPCOMUtils.defineLazyPreferenceGetter(
* Sent when a snapshot is removed.
*/
const Snapshots = new (class Snapshots {
constructor() {
// TODO: we should update the pagedata periodically. We first need a way to
// track when the last update happened, we may add an updated_at column to
// snapshots, though that requires some I/O to check it. Thus, we probably
// want to accumulate changes and update on idle, plus store a cache of the
// last notified pages to avoid hitting the same page continuously.
// PageDataService.on("page-data", this.#onPageData);
}
/**
* Supported data types.
*/
get DATA_TYPE() {
return {
PRODUCT: 1,
};
}
#notify(topic, urls) {
Services.obs.notifyObservers(null, topic, JSON.stringify(urls));
}
/**
* Fetches page data for the given urls and stores it with snapshots.
* @param {Array<Objects>} urls Array of {placeId, url} tuples.
*/
async #addPageData(urls) {
let index = 0;
let values = [];
let bindings = {};
for (let { placeId, url } of urls) {
let pageData = PageDataService.getCached(url);
if (pageData?.data.length) {
for (let data of pageData.data) {
if (Object.values(this.DATA_TYPE).includes(data.type)) {
bindings[`id${index}`] = placeId;
bindings[`type${index}`] = data.type;
// We store the whole data object that also includes type because
// it makes easier to query all the data at once and then build a
// Map from it.
bindings[`data${index}`] = JSON.stringify(data);
values.push(`(:id${index}, :type${index}, :data${index})`);
index++;
}
}
} else {
// TODO: queuing a fetch will notify page-data once done, if any data
// was found, but we're not yet handling that, see the constructor.
PageDataService.queueFetch(url).catch(console.error);
}
}
logConsole.debug(
`Inserting ${index} page data for: ${urls.map(u => u.url)}.`
);
if (index == 0) {
return;
}
await PlacesUtils.withConnectionWrapper(
"Snapshots.jsm::addPageData",
async db => {
await db.execute(
`
INSERT OR REPLACE INTO moz_places_metadata_snapshots_extra
(place_id, type, data)
VALUES ${values.join(", ")}
`,
bindings
);
}
);
}
/**
* Adds a new snapshot.
*
@ -201,7 +127,7 @@ const Snapshots = new (class Snapshots {
throw new Error("Missing url parameter to Snapshots.add()");
}
let placeId = await PlacesUtils.withConnectionWrapper(
let added = await PlacesUtils.withConnectionWrapper(
"Snapshots: add",
async db => {
let now = Date.now();
@ -216,30 +142,17 @@ const Snapshots = new (class Snapshots {
FROM moz_places_metadata
WHERE place_id = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url)
ON CONFLICT DO UPDATE SET user_persisted = :userPersisted, removed_at = NULL WHERE :userPersisted = 1
RETURNING place_id, created_at, user_persisted
RETURNING created_at
`,
{ createdAt: now, url, userPersisted }
);
// If the url did not exist in moz_places then rows will be empty.
if (rows.length) {
// If created_at doesn't match then this url was already a snapshot,
// and we only overwrite it when the new request is user_persisted.
if (
rows[0].getResultByName("created_at") != now &&
!rows[0].getResultByName("user_persisted")
) {
return null;
}
return rows[0].getResultByName("place_id");
}
return null;
return !!rows.length;
}
);
if (placeId) {
await this.#addPageData([{ placeId, url }]);
if (added) {
this.#notify("places-snapshots-added", [url]);
}
}
@ -253,20 +166,13 @@ const Snapshots = new (class Snapshots {
*/
async delete(url) {
await PlacesUtils.withConnectionWrapper("Snapshots: delete", async db => {
let placeId = (
await db.executeCached(
`UPDATE moz_places_metadata_snapshots
SET removed_at = :removedAt
WHERE place_id = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url)
RETURNING place_id`,
{ removedAt: Date.now(), url }
)
)[0].getResultByName("place_id");
// Remove orphan page data.
await db.executeCached(
`DELETE FROM moz_places_metadata_snapshots_extra
WHERE place_id = :placeId`,
{ placeId }
`
UPDATE moz_places_metadata_snapshots
SET removed_at = :removedAt
WHERE place_id = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url)
`,
{ removedAt: Date.now(), url }
);
});
@ -294,13 +200,10 @@ const Snapshots = new (class Snapshots {
`
SELECT h.url AS url, h.title AS title, created_at, removed_at,
document_type, first_interaction_at, last_interaction_at,
user_persisted, group_concat(e.data, ",") AS page_data
FROM moz_places_metadata_snapshots s
user_persisted FROM moz_places_metadata_snapshots s
JOIN moz_places h ON h.id = s.place_id
LEFT JOIN moz_places_metadata_snapshots_extra e ON e.place_id = s.place_id
WHERE h.url_hash = hash(:url) AND h.url = :url
${extraWhereCondition}
GROUP BY s.place_id
`,
{ url }
);
@ -320,47 +223,30 @@ const Snapshots = new (class Snapshots {
* A numerical limit to the number of snapshots to retrieve, defaults to 100.
* @param {boolean} [options.includeTombstones]
* Whether to include tombstones in the snapshots to obtain.
* @param {number} [options.type]
* Restrict the snapshots to those with a particular type of page data available.
* @returns {Snapshot[]}
* Returns snapshots in order of descending last interaction time.
*/
async query({
limit = 100,
includeTombstones = false,
type = undefined,
} = {}) {
async query({ limit = 100, includeTombstones = false } = {}) {
await this.#ensureVersionUpdates();
let db = await PlacesUtils.promiseDBConnection();
let clauses = [];
let bindings = { limit };
let whereStatement = "";
if (!includeTombstones) {
clauses.push("removed_at IS NULL");
whereStatement = " WHERE removed_at IS NULL";
}
if (type) {
clauses.push("type = :type");
bindings.type = type;
}
let whereStatement = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
let rows = await db.executeCached(
`
SELECT h.url AS url, h.title AS title, created_at, removed_at,
document_type, first_interaction_at, last_interaction_at,
user_persisted, group_concat(e.data, ",") AS page_data
FROM moz_places_metadata_snapshots s
user_persisted FROM moz_places_metadata_snapshots s
JOIN moz_places h ON h.id = s.place_id
LEFT JOIN moz_places_metadata_snapshots_extra e ON e.place_id = s.place_id
${whereStatement}
GROUP BY s.place_id
ORDER BY last_interaction_at DESC
LIMIT :limit
`,
bindings
{ limit }
);
return rows.map(row => this.#translateRow(row));
@ -405,18 +291,6 @@ const Snapshots = new (class Snapshots {
* @returns {Snapshot}
*/
#translateRow(row) {
// Maps data type to data.
let pageData = new Map();
let pageDataStr = row.getResultByName("page_data");
if (pageDataStr) {
try {
let dataArray = JSON.parse(`[${pageDataStr}]`);
dataArray.forEach(d => pageData.set(d.type, d.data));
} catch (e) {
logConsole.error(e);
}
}
return {
url: row.getResultByName("url"),
title: row.getResultByName("title"),
@ -430,7 +304,6 @@ const Snapshots = new (class Snapshots {
),
documentType: row.getResultByName("document_type"),
userPersisted: !!row.getResultByName("user_persisted"),
pageData,
};
}
@ -544,7 +417,7 @@ const Snapshots = new (class Snapshots {
INSERT OR IGNORE INTO moz_places_metadata_snapshots
(place_id, first_interaction_at, last_interaction_at, document_type, created_at)
${modelQueries.join(" UNION ")}
RETURNING place_id, (SELECT url FROM moz_places WHERE id=place_id) AS url, created_at
RETURNING (SELECT url FROM moz_places WHERE id=place_id) AS url, created_at
`;
let now = Date.now();
@ -558,10 +431,7 @@ const Snapshots = new (class Snapshots {
for (let row of results) {
// If created_at differs from the passed value then this snapshot already existed.
if (row.getResultByName("created_at") == now) {
newUrls.push({
placeId: row.getResultByName("place_id"),
url: row.getResultByName("url"),
});
newUrls.push(row.getResultByName("url"));
}
}
@ -571,11 +441,7 @@ const Snapshots = new (class Snapshots {
if (insertedUrls.length) {
logConsole.debug(`Inserted ${insertedUrls.length} snapshots.`);
await this.#addPageData(insertedUrls);
this.#notify(
"places-snapshots-added",
insertedUrls.map(result => result.url)
);
this.#notify("places-snapshots-added", insertedUrls);
}
}

150
browser/components/places/tests/unit/interactions/test_snapshots_pagedata.js
View File

@ -1,150 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/ */
/**
* Tests that adding a snapshot also adds related page data.
*/
XPCOMUtils.defineLazyModuleGetters(this, {
PageDataService: "resource:///modules/pagedata/PageDataService.jsm",
});
const TEST_URL1 = "https://example.com/";
const TEST_URL2 = "https://example.com/12345";
const TEST_URL3 = "https://example.com/14235";
add_task(async function pagedata() {
// Register some page data.
PageDataService.pageDataDiscovered(TEST_URL1, [
{
type: Snapshots.DATA_TYPE.PRODUCT,
data: {
price: 276,
},
},
{
// Unsupported page data.
type: 1234,
data: {
bacon: "good",
},
},
]);
PageDataService.pageDataDiscovered(TEST_URL2, [
{
type: Snapshots.DATA_TYPE.PRODUCT,
data: {
price: 384,
},
},
]);
let now = Date.now();
// Make snapshots for any page with a typing time greater than 30 seconds
// in any one visit.
Services.prefs.setCharPref(
"browser.places.interactions.snapshotCriteria",
JSON.stringify([
{
property: "total_view_time",
aggregator: "max",
cutoff: 30000,
},
])
);
await addInteractions([
{
url: TEST_URL1,
totalViewTime: 40000,
created_at: now - 1000,
},
{
url: TEST_URL2,
totalViewTime: 20000,
created_at: now - 2000,
},
{
url: TEST_URL2,
totalViewTime: 20000,
created_at: now - 3000,
},
{
url: TEST_URL3,
totalViewTime: 20000,
created_at: now - 4000,
},
]);
await assertSnapshots([
{
url: TEST_URL1,
userPersisted: false,
documentType: Interactions.DOCUMENT_TYPE.GENERIC,
},
]);
let snap = await Snapshots.get(TEST_URL1);
Assert.equal(snap.pageData.size, 1, "Should have some page data.");
Assert.equal(
snap.pageData.get(Snapshots.DATA_TYPE.PRODUCT).price,
276,
"Should have the right price."
);
await Snapshots.add({ url: TEST_URL2 });
snap = await Snapshots.get(TEST_URL2);
Assert.equal(snap.pageData.size, 1, "Should have some page data.");
Assert.equal(
snap.pageData.get(Snapshots.DATA_TYPE.PRODUCT).price,
384,
"Should have the right price."
);
await Snapshots.add({ url: TEST_URL3 });
snap = await Snapshots.get(TEST_URL3);
Assert.equal(snap.pageData.size, 0, "Should be no page data.");
await assertSnapshots(
[
{
url: TEST_URL1,
userPersisted: false,
documentType: Interactions.DOCUMENT_TYPE.GENERIC,
},
{
url: TEST_URL2,
userPersisted: false,
documentType: Interactions.DOCUMENT_TYPE.GENERIC,
},
],
{ type: Snapshots.DATA_TYPE.PRODUCT }
);
info("Ensure that removing a snapshot removes pagedata for it");
await Snapshots.delete(TEST_URL1);
await Snapshots.delete(TEST_URL2);
let db = await PlacesUtils.promiseDBConnection();
Assert.equal(
(
await db.execute(
"SELECT count(*) FROM moz_places_metadata_snapshots_extra"
)
)[0].getResultByIndex(0),
0,
"Ensure there's no leftover page data in the database"
);
info("Ensure adding back the snapshot adds pagedata for it");
await Snapshots.add({ url: TEST_URL1, userPersisted: true });
snap = await Snapshots.get(TEST_URL1);
Assert.equal(snap.pageData.size, 1, "Should have some page data.");
Assert.equal(
snap.pageData.get(Snapshots.DATA_TYPE.PRODUCT).price,
276,
"Should have the right price."
);
await reset();
});

5
browser/components/places/tests/unit/interactions/xpcshell.ini
View File

@ -1,13 +1,8 @@
[DEFAULT]
prefs =
browser.places.interactions.enabled=true
browser.places.interactions.log=true
browser.pagedata.log=true
head = head_interactions.js
firefox-appdir = browser
skip-if = toolkit == 'android'
[test_snapshots_basics.js]
[test_snapshots_create_criteria.js]
[test_snapshots_pagedata.js]
[test_snapshots_queries.js]