mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-08 19:04:45 +00:00
Bug 1573305 - Remove the check for unmerged synced bookmark changes. r=markh
When a local or remote item changed, we'd potentially scan three tables (with an expensive `LEFT JOIN`!) to check if anything changed...then scan the same tables again to build the local and remote trees. This check was originally meant to avoid unnecessary merges. However, the bottleneck isn't merging now; it's reading from the database. Since the merger has been rewritten in Rust, is synchronous, doesn't keep a transaction open for the entire merge (see the `total_sync_changes` check), and only emits ops for items that actually changed, it's more efficient to build and merge optimistically, and bail before applying if nothing changed. This commit also moves `validateLocalRoots` into Rust. Differential Revision: https://phabricator.services.mozilla.com/D41690 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Lina Cambridge
parent
13a7295574
commit
65dfa84ccd
6
Cargo.lock
generated
6
Cargo.lock
generated
@ -385,7 +385,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"atomic_refcell 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cstr 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"dogear 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"dogear 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"moz_task 0.1.0",
|
||||
@ -956,7 +956,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dogear"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -3989,7 +3989,7 @@ dependencies = [
|
||||
"checksum devd-rs 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0d009f166c0d9e9f9909dc751630b3a6411ab7f85a153d32d01deb364ffe52a7"
|
||||
"checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c"
|
||||
"checksum dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "88972de891f6118092b643d85a0b28e0678e0f948d7f879aa32f2d5aafe97d2a"
|
||||
"checksum dogear 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "99e2ed5f8036ceb83d951f769651ff0e2be5ddb0c62da87d50d27c22086db01c"
|
||||
"checksum dogear 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "251a15c9a597d70eb53cbb0c5473d8d8c6241aef615c092030ebab27fb5b26ef"
|
||||
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
|
||||
"checksum dtoa-short 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "068d4026697c1a18f0b0bb8cfcad1b0c151b90d8edb9bf4c235ad68128920d1d"
|
||||
"checksum dwrote 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0bd1369e02db5e9b842a9b67bce8a2fcc043beafb2ae8a799dd482d46ea1ff0d"
|
||||
|
||||
@ -346,9 +346,6 @@ add_task(async function test_folder_descendants() {
|
||||
);
|
||||
|
||||
_("Insert missing bookmarks locally to request later");
|
||||
// Note that the fact we insert the bookmarks via PlacesSyncUtils.bookmarks.insert
|
||||
// means that we are pretending Sync itself wrote them, hence they aren't
|
||||
// considered "changed" locally so never get uploaded.
|
||||
let childBmk = await PlacesSyncUtils.bookmarks.insert({
|
||||
kind: "bookmark",
|
||||
recordId: Utils.makeGUID(),
|
||||
@ -371,15 +368,22 @@ add_task(async function test_folder_descendants() {
|
||||
url: "https://mozilla.org",
|
||||
});
|
||||
|
||||
_("Sync again; server contents shouldn't change");
|
||||
_("Sync again");
|
||||
await Service.sync();
|
||||
deepEqual(
|
||||
getServerBookmarks(server)
|
||||
.keys()
|
||||
.sort(),
|
||||
initialRecordIds,
|
||||
"Second sync should not upload missing bookmarks"
|
||||
);
|
||||
{
|
||||
// The buffered engine will upload the missing records, since it does a full
|
||||
// tree merge. The old engine won't, since it relies on the Sync status
|
||||
// flag, and we used `PSU.b.i` to pretend that Sync added the bookmarks.
|
||||
let collection = getServerBookmarks(server);
|
||||
collection.remove(childBmk.recordId);
|
||||
collection.remove(grandChildBmk.recordId);
|
||||
collection.remove(grandChildSiblingBmk.recordId);
|
||||
deepEqual(
|
||||
collection.keys().sort(),
|
||||
initialRecordIds,
|
||||
"Second sync should not upload missing bookmarks"
|
||||
);
|
||||
}
|
||||
|
||||
// This assumes the parent record on the server is correct, and the server
|
||||
// is just missing the children. This isn't a correct assumption if the
|
||||
|
||||
2
third_party/rust/dogear/.cargo-checksum.json
vendored
2
third_party/rust/dogear/.cargo-checksum.json
vendored
@ -1 +1 @@
|
||||
{"files":{"CODE_OF_CONDUCT.md":"e85149c44f478f164f7d5f55f6e66c9b5ae236d4a11107d5e2a93fe71dd874b9","Cargo.toml":"8a1067e5d4af68c738884e76256033d638b938c40789eefb0c078a79f5238890","LICENSE":"c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4","README.md":"303ea5ec53d4e86f2c321056e8158e31aa061353a99e52de3d76859d40919efc","src/driver.rs":"913de08a578b38902f4eb0d78147289897af5d3b1ecdad818c4ea6f83da6c714","src/error.rs":"d4ef0cba5c7fc54959ed62da166f10435548d705e0a817eed449fb001fe4e21d","src/guid.rs":"c82af64fba3ad87948a9b599241e48753d17587e8c642f610949163be3d499bf","src/lib.rs":"0606e69b235650bf404ae0b03a1e85c2063bb4b7147fa4d5e8ff2c128a757453","src/merge.rs":"1fcdc00dcf47743e72372ba85b00e5f48d0fc9b8586f9b3bd5c0b0604628d8b3","src/store.rs":"e5ce4c358b05b2483198b37c90d2f6c740c8e349c70abcee3ba914cb80f66aae","src/tests.rs":"51bc77eb989974f7594dfa0cdf7741f8e26f2956eebc34d1f71cbfd137512940","src/tree.rs":"fcdc9a5ae0e3f1b0d62d01bf1024db2a0aa8660e6839ada607a6a20e0fe2ad83"},"package":"99e2ed5f8036ceb83d951f769651ff0e2be5ddb0c62da87d50d27c22086db01c"}
|
||||
{"files":{"CODE_OF_CONDUCT.md":"e85149c44f478f164f7d5f55f6e66c9b5ae236d4a11107d5e2a93fe71dd874b9","Cargo.toml":"cfc0416b0a1c11ade3b5c11ec579c8e26ed3fc3b5871dd99d4fbcc12bd2614d0","LICENSE":"c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4","README.md":"303ea5ec53d4e86f2c321056e8158e31aa061353a99e52de3d76859d40919efc","src/driver.rs":"913de08a578b38902f4eb0d78147289897af5d3b1ecdad818c4ea6f83da6c714","src/error.rs":"d4ef0cba5c7fc54959ed62da166f10435548d705e0a817eed449fb001fe4e21d","src/guid.rs":"c82af64fba3ad87948a9b599241e48753d17587e8c642f610949163be3d499bf","src/lib.rs":"0606e69b235650bf404ae0b03a1e85c2063bb4b7147fa4d5e8ff2c128a757453","src/merge.rs":"e74727171831585d304eee3a3c7ba24fec52a5e7c6999ed4e647fefa14b8be99","src/store.rs":"629410e44485c0473617911be0e06dc5ce504c7427782be02f05a2f4096b5351","src/tests.rs":"51bc77eb989974f7594dfa0cdf7741f8e26f2956eebc34d1f71cbfd137512940","src/tree.rs":"fcdc9a5ae0e3f1b0d62d01bf1024db2a0aa8660e6839ada607a6a20e0fe2ad83"},"package":"251a15c9a597d70eb53cbb0c5473d8d8c6241aef615c092030ebab27fb5b26ef"}
|
||||
2
third_party/rust/dogear/Cargo.toml
vendored
2
third_party/rust/dogear/Cargo.toml
vendored
@ -13,7 +13,7 @@
|
||||
[package]
|
||||
edition = "2018"
|
||||
name = "dogear"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
authors = ["Lina Cambridge <lina@mozilla.com>"]
|
||||
exclude = ["/.travis/**", ".travis.yml", "/docs/**", "book.toml"]
|
||||
description = "A library for merging bookmark trees."
|
||||
|
||||
14
third_party/rust/dogear/src/merge.rs
vendored
14
third_party/rust/dogear/src/merge.rs
vendored
@ -1857,6 +1857,20 @@ pub struct CompletionOps<'t> {
|
||||
pub upload: Vec<Upload<'t>>,
|
||||
}
|
||||
|
||||
impl<'t> CompletionOps<'t> {
|
||||
/// Returns `true` if there are no completion ops to apply.
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.change_guids.is_empty()
|
||||
&& self.apply_remote_items.is_empty()
|
||||
&& self.apply_new_local_structure.is_empty()
|
||||
&& self.flag_for_upload.is_empty()
|
||||
&& self.skip_upload.is_empty()
|
||||
&& self.flag_as_merged.is_empty()
|
||||
&& self.upload.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// A completion op to change the local GUID to the merged GUID. This is used
|
||||
/// to dedupe new local items to remote ones, as well as to fix up invalid
|
||||
/// GUIDs.
|
||||
|
||||
22
third_party/rust/dogear/src/store.rs
vendored
22
third_party/rust/dogear/src/store.rs
vendored
@ -24,23 +24,29 @@ use crate::tree::Tree;
|
||||
/// A store is the main interface to Dogear. It implements methods for building
|
||||
/// local and remote trees from a storage backend, fetching content info for
|
||||
/// matching items with similar contents, and persisting the merged tree.
|
||||
pub trait Store<E: From<Error>> {
|
||||
pub trait Store {
|
||||
/// The type returned from a successful merge.
|
||||
type Ok;
|
||||
|
||||
/// The type returned in the event of a store error.
|
||||
type Error: From<Error>;
|
||||
|
||||
/// Builds a fully rooted, consistent tree from the items and tombstones in
|
||||
/// the local store.
|
||||
fn fetch_local_tree(&self) -> Result<Tree, E>;
|
||||
fn fetch_local_tree(&self) -> Result<Tree, Self::Error>;
|
||||
|
||||
/// Builds a fully rooted, consistent tree from the items and tombstones in
|
||||
/// the mirror.
|
||||
fn fetch_remote_tree(&self) -> Result<Tree, E>;
|
||||
fn fetch_remote_tree(&self) -> Result<Tree, Self::Error>;
|
||||
|
||||
/// Applies the merged root to the local store, and stages items for
|
||||
/// upload. On Desktop, this method inserts the merged tree into a temp
|
||||
/// table, updates Places, and inserts outgoing items into another
|
||||
/// temp table.
|
||||
fn apply<'t>(&mut self, root: MergedRoot<'t>) -> Result<(), E>;
|
||||
fn apply<'t>(&mut self, root: MergedRoot<'t>) -> Result<Self::Ok, Self::Error>;
|
||||
|
||||
/// Builds and applies a merged tree using the default merge driver.
|
||||
fn merge(&mut self) -> Result<(), E> {
|
||||
fn merge(&mut self) -> Result<Self::Ok, Self::Error> {
|
||||
self.merge_with_driver(&DefaultDriver, &DefaultAbortSignal)
|
||||
}
|
||||
|
||||
@ -51,7 +57,7 @@ pub trait Store<E: From<Error>> {
|
||||
&mut self,
|
||||
driver: &impl Driver,
|
||||
signal: &impl AbortSignal,
|
||||
) -> Result<(), E> {
|
||||
) -> Result<Self::Ok, Self::Error> {
|
||||
signal.err_if_aborted()?;
|
||||
debug!(driver, "Building local tree");
|
||||
let (local_tree, time) = with_timing(|| self.fetch_local_tree())?;
|
||||
@ -95,10 +101,10 @@ pub trait Store<E: From<Error>> {
|
||||
|
||||
signal.err_if_aborted()?;
|
||||
debug!(driver, "Applying merged tree");
|
||||
let ((), time) = with_timing(|| self.apply(merged_root))?;
|
||||
let (result, time) = with_timing(|| self.apply(merged_root))?;
|
||||
driver.record_telemetry_event(TelemetryEvent::Apply(time));
|
||||
|
||||
Ok(())
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -75,49 +75,6 @@ const SyncedBookmarksMerger = Components.Constructor(
|
||||
"mozISyncedBookmarksMerger"
|
||||
);
|
||||
|
||||
/**
|
||||
* A common table expression for all local items in Places, to be included in a
|
||||
* `WITH RECURSIVE` clause. We start at the roots, excluding tags (bug 424160),
|
||||
* and work our way down.
|
||||
*
|
||||
* Note that syncable items (`isSyncable`) descend from the four syncable roots.
|
||||
* Any other roots and their descendants, like the left pane root, left pane
|
||||
* queries, and custom roots, are non-syncable.
|
||||
*
|
||||
* Newer Desktops should never reupload non-syncable items (bug 1274496), and
|
||||
* should have removed them in Places migrations (bug 1310295). However, these
|
||||
* items might be orphaned in "unfiled", in which case they're seen as syncable
|
||||
* locally. If the server has the missing parents and roots, we'll determine
|
||||
* that the items are non-syncable when merging, remove them from Places, and
|
||||
* upload tombstones to the server.
|
||||
*/
|
||||
XPCOMUtils.defineLazyGetter(
|
||||
this,
|
||||
"LocalItemsSQLFragment",
|
||||
() => `
|
||||
localItems(id, guid, parentId, parentGuid, position, type, title,
|
||||
parentTitle, placeId, dateAdded, lastModified, syncChangeCounter,
|
||||
isSyncable, level) AS (
|
||||
SELECT b.id, b.guid, p.id, p.guid, b.position, b.type, b.title, p.title,
|
||||
b.fk, b.dateAdded, b.lastModified, b.syncChangeCounter,
|
||||
b.guid IN (${PlacesUtils.bookmarks.userContentRoots
|
||||
.map(v => `'${v}'`)
|
||||
.join(",")}), 0
|
||||
FROM moz_bookmarks b
|
||||
JOIN moz_bookmarks p ON p.id = b.parent
|
||||
WHERE b.guid <> '${PlacesUtils.bookmarks.tagsGuid}' AND
|
||||
p.guid = '${PlacesUtils.bookmarks.rootGuid}'
|
||||
UNION ALL
|
||||
SELECT b.id, b.guid, s.id, s.guid, b.position, b.type, b.title, s.title,
|
||||
b.fk, b.dateAdded, b.lastModified, b.syncChangeCounter,
|
||||
s.isSyncable, s.level + 1
|
||||
FROM moz_bookmarks b
|
||||
JOIN localItems s ON s.id = b.parent
|
||||
WHERE b.guid <> '${PlacesUtils.bookmarks.rootGuid}'
|
||||
)
|
||||
`
|
||||
);
|
||||
|
||||
// These can be removed once they're exposed in a central location (bug
|
||||
// 1375896).
|
||||
const DB_URL_LENGTH_MAX = 65536;
|
||||
@ -597,30 +554,11 @@ class SyncedBookmarksMirror {
|
||||
* upload to the server, and to store in the mirror once upload
|
||||
* succeeds.
|
||||
*/
|
||||
async apply(options = {}) {
|
||||
let hasChanges =
|
||||
("weakUpload" in options && options.weakUpload.length > 0) ||
|
||||
(await this.hasChanges());
|
||||
if (!hasChanges) {
|
||||
MirrorLog.debug("No changes detected in both mirror and Places");
|
||||
let limit =
|
||||
"maxFrecenciesToRecalculate" in options
|
||||
? options.maxFrecenciesToRecalculate
|
||||
: DEFAULT_MAX_FRECENCIES_TO_RECALCULATE;
|
||||
await updateFrecencies(this.db, limit);
|
||||
return {};
|
||||
}
|
||||
let changeRecords = await this.forceApply(options);
|
||||
return changeRecords;
|
||||
}
|
||||
|
||||
// Forces a full merge, even if there are no local or remote changes, and
|
||||
// no items to weakly upload. Exposed for tests.
|
||||
async forceApply({
|
||||
localTimeSeconds = Date.now() / 1000,
|
||||
remoteTimeSeconds = 0,
|
||||
weakUpload = [],
|
||||
maxFrecenciesToRecalculate = DEFAULT_MAX_FRECENCIES_TO_RECALCULATE,
|
||||
async apply({
|
||||
localTimeSeconds,
|
||||
remoteTimeSeconds,
|
||||
weakUpload,
|
||||
maxFrecenciesToRecalculate,
|
||||
signal = null,
|
||||
} = {}) {
|
||||
// We intentionally don't use `executeBeforeShutdown` in this function,
|
||||
@ -632,25 +570,15 @@ class SyncedBookmarksMirror {
|
||||
this.finalizeController.signal,
|
||||
signal
|
||||
);
|
||||
let observersToNotify = new BookmarkObserverRecorder(this.db, {
|
||||
maxFrecenciesToRecalculate,
|
||||
signal: finalizeOrInterruptSignal,
|
||||
});
|
||||
|
||||
if (!(await this.validLocalRoots())) {
|
||||
throw new SyncedBookmarksMirror.MergeError(
|
||||
"Local tree has misparented root"
|
||||
);
|
||||
}
|
||||
|
||||
let changeRecords;
|
||||
try {
|
||||
changeRecords = await this.tryApply(
|
||||
finalizeOrInterruptSignal,
|
||||
localTimeSeconds,
|
||||
remoteTimeSeconds,
|
||||
observersToNotify,
|
||||
weakUpload,
|
||||
finalizeOrInterruptSignal
|
||||
maxFrecenciesToRecalculate
|
||||
);
|
||||
} finally {
|
||||
this.progress.reset();
|
||||
@ -660,111 +588,30 @@ class SyncedBookmarksMirror {
|
||||
}
|
||||
|
||||
async tryApply(
|
||||
signal,
|
||||
localTimeSeconds,
|
||||
remoteTimeSeconds,
|
||||
observersToNotify,
|
||||
weakUpload,
|
||||
signal
|
||||
maxFrecenciesToRecalculate = DEFAULT_MAX_FRECENCIES_TO_RECALCULATE
|
||||
) {
|
||||
await withTiming("Merging bookmarks in Rust", () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
let op = null;
|
||||
function onAbort() {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
op.cancel();
|
||||
}
|
||||
let callback = {
|
||||
QueryInterface: ChromeUtils.generateQI([
|
||||
Ci.mozISyncedBookmarksMirrorProgressListener,
|
||||
Ci.mozISyncedBookmarksMirrorCallback,
|
||||
]),
|
||||
// `mozISyncedBookmarksMirrorProgressListener` methods.
|
||||
onFetchLocalTree: (took, count, problems) => {
|
||||
this.progress.stepWithItemCount(
|
||||
ProgressTracker.STEPS.FETCH_LOCAL_TREE,
|
||||
took,
|
||||
count
|
||||
);
|
||||
// We don't record local tree problems in validation telemetry.
|
||||
},
|
||||
onFetchRemoteTree: (took, count, problemsBag) => {
|
||||
this.progress.stepWithItemCount(
|
||||
ProgressTracker.STEPS.FETCH_REMOTE_TREE,
|
||||
took,
|
||||
count
|
||||
);
|
||||
// Record validation telemetry for problems in the remote tree.
|
||||
let problems = bagToNamedCounts(problemsBag, [
|
||||
"orphans",
|
||||
"misparentedRoots",
|
||||
"multipleParents",
|
||||
"nonFolderParents",
|
||||
"parentChildDisagreements",
|
||||
"missingChildren",
|
||||
]);
|
||||
this.recordValidationTelemetry(took, count, problems);
|
||||
},
|
||||
onMerge: (took, countsBag) => {
|
||||
let counts = bagToNamedCounts(countsBag, [
|
||||
"items",
|
||||
"deletes",
|
||||
"dupes",
|
||||
"remoteRevives",
|
||||
"localDeletes",
|
||||
"localRevives",
|
||||
"remoteDeletes",
|
||||
]);
|
||||
this.progress.stepWithTelemetry(
|
||||
ProgressTracker.STEPS.MERGE,
|
||||
took,
|
||||
counts
|
||||
);
|
||||
},
|
||||
onApply: took => {
|
||||
this.progress.stepWithTelemetry(ProgressTracker.STEPS.APPLY, took);
|
||||
},
|
||||
// `mozISyncedBookmarksMirrorCallback` methods.
|
||||
handleSuccess(result) {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
resolve(result);
|
||||
},
|
||||
handleError(code, message) {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
switch (code) {
|
||||
case Cr.NS_ERROR_STORAGE_BUSY:
|
||||
reject(
|
||||
new SyncedBookmarksMirror.MergeConflictError(
|
||||
"Local tree changed during merge"
|
||||
)
|
||||
);
|
||||
break;
|
||||
let wasMerged = await withTiming("Merging bookmarks in Rust", () =>
|
||||
this.merge(signal, localTimeSeconds, remoteTimeSeconds, weakUpload)
|
||||
);
|
||||
|
||||
case Cr.NS_ERROR_ABORT:
|
||||
reject(new SyncedBookmarksMirror.InterruptedError(message));
|
||||
break;
|
||||
|
||||
default:
|
||||
reject(new SyncedBookmarksMirror.MergeError(message));
|
||||
}
|
||||
},
|
||||
};
|
||||
op = this.merger.merge(
|
||||
localTimeSeconds,
|
||||
remoteTimeSeconds,
|
||||
weakUpload,
|
||||
callback
|
||||
);
|
||||
if (signal.aborted) {
|
||||
op.cancel();
|
||||
} else {
|
||||
signal.addEventListener("abort", onAbort);
|
||||
}
|
||||
});
|
||||
});
|
||||
if (!wasMerged) {
|
||||
MirrorLog.debug("No changes detected in both mirror and Places");
|
||||
await updateFrecencies(this.db, maxFrecenciesToRecalculate);
|
||||
return {};
|
||||
}
|
||||
|
||||
// At this point, the database is consistent, so we can notify observers and
|
||||
// inflate records for outgoing items.
|
||||
|
||||
let observersToNotify = new BookmarkObserverRecorder(this.db, {
|
||||
maxFrecenciesToRecalculate,
|
||||
signal,
|
||||
});
|
||||
|
||||
await withTiming(
|
||||
"Notifying Places observers",
|
||||
async () => {
|
||||
@ -817,6 +664,107 @@ class SyncedBookmarksMirror {
|
||||
);
|
||||
}
|
||||
|
||||
merge(
|
||||
signal,
|
||||
localTimeSeconds = Date.now() / 1000,
|
||||
remoteTimeSeconds = 0,
|
||||
weakUpload = []
|
||||
) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let op = null;
|
||||
function onAbort() {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
op.cancel();
|
||||
}
|
||||
let callback = {
|
||||
QueryInterface: ChromeUtils.generateQI([
|
||||
Ci.mozISyncedBookmarksMirrorProgressListener,
|
||||
Ci.mozISyncedBookmarksMirrorCallback,
|
||||
]),
|
||||
// `mozISyncedBookmarksMirrorProgressListener` methods.
|
||||
onFetchLocalTree: (took, count, problems) => {
|
||||
this.progress.stepWithItemCount(
|
||||
ProgressTracker.STEPS.FETCH_LOCAL_TREE,
|
||||
took,
|
||||
count
|
||||
);
|
||||
// We don't record local tree problems in validation telemetry.
|
||||
},
|
||||
onFetchRemoteTree: (took, count, problemsBag) => {
|
||||
this.progress.stepWithItemCount(
|
||||
ProgressTracker.STEPS.FETCH_REMOTE_TREE,
|
||||
took,
|
||||
count
|
||||
);
|
||||
// Record validation telemetry for problems in the remote tree.
|
||||
let problems = bagToNamedCounts(problemsBag, [
|
||||
"orphans",
|
||||
"misparentedRoots",
|
||||
"multipleParents",
|
||||
"nonFolderParents",
|
||||
"parentChildDisagreements",
|
||||
"missingChildren",
|
||||
]);
|
||||
this.recordValidationTelemetry(took, count, problems);
|
||||
},
|
||||
onMerge: (took, countsBag) => {
|
||||
let counts = bagToNamedCounts(countsBag, [
|
||||
"items",
|
||||
"deletes",
|
||||
"dupes",
|
||||
"remoteRevives",
|
||||
"localDeletes",
|
||||
"localRevives",
|
||||
"remoteDeletes",
|
||||
]);
|
||||
this.progress.stepWithTelemetry(
|
||||
ProgressTracker.STEPS.MERGE,
|
||||
took,
|
||||
counts
|
||||
);
|
||||
},
|
||||
onApply: took => {
|
||||
this.progress.stepWithTelemetry(ProgressTracker.STEPS.APPLY, took);
|
||||
},
|
||||
// `mozISyncedBookmarksMirrorCallback` methods.
|
||||
handleSuccess(result) {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
resolve(result);
|
||||
},
|
||||
handleError(code, message) {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
switch (code) {
|
||||
case Cr.NS_ERROR_STORAGE_BUSY:
|
||||
reject(
|
||||
new SyncedBookmarksMirror.MergeConflictError(
|
||||
"Local tree changed during merge"
|
||||
)
|
||||
);
|
||||
break;
|
||||
|
||||
case Cr.NS_ERROR_ABORT:
|
||||
reject(new SyncedBookmarksMirror.InterruptedError(message));
|
||||
break;
|
||||
|
||||
default:
|
||||
reject(new SyncedBookmarksMirror.MergeError(message));
|
||||
}
|
||||
},
|
||||
};
|
||||
op = this.merger.merge(
|
||||
localTimeSeconds,
|
||||
remoteTimeSeconds,
|
||||
weakUpload,
|
||||
callback
|
||||
);
|
||||
if (signal.aborted) {
|
||||
op.cancel();
|
||||
} else {
|
||||
signal.addEventListener("abort", onAbort);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Discards the mirror contents. This is called when the user is node
|
||||
* reassigned, disables the bookmarks engine, or signs out.
|
||||
@ -1112,64 +1060,6 @@ class SyncedBookmarksMirror {
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
* Checks if Places or mirror have any unsynced/unmerged changes.
|
||||
*
|
||||
* @return {Boolean}
|
||||
* `true` if something has changed.
|
||||
*/
|
||||
async hasChanges() {
|
||||
// In the first subquery, we check incoming items with needsMerge = true
|
||||
// except the tombstones who don't correspond to any local bookmark because
|
||||
// we don't store them yet, hence never "merged" (see bug 1343103).
|
||||
let rows = await this.db.execute(`
|
||||
SELECT
|
||||
EXISTS (
|
||||
SELECT 1
|
||||
FROM items v
|
||||
LEFT JOIN moz_bookmarks b ON v.guid = b.guid
|
||||
WHERE v.needsMerge AND
|
||||
(NOT v.isDeleted OR b.guid NOT NULL)
|
||||
) OR EXISTS (
|
||||
WITH RECURSIVE
|
||||
${LocalItemsSQLFragment}
|
||||
SELECT 1
|
||||
FROM localItems
|
||||
WHERE syncChangeCounter > 0
|
||||
) OR EXISTS (
|
||||
SELECT 1
|
||||
FROM moz_bookmarks_deleted
|
||||
)
|
||||
AS hasChanges
|
||||
`);
|
||||
return !!rows[0].getResultByName("hasChanges");
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that all local roots are parented correctly. Misparented roots
|
||||
* (bug 1453994, bug 1472127) might produce an invalid tree, so we check
|
||||
* before merging, and rely on Places to reparent any invalid roots after
|
||||
* the next restart or maintenance run.
|
||||
*
|
||||
* @return {Boolean}
|
||||
* `true` if the Places root, and four syncable roots, are parented
|
||||
* correctly.
|
||||
*/
|
||||
async validLocalRoots() {
|
||||
let rows = await this.db.execute(`
|
||||
SELECT EXISTS(SELECT 1 FROM moz_bookmarks
|
||||
WHERE guid = '${PlacesUtils.bookmarks.rootGuid}' AND
|
||||
parent = 0) AND
|
||||
(SELECT COUNT(*) FROM moz_bookmarks b
|
||||
JOIN moz_bookmarks p ON p.id = b.parent
|
||||
WHERE b.guid IN (${PlacesUtils.bookmarks.userContentRoots.map(
|
||||
v => `'${v}'`
|
||||
)}) AND
|
||||
p.guid = '${PlacesUtils.bookmarks.rootGuid}') =
|
||||
${PlacesUtils.bookmarks.userContentRoots.length} AS areValid`);
|
||||
return !!rows[0].getResultByName("areValid");
|
||||
}
|
||||
|
||||
/**
|
||||
* Inflates Sync records for all staged outgoing items.
|
||||
*
|
||||
|
||||
@ -6,7 +6,7 @@ edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
atomic_refcell = "0.1"
|
||||
dogear = "0.3.0"
|
||||
dogear = "0.3.1"
|
||||
libc = "0.2"
|
||||
log = "0.4"
|
||||
cstr = "0.1"
|
||||
|
||||
@ -131,7 +131,7 @@ struct MergeTask {
|
||||
weak_uploads: Vec<nsString>,
|
||||
progress: Option<ThreadPtrHandle<mozISyncedBookmarksMirrorProgressListener>>,
|
||||
callback: ThreadPtrHandle<mozISyncedBookmarksMirrorCallback>,
|
||||
result: AtomicRefCell<error::Result<()>>,
|
||||
result: AtomicRefCell<error::Result<store::ApplyStatus>>,
|
||||
}
|
||||
|
||||
impl MergeTask {
|
||||
@ -200,22 +200,18 @@ impl Task for MergeTask {
|
||||
&self.weak_uploads,
|
||||
);
|
||||
*self.result.borrow_mut() = store
|
||||
.prepare()
|
||||
.validate()
|
||||
.and_then(|_| store.prepare())
|
||||
.and_then(|_| store.merge_with_driver(&driver, &*self.controller));
|
||||
}
|
||||
|
||||
fn done(&self) -> Result<(), nsresult> {
|
||||
let callback = self.callback.get().unwrap();
|
||||
match mem::replace(&mut *self.result.borrow_mut(), Err(error::Error::DidNotRun)) {
|
||||
Ok(()) => unsafe { callback.HandleSuccess() },
|
||||
Ok(status) => unsafe { callback.HandleSuccess(status.into()) },
|
||||
Err(err) => {
|
||||
let message = {
|
||||
let mut message = nsString::new();
|
||||
match write!(message, "{}", err) {
|
||||
Ok(_) => message,
|
||||
Err(_) => nsString::from("Merge failed with unknown error"),
|
||||
}
|
||||
};
|
||||
let mut message = nsString::new();
|
||||
write!(message, "{}", err).unwrap();
|
||||
unsafe { callback.HandleError(err.into(), &*message) }
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,6 +63,44 @@ impl<'s> Store<'s> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Ensures that all local roots are parented correctly.
|
||||
///
|
||||
/// The Places root can't be in another folder, or we'll recurse infinitely
|
||||
/// when we try to fetch the local tree.
|
||||
///
|
||||
/// The five built-in roots should be under the Places root, or we'll build
|
||||
/// and sync an invalid tree (bug 1453994, bug 1472127).
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
self.controller.err_if_aborted()?;
|
||||
let mut statement = self.db.prepare(format!(
|
||||
"SELECT NOT EXISTS(
|
||||
SELECT 1 FROM moz_bookmarks
|
||||
WHERE id = (SELECT parent FROM moz_bookmarks
|
||||
WHERE guid = '{0}')
|
||||
) AND NOT EXISTS(
|
||||
SELECT 1 FROM moz_bookmarks b
|
||||
JOIN moz_bookmarks p ON p.id = b.parent
|
||||
WHERE b.guid IN ('{1}', '{2}', '{3}', '{4}', '{5}') AND
|
||||
p.guid <> '{0}'
|
||||
)",
|
||||
dogear::ROOT_GUID,
|
||||
dogear::MENU_GUID,
|
||||
dogear::MOBILE_GUID,
|
||||
dogear::TAGS_GUID,
|
||||
dogear::TOOLBAR_GUID,
|
||||
dogear::UNFILED_GUID,
|
||||
))?;
|
||||
let has_valid_roots = match statement.step()? {
|
||||
Some(row) => row.get_by_index::<i64>(0)? == 1,
|
||||
None => false,
|
||||
};
|
||||
if has_valid_roots {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::InvalidLocalRoots)
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepares the mirror database for a merge.
|
||||
pub fn prepare(&self) -> Result<()> {
|
||||
// Sync associates keywords with bookmarks, and doesn't sync POST data;
|
||||
@ -73,6 +111,7 @@ impl<'s> Store<'s> {
|
||||
// bug 1328737). Just in case, we flag any remote bookmarks that have
|
||||
// different keywords for the same URL, or the same keyword for
|
||||
// different URLs, for reupload.
|
||||
self.controller.err_if_aborted()?;
|
||||
self.db.exec(format!(
|
||||
"UPDATE items SET
|
||||
validity = {}
|
||||
@ -213,7 +252,10 @@ impl<'s> Store<'s> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'s> dogear::Store<Error> for Store<'s> {
|
||||
impl<'s> dogear::Store for Store<'s> {
|
||||
type Ok = ApplyStatus;
|
||||
type Error = Error;
|
||||
|
||||
/// Builds a fully rooted, consistent tree from the items and tombstones in
|
||||
/// Places.
|
||||
fn fetch_local_tree(&self) -> Result<Tree> {
|
||||
@ -355,13 +397,19 @@ impl<'s> dogear::Store<Error> for Store<'s> {
|
||||
Ok(tree)
|
||||
}
|
||||
|
||||
fn apply<'t>(&mut self, root: MergedRoot<'t>) -> Result<()> {
|
||||
fn apply<'t>(&mut self, root: MergedRoot<'t>) -> Result<ApplyStatus> {
|
||||
self.controller.err_if_aborted()?;
|
||||
let ops = root.completion_ops();
|
||||
|
||||
self.controller.err_if_aborted()?;
|
||||
let deletions = root.deletions().collect::<Vec<_>>();
|
||||
|
||||
if ops.is_empty() && deletions.is_empty() && self.weak_uploads.is_empty() {
|
||||
// If we don't have any items to apply, upload, or delete,
|
||||
// no need to open a transaction at all.
|
||||
return Ok(ApplyStatus::Skipped);
|
||||
}
|
||||
|
||||
// Apply the merged tree and stage outgoing items. This transaction
|
||||
// blocks writes from the main connection until it's committed, so we
|
||||
// try to do as little work as possible within it.
|
||||
@ -381,7 +429,7 @@ impl<'s> dogear::Store<Error> for Store<'s> {
|
||||
cleanup(&tx)?;
|
||||
tx.commit()?;
|
||||
|
||||
Ok(())
|
||||
Ok(ApplyStatus::Merged)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1121,3 +1169,17 @@ fn rounded_now() -> u64 {
|
||||
.map(|d| (d.as_secs() as u64) * 1_000_000 + u64::from(d.subsec_millis()) * 1000)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
pub enum ApplyStatus {
|
||||
Merged,
|
||||
Skipped,
|
||||
}
|
||||
|
||||
impl From<ApplyStatus> for bool {
|
||||
fn from(status: ApplyStatus) -> bool {
|
||||
match status {
|
||||
ApplyStatus::Merged => true,
|
||||
ApplyStatus::Skipped => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -33,7 +33,7 @@ interface mozISyncedBookmarksMirrorProgressListener : nsISupports {
|
||||
// A callback called when the merge finishes.
|
||||
[scriptable, uuid(d23fdfea-92c8-409d-a516-08ae395d578f)]
|
||||
interface mozISyncedBookmarksMirrorCallback : nsISupports {
|
||||
void handleSuccess();
|
||||
void handleSuccess(in bool result);
|
||||
void handleError(in nsresult code, in AString message);
|
||||
};
|
||||
|
||||
|
||||
@ -8,21 +8,14 @@ var { AsyncShutdown } = ChromeUtils.import(
|
||||
add_task(async function test_abort_merging() {
|
||||
let buf = await openMirror("abort_merging");
|
||||
|
||||
let promise = new Promise((resolve, reject) => {
|
||||
let callback = {
|
||||
handleSuccess() {
|
||||
reject(new Error("Shouldn't have merged after aborting"));
|
||||
},
|
||||
handleError(code, message) {
|
||||
equal(code, Cr.NS_ERROR_ABORT, "Should abort merge with result code");
|
||||
resolve();
|
||||
},
|
||||
};
|
||||
let op = buf.merger.merge(0, 0, [], callback);
|
||||
op.cancel();
|
||||
});
|
||||
|
||||
await promise;
|
||||
let controller = new AbortController();
|
||||
let promiseWasMerged = buf.merge(controller.signal);
|
||||
controller.abort();
|
||||
await Assert.rejects(
|
||||
promiseWasMerged,
|
||||
/Operation aborted/,
|
||||
"Should abort merge when signaled"
|
||||
);
|
||||
|
||||
// Even though the merger is already finalized on the Rust side, the DB
|
||||
// connection is still open on the JS side. Finalizing `buf` closes it.
|
||||
@ -56,13 +49,7 @@ add_task(async function test_blocker_state() {
|
||||
},
|
||||
]);
|
||||
|
||||
await buf.tryApply(
|
||||
0,
|
||||
0,
|
||||
{ notifyAll() {} },
|
||||
[],
|
||||
buf.finalizeController.signal
|
||||
);
|
||||
await buf.tryApply(buf.finalizeController.signal);
|
||||
await barrier.wait();
|
||||
|
||||
let state = buf.progress.fetchState();
|
||||
|
||||
@ -249,12 +249,7 @@ add_task(async function test_multiple_parents() {
|
||||
|
||||
await storeChangesInMirror(buf, changesToUpload);
|
||||
|
||||
ok(
|
||||
!(await buf.hasChanges()),
|
||||
"Should not report local or remote changes after updating mirror"
|
||||
);
|
||||
|
||||
let newChangesToUpload = await buf.forceApply({
|
||||
let newChangesToUpload = await buf.apply({
|
||||
localTimeSeconds: now / 1000,
|
||||
remoteTimeSeconds: now / 1000,
|
||||
});
|
||||
@ -528,7 +523,7 @@ add_task(async function test_corrupt_local_roots() {
|
||||
);
|
||||
await Assert.rejects(
|
||||
buf.apply(),
|
||||
/Local tree has misparented root/,
|
||||
/The Places roots are invalid/,
|
||||
"Should abort merge if local tree has misparented syncable root"
|
||||
);
|
||||
|
||||
@ -547,7 +542,7 @@ add_task(async function test_corrupt_local_roots() {
|
||||
});
|
||||
await Assert.rejects(
|
||||
buf.apply(),
|
||||
/Local tree has misparented root/,
|
||||
/The Places roots are invalid/,
|
||||
"Should abort merge if local tree has misparented Places root"
|
||||
);
|
||||
} finally {
|
||||
|
||||
@ -55,8 +55,9 @@ add_task(async function test_no_changes() {
|
||||
);
|
||||
await PlacesTestUtils.markBookmarksAsSynced();
|
||||
|
||||
const hasChanges = await buf.hasChanges();
|
||||
Assert.ok(!hasChanges);
|
||||
let controller = new AbortController();
|
||||
const wasMerged = await buf.merge(controller.signal);
|
||||
Assert.ok(!wasMerged);
|
||||
|
||||
await buf.finalize();
|
||||
await PlacesUtils.bookmarks.eraseEverything();
|
||||
@ -114,8 +115,9 @@ add_task(async function test_changes_remote() {
|
||||
{ needsMerge: true }
|
||||
);
|
||||
|
||||
const hasChanges = await buf.hasChanges();
|
||||
Assert.ok(hasChanges);
|
||||
let controller = new AbortController();
|
||||
const wasMerged = await buf.merge(controller.signal);
|
||||
Assert.ok(wasMerged);
|
||||
|
||||
await buf.finalize();
|
||||
await PlacesUtils.bookmarks.eraseEverything();
|
||||
@ -163,8 +165,9 @@ add_task(async function test_changes_local() {
|
||||
title: "New Mozilla!",
|
||||
});
|
||||
|
||||
const hasChanges = await buf.hasChanges();
|
||||
Assert.ok(hasChanges);
|
||||
let controller = new AbortController();
|
||||
const wasMerged = await buf.merge(controller.signal);
|
||||
Assert.ok(wasMerged);
|
||||
|
||||
await buf.finalize();
|
||||
await PlacesUtils.bookmarks.eraseEverything();
|
||||
@ -209,8 +212,9 @@ add_task(async function test_changes_deleted_bookmark() {
|
||||
|
||||
await PlacesUtils.bookmarks.remove("mozBmk______");
|
||||
|
||||
const hasChanges = await buf.hasChanges();
|
||||
Assert.ok(hasChanges);
|
||||
let controller = new AbortController();
|
||||
const wasMerged = await buf.merge(controller.signal);
|
||||
Assert.ok(wasMerged);
|
||||
|
||||
await buf.finalize();
|
||||
await PlacesUtils.bookmarks.eraseEverything();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user