gecko-dev/toolkit/components/places/PlacesBackups.jsm
Nicholas Nethercote 2d6a2227fb Bug 996438 - Import BookmarkJSONUtils.jsm lazily into PlacesBackups.jsm. r=mak.
--HG--
extra : rebase_source : 244d30f479ddf9d79dfe216bd9024ba7f45c662a
2014-04-15 18:20:28 -07:00

680 lines
25 KiB
JavaScript

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: sw=2 ts=2 sts=2 expandtab filetype=javascript
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
this.EXPORTED_SYMBOLS = ["PlacesBackups"];
const Ci = Components.interfaces;
const Cu = Components.utils;
const Cc = Components.classes;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/PlacesUtils.jsm");
Cu.import("resource://gre/modules/Task.jsm");
Cu.import("resource://gre/modules/osfile.jsm");
Cu.import("resource://gre/modules/NetUtil.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "BookmarkJSONUtils",
"resource://gre/modules/BookmarkJSONUtils.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Deprecated",
"resource://gre/modules/Deprecated.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "OS",
"resource://gre/modules/osfile.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Sqlite",
"resource://gre/modules/Sqlite.jsm");
XPCOMUtils.defineLazyGetter(this, "localFileCtor",
() => Components.Constructor("@mozilla.org/file/local;1",
"nsILocalFile", "initWithPath"));
XPCOMUtils.defineLazyGetter(this, "filenamesRegex",
() => new RegExp("^bookmarks-([0-9\-]+)(?:_([0-9]+)){0,1}(?:_([a-z0-9=\+\-]{24})){0,1}\.(json|html)", "i")
);
/**
* Appends meta-data information to a given filename.
*/
function appendMetaDataToFilename(aFilename, aMetaData) {
let matches = aFilename.match(filenamesRegex);
return "bookmarks-" + matches[1] +
"_" + aMetaData.count +
"_" + aMetaData.hash +
"." + matches[4];
}
/**
* Gets the hash from a backup filename.
*
* @return the extracted hash or null.
*/
function getHashFromFilename(aFilename) {
let matches = aFilename.match(filenamesRegex);
if (matches && matches[3])
return matches[3];
return null;
}
/**
* Given two filenames, checks if they contain the same date.
*/
function isFilenameWithSameDate(aSourceName, aTargetName) {
let sourceMatches = aSourceName.match(filenamesRegex);
let targetMatches = aTargetName.match(filenamesRegex);
return sourceMatches && targetMatches &&
sourceMatches[1] == targetMatches[1] &&
sourceMatches[4] == targetMatches[4];
}
/**
* Given a filename, searches for another backup with the same date.
*
* @return OS.File path string or null.
*/
function getBackupFileForSameDate(aFilename) {
return Task.spawn(function* () {
let backupFiles = yield PlacesBackups.getBackupFiles();
for (let backupFile of backupFiles) {
if (isFilenameWithSameDate(OS.Path.basename(backupFile), aFilename))
return backupFile;
}
return null;
});
}
this.PlacesBackups = {
/**
* Matches the backup filename:
* 0: file name
* 1: date in form Y-m-d
* 2: bookmarks count
* 3: contents hash
* 4: file extension
*/
get filenamesRegex() filenamesRegex,
get folder() {
Deprecated.warning(
"PlacesBackups.folder is deprecated and will be removed in a future version",
"https://bugzilla.mozilla.org/show_bug.cgi?id=859695");
return this._folder;
},
/**
* This exists just to avoid spamming deprecate warnings from internal calls
* needed to support deprecated methods themselves.
*/
get _folder() {
let bookmarksBackupDir = Services.dirsvc.get("ProfD", Ci.nsILocalFile);
bookmarksBackupDir.append(this.profileRelativeFolderPath);
if (!bookmarksBackupDir.exists()) {
bookmarksBackupDir.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0700", 8));
if (!bookmarksBackupDir.exists())
throw("Unable to create bookmarks backup folder");
}
delete this._folder;
return this._folder = bookmarksBackupDir;
},
/**
* Gets backup folder asynchronously.
* @return {Promise}
* @resolve the folder (the folder string path).
*/
getBackupFolder: function PB_getBackupFolder() {
return Task.spawn(function* () {
if (this._backupFolder) {
return this._backupFolder;
}
let profileDir = OS.Constants.Path.profileDir;
let backupsDirPath = OS.Path.join(profileDir, this.profileRelativeFolderPath);
yield OS.File.makeDir(backupsDirPath, { ignoreExisting: true });
return this._backupFolder = backupsDirPath;
}.bind(this));
},
get profileRelativeFolderPath() "bookmarkbackups",
/**
* Cache current backups in a sorted (by date DESC) array.
*/
get entries() {
Deprecated.warning(
"PlacesBackups.entries is deprecated and will be removed in a future version",
"https://bugzilla.mozilla.org/show_bug.cgi?id=859695");
return this._entries;
},
/**
* This exists just to avoid spamming deprecate warnings from internal calls
* needed to support deprecated methods themselves.
*/
get _entries() {
delete this._entries;
this._entries = [];
let files = this._folder.directoryEntries;
while (files.hasMoreElements()) {
let entry = files.getNext().QueryInterface(Ci.nsIFile);
// A valid backup is any file that matches either the localized or
// not-localized filename (bug 445704).
if (!entry.isHidden() && filenamesRegex.test(entry.leafName)) {
// Remove bogus backups in future dates.
if (this.getDateForFile(entry) > new Date()) {
entry.remove(false);
continue;
}
this._entries.push(entry);
}
}
this._entries.sort((a, b) => {
let aDate = this.getDateForFile(a);
let bDate = this.getDateForFile(b);
return aDate < bDate ? 1 : aDate > bDate ? -1 : 0;
});
return this._entries;
},
/**
* Cache current backups in a sorted (by date DESC) array.
* @return {Promise}
* @resolve a sorted array of string paths.
*/
getBackupFiles: function PB_getBackupFiles() {
return Task.spawn(function* () {
if (this._backupFiles)
return this._backupFiles;
this._backupFiles = [];
let backupFolderPath = yield this.getBackupFolder();
let iterator = new OS.File.DirectoryIterator(backupFolderPath);
yield iterator.forEach(function(aEntry) {
// Since this is a lazy getter and OS.File I/O is serialized, we can
// safely remove .tmp files without risking to remove ongoing backups.
if (aEntry.name.endsWith(".tmp")) {
OS.File.remove(aEntry.path);
return;
}
if (filenamesRegex.test(aEntry.name)) {
// Remove bogus backups in future dates.
let filePath = aEntry.path;
if (this.getDateForFile(filePath) > new Date()) {
return OS.File.remove(filePath);
} else {
this._backupFiles.push(filePath);
}
}
}.bind(this));
iterator.close();
this._backupFiles.sort((a, b) => {
let aDate = this.getDateForFile(a);
let bDate = this.getDateForFile(b);
return aDate < bDate ? 1 : aDate > bDate ? -1 : 0;
});
return this._backupFiles;
}.bind(this));
},
/**
* Creates a filename for bookmarks backup files.
*
* @param [optional] aDateObj
* Date object used to build the filename.
* Will use current date if empty.
* @return A bookmarks backup filename.
*/
getFilenameForDate: function PB_getFilenameForDate(aDateObj) {
let dateObj = aDateObj || new Date();
// Use YYYY-MM-DD (ISO 8601) as it doesn't contain illegal characters
// and makes the alphabetical order of multiple backup files more useful.
return "bookmarks-" + dateObj.toLocaleFormat("%Y-%m-%d") + ".json";
},
/**
* Creates a Date object from a backup file. The date is the backup
* creation date.
*
* @param aBackupFile
* nsIFile or string path of the backup.
* @return A Date object for the backup's creation time.
*/
getDateForFile: function PB_getDateForFile(aBackupFile) {
let filename = (aBackupFile instanceof Ci.nsIFile) ? aBackupFile.leafName
: OS.Path.basename(aBackupFile);
let matches = filename.match(filenamesRegex);
if (!matches)
throw("Invalid backup file name: " + filename);
return new Date(matches[1].replace(/-/g, "/"));
},
/**
* Get the most recent backup file.
*
* @param [optional] aFileExt
* Force file extension. Either "html" or "json".
* Will check for both if not defined.
* @returns nsIFile backup file
*/
getMostRecent: function PB_getMostRecent(aFileExt) {
Deprecated.warning(
"PlacesBackups.getMostRecent is deprecated and will be removed in a future version",
"https://bugzilla.mozilla.org/show_bug.cgi?id=859695");
let fileExt = aFileExt || "(json|html)";
for (let i = 0; i < this._entries.length; i++) {
let rx = new RegExp("\." + fileExt + "$");
if (this._entries[i].leafName.match(rx))
return this._entries[i];
}
return null;
},
/**
* Get the most recent backup file.
*
* @param [optional] aFileExt
* Force file extension. Either "html" or "json".
* Will check for both if not defined.
* @return {Promise}
* @result the path to the file.
*/
getMostRecentBackup: function PB_getMostRecentBackup(aFileExt) {
return Task.spawn(function* () {
let fileExt = aFileExt || "(json|html)";
let entries = yield this.getBackupFiles();
for (let entry of entries) {
let rx = new RegExp("\." + fileExt + "$");
if (OS.Path.basename(entry).match(rx)) {
return entry;
}
}
return null;
}.bind(this));
},
/**
* Serializes bookmarks using JSON, and writes to the supplied file.
* Note: any item that should not be backed up must be annotated with
* "places/excludeFromBackup".
*
* @param aFilePath
* OS.File path for the "bookmarks.json" file to be created.
* @return {Promise}
* @resolves the number of serialized uri nodes.
* @deprecated passing an nsIFile is deprecated
*/
saveBookmarksToJSONFile: function PB_saveBookmarksToJSONFile(aFilePath) {
if (aFilePath instanceof Ci.nsIFile) {
Deprecated.warning("Passing an nsIFile to PlacesBackups.saveBookmarksToJSONFile " +
"is deprecated. Please use an OS.File path instead.",
"https://developer.mozilla.org/docs/JavaScript_OS.File");
aFilePath = aFilePath.path;
}
return Task.spawn(function* () {
let { count: nodeCount, hash: hash } =
yield BookmarkJSONUtils.exportToFile(aFilePath);
let backupFolderPath = yield this.getBackupFolder();
if (OS.Path.dirname(aFilePath) == backupFolderPath) {
// We are creating a backup in the default backups folder,
// so just update the internal cache.
this._entries.unshift(new localFileCtor(aFilePath));
if (!this._backupFiles) {
yield this.getBackupFiles();
}
this._backupFiles.unshift(aFilePath);
} else {
// If we are saving to a folder different than our backups folder, then
// we also want to copy this new backup to it.
// This way we ensure the latest valid backup is the same saved by the
// user. See bug 424389.
let mostRecentBackupFile = yield this.getMostRecentBackup("json");
if (!mostRecentBackupFile ||
hash != getHashFromFilename(OS.Path.basename(mostRecentBackupFile))) {
let name = this.getFilenameForDate();
let newFilename = appendMetaDataToFilename(name,
{ count: nodeCount,
hash: hash });
let newFilePath = OS.Path.join(backupFolderPath, newFilename);
let backupFile = yield getBackupFileForSameDate(name);
if (backupFile) {
// There is already a backup for today, replace it.
yield OS.File.remove(backupFile, { ignoreAbsent: true });
if (!this._backupFiles)
yield this.getBackupFiles();
else
this._backupFiles.shift();
this._backupFiles.unshift(newFilePath);
} else {
// There is no backup for today, add the new one.
this._entries.unshift(new localFileCtor(newFilePath));
if (!this._backupFiles)
yield this.getBackupFiles();
this._backupFiles.unshift(newFilePath);
}
yield OS.File.copy(aFilePath, newFilePath);
}
}
return nodeCount;
}.bind(this));
},
/**
* Creates a dated backup in <profile>/bookmarkbackups.
* Stores the bookmarks using JSON.
* Note: any item that should not be backed up must be annotated with
* "places/excludeFromBackup".
*
* @param [optional] int aMaxBackups
* The maximum number of backups to keep. If set to 0
* all existing backups are removed and aForceBackup is
* ignored, so a new one won't be created.
* @param [optional] bool aForceBackup
* Forces creating a backup even if one was already
* created that day (overwrites).
* @return {Promise}
*/
create: function PB_create(aMaxBackups, aForceBackup) {
let limitBackups = function* () {
let backupFiles = yield this.getBackupFiles();
if (typeof aMaxBackups == "number" && aMaxBackups > -1 &&
backupFiles.length >= aMaxBackups) {
let numberOfBackupsToDelete = backupFiles.length - aMaxBackups;
while (numberOfBackupsToDelete--) {
this._entries.pop();
let oldestBackup = this._backupFiles.pop();
yield OS.File.remove(oldestBackup);
}
}
}.bind(this);
return Task.spawn(function* () {
if (aMaxBackups === 0) {
// Backups are disabled, delete any existing one and bail out.
yield limitBackups(0);
return;
}
// Ensure to initialize _backupFiles
if (!this._backupFiles)
yield this.getBackupFiles();
let newBackupFilename = this.getFilenameForDate();
// If we already have a backup for today we should do nothing, unless we
// were required to enforce a new backup.
let backupFile = yield getBackupFileForSameDate(newBackupFilename);
if (backupFile && !aForceBackup)
return;
if (backupFile) {
// In case there is a backup for today we should recreate it.
this._backupFiles.shift();
this._entries.shift();
yield OS.File.remove(backupFile, { ignoreAbsent: true });
}
// Now check the hash of the most recent backup, and try to create a new
// backup, if that fails due to hash conflict, just rename the old backup.
let mostRecentBackupFile = yield this.getMostRecentBackup();
let mostRecentHash = mostRecentBackupFile &&
getHashFromFilename(OS.Path.basename(mostRecentBackupFile));
// Save bookmarks to a backup file.
let backupFolder = yield this.getBackupFolder();
let newBackupFile = OS.Path.join(backupFolder, newBackupFilename);
let newFilenameWithMetaData;
try {
let { count: nodeCount, hash: hash } =
yield BookmarkJSONUtils.exportToFile(newBackupFile,
{ failIfHashIs: mostRecentHash });
newFilenameWithMetaData = appendMetaDataToFilename(newBackupFilename,
{ count: nodeCount,
hash: hash });
} catch (ex if ex.becauseSameHash) {
// The last backup already contained up-to-date information, just
// rename it as if it was today's backup.
this._backupFiles.shift();
this._entries.shift();
newBackupFile = mostRecentBackupFile;
newFilenameWithMetaData = appendMetaDataToFilename(
newBackupFilename,
{ count: this.getBookmarkCountForFile(mostRecentBackupFile),
hash: mostRecentHash });
}
// Append metadata to the backup filename.
let newBackupFileWithMetadata = OS.Path.join(backupFolder, newFilenameWithMetaData);
yield OS.File.move(newBackupFile, newBackupFileWithMetadata);
this._entries.unshift(new localFileCtor(newBackupFileWithMetadata));
this._backupFiles.unshift(newBackupFileWithMetadata);
// Limit the number of backups.
yield limitBackups(aMaxBackups);
}.bind(this));
},
/**
* Gets the bookmark count for backup file.
*
* @param aFilePath
* File path The backup file.
*
* @return the bookmark count or null.
*/
getBookmarkCountForFile: function PB_getBookmarkCountForFile(aFilePath) {
let count = null;
let filename = OS.Path.basename(aFilePath);
let matches = filename.match(filenamesRegex);
if (matches && matches[2])
count = matches[2];
return count;
},
/**
* Gets a bookmarks tree representation usable to create backups in different
* file formats. The root or the tree is PlacesUtils.placesRootId.
* Items annotated with PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO and all of their
* descendants are excluded.
*
* @return an object representing a tree with the places root as its root.
* Each bookmark is represented by an object having these properties:
* * id: the item id (make this not enumerable after bug 824502)
* * title: the title
* * guid: unique id
* * parent: item id of the parent folder, not enumerable
* * index: the position in the parent
* * dateAdded: microseconds from the epoch
* * lastModified: microseconds from the epoch
* * type: type of the originating node as defined in PlacesUtils
* The following properties exist only for a subset of bookmarks:
* * annos: array of annotations
* * uri: url
* * iconuri: favicon's url
* * keyword: associated keyword
* * charset: last known charset
* * tags: csv string of tags
* * root: string describing whether this represents a root
* * children: array of child items in a folder
*/
getBookmarksTree: function () {
return Task.spawn(function* () {
let dbFilePath = OS.Path.join(OS.Constants.Path.profileDir, "places.sqlite");
let conn = yield Sqlite.openConnection({ path: dbFilePath,
sharedMemoryCache: false });
let rows = [];
try {
rows = yield conn.execute(
"SELECT b.id, h.url, IFNULL(b.title, '') AS title, b.parent, " +
"b.position AS [index], b.type, b.dateAdded, b.lastModified, " +
"b.guid, f.url AS iconuri, " +
"( SELECT GROUP_CONCAT(t.title, ',') " +
"FROM moz_bookmarks b2 " +
"JOIN moz_bookmarks t ON t.id = +b2.parent AND t.parent = :tags_folder " +
"WHERE b2.fk = h.id " +
") AS tags, " +
"EXISTS (SELECT 1 FROM moz_items_annos WHERE item_id = b.id LIMIT 1) AS has_annos, " +
"( SELECT a.content FROM moz_annos a " +
"JOIN moz_anno_attributes n ON a.anno_attribute_id = n.id " +
"WHERE place_id = h.id AND n.name = :charset_anno " +
") AS charset " +
"FROM moz_bookmarks b " +
"LEFT JOIN moz_bookmarks p ON p.id = b.parent " +
"LEFT JOIN moz_places h ON h.id = b.fk " +
"LEFT JOIN moz_favicons f ON f.id = h.favicon_id " +
"WHERE b.id <> :tags_folder AND b.parent <> :tags_folder AND p.parent <> :tags_folder " +
"ORDER BY b.parent, b.position",
{ tags_folder: PlacesUtils.tagsFolderId,
charset_anno: PlacesUtils.CHARSET_ANNO });
} catch(e) {
Cu.reportError("Unable to query the database " + e);
} finally {
yield conn.close();
}
let startTime = Date.now();
// Create a Map for lookup and recursive building of the tree.
let itemsMap = new Map();
for (let row of rows) {
let id = row.getResultByName("id");
try {
let bookmark = sqliteRowToBookmarkObject(row);
if (itemsMap.has(id)) {
// Since children may be added before parents, we should merge with
// the existing object.
let original = itemsMap.get(id);
for (prop in bookmark) {
original[prop] = bookmark[prop];
}
bookmark = original;
}
else {
itemsMap.set(id, bookmark);
}
// Append bookmark to its parent.
if (!itemsMap.has(bookmark.parent))
itemsMap.set(bookmark.parent, {});
let parent = itemsMap.get(bookmark.parent);
if (!("children" in parent))
parent.children = [];
parent.children.push(bookmark);
} catch (e) {
Cu.reportError("Error while reading node " + id + " " + e);
}
}
// Handle excluded items, by removing entire subtrees pointed by them.
function removeFromMap(id) {
// Could have been removed by a previous call, since we can't
// predict order of items in EXCLUDE_FROM_BACKUP_ANNO.
if (itemsMap.has(id)) {
let excludedItem = itemsMap.get(id);
if (excludedItem.children) {
for (let child of excludedItem.children) {
removeFromMap(child.id);
}
}
// Remove the excluded item from its parent's children...
let parentItem = itemsMap.get(excludedItem.parent);
parentItem.children = parentItem.children.filter(aChild => aChild.id != id);
// ...then remove it from the map.
itemsMap.delete(id);
}
}
for (let id of PlacesUtils.annotations.getItemsWithAnnotation(
PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO)) {
removeFromMap(id);
}
// Report the time taken to build the tree. This doesn't take into
// account the time spent in the query since that's off the main-thread.
try {
Services.telemetry
.getHistogramById("PLACES_BACKUPS_BOOKMARKSTREE_MS")
.add(Date.now() - startTime);
} catch (ex) {
Components.utils.reportError("Unable to report telemetry.");
}
return [itemsMap.get(PlacesUtils.placesRootId), itemsMap.size];
});
}
}
/**
* Helper function to convert a Sqlite.jsm row to a bookmark object
* representation.
*
* @param aRow The Sqlite.jsm result row.
*/
function sqliteRowToBookmarkObject(aRow) {
let bookmark = {};
for (let p of [ "id" ,"guid", "title", "index", "dateAdded", "lastModified" ]) {
bookmark[p] = aRow.getResultByName(p);
}
Object.defineProperty(bookmark, "parent",
{ value: aRow.getResultByName("parent") });
let type = aRow.getResultByName("type");
// Add annotations.
if (aRow.getResultByName("has_annos")) {
try {
bookmark.annos = PlacesUtils.getAnnotationsForItem(bookmark.id);
} catch (e) {
Cu.reportError("Unexpected error while reading annotations " + e);
}
}
switch (type) {
case Ci.nsINavBookmarksService.TYPE_BOOKMARK:
// TODO: What about shortcuts?
bookmark.type = PlacesUtils.TYPE_X_MOZ_PLACE;
// This will throw if we try to serialize an invalid url and the node will
// just be skipped.
bookmark.uri = NetUtil.newURI(aRow.getResultByName("url")).spec;
// Keywords are cached, so this should be decently fast.
let keyword = PlacesUtils.bookmarks.getKeywordForBookmark(bookmark.id);
if (keyword)
bookmark.keyword = keyword;
let charset = aRow.getResultByName("charset");
if (charset)
bookmark.charset = charset;
let tags = aRow.getResultByName("tags");
if (tags)
bookmark.tags = tags;
let iconuri = aRow.getResultByName("iconuri");
if (iconuri)
bookmark.iconuri = iconuri;
break;
case Ci.nsINavBookmarksService.TYPE_FOLDER:
bookmark.type = PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER;
// Mark root folders.
if (bookmark.id == PlacesUtils.placesRootId)
bookmark.root = "placesRoot";
else if (bookmark.id == PlacesUtils.bookmarksMenuFolderId)
bookmark.root = "bookmarksMenuFolder";
else if (bookmark.id == PlacesUtils.unfiledBookmarksFolderId)
bookmark.root = "unfiledBookmarksFolder";
else if (bookmark.id == PlacesUtils.toolbarFolderId)
bookmark.root = "toolbarFolder";
break;
case Ci.nsINavBookmarksService.TYPE_SEPARATOR:
bookmark.type = PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR;
break;
default:
Cu.reportError("Unexpected bookmark type");
break;
}
return bookmark;
}