commit acb45593e723b7241ea4646cae4632339819a924
parent 7fbfdce00e2ffd1a9a51e2b76eb78a43aefbe4a3
Author: Dan Stillman <dstillman@zotero.org>
Date: Tue, 26 Mar 2013 02:28:33 -0400
Fix WebDAV file purging
Deleted files are purged at the end of every sync, without any delay.
(If there's a conflict, it will be resolved before the file is deleted.)
Orphaned files are deleted once every 10 days, since it's a potentially
expensive operation for the server.
Diffstat:
6 files changed, 292 insertions(+), 274 deletions(-)
diff --git a/chrome/content/zotero/preferences/preferences_sync.js b/chrome/content/zotero/preferences/preferences_sync.js
@@ -93,21 +93,23 @@ Zotero_Preferences.Sync = {
var sql = "INSERT OR IGNORE INTO settings VALUES (?,?,?)";
Zotero.DB.query(sql, ['storage', 'zfsPurge', 'user']);
- Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles(function (success) {
- if (success) {
- ps.alert(
- null,
- Zotero.getString("general.success"),
- "Attachment files from your personal library have been removed from the Zotero servers."
- );
- }
- else {
- ps.alert(
- null,
- Zotero.getString("general.error"),
- "An error occurred. Please try again later."
- );
- }
+ Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
+ .then(function () {
+ ps.alert(
+ null,
+ Zotero.getString("general.success"),
+ "Attachment files from your personal library have been removed from the Zotero servers."
+ );
+ })
+ .catch(function (e) {
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+
+ ps.alert(
+ null,
+ Zotero.getString("general.error"),
+ "An error occurred. Please try again later."
+ );
});
}
}
diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js
@@ -326,6 +326,24 @@ Zotero.Sync.Storage = new function () {
Zotero.debug("File sync failed for library " + libraryID);
finalPromises.push([libraryID, libraryQueues]);
}
+
+ // If WebDAV sync enabled, purge deleted and orphaned files
+ if (libraryID == 0 && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
+ Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles()
+ .then(function () {
+ return Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles();
+ })
+ .catch(function (e) {
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+ });
+ }
+ });
+
+ Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
+ .catch(function (e) {
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
});
if (promises.length && !changedLibraries.length) {
@@ -1755,20 +1773,11 @@ Zotero.Sync.Storage = new function () {
/**
* @inner
- * @param {Integer} [days=pref:e.z.sync.storage.deleteDelayDays]
- * Number of days old files have to be
* @return {String[]|FALSE} Array of keys, or FALSE if none
*/
- this.getDeletedFiles = function (days) {
- if (!days) {
- days = Zotero.Prefs.get("sync.storage.deleteDelayDays");
- }
-
- var ts = Zotero.Date.getUnixTimestamp();
- ts = ts - (86400 * days);
-
- var sql = "SELECT key FROM storageDeleteLog WHERE timestamp<?";
- return Zotero.DB.columnQuery(sql, ts);
+ this.getDeletedFiles = function () {
+ var sql = "SELECT key FROM storageDeleteLog";
+ return Zotero.DB.columnQuery(sql);
}
diff --git a/chrome/content/zotero/xpcom/storage/webdav.js b/chrome/content/zotero/xpcom/storage/webdav.js
@@ -473,21 +473,21 @@ Zotero.Sync.Storage.WebDAV = (function () {
};
if (files.length == 0) {
- return Q.resolve(results);
+ return Q(results);
}
let deleteURI = _rootURI.clone();
// This should never happen, but let's be safe
if (!deleteURI.spec.match(/\/$/)) {
- throw new Error(
- "Root URI does not end in slash in "
- + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()"
- );
+ return Q.reject("Root URI does not end in slash in "
+ + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()");
}
- results = Q.resolve(results);
- files.forEach(function (fileName) {
- results = results.then(function (results) {
+ var funcs = [];
+ for (let i=0; i<files.length; i++) {
+ let fileName = files[i];
+ let baseName = fileName.match(/^([^\.]+)/)[1];
+ funcs.push(function () {
let deleteURI = _rootURI.clone();
deleteURI.QueryInterface(Components.interfaces.nsIURL);
deleteURI.fileName = fileName;
@@ -502,30 +502,25 @@ Zotero.Sync.Storage.WebDAV = (function () {
break;
case 404:
- var fileDeleted = false;
+ var fileDeleted = true;
break;
}
// If an item file URI, get the property URI
var deletePropURI = getPropertyURIFromItemURI(deleteURI);
- if (!deletePropURI) {
+
+ // If we already deleted the prop file, skip it
+ if (!deletePropURI || results.deleted.indexOf(deletePropURI.fileName) != -1) {
if (fileDeleted) {
- results.deleted.push(fileName);
+ results.deleted.push(baseName);
}
else {
- results.missing.push(fileName);
+ results.missing.push(baseName);
}
- return results;
+ return;
}
- // If property file appears separately in delete queue,
- // remove it, since we're taking care of it here
- var propIndex = files.indexOf(deletePropURI.fileName);
- if (propIndex > i) {
- delete files[propIndex];
- i--;
- last = (i == files.length - 1);
- }
+ let propFileName = deletePropURI.fileName;
// Delete property file
return Zotero.HTTP.promise("DELETE", deletePropURI, { successCodes: [200, 204, 404] })
@@ -534,29 +529,40 @@ Zotero.Sync.Storage.WebDAV = (function () {
case 204:
// IIS 5.1 and Sakai return 200
case 200:
- results.deleted.push(fileName);
+ results.deleted.push(baseName);
break;
case 404:
if (fileDeleted) {
- results.deleted.push(fileName);
+ results.deleted.push(baseName);
}
else {
- results.missing.push(fileName);
+ results.missing.push(baseName);
}
break;
}
});
})
.catch(function (e) {
- results.error.push(fileName);
- var msg = "An error occurred attempting to delete "
- + "'" + fileName
- + "' (" + e.status + " " + e.xmlhttp.statusText + ").";
+ results.error.push(baseName);
+ throw e;
});
});
+ }
+
+ Components.utils.import("resource://zotero/concurrent-caller.js");
+ var caller = new ConcurrentCaller(4);
+ caller.stopOnError = true;
+ caller.setLogger(function (msg) {
+ Zotero.debug("[ConcurrentCaller] " + msg);
+ });
+ caller.setErrorLogger(function (msg) {
+ Components.utils.reportError(msg);
+ });
+ return caller.fcall(funcs)
+ .then(function () {
+ return results;
});
- return results;
}
@@ -881,7 +887,8 @@ Zotero.Sync.Storage.WebDAV = (function () {
deleteStorageFiles([item.key + ".prop"])
.finally(function (results) {
deferred.resolve(false);
- });
+ })
+ .done();
return;
}
else if (status != 200) {
@@ -1457,182 +1464,199 @@ Zotero.Sync.Storage.WebDAV = (function () {
/**
- * Remove files on storage server that were deleted locally more than
- * sync.storage.deleteDelayDays days ago
+ * Remove files on storage server that were deleted locally
*
* @param {Function} callback Passed number of files deleted
*/
obj._purgeDeletedStorageFiles = function () {
- if (!this._active) {
- return Q(false);
- }
-
- Zotero.debug("Purging deleted storage files");
- var files = Zotero.Sync.Storage.getDeletedFiles();
- if (!files) {
- Zotero.debug("No files to delete remotely");
- return Q(false);
- }
-
- // Add .zip extension
- var files = files.map(function (file) file + ".zip");
-
- return deleteStorageFiles(files)
- .then(function (results) {
- // Remove deleted and nonexistent files from storage delete log
- var toPurge = results.deleted.concat(results.missing);
- if (toPurge.length > 0) {
- var done = 0;
- var maxFiles = 999;
- var numFiles = toPurge.length;
-
- Zotero.DB.beginTransaction();
-
- do {
- var chunk = toPurge.splice(0, maxFiles);
- var sql = "DELETE FROM storageDeleteLog WHERE key IN ("
- + chunk.map(function () '?').join() + ")";
- Zotero.DB.query(sql, chunk);
- done += chunk.length;
- }
- while (done < numFiles);
-
- Zotero.DB.commitTransaction();
+ return Q.fcall(function () {
+ if (!this.includeUserFiles) {
+ return false;
}
- return results.deleted.length;
- });
+ Zotero.debug("Purging deleted storage files");
+ var files = Zotero.Sync.Storage.getDeletedFiles();
+ if (!files) {
+ Zotero.debug("No files to delete remotely");
+ return false;
+ }
+
+ // Add .zip extension
+ var files = files.map(function (file) file + ".zip");
+
+ return deleteStorageFiles(files)
+ .then(function (results) {
+ // Remove deleted and nonexistent files from storage delete log
+ var toPurge = results.deleted.concat(results.missing);
+ if (toPurge.length > 0) {
+ var done = 0;
+ var maxFiles = 999;
+ var numFiles = toPurge.length;
+
+ Zotero.DB.beginTransaction();
+
+ do {
+ var chunk = toPurge.splice(0, maxFiles);
+ var sql = "DELETE FROM storageDeleteLog WHERE key IN ("
+ + chunk.map(function () '?').join() + ")";
+ Zotero.DB.query(sql, chunk);
+ done += chunk.length;
+ }
+ while (done < numFiles);
+
+ Zotero.DB.commitTransaction();
+ }
+
+ Zotero.debug(results);
+
+ return results.deleted.length;
+ });
+ }.bind(this));
};
/**
* Delete orphaned storage files older than a day before last sync time
- *
- * @param {Function} callback
*/
- obj._purgeOrphanedStorageFiles = function (callback) {
- const daysBeforeSyncTime = 1;
-
- if (!this._active) {
- return false;
- }
-
- // If recently purged, skip
- var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge');
- var days = 10;
- if (lastpurge && new Date(lastpurge * 1000) > (new Date() - (1000 * 60 * 60 * 24 * days))) {
- return false;
- }
-
- Zotero.debug("Purging orphaned storage files");
-
- var uri = this.rootURI;
- var path = uri.path;
-
- var xmlstr = "<propfind xmlns='DAV:'><prop>"
- + "<getlastmodified/>"
- + "</prop></propfind>";
-
- var lastSyncDate = new Date(Zotero.Sync.Server.lastLocalSyncTime * 1000);
-
- Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (req) {
- Zotero.debug(req.responseText);
-
- var funcName = "Zotero.Sync.Storage.purgeOrphanedStorageFiles()";
+ obj._purgeOrphanedStorageFiles = function () {
+ return Q.fcall(function () {
+ const daysBeforeSyncTime = 1;
- var responseNode = req.responseXML.documentElement;
- responseNode.xpath = function (path) {
- return Zotero.Utilities.xpath(this, path, { D: 'DAV:' });
- };
+ if (!this.includeUserFiles) {
+ return false;
+ }
- var deleteFiles = [];
- var trailingSlash = !!path.match(/\/$/);
- for each(var response in responseNode.xpath("response")) {
- var href = Zotero.Utilities.xpath(response, "href", { D: 'DAV:' });
- href = href.length ? href[0] : ''
-
- // Strip trailing slash if there isn't one on the root path
- if (!trailingSlash) {
- href = href.replace(/\/$/, "")
- }
-
- // Absolute
- if (href.match(/^https?:\/\//)) {
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var href = ios.newURI(href, null, null);
- href = href.path;
- }
-
- // Skip root URI
- if (href == path
- // Some Apache servers respond with a "/zotero" href
- // even for a "/zotero/" request
- || (trailingSlash && href + '/' == path)
- // Try URL-encoded as well, as above
- || decodeURIComponent(href) == path) {
- continue;
- }
-
- if (href.indexOf(path) == -1
- // Try URL-encoded as well, in case there's a '~' or similar
- // character in the URL and the server (e.g., Sakai) is
- // encoding the value
- && decodeURIComponent(href).indexOf(path) == -1) {
- Zotero.Sync.Storage.EventManager.error(
- "DAV:href '" + href + "' does not begin with path '"
- + path + "' in " + funcName
- );
- }
-
- var matches = href.match(/[^\/]+$/);
- if (!matches) {
- Zotero.Sync.Storage.EventManager.error(
- "Unexpected href '" + href + "' in " + funcName
- )
- }
- var file = matches[0];
-
- if (file.indexOf('.') == 0) {
- Zotero.debug("Skipping hidden file " + file);
- continue;
- }
- if (!file.match(/\.zip$/) && !file.match(/\.prop$/)) {
- Zotero.debug("Skipping file " + file);
- continue;
- }
-
- var key = file.replace(/\.(zip|prop)$/, '');
- var item = Zotero.Items.getByLibraryAndKey(null, key);
- if (item) {
- Zotero.debug("Skipping existing file " + file);
- continue;
- }
-
- Zotero.debug("Checking orphaned file " + file);
-
- // TODO: Parse HTTP date properly
- var lastModified = Zotero.Utilities.xpath(
- response, "//getlastmodified", { D: 'DAV:' }
- );
- lastModified = lastModified.length ? lastModified[0] : ''
- lastModified = Zotero.Date.strToISO(lastModified);
- lastModified = Zotero.Date.sqlToDate(lastModified);
-
- // Delete files older than a day before last sync time
- var days = (lastSyncDate - lastModified) / 1000 / 60 / 60 / 24;
-
- if (days > daysBeforeSyncTime) {
- deleteFiles.push(file);
- }
+ // If recently purged, skip
+ var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge');
+ var days = 10;
+ if (lastpurge && new Date(lastpurge * 1000) > (new Date() - (1000 * 60 * 60 * 24 * days))) {
+ return false;
}
- deleteStorageFiles(deleteFiles)
- .then(function (results) {
- Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000))
- Zotero.debug(results);
- });
- }, { Depth: 1 });
+ Zotero.debug("Purging orphaned storage files");
+
+ var uri = this.rootURI;
+ var path = uri.path;
+
+ var xmlstr = "<propfind xmlns='DAV:'><prop>"
+ + "<getlastmodified/>"
+ + "</prop></propfind>";
+
+ var lastSyncDate = new Date(Zotero.Sync.Server.lastLocalSyncTime * 1000);
+
+ var deferred = Q.defer();
+
+ Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (xmlhttp) {
+ Q.fcall(function () {
+ Zotero.debug(xmlhttp.responseText);
+
+ var funcName = "Zotero.Sync.Storage.purgeOrphanedStorageFiles()";
+
+ var responseNode = xmlhttp.responseXML.documentElement;
+ responseNode.xpath = function (path) {
+ return Zotero.Utilities.xpath(this, path, { D: 'DAV:' });
+ };
+
+ var deleteFiles = [];
+ var trailingSlash = !!path.match(/\/$/);
+ for each(var response in responseNode.xpath("D:response")) {
+ var href = Zotero.Utilities.xpathText(
+ response, "D:href", { D: 'DAV:' }
+ ) || "";
+ Zotero.debug(href);
+
+ // Strip trailing slash if there isn't one on the root path
+ if (!trailingSlash) {
+ href = href.replace(/\/$/, "");
+ }
+
+ // Absolute
+ if (href.match(/^https?:\/\//)) {
+ var ios = Components.classes["@mozilla.org/network/io-service;1"].
+ getService(Components.interfaces.nsIIOService);
+ var href = ios.newURI(href, null, null);
+ href = href.path;
+ }
+
+ // Skip root URI
+ if (href == path
+ // Some Apache servers respond with a "/zotero" href
+ // even for a "/zotero/" request
+ || (trailingSlash && href + '/' == path)
+ // Try URL-encoded as well, as above
+ || decodeURIComponent(href) == path) {
+ continue;
+ }
+
+ if (href.indexOf(path) == -1
+ // Try URL-encoded as well, in case there's a '~' or similar
+ // character in the URL and the server (e.g., Sakai) is
+ // encoding the value
+ && decodeURIComponent(href).indexOf(path) == -1) {
+ throw new Error(
+ "DAV:href '" + href + "' does not begin with path '"
+ + path + "' in " + funcName
+ );
+ }
+
+ var matches = href.match(/[^\/]+$/);
+ if (!matches) {
+ throw new Error(
+ "Unexpected href '" + href + "' in " + funcName
+ );
+ }
+ var file = matches[0];
+
+ if (file.indexOf('.') == 0) {
+ Zotero.debug("Skipping hidden file " + file);
+ continue;
+ }
+ if (!file.match(/\.zip$/) && !file.match(/\.prop$/)) {
+ Zotero.debug("Skipping file " + file);
+ continue;
+ }
+
+ var key = file.replace(/\.(zip|prop)$/, '');
+ var item = Zotero.Items.getByLibraryAndKey(null, key);
+ if (item) {
+ Zotero.debug("Skipping existing file " + file);
+ continue;
+ }
+
+ Zotero.debug("Checking orphaned file " + file);
+
+ // TODO: Parse HTTP date properly
+ Zotero.debug(response.innerHTML);
+ var lastModified = Zotero.Utilities.xpathText(
+ response, ".//D:getlastmodified", { D: 'DAV:' }
+ );
+ lastModified = Zotero.Date.strToISO(lastModified);
+ lastModified = Zotero.Date.sqlToDate(lastModified);
+
+ // Delete files older than a day before last sync time
+ var days = (lastSyncDate - lastModified) / 1000 / 60 / 60 / 24;
+
+ if (days > daysBeforeSyncTime) {
+ deleteFiles.push(file);
+ }
+ }
+
+ return deleteStorageFiles(deleteFiles)
+ .then(function (results) {
+ Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000))
+ Zotero.debug(results);
+ });
+ })
+ .catch(function (e) {
+ deferred.reject(e);
+ })
+ .then(function () {
+ deferred.resolve();
+ });
+ }, { Depth: 1 });
+
+ return deferred.promise;
+ }.bind(this));
};
return obj;
diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js
@@ -1006,7 +1006,7 @@ Zotero.Sync.Storage.ZFS = (function () {
Zotero.debug("Credentials are cached");
_cachedCredentials = true;
})
- .fail(function (e) {
+ .catch(function (e) {
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
if (e.status == 401) {
var msg = "File sync login failed\n\n"
@@ -1030,57 +1030,49 @@ Zotero.Sync.Storage.ZFS = (function () {
/**
* Remove all synced files from the server
*/
- obj._purgeDeletedStorageFiles = function (callback) {
- // If we don't have a user id we've never synced and don't need to bother
- if (!Zotero.userID) {
- return false;
- }
-
- var sql = "SELECT value FROM settings WHERE setting=? AND key=?";
- var values = Zotero.DB.columnQuery(sql, ['storage', 'zfsPurge']);
- if (!values) {
- return false;
- }
-
- // TODO: promisify
-
- Zotero.debug("Unlinking synced files on ZFS");
-
- var uri = this.userURI;
- uri.spec += "removestoragefiles?";
- // Unused
- for each(var value in values) {
- switch (value) {
- case 'user':
- uri.spec += "user=1&";
- break;
-
- case 'group':
- uri.spec += "group=1&";
- break;
-
- default:
- throw "Invalid zfsPurge value '" + value
- + "' in ZFS purgeDeletedStorageFiles()";
+ obj._purgeDeletedStorageFiles = function () {
+ return Q.fcall(function () {
+ // If we don't have a user id we've never synced and don't need to bother
+ if (!Zotero.userID) {
+ return false;
}
- }
- uri.spec = uri.spec.substr(0, uri.spec.length - 1);
-
- Zotero.HTTP.doPost(uri, "", function (xmlhttp) {
- if (xmlhttp.status != 204) {
- if (callback) {
- callback(false);
- }
- throw "Unexpected status code " + xmlhttp.status + " purging ZFS files";
+
+ var sql = "SELECT value FROM settings WHERE setting=? AND key=?";
+ var values = Zotero.DB.columnQuery(sql, ['storage', 'zfsPurge']);
+ if (!values) {
+ return false;
}
- var sql = "DELETE FROM settings WHERE setting=? AND key=?";
- Zotero.DB.query(sql, ['storage', 'zfsPurge']);
+ // TODO: promisify
+
+ Zotero.debug("Unlinking synced files on ZFS");
- if (callback) {
- callback(true);
+ var uri = this.userURI;
+ uri.spec += "removestoragefiles?";
+ // Unused
+ for each(var value in values) {
+ switch (value) {
+ case 'user':
+ uri.spec += "user=1&";
+ break;
+
+ case 'group':
+ uri.spec += "group=1&";
+ break;
+
+ default:
+ throw "Invalid zfsPurge value '" + value
+ + "' in ZFS purgeDeletedStorageFiles()";
+ }
}
- });
+ uri.spec = uri.spec.substr(0, uri.spec.length - 1);
+
+ return Zotero.HTTP.promise("POST", uri, "")
+ .then(function (req) {
+ var sql = "DELETE FROM settings WHERE setting=? AND key=?";
+ Zotero.DB.query(sql, ['storage', 'zfsPurge']);
+ });
+ }.bind(this));
};
return obj;
diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js
@@ -429,7 +429,7 @@ Zotero.Sync.EventListener = new function () {
var sql = "REPLACE INTO syncDeleteLog VALUES (?, ?, ?, ?)";
var syncStatement = Zotero.DB.getStatement(sql);
- if (isItem && Zotero.Sync.Storage.WebDAV.active) {
+ if (isItem && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
var storageEnabled = true;
var sql = "INSERT INTO storageDeleteLog VALUES (?, ?, ?)";
var storageStatement = Zotero.DB.getStatement(sql);
diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js
@@ -1844,15 +1844,6 @@ Components.utils.import("resource://gre/modules/Services.jsm");
Zotero.Items.purge();
// DEBUG: this might not need to be permanent
Zotero.Relations.purge();
-
- if (!skipStoragePurge && Math.random() < 1/10) {
- Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles();
- Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles();
- }
-
- if (!skipStoragePurge) {
- Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles();
- }
}
