commit 4c30e6738f42f6655caad218c0d0becb4ca9755e
parent 673a22430b1381b86099ffcbeb14941dd15224ed
Author: Dan Stillman <dstillman@zotero.org>
Date: Wed, 3 Feb 2010 04:43:20 +0000
- Use millisecond resolution for file sync timestamps (fixes problem with slow sync initialization on some systems)
- Fix errors syncing files with filenames containing extended characters
This will not sync with the server until a server update is pushed.
Diffstat:
6 files changed, 184 insertions(+), 92 deletions(-)
diff --git a/chrome/content/zotero/xpcom/schema.js b/chrome/content/zotero/xpcom/schema.js
@@ -1056,10 +1056,6 @@ Zotero.Schema = new function(){
Zotero.DB.query(sql);
var sql = "INSERT INTO itemData VALUES (1, 1, 2)";
Zotero.DB.query(sql);
- var sql = "INSERT INTO itemDataValues VALUES (3, CURRENT_TIMESTAMP)";
- Zotero.DB.query(sql);
- var sql = "INSERT INTO itemData VALUES (1, 27, 3)";
- Zotero.DB.query(sql);
// CHNM as creator
var sql = "INSERT INTO creatorData VALUES (1, '', 'Center for History and New Media', '', 1, NULL)";
@@ -2793,6 +2789,10 @@ Zotero.Schema = new function(){
Zotero.DB.query("UPDATE itemData SET fieldID=121 WHERE itemID IN (SELECT itemID FROM items WHERE itemTypeID=19) AND fieldID=14");
}
+ if (i==71) {
+ Zotero.DB.query("UPDATE itemAttachments SET storageModTime=storageModTime*1000 WHERE storageModTime<10000000000");
+ }
+
Zotero.wait();
}
diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js
@@ -218,7 +218,7 @@ Zotero.Sync.Storage = new function () {
/**
* @param {Integer} itemID
- * @return {Integer|NULL} Mod time as Unix timestamp,
+ * @return {Integer|NULL} Mod time as timestamp in ms,
* or NULL if never synced
*/
this.getSyncedModificationTime = function (itemID) {
@@ -235,7 +235,7 @@ Zotero.Sync.Storage = new function () {
/**
* @param {Integer} itemID
* @param {Integer} mtime File modification time as
- * Unix timestamp
+ * timestamp in ms
* @param {Boolean} [updateItem=FALSE] Update dateModified field of
* attachment item
*/
@@ -459,10 +459,10 @@ Zotero.Sync.Storage = new function () {
continue;
}
- var fileModTime = Math.round(file.lastModifiedTime / 1000);
+ var fmtime = file.lastModifiedTime;
//Zotero.debug("Stored mtime is " + attachmentData[item.id].mtime);
- //Zotero.debug("File mtime is " + fileModTime);
+ //Zotero.debug("File mtime is " + fmtime);
// Download-marking mode
if (itemModTimes) {
@@ -481,8 +481,29 @@ Zotero.Sync.Storage = new function () {
continue;
}
+ var mtime = attachmentData[item.id].mtime;
+
// If stored time matches file, it hasn't changed
- if (attachmentData[item.id].mtime == fileModTime) {
+ if (mtime == fmtime) {
+ continue;
+ }
+
+ // Allow floored timestamps for filesystems that don't support
+ // millisecond precision (e.g., HFS+)
+ if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
+ Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+ + "for " + file.leafName + " -- ignoring");
+ continue;
+ }
+
+ // Allow timestamp to be exactly one hour off to get around
+ // time zone issues -- there may be a proper way to fix this
+ if (Math.abs(fmtime - mtime) == 3600000
+ // And check with one-second precision as well
+ || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+ || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+ Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ + "-- assuming time zone issue and skipping upload");
continue;
}
@@ -501,13 +522,13 @@ Zotero.Sync.Storage = new function () {
}
var fileHash = item.attachmentHash;
if (attachmentData[item.id].hash && attachmentData[item.id].hash == fileHash) {
- Zotero.debug("Mod time didn't match (" + fileModTime + "!=" + attachmentData[item.id].mtime + ") "
+ Zotero.debug("Mod time didn't match (" + fmtime + "!=" + mtime + ") "
+ "but hash did for " + file.leafName + " -- ignoring");
continue;
}
Zotero.debug("Marking attachment " + item.id + " as changed ("
- + attachmentData[item.id].mtime + " != " + fileModTime + ")");
+ + mtime + " != " + fmtime + ")");
updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
}
@@ -603,7 +624,7 @@ Zotero.Sync.Storage = new function () {
}
if (!data.compressed && !data.syncHash) {
- _error("|data.storageHash| is required if |data.compressed| is false in " + funcName);
+ _error("|data.syncHash| is required if |data.compressed| is false in " + funcName);
}
var item = data.item;
@@ -658,12 +679,12 @@ Zotero.Sync.Storage = new function () {
Zotero.Sync.Storage.resyncOnFinish = true;
}
else {
- file.lastModifiedTime = syncModTime * 1000;
-
- // Only save hash if file isn't compressed
- if (!data.compressed) {
- Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
+ file.lastModifiedTime = syncModTime;
+ // If hash not provided (e.g., WebDAV), calculate it now
+ if (!syncHash) {
+ syncHash = item.attachmentHash;
}
+ Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
}
@@ -817,23 +838,16 @@ Zotero.Sync.Storage = new function () {
throw ("Empty path for item " + item.key + " in " + funcName);
}
- var itemFileName = item.getFilename();
var fileName = file.leafName;
var renamed = false;
- // If file doesn't match the known filename, use that name
- if (itemFileName != fileName) {
- Zotero.debug("Renaming file '" + fileName + "' to known filename '" + itemFileName + "'");
- fileName = itemFileName;
- renamed = true;
- }
-
// Make sure the new filename is valid, in case an invalid character made it over
// (e.g., from before we checked for them)
var filteredName = Zotero.File.getValidFileName(fileName);
if (filteredName != fileName) {
Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
fileName = filteredName;
+ file.leafName = fileName;
renamed = true;
}
@@ -842,9 +856,7 @@ Zotero.Sync.Storage = new function () {
tempFile.moveTo(parentDir, fileName);
}
catch (e) {
- var destFile = parentDir.clone();
- destFile.QueryInterface(Components.interfaces.nsILocalFile);
- destFile.setRelativeDescriptor(parentDir, fileName);
+ var destFile = file.clone();
var windowsLength = false;
var nameLength = false;
@@ -910,10 +922,7 @@ Zotero.Sync.Storage = new function () {
var returnFile = null;
// processDownload() needs to know that we're renaming the file
if (renamed) {
- destFile = parentDir.clone();
- destFile.QueryInterface(Components.interfaces.nsILocalFile);
- destFile.setRelativeDescriptor(parentDir, fileName);
- returnFile = destFile;
+ var returnFile = file.clone();
}
return returnFile;
@@ -966,7 +975,6 @@ Zotero.Sync.Storage = new function () {
var entries = zipReader.findEntries(null);
while (entries.hasMore()) {
- var renamed = false;
count++;
var entryName = entries.getNext();
var b64re = /%ZB64$/;
@@ -984,35 +992,54 @@ Zotero.Sync.Storage = new function () {
continue;
}
- var itemFileName = item.getFilename();
+ Zotero.debug("Extracting " + fileName);
- // If only one file in zip and it doesn't match the known filename,
- // take our chances and use that name
- if (count == 1 && !entries.hasMore()) {
- if (itemFileName != fileName) {
- Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
- fileName = itemFileName;
- renamed = true;
- }
- }
+ var primaryFile = false;
+ var filtered = false;
+ var renamed = false;
- var primaryFile = itemFileName == fileName;
+ // Get the old filename
+ var itemFileName = item.getFilename();
// Make sure the new filename is valid, in case an invalid character
// somehow make it into the ZIP (e.g., from before we checked for them)
+ //
+ // Do this before trying to use the relative descriptor, since otherwise
+ // it might fail silently and select the parent directory
var filteredName = Zotero.File.getValidFileName(fileName);
if (filteredName != fileName) {
Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
fileName = filteredName;
- if (primaryFile) {
- renamed = true;
- }
+ filtered = true;
}
- Zotero.debug("Extracting " + fileName);
+ // Name in ZIP is a relative descriptor, so file has to be reconstructed
+ // using setRelativeDescriptor()
var destFile = parentDir.clone();
destFile.QueryInterface(Components.interfaces.nsILocalFile);
destFile.setRelativeDescriptor(parentDir, fileName);
+
+ fileName = destFile.leafName;
+
+ // If only one file in zip and it doesn't match the known filename,
+ // take our chances and use that name
+ if (count == 1 && !entries.hasMore()) {
+ // May not be necessary, but let's be safe
+ itemFileName = Zotero.File.getValidFileName(itemFileName);
+ if (itemFileName != fileName) {
+ Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2);
+ Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
+ fileName = itemFileName;
+ destFile.leafName = fileName;
+ renamed = true;
+ }
+ }
+
+ var primaryFile = itemFileName == fileName;
+ if (primaryFile && filtered) {
+ renamed = true;
+ }
+
if (destFile.exists()) {
var msg = "ZIP entry '" + fileName + "' " + "already exists";
Zotero.debug(msg, 2);
@@ -1680,10 +1707,10 @@ Zotero.Sync.Storage.QueueManager = new function () {
var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name);
var item1 = item.clone(false, false, true);
item1.setField('dateModified',
- Zotero.Date.dateToSQL(new Date(conflict.localData.modTime * 1000), true));
+ Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true));
var item2 = item.clone(false, false, true);
item2.setField('dateModified',
- Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime * 1000), true));
+ Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true));
objectPairs.push([item1, item2]);
}
diff --git a/chrome/content/zotero/xpcom/storage/webdav.js b/chrome/content/zotero/xpcom/storage/webdav.js
@@ -241,27 +241,52 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct
var xml = new XML(req.responseText);
}
catch (e) {
+ Zotero.debug(e);
var xml = null;
}
- if (xml && xml.childNodes.length()) {
+
+ if (xml) {
+ Zotero.debug(xml.children().length());
+ }
+
+ if (xml && xml.children().length()) {
// TODO: other stuff, but this makes us forward-compatible
mtime = xml.mtime.toString();
+ var seconds = false;
}
else {
mtime = req.responseText;
+ var seconds = true;
+ }
+
+ var invalid = false;
+
+ // Unix timestamps need to be converted to ms-based timestamps
+ if (seconds) {
+ if (mtime.match(/^[0-9]{1,10}$/)) {
+ Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
+ mtime = mtime * 1000;
+ }
+ else {
+ invalid = true;
+ }
+ }
+ else if (!mtime.match(/^[0-9]{1,13}$/)) {
+ invalid = true;
}
// Delete invalid .prop files
- if (!(mtime + '').match(/^[0-9]{10}$/)) {
- var msg = "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20)
- + "' for item " + Zotero.Items.getLibraryKeyHash(item) + " in " + funcName;
+ if (invalid) {
+ var msg = "An error occurred during file syncing. Try the sync again.\n\n"
+ + "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20)
+ + "' for item " + Zotero.Items.getLibraryKeyHash(item);
Zotero.debug(msg, 1);
self._deleteStorageFiles([item.key + ".prop"], null, self);
self.onError(msg);
return;
}
- var mdate = new Date(mtime * 1000);
+ var mdate = new Date(parseInt(mtime));
callback(item, mdate);
});
}
@@ -271,12 +296,20 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct
* Set mod time of file on storage server
*
* @param {Zotero.Item} item
- * @param {Function} callback Callback f(item, mtime)
+ * @param {Function} callback Callback f(item, props)
*/
Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = function (item, callback) {
var uri = this._getItemPropertyURI(item);
- Zotero.Utilities.HTTP.WebDAV.doPut(uri, item.attachmentModificationTime + '', function (req) {
+ var mtime = item.attachmentModificationTime;
+ var hash = item.attachmentHash;
+
+ var prop = <properties version="1">
+ <mtime>{mtime}</mtime>
+ <hash>{hash}</hash>
+ </properties>;
+
+ Zotero.Utilities.HTTP.WebDAV.doPut(uri, prop.toXMLString(), function (req) {
switch (req.status) {
case 200:
case 201:
@@ -288,7 +321,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = funct
throw ("Unexpected status code " + req.status + " in "
+ "Zotero.Sync.Storage._setStorageModificationTime()");
}
- callback(item, item.attachmentModificationTime);
+ callback(item, { mtime: mtime, hash: hash });
});
}
@@ -324,12 +357,11 @@ Zotero.Sync.Storage.Session.WebDAV.prototype.downloadFile = function (request) {
}
try {
- var syncModTime = Zotero.Date.toUnixTimestamp(mdate);
+ var syncModTime = mdate.getTime();
// Skip download if local file exists and matches mod time
var file = item.getFile();
- if (file && file.exists()
- && syncModTime == Math.round(file.lastModifiedTime / 1000)) {
+ if (file && file.exists() && syncModTime == file.lastModifiedTime) {
Zotero.debug("File mod time matches remote file -- skipping download");
Zotero.DB.beginTransaction();
@@ -461,21 +493,35 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._processUploadFile = function (data
!= Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
if (mdate) {
// Remote prop time
- var mtime = Zotero.Date.toUnixTimestamp(mdate);
+ var mtime = mdate.getTime();
+
// Local file time
var fmtime = item.attachmentModificationTime;
+ var same = false;
+ if (fmtime == mtime) {
+ same = true;
+ Zotero.debug("File mod time matches remote file -- skipping upload");
+ }
+ // Allow floored timestamps for filesystems that don't support
+ // millisecond precision (e.g., HFS+)
+ else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
+ same = true;
+ Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+ + "-- skipping upload");
+ }
// Allow timestamp to be exactly one hour off to get around
// time zone issues -- there may be a proper way to fix this
- if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) {
- if (fmtime == mtime) {
- Zotero.debug("File mod time matches remote file -- skipping upload");
- }
- else {
- Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
- + "-- assuming time zone issue and skipping upload");
- }
-
+ else if (Math.abs(fmtime - mtime) == 3600000
+ // And check with one-second precision as well
+ || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+ || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+ same = true;
+ Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ + "-- assuming time zone issue and skipping upload");
+ }
+
+ if (same) {
Zotero.DB.beginTransaction();
var syncState = Zotero.Sync.Storage.getSyncState(item.id);
Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
@@ -592,7 +638,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR
var self = this;
- this._setStorageModificationTime(item, function (item, mtime) {
+ this._setStorageModificationTime(item, function (item, props) {
if (!request.isRunning()) {
Zotero.debug("Upload request '" + request.name
+ "' is no longer running after getting mod time");
@@ -602,10 +648,8 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR
Zotero.DB.beginTransaction();
Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, mtime, true);
-
- var hash = item.attachmentHash;
- Zotero.Sync.Storage.setSyncedHash(item.id, hash);
+ Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true);
+ Zotero.Sync.Storage.setSyncedHash(item.id, props.hash);
Zotero.DB.commitTransaction();
diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js
@@ -130,7 +130,8 @@ Zotero.Sync.Storage.Session.ZFS.prototype._getStorageFileInfo = function (item,
var info = {};
info.hash = req.getResponseHeader('ETag');
info.filename = req.getResponseHeader('X-Zotero-Filename');
- info.mtime = req.getResponseHeader('X-Zotero-Modification-Time');
+ var mtime = req.getResponseHeader('X-Zotero-Modification-Time');
+ info.mtime = parseInt(mtime);
info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes';
Zotero.debug(info);
@@ -184,7 +185,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.downloadFile = function (request) {
var file = item.getFile();
// Skip download if local file exists and matches mod time
if (file && file.exists()) {
- if (syncModTime == Math.round(file.lastModifiedTime / 1000)) {
+ if (syncModTime == file.lastModifiedTime) {
Zotero.debug("File mod time matches remote file -- skipping download");
Zotero.DB.beginTransaction();
@@ -346,20 +347,34 @@ Zotero.Sync.Storage.Session.ZFS.prototype._processUploadFile = function (data) {
if (info) {
// Remote mod time
var mtime = info.mtime;
+
// Local file time
var fmtime = item.attachmentModificationTime;
+ var same = false;
+ if (fmtime == mtime) {
+ same = true;
+ Zotero.debug("File mod time matches remote file -- skipping upload");
+ }
+ // Allow floored timestamps for filesystems that don't support
+ // millisecond precision (e.g., HFS+)
+ else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
+ same = true;
+ Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+ + "-- skipping upload");
+ }
// Allow timestamp to be exactly one hour off to get around
// time zone issues -- there may be a proper way to fix this
- if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) {
- if (fmtime == mtime) {
- Zotero.debug("File mod time matches remote file -- skipping upload");
- }
- else {
- Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
- + "-- assuming time zone issue and skipping upload");
- }
-
+ else if (Math.abs(fmtime - mtime) == 3600000
+ // And check with one-second precision as well
+ || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+ || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+ same = true;
+ Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ + "-- assuming time zone issue and skipping upload");
+ }
+
+ if (same) {
Zotero.debug(Zotero.Sync.Storage.getSyncedModificationTime(item.id));
Zotero.DB.beginTransaction();
@@ -804,7 +819,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.getLastSyncTime = function (callback)
if (req.status == 200) {
var ts = req.responseText;
- var date = new Date(req.responseText * 1000);
+ var date = new Date(ts * 1000);
Zotero.debug("Last successful storage sync was " + date);
self._lastSyncTime = ts;
}
@@ -934,7 +949,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.purgeDeletedStorageFiles = function (c
*/
Zotero.Sync.Storage.Session.ZFS.prototype._getItemURI = function (item) {
var uri = this.rootURI;
- uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1';
+ uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1';
return uri;
}
diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js
@@ -1133,7 +1133,7 @@ Zotero.Sync.Server = new function () {
this.canAutoResetClient = true;
this.manualSyncRequired = false;
this.nextLocalSyncDate = false;
- this.apiVersion = 7;
+ this.apiVersion = 8;
default xml namespace = '';
@@ -2450,7 +2450,8 @@ Zotero.Sync.Server.Data = new function() {
continue;
case 'item':
- var diff = obj.diff(remoteObj, false, ["dateModified"]);
+ var diff = obj.diff(remoteObj, false, ["dateAdded", "dateModified"]);
+ Zotero.debug('Diff:');
Zotero.debug(diff);
if (!diff) {
// Check if creators changed
@@ -2655,6 +2656,11 @@ Zotero.Sync.Server.Data = new function() {
var mtime = xmlNode.@storageModTime.toString();
if (mtime) {
var lk = Zotero.Items.getLibraryKeyHash(obj)
+ // Convert previously used Unix timestamps to ms-based timestamps
+ if (mtime < 10000000000) {
+ Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
+ mtime = mtime * 1000;
+ }
itemStorageModTimes[lk] = parseInt(mtime);
}
}
diff --git a/userdata.sql b/userdata.sql
@@ -1,4 +1,4 @@
--- 70
+-- 71
-- Copyright (c) 2009 Center for History and New Media
-- George Mason University, Fairfax, Virginia, USA
