commit 73f4d28ab26de537efc8e7d1a3e38d2ecdc01e58
parent 6d46b06617f4a943ea27bcb4cc526aedbfc4af46
Author: Dan Stillman <dstillman@zotero.org>
Date: Thu, 29 Oct 2015 03:41:54 -0400
ZFS file sync overhaul for API syncing
This mostly gets ZFS file syncing and file conflict resolution working
with the API sync process. WebDAV will need to be updated separately.
Known issues:
- File sync progress is temporarily gone
- File uploads can result in an unnecessary 412 loop on the next data
sync
- This causes Firefox to crash on one of my computers during tests,
which would be easier to debug if it produced a crash log.
Also:
- Adds httpd.js for use in tests when FakeXMLHttpRequest can't be used
(e.g., saveURI()).
- Adds some additional test data files for attachment tests
Diffstat:
45 files changed, 12152 insertions(+), 6165 deletions(-)
diff --git a/chrome/content/zotero/bindings/attachmentbox.xml b/chrome/content/zotero/bindings/attachmentbox.xml
@@ -57,6 +57,7 @@
Zotero.debug("Setting mode to '" + val + "'");
this.editable = false;
+ this.synchronous = false;
this.displayURL = false;
this.displayFileName = false;
this.clickableLink = false;
@@ -93,6 +94,7 @@
break;
case 'merge':
+ this.synchronous = true;
this.displayURL = true;
this.displayFileName = true;
this.displayAccessed = true;
@@ -102,6 +104,7 @@
break;
case 'mergeedit':
+ this.synchronous = true;
this.editable = true;
this.displayURL = true;
this.displayFileName = true;
@@ -112,6 +115,13 @@
this.displayDateModified = true;
break;
+ case 'filemerge':
+ this.synchronous = true;
+ this.displayURL = true;
+ this.displayFileName = true;
+ this.displayDateModified = true;
+ break;
+
default:
throw ("Invalid mode '" + val + "' in attachmentbox.xml");
}
@@ -123,18 +133,16 @@
</property>
<field name="_item"/>
- <property name="item"
- onget="return this._item;"
- onset="this._item = val; this.refresh();">
- </property>
-
- <!-- .ref is an alias for .item -->
- <property name="ref"
- onget="return this._item;"
- onset="this._item = val; this.refresh();">
+ <property name="item" onget="return this._item;">
+ <setter><![CDATA[
+ if (!(val instanceof Zotero.Item)) {
+ throw new Error("'item' must be a Zotero.Item");
+ }
+ this._item = val;
+ this.refresh();
+ ]]></setter>
</property>
-
<!-- Methods -->
<constructor>
@@ -167,125 +175,122 @@
<method name="refresh">
<body><![CDATA[
- Zotero.spawn(function* () {
- Zotero.debug('Refreshing attachment box');
-
- yield Zotero.Promise.all([this.item.loadItemData(), this.item.loadNote()])
- .tap(() => Zotero.Promise.check(this.item));
-
- var attachmentBox = document.getAnonymousNodes(this)[0];
- var title = this._id('title');
- var fileNameRow = this._id('fileNameRow');
- var urlField = this._id('url');
- var accessed = this._id('accessedRow');
- var pagesRow = this._id('pagesRow');
- var dateModifiedRow = this._id('dateModifiedRow');
- var indexStatusRow = this._id('indexStatusRow');
- var selectButton = this._id('select-button');
-
- // DEBUG: this is annoying -- we really want to use an abstracted
- // version of createValueElement() from itemPane.js
- // (ideally in an XBL binding)
-
- // Wrap title to multiple lines if necessary
- while (title.hasChildNodes()) {
- title.removeChild(title.firstChild);
- }
- var val = this.item.getField('title');
-
- if (typeof val != 'string') {
- val += "";
- }
-
- var firstSpace = val.indexOf(" ");
- // Crop long uninterrupted text
- if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
- title.setAttribute('crop', 'end');
- title.setAttribute('value', val);
- }
- // Create a <description> element, essentially
- else {
- title.removeAttribute('value');
- title.appendChild(document.createTextNode(val));
- }
-
- if (this.editable) {
- title.className = 'zotero-clicky';
-
- // For the time being, use a silly little popup
- title.addEventListener('click', this.editTitle, false);
- }
+ Zotero.debug('Refreshing attachment box');
+
+ var attachmentBox = document.getAnonymousNodes(this)[0];
+ var title = this._id('title');
+ var fileNameRow = this._id('fileNameRow');
+ var urlField = this._id('url');
+ var accessed = this._id('accessedRow');
+ var pagesRow = this._id('pagesRow');
+ var dateModifiedRow = this._id('dateModifiedRow');
+ var indexStatusRow = this._id('indexStatusRow');
+ var selectButton = this._id('select-button');
+
+ // DEBUG: this is annoying -- we really want to use an abstracted
+ // version of createValueElement() from itemPane.js
+ // (ideally in an XBL binding)
+
+ // Wrap title to multiple lines if necessary
+ while (title.hasChildNodes()) {
+ title.removeChild(title.firstChild);
+ }
+ var val = this.item.getField('title');
+
+ if (typeof val != 'string') {
+ val += "";
+ }
+
+ var firstSpace = val.indexOf(" ");
+ // Crop long uninterrupted text
+ if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
+ title.setAttribute('crop', 'end');
+ title.setAttribute('value', val);
+ }
+ // Create a <description> element, essentially
+ else {
+ title.removeAttribute('value');
+ title.appendChild(document.createTextNode(val));
+ }
+
+ if (this.editable) {
+ title.className = 'zotero-clicky';
- var isImportedURL = this.item.attachmentLinkMode ==
- Zotero.Attachments.LINK_MODE_IMPORTED_URL;
+ // For the time being, use a silly little popup
+ title.addEventListener('click', this.editTitle, false);
+ }
+
+ var isImportedURL = this.item.attachmentLinkMode ==
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL;
+
+ // Metadata for URL's
+ if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
+ || isImportedURL) {
- // Metadata for URL's
- if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
- || isImportedURL) {
-
- // URL
- if (this.displayURL) {
- var urlSpec = this.item.getField('url');
- urlField.setAttribute('value', urlSpec);
- urlField.setAttribute('hidden', false);
- if (this.clickableLink) {
- urlField.onclick = function (event) {
- ZoteroPane_Local.loadURI(this.value, event)
- };
- urlField.className = 'zotero-text-link';
- }
- else {
- urlField.className = '';
- }
- urlField.hidden = false;
- }
- else {
- urlField.hidden = true;
- }
-
- // Access date
- if (this.displayAccessed) {
- this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
- + Zotero.getString('punctuation.colon');
- this._id("accessed").value = Zotero.Date.sqlToDate(
- this.item.getField('accessDate'), true
- ).toLocaleString();
- accessed.hidden = false;
+ // URL
+ if (this.displayURL) {
+ var urlSpec = this.item.getField('url');
+ urlField.setAttribute('value', urlSpec);
+ urlField.setAttribute('hidden', false);
+ if (this.clickableLink) {
+ urlField.onclick = function (event) {
+ ZoteroPane_Local.loadURI(this.value, event)
+ };
+ urlField.className = 'zotero-text-link';
}
else {
- accessed.hidden = true;
+ urlField.className = '';
}
+ urlField.hidden = false;
}
- // Metadata for files
else {
urlField.hidden = true;
+ }
+
+ // Access date
+ if (this.displayAccessed) {
+ this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
+ + Zotero.getString('punctuation.colon');
+ this._id("accessed").value = Zotero.Date.sqlToDate(
+ this.item.getField('accessDate'), true
+ ).toLocaleString();
+ accessed.hidden = false;
+ }
+ else {
accessed.hidden = true;
}
+ }
+ // Metadata for files
+ else {
+ urlField.hidden = true;
+ accessed.hidden = true;
+ }
+
+ if (this.item.attachmentLinkMode
+ != Zotero.Attachments.LINK_MODE_LINKED_URL
+ && this.displayFileName) {
+ var fileName = this.item.attachmentFilename;
- if (this.item.attachmentLinkMode
- != Zotero.Attachments.LINK_MODE_LINKED_URL
- && this.displayFileName) {
- var fileName = this.item.getFilename();
-
- if (fileName) {
- this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename')
- + Zotero.getString('punctuation.colon');
- this._id("fileName").value = fileName;
- fileNameRow.hidden = false;
- }
- else {
- fileNameRow.hidden = true;
- }
+ if (fileName) {
+ this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename')
+ + Zotero.getString('punctuation.colon');
+ this._id("fileName").value = fileName;
+ fileNameRow.hidden = false;
}
else {
fileNameRow.hidden = true;
}
-
- // Page count
- if (this.displayPages) {
- var pages = yield Zotero.Fulltext.getPages(this.item.id)
- .tap(() => Zotero.Promise.check(this.item));
- var pages = pages ? pages.total : null;
+ }
+ else {
+ fileNameRow.hidden = true;
+ }
+
+ // Page count
+ if (this.displayPages) {
+ Zotero.Fulltext.getPages(this.item.id)
+ .tap(() => Zotero.Promise.check(this.item))
+ .then(function (pages) {
+ pages = pages ? pages.total : null;
if (pages) {
this._id("pages-label").value = Zotero.getString('itemFields.pages')
+ Zotero.getString('punctuation.colon');
@@ -295,77 +300,85 @@
else {
pagesRow.hidden = true;
}
- }
- else {
- pagesRow.hidden = true;
- }
-
- if (this.displayDateModified) {
- this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified')
- + Zotero.getString('punctuation.colon');
- var mtime = yield this.item.attachmentModificationTime
- .tap(() => Zotero.Promise.check(this.item));
- if (mtime) {
- this._id("dateModified").value = new Date(mtime).toLocaleString();
- }
- // Use the item's mod time as a backup (e.g., when sync
- // passes in the mod time for the nonexistent remote file)
- else {
- this._id("dateModified").value = Zotero.Date.sqlToDate(
- this.item.getField('dateModified'), true
- ).toLocaleString();
- }
+ });
+ }
+ else {
+ pagesRow.hidden = true;
+ }
+
+ if (this.displayDateModified) {
+ this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified')
+ + Zotero.getString('punctuation.colon');
+ // Conflict resolution uses a modal window, so promises won't work, but
+ // the sync process passes in the file mod time as dateModified
+ if (this.synchronous) {
+ this._id("dateModified").value = Zotero.Date.sqlToDate(
+ this.item.getField('dateModified'), true
+ ).toLocaleString();
dateModifiedRow.hidden = false;
}
else {
- dateModifiedRow.hidden = true;
+ this.item.attachmentModificationTime
+ .tap(() => Zotero.Promise.check(this._id))
+ .then(function (mtime) {
+ if (!this._id) return;
+ if (mtime) {
+ this._id("dateModified").value = new Date(mtime).toLocaleString();
+ }
+ dateModifiedRow.hidden = false;
+ });
}
-
- // Full-text index information
- if (this.displayIndexed) {
- yield this.updateItemIndexedState()
- .tap(() => Zotero.Promise.check(this.item));
+ }
+ else {
+ dateModifiedRow.hidden = true;
+ }
+
+ // Full-text index information
+ if (this.displayIndexed) {
+ this.updateItemIndexedState()
+ .tap(() => Zotero.Promise.check(this.item))
+ .then(function () {
indexStatusRow.hidden = false;
- }
- else {
- indexStatusRow.hidden = true;
- }
-
- // Note editor
- var noteEditor = this._id('attachment-note-editor');
- if (this.displayNote) {
- if (this.displayNoteIfEmpty || this.item.getNote() != '') {
- Zotero.debug("setting links on top");
- noteEditor.linksOnTop = true;
- noteEditor.hidden = false;
-
- // Don't make note editable (at least for now)
- if (this.mode == 'merge' || this.mode == 'mergeedit') {
- noteEditor.mode = 'merge';
- noteEditor.displayButton = false;
- }
- else {
- noteEditor.mode = this.mode;
- }
- noteEditor.parent = null;
- noteEditor.item = this.item;
+ });
+ }
+ else {
+ indexStatusRow.hidden = true;
+ }
+
+ // Note editor
+ var noteEditor = this._id('attachment-note-editor');
+ if (this.displayNote) {
+ if (this.displayNoteIfEmpty || this.item.getNote() != '') {
+ Zotero.debug("setting links on top");
+ noteEditor.linksOnTop = true;
+ noteEditor.hidden = false;
+
+ // Don't make note editable (at least for now)
+ if (this.mode == 'merge' || this.mode == 'mergeedit') {
+ noteEditor.mode = 'merge';
+ noteEditor.displayButton = false;
}
+ else {
+ noteEditor.mode = this.mode;
+ }
+ noteEditor.parent = null;
+ noteEditor.item = this.item;
}
- else {
- noteEditor.hidden = true;
- }
-
+ }
+ else {
+ noteEditor.hidden = true;
+ }
- if (this.displayButton) {
- selectButton.label = this.buttonCaption;
- selectButton.hidden = false;
- selectButton.setAttribute('oncommand',
- 'document.getBindingParent(this).clickHandler(this)');
- }
- else {
- selectButton.hidden = true;
- }
- }, this);
+
+ if (this.displayButton) {
+ selectButton.label = this.buttonCaption;
+ selectButton.hidden = false;
+ selectButton.setAttribute('oncommand',
+ 'document.getBindingParent(this).clickHandler(this)');
+ }
+ else {
+ selectButton.hidden = true;
+ }
]]></body>
</method>
diff --git a/chrome/content/zotero/bindings/itembox.xml b/chrome/content/zotero/bindings/itembox.xml
@@ -71,6 +71,7 @@
switch (val) {
case 'view':
+ case 'merge':
break;
case 'edit':
@@ -99,10 +100,9 @@
<field name="_item"/>
<property name="item" onget="return this._item;">
- <setter>
- <![CDATA[
+ <setter><![CDATA[
if (!(val instanceof Zotero.Item)) {
- throw ("<zoteroitembox>.item must be a Zotero.Item");
+ throw new Error("'item' must be a Zotero.Item");
}
// When changing items, reset truncation of creator list
@@ -112,8 +112,7 @@
this._item = val;
this.refresh();
- ]]>
- </setter>
+ ]]></setter>
</property>
<!-- .ref is an alias for .item -->
diff --git a/chrome/content/zotero/bindings/merge.xml b/chrome/content/zotero/bindings/merge.xml
@@ -99,9 +99,11 @@
}
// Check for note or attachment
- this.type = this._getTypeFromObject(
- this._data.left.deleted ? this._data.right : this._data.left
- );
+ if (!this.type) {
+ this.type = this._getTypeFromObject(
+ this._data.left.deleted ? this._data.right : this._data.left
+ );
+ }
var showButton = this.type != 'item';
@@ -109,7 +111,6 @@
this._rightpane.showButton = showButton;
this._leftpane.data = this._data.left;
this._rightpane.data = this._data.right;
- this._mergepane.type = this.type;
this._mergepane.data = this._data.merge;
if (this._data.selected == 'left') {
@@ -313,6 +314,7 @@
break;
case 'attachment':
+ case 'file':
elementName = 'zoteroattachmentbox';
break;
@@ -320,13 +322,8 @@
elementName = 'zoteronoteeditor';
break;
- case 'file':
- elementName = 'zoterostoragefilebox';
- break;
-
default:
- throw ("Object type '" + this.type
- + "' not supported in <zoteromergepane>.ref");
+ throw new Error("Object type '" + this.type + "' not supported");
}
var objbox = document.createElement(elementName);
@@ -342,8 +339,7 @@
objbox.setAttribute("anonid", "objectbox");
objbox.setAttribute("flex", "1");
-
- objbox.mode = 'view';
+ objbox.mode = this.type == 'file' ? 'filemerge' : 'merge';
var button = this._id('choose-button');
if (this.showButton) {
@@ -363,7 +359,7 @@
// Create item from JSON for metadata box
var item = new Zotero.Item(val.itemType);
item.fromJSON(val);
- objbox.ref = item;
+ objbox.item = item;
]]>
</setter>
</property>
diff --git a/chrome/content/zotero/bindings/noteeditor.xml b/chrome/content/zotero/bindings/noteeditor.xml
@@ -64,6 +64,7 @@
switch (val) {
case 'view':
+ case 'merge':
break;
case 'edit':
diff --git a/chrome/content/zotero/bindings/relatedbox.xml b/chrome/content/zotero/bindings/relatedbox.xml
@@ -109,6 +109,7 @@
]]>
</destructor>
+ <!-- TODO: Asyncify -->
<method name="notify">
<parameter name="event"/>
<parameter name="type"/>
diff --git a/chrome/content/zotero/merge.js b/chrome/content/zotero/merge.js
@@ -47,13 +47,20 @@ var Zotero_Merge_Window = new function () {
_wizard.getButton('cancel').setAttribute('label', Zotero.getString('sync.cancel'));
- _io = window.arguments[0].wrappedJSObject;
+ _io = window.arguments[0];
+ // Not totally clear when this is necessary
+ if (window.arguments[0].wrappedJSObject) {
+ _io = window.arguments[0].wrappedJSObject;
+ }
_conflicts = _io.dataIn.conflicts;
if (!_conflicts.length) {
// TODO: handle no conflicts
return;
}
+ if (_io.dataIn.type) {
+ _mergeGroup.type = _io.dataIn.type;
+ }
_mergeGroup.leftCaption = _io.dataIn.captions[0];
_mergeGroup.rightCaption = _io.dataIn.captions[1];
_mergeGroup.mergeCaption = _io.dataIn.captions[2];
@@ -240,7 +247,7 @@ var Zotero_Merge_Window = new function () {
}
// Apply changes from each side and pick most recent version for conflicting fields
var mergeInfo = {
- data: {}
+ data: {}
};
Object.assign(mergeInfo.data, _conflicts[pos].left)
Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].changes);
@@ -251,7 +258,9 @@ var Zotero_Merge_Window = new function () {
else {
var side = 1;
}
- Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side]));
+ Zotero.DataObjectUtilities.applyChanges(
+ mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side])
+ );
mergeInfo.selected = side ? 'right' : 'left';
return mergeInfo;
}
@@ -284,13 +293,22 @@ var Zotero_Merge_Window = new function () {
function _updateResolveAllCheckbox() {
- if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
- var label = 'resolveAllRemoteFields';
+ if (_mergeGroup.type == 'file') {
+ if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
+ var label = 'resolveAllRemote';
+ }
+ else {
+ var label = 'resolveAllLocal';
+ }
}
else {
- var label = 'resolveAllLocalFields';
+ if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
+ var label = 'resolveAllRemoteFields';
+ }
+ else {
+ var label = 'resolveAllLocalFields';
+ }
}
- // TODO: files
_resolveAllCheckbox.label = Zotero.getString('sync.conflict.' + label);
}
diff --git a/chrome/content/zotero/xpcom/data/item.js b/chrome/content/zotero/xpcom/data/item.js
@@ -50,7 +50,6 @@ Zotero.Item = function(itemTypeOrID) {
this._attachmentLinkMode = null;
this._attachmentContentType = null;
this._attachmentPath = null;
- this._attachmentSyncState = null;
// loadCreators
this._creators = [];
@@ -1453,14 +1452,13 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
if (this._changed.attachmentData) {
let sql = "REPLACE INTO itemAttachments (itemID, parentItemID, linkMode, "
- + "contentType, charsetID, path, syncState) VALUES (?,?,?,?,?,?,?)";
+ + "contentType, charsetID, path) VALUES (?,?,?,?,?,?)";
let linkMode = this.attachmentLinkMode;
let contentType = this.attachmentContentType;
let charsetID = this.attachmentCharset
? Zotero.CharacterSets.getID(this.attachmentCharset)
: null;
let path = this.attachmentPath;
- let syncState = this.attachmentSyncState;
if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE && libraryType != 'user') {
throw new Error("Linked files can only be added to user library");
@@ -1472,8 +1470,7 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
{ int: linkMode },
contentType ? { string: contentType } : null,
charsetID ? { int: charsetID } : null,
- path ? { string: path } : null,
- syncState ? { int: syncState } : 0
+ path ? { string: path } : null
];
yield Zotero.DB.queryAsync(sql, params);
@@ -2295,8 +2292,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
yield this.relinkAttachmentFile(destPath);
yield Zotero.DB.executeTransaction(function* () {
- yield Zotero.Sync.Storage.setSyncedHash(this.id, null, false);
- yield Zotero.Sync.Storage.setSyncState(this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD);
+ yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
+ );
}.bind(this));
return true;
@@ -2317,11 +2316,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
/**
* @param {string} path File path
- * @param {Boolean} [skipItemUpdate] Don't update attachment item mod time,
- * so that item doesn't sync. Used when a file
- * needs to be renamed to be accessible but the
- * user doesn't have access to modify the
- * attachment metadata
+ * @param {Boolean} [skipItemUpdate] Don't update attachment item mod time, so that item doesn't
+ * sync. Used when a file needs to be renamed to be accessible but the user doesn't have
+ * access to modify the attachment metadata. This also allows a save when the library is
+ * read-only.
*/
Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function* (path, skipItemUpdate) {
if (path instanceof Components.interfaces.nsIFile) {
@@ -2382,7 +2380,8 @@ Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function*
yield this.saveTx({
skipDateModifiedUpdate: true,
- skipClientDateModifiedUpdate: skipItemUpdate
+ skipClientDateModifiedUpdate: skipItemUpdate,
+ skipEditCheck: skipItemUpdate
});
return true;
@@ -3606,9 +3605,6 @@ Zotero.Item.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, ski
if (this.attachmentPath) {
newItem.attachmentPath = this.attachmentPath;
}
- if (this.attachmentSyncState) {
- newItem.attachmentSyncState = this.attachmentSyncState;
- }
}
}
}
diff --git a/chrome/content/zotero/xpcom/data/items.js b/chrome/content/zotero/xpcom/data/items.js
@@ -84,8 +84,7 @@ Zotero.Items = function() {
attachmentCharset: "CS.charset AS attachmentCharset",
attachmentLinkMode: "IA.linkMode AS attachmentLinkMode",
attachmentContentType: "IA.contentType AS attachmentContentType",
- attachmentPath: "IA.path AS attachmentPath",
- attachmentSyncState: "IA.syncState AS attachmentSyncState"
+ attachmentPath: "IA.path AS attachmentPath"
};
}
}, {lazy: true});
diff --git a/chrome/content/zotero/xpcom/file.js b/chrome/content/zotero/xpcom/file.js
@@ -48,7 +48,7 @@ Zotero.File = new function(){
else if (pathOrFile instanceof Ci.nsIFile) {
return pathOrFile;
}
- throw new Error('Unexpected value provided to Zotero.File.pathToFile() (' + pathOrFile + ')');
+ throw new Error("Unexpected value '" + pathOrFile + "'");
}
@@ -348,7 +348,7 @@ Zotero.File = new function(){
* @param {String} [charset] - The character set; defaults to UTF-8
* @return {Promise} - A promise that is resolved when the file has been written
*/
- this.putContentsAsync = function putContentsAsync(path, data, charset) {
+ this.putContentsAsync = function (path, data, charset) {
if (path instanceof Ci.nsIFile) {
path = path.path;
}
@@ -424,18 +424,17 @@ Zotero.File = new function(){
* iterator when done
*
* The DirectoryInterator is passed as the first parameter to the generator.
- * A StopIteration error will be caught automatically.
*
* Zotero.File.iterateDirectory(path, function* (iterator) {
* while (true) {
* var entry = yield iterator.next();
* [...]
* }
- * }).done()
+ * })
*
* @return {Promise}
*/
- this.iterateDirectory = function iterateDirectory(path, generator) {
+ this.iterateDirectory = function (path, generator) {
var iterator = new OS.File.DirectoryIterator(path);
return Zotero.Promise.coroutine(generator)(iterator)
.catch(function (e) {
@@ -470,6 +469,8 @@ Zotero.File = new function(){
this.createShortened = function (file, type, mode, maxBytes) {
+ file = this.pathToFile(file);
+
if (!maxBytes) {
maxBytes = 255;
}
@@ -575,6 +576,8 @@ Zotero.File = new function(){
}
break;
}
+
+ return file.leafName;
}
@@ -902,29 +905,28 @@ Zotero.File = new function(){
this.checkFileAccessError = function (e, file, operation) {
+ var str = 'file.accessError.';
if (file) {
- var str = Zotero.getString('file.accessError.theFile', file.path);
+ str += 'theFile'
}
else {
- var str = Zotero.getString('file.accessError.aFile');
+ str += 'aFile'
}
+ str += 'CannotBe';
switch (operation) {
case 'create':
- var opWord = Zotero.getString('file.accessError.created');
- break;
-
- case 'update':
- var opWord = Zotero.getString('file.accessError.updated');
+ str += 'Created';
break;
case 'delete':
- var opWord = Zotero.getString('file.accessError.deleted');
+ str += 'Deleted';
break;
default:
- var opWord = Zotero.getString('file.accessError.updated');
+ str += 'Updated';
}
+ str = Zotero.getString(str, file.path ? file.path : undefined);
Zotero.debug(file.path);
Zotero.debug(e, 1);
@@ -962,4 +964,64 @@ Zotero.File = new function(){
throw (e);
}
+
+
+ this.checkPathAccessError = function (e, path, operation) {
+ var str = 'file.accessError.';
+ if (path) {
+ str += 'theFile'
+ }
+ else {
+ str += 'aFile'
+ }
+ str += 'CannotBe';
+
+ switch (operation) {
+ case 'create':
+ str += 'Created';
+ break;
+
+ case 'delete':
+ str += 'Deleted';
+ break;
+
+ default:
+ str += 'Updated';
+ }
+ str = Zotero.getString(str, path ? path : undefined);
+
+ Zotero.debug(path);
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+
+ // TODO: Check for specific errors?
+ if (e instanceof OS.File.Error) {
+ let checkFileWindows = Zotero.getString('file.accessError.message.windows');
+ let checkFileOther = Zotero.getString('file.accessError.message.other');
+ var msg = str + "\n\n"
+ + (Zotero.isWin ? checkFileWindows : checkFileOther)
+ + "\n\n"
+ + Zotero.getString('file.accessError.restart');
+
+ var e = new Zotero.Error(
+ msg,
+ 0,
+ {
+ dialogButtonText: Zotero.getString('file.accessError.showParentDir'),
+ dialogButtonCallback: function () {
+ try {
+ file.parent.QueryInterface(Components.interfaces.nsILocalFile);
+ file.parent.reveal();
+ }
+ // Unsupported on some platforms
+ catch (e2) {
+ Zotero.launchFile(file.parent);
+ }
+ }
+ }
+ );
+ }
+
+ throw e;
+ }
}
diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js
@@ -33,6 +33,7 @@ Zotero.Sync.Storage = new function () {
this.SYNC_STATE_IN_SYNC = 2;
this.SYNC_STATE_FORCE_UPLOAD = 3;
this.SYNC_STATE_FORCE_DOWNLOAD = 4;
+ this.SYNC_STATE_IN_CONFLICT = 5;
this.SUCCESS = 1;
this.ERROR_NO_URL = -1;
@@ -57,14 +58,11 @@ Zotero.Sync.Storage = new function () {
this.__defineGetter__("defaultError", function () Zotero.getString('sync.storage.error.default', Zotero.appName));
this.__defineGetter__("defaultErrorRestart", function () Zotero.getString('sync.storage.error.defaultRestart', Zotero.appName));
+ var _itemDownloadPercentages = {};
+
//
// Public properties
//
-
-
- this.__defineGetter__("syncInProgress", function () _syncInProgress);
- this.__defineGetter__("updatesInProgress", function () _updatesInProgress);
-
this.compressionTracker = {
compressed: 0,
uncompressed: 0,
@@ -75,539 +73,6 @@ Zotero.Sync.Storage = new function () {
}
}
- Zotero.Notifier.registerObserver(this, ['file']);
-
-
- //
- // Private properties
- //
- var _maxCheckAgeInSeconds = 10800; // maximum age for upload modification check (3 hours)
- var _syncInProgress;
- var _updatesInProgress;
- var _itemDownloadPercentages = {};
- var _uploadCheckFiles = [];
- var _lastFullFileCheck = {};
-
-
- this.sync = function (options) {
- if (options.libraries) {
- Zotero.debug("Starting file sync for libraries " + options.libraries);
- }
- else {
- Zotero.debug("Starting file sync");
- }
-
- var self = this;
-
- var libraryModes = {};
- var librarySyncTimes = {};
-
- // Get personal library file sync mode
- return Zotero.Promise.try(function () {
- // TODO: Make sure modes are active
-
- if (options.libraries && options.libraries.indexOf(0) == -1) {
- return;
- }
-
- if (Zotero.Sync.Storage.ZFS.includeUserFiles) {
- libraryModes[0] = Zotero.Sync.Storage.ZFS;
- }
- else if (Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- libraryModes[0] = Zotero.Sync.Storage.WebDAV;
- }
- })
- .then(function () {
- // Get group library file sync modes
- if (Zotero.Sync.Storage.ZFS.includeGroupFiles) {
- var groups = Zotero.Groups.getAll();
- for each(var group in groups) {
- if (options.libraries && options.libraries.indexOf(group.libraryID) == -1) {
- continue;
- }
- // TODO: if library file syncing enabled
- libraryModes[group.libraryID] = Zotero.Sync.Storage.ZFS;
- }
- }
-
- // Cache auth credentials for each mode
- var modes = [];
- var promises = [];
- for each(var mode in libraryModes) {
- if (modes.indexOf(mode) == -1) {
- modes.push(mode);
-
- // Try to verify WebDAV server first if it hasn't been
- if (mode == Zotero.Sync.Storage.WebDAV
- && !Zotero.Sync.Storage.WebDAV.verified) {
- Zotero.debug("WebDAV file sync is not active");
- var promise = Zotero.Sync.Storage.checkServerPromise(Zotero.Sync.Storage.WebDAV)
- .then(function () {
- return mode.cacheCredentials();
- });
- }
- else {
- var promise = mode.cacheCredentials();
- }
- promises.push(Zotero.Promise.allSettled([mode, promise]));
- }
- }
-
- return Zotero.Promise.all(promises)
- // Get library last-sync times
- .then(function (cacheCredentialsPromises) {
- var promises = [];
-
- // Mark WebDAV verification failure as user library error.
- // We ignore credentials-caching errors for ZFS and let the
- // later requests fail.
- cacheCredentialsPromises.forEach(function (results) {
- let mode = results[0].value;
- if (mode == Zotero.Sync.Storage.WebDAV) {
- if (results[1].state == "rejected") {
- promises.push(Zotero.Promise.allSettled(
- [0, Zotero.Promise.reject(results[1].reason)]
- ));
- // Skip further syncing of user library
- delete libraryModes[0];
- }
- }
- });
-
- for (var libraryID in libraryModes) {
- libraryID = parseInt(libraryID);
-
- // Get the last sync time for each library
- if (self.downloadOnSync(libraryID)) {
- promises.push(Zotero.Promise.allSettled(
- [libraryID, libraryModes[libraryID].getLastSyncTime(libraryID)]
- ));
- }
- // If download-as-needed, we don't need the last sync time
- else {
- promises.push(Zotero.Promise.allSettled([libraryID, null]));
- }
- }
- return Zotero.Promise.all(promises);
- });
- })
- .then(function (promises) {
- if (!promises.length) {
- Zotero.debug("No libraries are active for file sync");
- return [];
- }
-
- var libraryQueues = [];
-
- // Get the libraries we have sync times for
- promises.forEach(function (results) {
- let libraryID = results[0].value;
- let lastSyncTime = results[1].value;
- if (results[1].state == "fulfilled") {
- librarySyncTimes[libraryID] = lastSyncTime;
- }
- else {
- Zotero.debug(lastSyncTime.reason);
- Components.utils.reportError(lastSyncTime.reason);
- // Pass rejected promise through
- libraryQueues.push(results);
- }
- });
-
- // Check for updated files to upload in each library
- var promises = [];
- for (let libraryID in librarySyncTimes) {
- let promise;
- libraryID = parseInt(libraryID);
-
- if (!Zotero.Libraries.isFilesEditable(libraryID)) {
- Zotero.debug("No file editing access -- skipping file "
- + "modification check for library " + libraryID);
- continue;
- }
- // If this is a background sync, it's not the first sync of
- // the session, the library has had at least one full check
- // this session, and it's been less than _maxCheckAgeInSeconds
- // since the last full check of this library, check only files
- // that were previously modified or opened recently
- else if (options.background
- && !options.firstInSession
- && _lastFullFileCheck[libraryID]
- && (_lastFullFileCheck[libraryID] + (_maxCheckAgeInSeconds * 1000))
- > new Date().getTime()) {
- let itemIDs = _getFilesToCheck(libraryID);
- promise = self.checkForUpdatedFiles(libraryID, itemIDs);
- }
- // Otherwise check all files in the library
- else {
- _lastFullFileCheck[libraryID] = new Date().getTime();
- promise = self.checkForUpdatedFiles(libraryID);
- }
- promises.push(promise);
- }
- return Zotero.Promise.all(promises)
- .then(function () {
- // Queue files to download and upload from each library
- for (let libraryID in librarySyncTimes) {
- libraryID = parseInt(libraryID);
-
- var downloadAll = self.downloadOnSync(libraryID);
-
- // Forced downloads happen even in on-demand mode
- var sql = "SELECT COUNT(*) FROM items "
- + "JOIN itemAttachments USING (itemID) "
- + "WHERE libraryID=? AND syncState=?";
- var downloadForced = !!Zotero.DB.valueQuery(
- sql,
- [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]
- );
-
- // If we don't have any forced downloads, we can skip
- // downloads if the last sync time hasn't changed
- // or doesn't exist on the server (meaning there are no files)
- if (downloadAll && !downloadForced) {
- let lastSyncTime = librarySyncTimes[libraryID];
- if (lastSyncTime) {
- var version = self.getStoredLastSyncTime(
- libraryModes[libraryID], libraryID
- );
- if (version == lastSyncTime) {
- Zotero.debug("Last " + libraryModes[libraryID].name
- + " sync id hasn't changed for library "
- + libraryID + " -- skipping file downloads");
- downloadAll = false;
- }
- }
- else {
- Zotero.debug("No last " + libraryModes[libraryID].name
- + " sync time for library " + libraryID
- + " -- skipping file downloads");
- downloadAll = false;
- }
- }
-
- if (downloadAll || downloadForced) {
- for each(var itemID in _getFilesToDownload(libraryID, !downloadAll)) {
- var item = Zotero.Items.get(itemID);
- self.queueItem(item);
- }
- }
-
- // Get files to upload
- if (Zotero.Libraries.isFilesEditable(libraryID)) {
- for each(var itemID in _getFilesToUpload(libraryID)) {
- var item = Zotero.Items.get(itemID);
- self.queueItem(item);
- }
- }
- else {
- Zotero.debug("No file editing access -- skipping file uploads for library " + libraryID);
- }
- }
-
- // Start queues for each library
- for (let libraryID in librarySyncTimes) {
- libraryID = parseInt(libraryID);
- libraryQueues.push(Zotero.Promise.allSettled(
- [libraryID, Zotero.Sync.Storage.QueueManager.start(libraryID)]
- ));
- }
-
- // The promise is done when all libraries are done
- return Zotero.Promise.all(libraryQueues);
- });
- })
- .then(function (promises) {
- Zotero.debug('Queue manager is finished');
-
- var changedLibraries = [];
- var finalPromises = [];
-
- promises.forEach(function (results) {
- var libraryID = results[0].value;
- var libraryQueues = results[1].value;
-
- if (results[1].state == "fulfilled") {
- libraryQueues.forEach(function (queuePromise) {
- if (queueZotero.Promise.isFulfilled()) {
- let result = queueZotero.Promise.value();
- Zotero.debug("File " + result.type + " sync finished "
- + "for library " + libraryID);
- if (result.localChanges) {
- changedLibraries.push(libraryID);
- }
- finalPromises.push(Zotero.Promise.allSettled([
- libraryID,
- libraryModes[libraryID].setLastSyncTime(
- libraryID,
- result.remoteChanges ? false : librarySyncTimes[libraryID]
- )
- ]));
- }
- else {
- let e = queueZotero.Promise.reason();
- Zotero.debug("File " + e.type + " sync failed "
- + "for library " + libraryID);
- finalPromises.push(Zotero.Promise.allSettled(
- [libraryID, Zotero.Promise.reject(e)]
- ));
- }
- });
- }
- else {
- Zotero.debug("File sync failed for library " + libraryID);
- finalPromises.push([libraryID, libraryQueues]);
- }
-
- // If WebDAV sync enabled, purge deleted and orphaned files
- if (libraryID == Zotero.Libraries.userLibraryID
- && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles()
- .then(function () {
- return Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles();
- })
- .catch(function (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- });
- }
- });
-
- Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
- .catch(function (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- });
-
- if (promises.length && !changedLibraries.length) {
- Zotero.debug("No local changes made during file sync");
- }
-
- return Zotero.Promise.all(finalPromises)
- .then(function (promises) {
- var results = {
- changesMade: !!changedLibraries.length,
- errors: []
- };
-
- promises.forEach(function (promiseResults) {
- var libraryID = promiseResults[0].value;
- if (promiseResults[1].state == "rejected") {
- let e = promiseResults[1].reason;
- if (typeof e == 'string') {
- e = new Error(e);
- }
- e.libraryID = libraryID;
- results.errors.push(e);
- }
- });
-
- return results;
- });
- });
- }
-
-
- //
- // Public methods
- //
- this.queueItem = function (item, highPriority) {
- var library = item.libraryID;
- if (libraryID) {
- var mode = Zotero.Sync.Storage.ZFS;
- }
- else {
- var mode = Zotero.Sync.Storage.ZFS.includeUserFiles
- ? Zotero.Sync.Storage.ZFS : Zotero.Sync.Storage.WebDAV;
- }
- switch (Zotero.Sync.Storage.getSyncState(item.id)) {
- case this.SYNC_STATE_TO_DOWNLOAD:
- case this.SYNC_STATE_FORCE_DOWNLOAD:
- var queue = 'download';
- var callbacks = {
- onStart: function (request) {
- return mode.downloadFile(request);
- }
- };
- break;
-
- case this.SYNC_STATE_TO_UPLOAD:
- case this.SYNC_STATE_FORCE_UPLOAD:
- var queue = 'upload';
- var callbacks = {
- onStart: function (request) {
- return mode.uploadFile(request);
- }
- };
- break;
-
- case false:
- Zotero.debug("Sync state for item " + item.id + " not found", 2);
- return;
- }
-
- var queue = Zotero.Sync.Storage.QueueManager.get(queue, library);
- var request = new Zotero.Sync.Storage.Request(
- item.libraryID + '/' + item.key, callbacks
- );
- if (queue.type == 'upload') {
- try {
- request.setMaxSize(Zotero.Attachments.getTotalFileSize(item));
- }
- // If this fails, ignore it, though we might fail later
- catch (e) {
- // But if the file doesn't exist yet, don't try to upload it
- //
- // This isn't a perfect test, because the file could still be
- // in the process of being downloaded. It'd be better to
- // download files to a temp directory and move them into place.
- if (!item.getFile()) {
- Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping");
- return;
- }
-
- Components.utils.reportError(e);
- Zotero.debug(e, 1);
- }
- }
- queue.addRequest(request, highPriority);
- };
-
-
- this.getStoredLastSyncTime = function (mode, libraryID) {
- var sql = "SELECT version FROM version WHERE schema=?";
- return Zotero.DB.valueQuery(
- sql, "storage_" + mode.name.toLowerCase() + "_" + libraryID
- );
- };
-
-
- this.setStoredLastSyncTime = function (mode, libraryID, time) {
- var sql = "REPLACE INTO version SET version=? WHERE schema=?";
- Zotero.DB.query(
- sql,
- [
- time,
- "storage_" + mode.name.toLowerCase() + "_" + libraryID
- ]
- );
- };
-
-
- /**
- * @param {Integer} itemID
- */
- this.getSyncState = function (itemID) {
- var sql = "SELECT syncState FROM itemAttachments WHERE itemID=?";
- return Zotero.DB.valueQueryAsync(sql, itemID);
- }
-
-
- /**
- * @param {Integer} itemID
- * @param {Integer} syncState Constant from Zotero.Sync.Storage
- */
- this.setSyncState = Zotero.Promise.method(function (itemID, syncState) {
- switch (syncState) {
- case this.SYNC_STATE_TO_UPLOAD:
- case this.SYNC_STATE_TO_DOWNLOAD:
- case this.SYNC_STATE_IN_SYNC:
- case this.SYNC_STATE_FORCE_UPLOAD:
- case this.SYNC_STATE_FORCE_DOWNLOAD:
- break;
-
- default:
- throw new Error("Invalid sync state '" + syncState);
- }
-
- var sql = "UPDATE itemAttachments SET syncState=? WHERE itemID=?";
- return Zotero.DB.valueQueryAsync(sql, [syncState, itemID]);
- });
-
-
- /**
- * @param {Integer} itemID
- * @return {Integer|NULL} Mod time as timestamp in ms,
- * or NULL if never synced
- */
- this.getSyncedModificationTime = function (itemID) {
- var sql = "SELECT storageModTime FROM itemAttachments WHERE itemID=?";
- var mtime = Zotero.DB.valueQuery(sql, itemID);
- if (mtime === false) {
- throw "Item " + itemID + " not found in "
- + "Zotero.Sync.Storage.getSyncedModificationTime()";
- }
- return mtime;
- }
-
-
- /**
- * @param {Integer} itemID
- * @param {Integer} mtime File modification time as
- * timestamp in ms
- * @param {Boolean} [updateItem=FALSE] Update dateModified field of
- * attachment item
- */
- this.setSyncedModificationTime = function (itemID, mtime, updateItem) {
- if (mtime < 0) {
- Components.utils.reportError("Invalid file mod time " + mtime
- + " in Zotero.Storage.setSyncedModificationTime()");
- mtime = 0;
- }
-
- Zotero.DB.beginTransaction();
-
- var sql = "UPDATE itemAttachments SET storageModTime=? WHERE itemID=?";
- Zotero.DB.valueQuery(sql, [mtime, itemID]);
-
- if (updateItem) {
- // Update item date modified so the new mod time will be synced
- var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
- Zotero.DB.query(sql, [Zotero.DB.transactionDateTime, itemID]);
- }
-
- Zotero.DB.commitTransaction();
- }
-
-
- /**
- * @param {Integer} itemID
- * @return {Promise<String|null|false>} - File hash, null if never synced, if false if
- * file doesn't exist
- */
- this.getSyncedHash = Zotero.Promise.coroutine(function* (itemID) {
- var sql = "SELECT storageHash FROM itemAttachments WHERE itemID=?";
- var hash = yield Zotero.DB.valueQueryAsync(sql, itemID);
- if (hash === false) {
- throw new Error("Item " + itemID + " not found");
- }
- return hash;
- })
-
-
- /**
- * @param {Integer} itemID
- * @param {String} hash File hash
- * @param {Boolean} [updateItem=FALSE] Update dateModified field of
- * attachment item
- */
- this.setSyncedHash = Zotero.Promise.coroutine(function* (itemID, hash, updateItem) {
- if (hash !== null && hash.length != 32) {
- throw ("Invalid file hash '" + hash + "' in Zotero.Storage.setSyncedHash()");
- }
-
- Zotero.DB.requireTransaction();
-
- var sql = "UPDATE itemAttachments SET storageHash=? WHERE itemID=?";
- yield Zotero.DB.queryAsync(sql, [hash, itemID]);
-
- if (updateItem) {
- // Update item date modified so the new mod time will be synced
- var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
- yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
- }
- });
-
/**
* Check if modification time of file on disk matches the mod time
@@ -646,518 +111,6 @@ Zotero.Sync.Storage = new function () {
}
- /**
- * @param {Integer|'groups'} [libraryID]
- */
- this.downloadAsNeeded = function (libraryID) {
- // Personal library
- if (!libraryID) {
- return Zotero.Prefs.get('sync.storage.downloadMode.personal') == 'on-demand';
- }
- // Group library (groupID or 'groups')
- else {
- return Zotero.Prefs.get('sync.storage.downloadMode.groups') == 'on-demand';
- }
- }
-
-
- /**
- * @param {Integer|'groups'} [libraryID]
- */
- this.downloadOnSync = function (libraryID) {
- // Personal library
- if (!libraryID) {
- return Zotero.Prefs.get('sync.storage.downloadMode.personal') == 'on-sync';
- }
- // Group library (groupID or 'groups')
- else {
- return Zotero.Prefs.get('sync.storage.downloadMode.groups') == 'on-sync';
- }
- }
-
-
-
- /**
- * Scans local files and marks any that have changed for uploading
- * and any that are missing for downloading
- *
- * @param {Integer} [libraryID]
- * @param {Integer[]} [itemIDs]
- * @param {Object} [itemModTimes] Item mod times indexed by item ids;
- * items with stored mod times
- * that differ from the provided
- * time but file mod times
- * matching the stored time will
- * be marked for download
- * @return {Promise} Promise resolving to TRUE if any items changed state,
- * FALSE otherwise
- */
- this.checkForUpdatedFiles = function (libraryID, itemIDs, itemModTimes) {
- return Zotero.Promise.try(function () {
- libraryID = parseInt(libraryID);
- if (isNaN(libraryID)) {
- libraryID = false;
- }
-
- var msg = "Checking for locally changed attachment files";
-
- var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"]
- .getService(Components.interfaces.nsIMemoryReporterManager);
- memmgr.init();
- //Zotero.debug("Memory usage: " + memmgr.resident);
-
- if (libraryID !== false) {
- if (itemIDs) {
- if (!itemIDs.length) {
- var msg = "No files to check for local changes in library " + libraryID;
- Zotero.debug(msg);
- return false;
- }
- }
- if (itemModTimes) {
- throw new Error("itemModTimes is not allowed when libraryID is set");
- }
-
- msg += " in library " + libraryID;
- }
- else if (itemIDs) {
- throw new Error("libraryID not provided");
- }
- else if (itemModTimes) {
- if (!Object.keys(itemModTimes).length) {
- return false;
- }
- msg += " in download-marking mode";
- }
- else {
- throw new Error("libraryID, itemIDs, or itemModTimes must be provided");
- }
- Zotero.debug(msg);
-
- var changed = false;
-
- if (!itemIDs) {
- itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
- }
-
- // Can only handle a certain number of bound parameters at a time
- var numIDs = itemIDs.length;
- var maxIDs = Zotero.DB.MAX_BOUND_PARAMETERS - 10;
- var done = 0;
- var rows = [];
-
- Zotero.DB.beginTransaction();
-
- do {
- var chunk = itemIDs.splice(0, maxIDs);
- var sql = "SELECT itemID, linkMode, path, storageModTime, storageHash, syncState "
- + "FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE linkMode IN (?,?) AND syncState IN (?,?)";
- var params = [];
- params.push(
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL,
- Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
- Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
- );
- if (libraryID !== false) {
- sql += " AND libraryID=?";
- params.push(libraryID);
- }
- if (chunk.length) {
- sql += " AND itemID IN (" + chunk.map(function () '?').join() + ")";
- params = params.concat(chunk);
- }
- var chunkRows = Zotero.DB.query(sql, params);
- if (chunkRows) {
- rows = rows.concat(chunkRows);
- }
- done += chunk.length;
- }
- while (done < numIDs);
-
- Zotero.DB.commitTransaction();
-
- // If no files, or everything is already marked for download,
- // we don't need to do anything
- if (!rows.length) {
- var msg = "No in-sync or to-upload files found";
- if (libraryID !== false) {
- msg += " in library " + libraryID;
- }
- Zotero.debug(msg);
- return false;
- }
-
- // Index attachment data by item id
- itemIDs = [];
- var attachmentData = {};
- for each(let row in rows) {
- var id = row.itemID;
- itemIDs.push(id);
- attachmentData[id] = {
- linkMode: row.linkMode,
- path: row.path,
- mtime: row.storageModTime,
- hash: row.storageHash,
- state: row.syncState
- };
- }
- rows = null;
-
- var t = new Date();
- var items = Zotero.Items.get(itemIDs);
- var numItems = items.length;
- var updatedStates = {};
-
- let checkItems = function () {
- if (!items.length) return Zotero.Promise.resolve();
-
- //Zotero.debug("Memory usage: " + memmgr.resident);
-
- let item = items.shift();
- let row = attachmentData[item.id];
- let lk = item.libraryKey;
- Zotero.debug("Checking attachment file for item " + lk);
-
- let nsIFile = item.getFile(row, true);
- if (!nsIFile) {
- Zotero.debug("Marking pathless attachment " + lk + " as in-sync");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_IN_SYNC;
- return checkItems();
- }
- let file = null;
- return Zotero.Promise.resolve(OS.File.open(nsIFile.path))
- .then(function (promisedFile) {
- file = promisedFile;
- return file.stat()
- .then(function (info) {
- //Zotero.debug("Memory usage: " + memmgr.resident);
-
- var fmtime = info.lastModificationDate.getTime();
- //Zotero.debug("File modification time for item " + lk + " is " + fmtime);
-
- if (fmtime < 1) {
- Zotero.debug("File mod time " + fmtime + " is less than 1 -- interpreting as 1", 2);
- fmtime = 1;
- }
-
- // If file is already marked for upload, skip check. Even if this
- // is download-marking mode (itemModTimes) and the file was
- // changed remotely, conflicts are checked at upload time, so we
- // don't need to worry about it here.
- if (row.state == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) {
- return;
- }
-
- //Zotero.debug("Stored mtime is " + row.mtime);
- //Zotero.debug("File mtime is " + fmtime);
-
- // Download-marking mode
- if (itemModTimes) {
- Zotero.debug("Remote mod time for item " + lk + " is " + itemModTimes[item.id]);
-
- // Ignore attachments whose stored mod times haven't changed
- if (row.storageModTime == itemModTimes[item.id]) {
- Zotero.debug("Storage mod time (" + row.storageModTime + ") "
- + "hasn't changed for item " + lk);
- return;
- }
-
- Zotero.debug("Marking attachment " + lk + " for download "
- + "(stored mtime: " + itemModTimes[item.id] + ")");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
- }
-
- var mtime = row.mtime;
-
- // If stored time matches file, it hasn't changed locally
- if (mtime == fmtime) {
- return;
- }
-
- // Allow floored timestamps for filesystems that don't support
- // millisecond precision (e.g., HFS+)
- if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
- Zotero.debug("File mod times are within one-second precision "
- + "(" + fmtime + " ≅ " + mtime + ") for " + file.leafName
- + " for item " + lk + " -- ignoring");
- return;
- }
-
- // Allow timestamp to be exactly one hour off to get around
- // time zone issues -- there may be a proper way to fix this
- if (Math.abs(fmtime - mtime) == 3600000
- // And check with one-second precision as well
- || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
- || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
- Zotero.debug("File mod time (" + fmtime + ") is exactly one "
- + "hour off remote file (" + mtime + ") for item " + lk
- + "-- assuming time zone issue and skipping upload");
- return;
- }
-
- // If file hash matches stored hash, only the mod time changed, so skip
- return Zotero.Utilities.Internal.md5Async(file)
- .then(function (fileHash) {
- if (row.hash && row.hash == fileHash) {
- // We have to close the file before modifying it from the main
- // thread (at least on Windows, where assigning lastModifiedTime
- // throws an NS_ERROR_FILE_IS_LOCKED otherwise)
- return Zotero.Promise.resolve(file.close())
- .then(function () {
- Zotero.debug("Mod time didn't match (" + fmtime + "!=" + mtime + ") "
- + "but hash did for " + nsIFile.leafName + " for item " + lk
- + " -- updating file mod time");
- try {
- nsIFile.lastModifiedTime = row.mtime;
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, nsIFile, 'update');
- }
- });
- }
-
- // Mark file for upload
- Zotero.debug("Marking attachment " + lk + " as changed "
- + "(" + mtime + " != " + fmtime + ")");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
- });
- });
- })
- .finally(function () {
- if (file) {
- //Zotero.debug("Closing file for item " + lk);
- file.close();
- }
- })
- .catch(function (e) {
- if (e instanceof OS.File.Error &&
- (e.becauseNoSuchFile
- // This can happen if a path is too long on Windows,
- // e.g. a file is being accessed on a VM through a share
- // (and probably in other cases).
- || (e.winLastError && e.winLastError == 3)
- // Handle long filenames on OS X/Linux
- || (e.unixErrno && e.unixErrno == 63))) {
- Zotero.debug("Marking attachment " + lk + " as missing");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD;
- return;
- }
-
- if (e instanceof OS.File.Error) {
- if (e.becauseClosed) {
- Zotero.debug("File was closed", 2);
- }
- Zotero.debug(e);
- Zotero.debug(e.toString());
- throw new Error("Error for operation '" + e.operation + "' for " + nsIFile.path);
- }
-
- throw e;
- })
- .then(function () {
- return checkItems();
- });
- };
-
- return checkItems()
- .then(function () {
- for (let itemID in updatedStates) {
- Zotero.Sync.Storage.setSyncState(itemID, updatedStates[itemID]);
- changed = true;
- }
-
- if (!changed) {
- Zotero.debug("No synced files have changed locally");
- }
-
- let msg = "Checked " + numItems + " files in ";
- if (libraryID !== false) {
- msg += "library " + libraryID + " in ";
- }
- msg += (new Date() - t) + "ms";
- Zotero.debug(msg);
-
- return changed;
- });
- });
- };
-
-
- /**
- * Download a single file
- *
- * If no queue is active, start one. Otherwise, add to existing queue.
- */
- this.downloadFile = function (item, requestCallbacks) {
- var itemID = item.id;
- var mode = getModeFromLibrary(item.libraryID);
-
- // TODO: verify WebDAV on-demand?
- if (!mode || !mode.verified) {
- Zotero.debug("File syncing is not active for item's library -- skipping download");
- return false;
- }
-
- if (!item.isImportedAttachment()) {
- throw new Error("Not an imported attachment");
- }
-
- if (item.getFile()) {
- Zotero.debug("File already exists -- replacing");
- }
-
- // TODO: start sync icon in cacheCredentials
- return Zotero.Promise.try(function () {
- return mode.cacheCredentials();
- })
- .then(function () {
- // TODO: start sync icon
- var library = item.libraryID;
- var queue = Zotero.Sync.Storage.QueueManager.get(
- 'download', library
- );
-
- if (!requestCallbacks) {
- requestCallbacks = {};
- }
- var onStart = function (request) {
- return mode.downloadFile(request);
- };
- requestCallbacks.onStart = requestCallbacks.onStart
- ? [onStart, requestCallbacks.onStart]
- : onStart;
-
- var request = new Zotero.Sync.Storage.Request(
- library + '/' + item.key, requestCallbacks
- );
-
- queue.addRequest(request, true);
- queue.start();
-
- return request.promise;
- });
- }
-
-
- /**
- * Extract a downloaded file and update the database metadata
- *
- * This is called from Zotero.Sync.Server.StreamListener.onStopRequest()
- *
- * @return {Promise<Object>} data - Promise for object with properties 'request', 'item',
- * 'compressed', 'syncModTime', 'syncHash'
- */
- this.processDownload = Zotero.Promise.coroutine(function* (data) {
- var funcName = "Zotero.Sync.Storage.processDownload()";
-
- if (!data) {
- throw "'data' not set in " + funcName;
- }
-
- if (!data.item) {
- throw "'data.item' not set in " + funcName;
- }
-
- if (!data.syncModTime) {
- throw "'data.syncModTime' not set in " + funcName;
- }
-
- if (!data.compressed && !data.syncHash) {
- throw "'data.syncHash' is required if 'data.compressed' is false in " + funcName;
- }
-
- var item = data.item;
- var syncModTime = data.syncModTime;
- var syncHash = data.syncHash;
-
- // TODO: Test file hash
-
- if (data.compressed) {
- var newFile = yield _processZipDownload(item);
- }
- else {
- var newFile = yield _processDownload(item);
- }
-
- // If |newFile| is set, the file was renamed, so set item filename to that
- // and mark for updated
- var file = item.getFile();
- if (newFile && file.leafName != newFile.leafName) {
- // Bypass library access check
- _updatesInProgress = true;
-
- // If library isn't editable but filename was changed, update
- // database without updating the item's mod time, which would result
- // in a library access error
- if (!Zotero.Items.isEditable(item)) {
- Zotero.debug("File renamed without library access -- "
- + "updating itemAttachments path", 3);
- item.relinkAttachmentFile(newFile, true);
- var useCurrentModTime = false;
- }
- else {
- item.relinkAttachmentFile(newFile);
-
- // TODO: use an integer counter instead of mod time for change detection
- var useCurrentModTime = true;
- }
-
- file = item.getFile();
- _updatesInProgress = false;
- }
- else {
- var useCurrentModTime = false;
- }
-
- if (!file) {
- // This can happen if an HTML snapshot filename was changed and synced
- // elsewhere but the renamed file wasn't synced, so the ZIP doesn't
- // contain a file with the known name
- var missingFile = item.getFile(null, true);
- Components.utils.reportError("File '" + missingFile.leafName
- + "' not found after processing download "
- + item.libraryID + "/" + item.key + " in " + funcName);
- return false;
- }
-
- Zotero.DB.beginTransaction();
-
- //var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //var updateItem = syncState != this.SYNC_STATE_TO_DOWNLOAD;
- var updateItem = false;
-
- try {
- if (useCurrentModTime) {
- file.lastModifiedTime = new Date();
-
- // Reset hash and sync state
- Zotero.Sync.Storage.setSyncedHash(item.id, null);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD);
- this.queueItem(item);
- }
- else {
- file.lastModifiedTime = syncModTime;
- // If hash not provided (e.g., WebDAV), calculate it now
- if (!syncHash) {
- syncHash = item.attachmentHash;
- }
- Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- }
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, file, 'update');
- }
-
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
- Zotero.DB.commitTransaction();
-
- return true;
- });
-
-
this.checkServerPromise = function (mode) {
return mode.checkServer()
.spread(function (uri, status) {
@@ -1277,570 +230,16 @@ Zotero.Sync.Storage = new function () {
}
- this.getItemFromRequestName = function (name) {
- var [libraryID, key] = name.split('/');
- return Zotero.Items.getByLibraryAndKey(libraryID, key);
- }
-
-
- this.notify = function(event, type, ids, extraData) {
- if (event == 'open' && type == 'file') {
- let timestamp = new Date().getTime();
-
- for each(let id in ids) {
- _uploadCheckFiles.push({
- itemID: id,
- timestamp: timestamp
- });
- }
- }
- }
-
-
- //
- // Private methods
- //
- function getModeFromLibrary(libraryID) {
- if (libraryID === undefined) {
- throw new Error("libraryID not provided");
- }
-
- // Personal library
- if (!libraryID) {
- if (Zotero.Sync.Storage.ZFS.includeUserFiles) {
- return Zotero.Sync.Storage.ZFS;
- }
- if (Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- return Zotero.Sync.Storage.WebDAV;
- }
- return false;
- }
-
- // Group library
- else {
- if (Zotero.Sync.Storage.ZFS.includeGroupFiles) {
- return Zotero.Sync.Storage.ZFS;
- }
- return false;
- }
- }
-
-
- var _processDownload = Zotero.Promise.coroutine(function* (item) {
- var funcName = "Zotero.Sync.Storage._processDownload()";
-
- var tempFile = Zotero.getTempDirectory();
- tempFile.append(item.key + '.tmp');
-
- if (!tempFile.exists()) {
- Zotero.debug(tempFile.path);
- throw ("Downloaded file not found in " + funcName);
- }
-
- var parentDir = Zotero.Attachments.getStorageDirectory(item);
- if (!parentDir.exists()) {
- yield Zotero.Attachments.createDirectoryForItem(item);
- }
-
- _deleteExistingAttachmentFiles(item);
-
- var file = item.getFile(null, true);
- if (!file) {
- throw ("Empty path for item " + item.key + " in " + funcName);
- }
- // Don't save Windows aliases
- if (file.leafName.endsWith('.lnk')) {
- return false;
- }
-
- var fileName = file.leafName;
- var renamed = false;
-
- // Make sure the new filename is valid, in case an invalid character made it over
- // (e.g., from before we checked for them)
- var filteredName = Zotero.File.getValidFileName(fileName);
- if (filteredName != fileName) {
- Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
- fileName = filteredName;
- file.leafName = fileName;
- renamed = true;
- }
-
- Zotero.debug("Moving download file " + tempFile.leafName + " into attachment directory as '" + fileName + "'");
- try {
- var destFile = parentDir.clone();
- destFile.append(fileName);
- Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- if (destFile.leafName != fileName) {
- Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
-
- // Abort if Windows path limitation would cause filenames to be overly truncated
- if (Zotero.isWin && destFile.leafName.length < 40) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
- // TODO: localize
- var msg = "Due to a Windows path length limitation, your Zotero data directory "
- + "is too deep in the filesystem for syncing to work reliably. "
- + "Please relocate your Zotero data to a higher directory.";
- Zotero.debug(msg, 1);
- throw new Error(msg);
- }
-
- renamed = true;
- }
-
- try {
- tempFile.moveTo(parentDir, destFile.leafName);
- }
- catch (e) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
-
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- var returnFile = null;
- // processDownload() needs to know that we're renaming the file
- if (renamed) {
- returnFile = destFile.clone();
- }
- return returnFile;
- });
-
-
- var _processZipDownload = Zotero.Promise.coroutine(function* (item) {
- var funcName = "Zotero.Sync.Storage._processDownloadedZip()";
-
- var zipFile = Zotero.getTempDirectory();
- zipFile.append(item.key + '.zip.tmp');
-
- if (!zipFile.exists()) {
- throw ("Downloaded ZIP file not found in " + funcName);
- }
-
- var zipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"].
- createInstance(Components.interfaces.nsIZipReader);
- try {
- zipReader.open(zipFile);
- zipReader.test(null);
-
- Zotero.debug("ZIP file is OK");
- }
- catch (e) {
- Zotero.debug(zipFile.leafName + " is not a valid ZIP file", 2);
- zipReader.close();
-
- try {
- zipFile.remove(false);
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, zipFile, 'delete');
- }
-
- // TODO: Remove prop file to trigger reuploading, in case it was an upload error?
-
- return false;
- }
-
- var parentDir = Zotero.Attachments.getStorageDirectory(item);
- if (!parentDir.exists()) {
- yield Zotero.Attachments.createDirectoryForItem(item);
- }
-
- try {
- _deleteExistingAttachmentFiles(item);
- }
- catch (e) {
- zipReader.close();
- throw (e);
- }
-
- var returnFile = null;
- var count = 0;
-
- var entries = zipReader.findEntries(null);
- while (entries.hasMore()) {
- count++;
- var entryName = entries.getNext();
- var b64re = /%ZB64$/;
- if (entryName.match(b64re)) {
- var fileName = Zotero.Utilities.Internal.Base64.decode(
- entryName.replace(b64re, '')
- );
- }
- else {
- var fileName = entryName;
- }
-
- if (fileName.startsWith('.zotero')) {
- Zotero.debug("Skipping " + fileName);
- continue;
- }
-
- Zotero.debug("Extracting " + fileName);
-
- var primaryFile = false;
- var filtered = false;
- var renamed = false;
-
- // Get the old filename
- var itemFileName = item.getFilename();
-
- // Make sure the new filename is valid, in case an invalid character
- // somehow make it into the ZIP (e.g., from before we checked for them)
- //
- // Do this before trying to use the relative descriptor, since otherwise
- // it might fail silently and select the parent directory
- var filteredName = Zotero.File.getValidFileName(fileName);
- if (filteredName != fileName) {
- Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
- fileName = filteredName;
- filtered = true;
- }
-
- // Name in ZIP is a relative descriptor, so file has to be reconstructed
- // using setRelativeDescriptor()
- var destFile = parentDir.clone();
- destFile.QueryInterface(Components.interfaces.nsILocalFile);
- destFile.setRelativeDescriptor(parentDir, fileName);
-
- fileName = destFile.leafName;
-
- // If only one file in zip and it doesn't match the known filename,
- // take our chances and use that name
- if (count == 1 && !entries.hasMore() && itemFileName) {
- // May not be necessary, but let's be safe
- itemFileName = Zotero.File.getValidFileName(itemFileName);
- if (itemFileName != fileName) {
- Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2);
- Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
- fileName = itemFileName;
- destFile.leafName = fileName;
- renamed = true;
- }
- }
-
- var primaryFile = itemFileName == fileName;
- if (primaryFile && filtered) {
- renamed = true;
- }
-
- if (destFile.exists()) {
- var msg = "ZIP entry '" + fileName + "' " + "already exists";
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg + " in " + funcName);
- continue;
- }
-
- try {
- Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
- }
- catch (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
-
- zipReader.close();
-
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- if (destFile.leafName != fileName) {
- Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
-
- // Abort if Windows path limitation would cause filenames to be overly truncated
- if (Zotero.isWin && destFile.leafName.length < 40) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
- zipReader.close();
- // TODO: localize
- var msg = "Due to a Windows path length limitation, your Zotero data directory "
- + "is too deep in the filesystem for syncing to work reliably. "
- + "Please relocate your Zotero data to a higher directory.";
- Zotero.debug(msg, 1);
- throw new Error(msg);
- }
-
- if (primaryFile) {
- renamed = true;
- }
- }
-
- try {
- zipReader.extract(entryName, destFile);
- }
- catch (e) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
-
- // For advertising junk files, ignore a bug on Windows where
- // destFile.create() works but zipReader.extract() doesn't
- // when the path length is close to 255.
- if (destFile.leafName.match(/[a-zA-Z0-9+=]{130,}/)) {
- var msg = "Ignoring error extracting '" + destFile.path + "'";
- Zotero.debug(msg, 2);
- Zotero.debug(e, 2);
- Components.utils.reportError(msg + " in " + funcName);
- continue;
- }
-
- zipReader.close();
-
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- destFile.permissions = 0644;
-
- // If we're renaming the main file, processDownload() needs to know
- if (renamed) {
- returnFile = destFile;
- }
- }
- zipReader.close();
- zipFile.remove(false);
-
- return returnFile;
- });
-
-
- function _deleteExistingAttachmentFiles(item) {
- var funcName = "Zotero.Sync.Storage._deleteExistingAttachmentFiles()";
-
- var parentDir = Zotero.Attachments.getStorageDirectory(item);
-
- // Delete existing files
- var otherFiles = parentDir.directoryEntries;
- otherFiles.QueryInterface(Components.interfaces.nsIDirectoryEnumerator);
- var filesToDelete = [];
- var file;
- while (file = otherFiles.nextFile) {
- if (file.leafName.startsWith('.zotero')) {
- continue;
- }
-
- // Check symlink awareness, just to be safe
- if (!parentDir.contains(file, false)) {
- var msg = "Storage directory doesn't contain '" + file.leafName + "'";
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg + " in " + funcName);
- continue;
- }
-
- filesToDelete.push(file);
- }
- otherFiles.close();
-
- // Do deletes outside of the enumerator to avoid an access error on Windows
- for each(var file in filesToDelete) {
- try {
- if (file.isFile()) {
- Zotero.debug("Deleting existing file " + file.leafName);
- file.remove(false);
- }
- else if (file.isDirectory()) {
- Zotero.debug("Deleting existing directory " + file.leafName);
- file.remove(true);
- }
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, file, 'delete');
- }
- }
- }
-
-
- /**
- * Create zip file of attachment directory
- *
- * @param {Zotero.Sync.Storage.Request} request
- * @param {Function} callback
- * @return {Boolean} TRUE if zip process started,
- * FALSE if storage was empty
- */
- this.createUploadFile = function (request, callback) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- Zotero.debug("Creating zip file for item " + item.libraryID + "/" + item.key);
-
- try {
- switch (item.attachmentLinkMode) {
- case Zotero.Attachments.LINK_MODE_LINKED_FILE:
- case Zotero.Attachments.LINK_MODE_LINKED_URL:
- throw (new Error(
- "Upload file must be an imported snapshot or file in "
- + "Zotero.Sync.Storage.createUploadFile()"
- ));
- }
-
- var dir = Zotero.Attachments.getStorageDirectory(item);
-
- var tmpFile = Zotero.getTempDirectory();
- tmpFile.append(item.key + '.zip');
-
- var zw = Components.classes["@mozilla.org/zipwriter;1"]
- .createInstance(Components.interfaces.nsIZipWriter);
- zw.open(tmpFile, 0x04 | 0x08 | 0x20); // open rw, create, truncate
- var fileList = _zipDirectory(dir, dir, zw);
- if (fileList.length == 0) {
- Zotero.debug('No files to add -- removing zip file');
- zw.close();
- tmpFile.remove(null);
- return false;
- }
-
- Zotero.debug('Creating ' + tmpFile.leafName + ' with ' + fileList.length + ' file(s)');
-
- var observer = new Zotero.Sync.Storage.ZipWriterObserver(
- zw, callback, { request: request, files: fileList }
- );
- zw.processQueue(observer, null);
- return true;
- }
- // DEBUG: Do we want to catch this?
- catch (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- return false;
- }
- }
-
- function _zipDirectory(rootDir, dir, zipWriter) {
- var fileList = [];
- dir = dir.directoryEntries;
- while (dir.hasMoreElements()) {
- var file = dir.getNext();
- file.QueryInterface(Components.interfaces.nsILocalFile);
- if (file.isDirectory()) {
- //Zotero.debug("Recursing into directory " + file.leafName);
- fileList.concat(_zipDirectory(rootDir, file, zipWriter));
- continue;
- }
- var fileName = file.getRelativeDescriptor(rootDir);
- if (fileName.startsWith('.zotero')) {
- Zotero.debug('Skipping file ' + fileName);
- continue;
- }
-
- //Zotero.debug("Adding file " + fileName);
-
- zipWriter.addEntryFile(
- fileName,
- Components.interfaces.nsIZipWriter.COMPRESSION_DEFAULT,
- file,
- true
- );
- fileList.push(fileName);
- }
- return fileList;
- }
- /**
- * Get files marked as ready to download
- *
- * @inner
- * @return {Number[]} Array of attachment itemIDs
- */
- function _getFilesToDownload(libraryID, forcedOnly) {
- var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE libraryID=? AND syncState IN (?";
- var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD];
- if (!forcedOnly) {
- sql += ",?";
- params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD);
- }
- sql += ") "
- // Skip attachments with empty path, which can't be saved, and files with .zotero*
- // paths, which have somehow ended up in some users' libraries
- + "AND path!='' AND path NOT LIKE 'storage:.zotero%'";
- var itemIDs = Zotero.DB.columnQuery(sql, params);
- if (!itemIDs) {
- return [];
- }
- return itemIDs;
- }
- /**
- * Get files marked as ready to upload
- *
- * @inner
- * @return {Number[]} Array of attachment itemIDs
- */
- function _getFilesToUpload(libraryID) {
- var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE syncState IN (?,?) AND linkMode IN (?,?)";
- var params = [
- Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
- Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD,
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL
- ];
- if (typeof libraryID != 'undefined') {
- sql += " AND libraryID=?";
- params.push(libraryID);
- }
- else {
- throw new Error("libraryID not specified");
- }
- var itemIDs = Zotero.DB.columnQuery(sql, params);
- if (!itemIDs) {
- return [];
- }
- return itemIDs;
- }
- /**
- * Get files to check for local modifications for uploading
- *
- * This includes files previously modified and files opened externally
- * via Zotero within _maxCheckAgeInSeconds.
- */
- function _getFilesToCheck(libraryID) {
- var minTime = new Date().getTime() - (_maxCheckAgeInSeconds * 1000);
-
- // Get files by modification time
- var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND "
- + "storageModTime>=?";
- var params = [
- libraryID,
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL,
- Zotero.Sync.Storage.SYNC_STATE_IN_SYNC,
- minTime
- ];
- var itemIDs = Zotero.DB.columnQuery(sql, params) || [];
-
- // Get files by open time
- _uploadCheckFiles.filter(function (x) x.timestamp >= minTime);
- itemIDs = itemIDs.concat([x.itemID for each(x in _uploadCheckFiles)])
-
- return Zotero.Utilities.arrayUnique(itemIDs);
- }
- /**
- * @inner
- * @return {String[]|FALSE} Array of keys, or FALSE if none
- */
- this.getDeletedFiles = function () {
- var sql = "SELECT key FROM storageDeleteLog";
- return Zotero.DB.columnQuery(sql);
- }
+
function error(e) {
@@ -1892,56 +291,4 @@ Zotero.Sync.Storage = new function () {
}
-/**
- * Request observer for zip writing
- *
- * Implements nsIRequestObserver
- *
- * @param {nsIZipWriter} zipWriter
- * @param {Function} callback
- * @param {Object} data
- */
-Zotero.Sync.Storage.ZipWriterObserver = function (zipWriter, callback, data) {
- this._zipWriter = zipWriter;
- this._callback = callback;
- this._data = data;
-}
-Zotero.Sync.Storage.ZipWriterObserver.prototype = {
- onStartRequest: function () {},
-
- onStopRequest: function(req, context, status) {
- var zipFileName = this._zipWriter.file.leafName;
-
- var originalSize = 0;
- for each(var fileName in this._data.files) {
- var entry = this._zipWriter.getEntry(fileName);
- if (!entry) {
- var msg = "ZIP entry '" + fileName + "' not found for request '" + this._data.request.name + "'";
- Components.utils.reportError(msg);
- Zotero.debug(msg, 1);
- this._zipWriter.close();
- this._callback(false);
- return;
- }
- originalSize += entry.realSize;
- }
- delete this._data.files;
-
- this._zipWriter.close();
-
- Zotero.debug("Zip of " + zipFileName + " finished with status " + status
- + " (original " + Math.round(originalSize / 1024) + "KB, "
- + "compressed " + Math.round(this._zipWriter.file.fileSize / 1024) + "KB, "
- + Math.round(
- ((originalSize - this._zipWriter.file.fileSize) / originalSize) * 100
- ) + "% reduction)");
-
- Zotero.Sync.Storage.compressionTracker.compressed += this._zipWriter.file.fileSize;
- Zotero.Sync.Storage.compressionTracker.uncompressed += originalSize;
- Zotero.debug("Average compression so far: "
- + Math.round(Zotero.Sync.Storage.compressionTracker.ratio * 100) + "%");
-
- this._callback(this._data);
- }
-}
diff --git a/chrome/content/zotero/xpcom/storage/mode.js b/chrome/content/zotero/xpcom/storage/mode.js
@@ -1,87 +0,0 @@
-/*
- ***** BEGIN LICENSE BLOCK *****
-
- Copyright © 2009 Center for History and New Media
- George Mason University, Fairfax, Virginia, USA
- http://zotero.org
-
- This file is part of Zotero.
-
- Zotero is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Zotero is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
- ***** END LICENSE BLOCK *****
-*/
-
-
-Zotero.Sync.Storage.Mode = function () {};
-
-Zotero.Sync.Storage.Mode.prototype.__defineGetter__('verified', function () {
- return this._verified;
-});
-
-Zotero.Sync.Storage.Mode.prototype.__defineGetter__('username', function () {
- return this._username;
-});
-
-Zotero.Sync.Storage.Mode.prototype.__defineGetter__('password', function () {
- return this._password;
-});
-
-Zotero.Sync.Storage.Mode.prototype.__defineSetter__('password', function (val) {
- this._password = val;
-});
-
-Zotero.Sync.Storage.Mode.prototype.init = function () {
- return this._init();
-}
-
-Zotero.Sync.Storage.Mode.prototype.sync = function (observer) {
- return Zotero.Sync.Storage.sync(this.name, observer);
-}
-
-Zotero.Sync.Storage.Mode.prototype.downloadFile = function (request) {
- return this._downloadFile(request);
-}
-
-Zotero.Sync.Storage.Mode.prototype.uploadFile = function (request) {
- return this._uploadFile(request);
-}
-
-Zotero.Sync.Storage.Mode.prototype.getLastSyncTime = function (libraryID) {
- return this._getLastSyncTime(libraryID);
-}
-
-Zotero.Sync.Storage.Mode.prototype.setLastSyncTime = function (callback, useLastSyncTime) {
- return this._setLastSyncTime(callback, useLastSyncTime);
-}
-
-Zotero.Sync.Storage.Mode.prototype.checkServer = function (callback) {
- return this._checkServer(callback);
-}
-
-Zotero.Sync.Storage.Mode.prototype.checkServerCallback = function (uri, status, window, skipSuccessMessage) {
- return this._checkServerCallback(uri, status, window, skipSuccessMessage);
-}
-
-Zotero.Sync.Storage.Mode.prototype.cacheCredentials = function () {
- return this._cacheCredentials();
-}
-
-Zotero.Sync.Storage.Mode.prototype.purgeDeletedStorageFiles = function (callback) {
- return this._purgeDeletedStorageFiles(callback);
-}
-
-Zotero.Sync.Storage.Mode.prototype.purgeOrphanedStorageFiles = function (callback) {
- return this._purgeOrphanedStorageFiles(callback);
-}
diff --git a/chrome/content/zotero/xpcom/storage/queue.js b/chrome/content/zotero/xpcom/storage/queue.js
@@ -1,427 +0,0 @@
-/*
- ***** BEGIN LICENSE BLOCK *****
-
- Copyright © 2009 Center for History and New Media
- George Mason University, Fairfax, Virginia, USA
- http://zotero.org
-
- This file is part of Zotero.
-
- Zotero is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Zotero is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
- ***** END LICENSE BLOCK *****
-*/
-
-/**
- * Queue for storage sync transfer requests
- *
- * @param {String} type Queue type (e.g., 'download' or 'upload')
- */
-Zotero.Sync.Storage.Queue = function (type, libraryID) {
- Zotero.debug("Initializing " + type + " queue for library " + libraryID);
-
- // Public properties
- this.type = type;
- this.libraryID = libraryID;
- this.maxConcurrentRequests = 1;
- this.activeRequests = 0;
- this.totalRequests = 0;
-
- // Private properties
- this._requests = {};
- this._highPriority = [];
- this._running = false;
- this._stopping = false;
- this._finished = false;
- this._error = false;
- this._finishedReqs = 0;
- this._localChanges = false;
- this._remoteChanges = false;
- this._conflicts = [];
- this._cachedPercentage;
- this._cachedPercentageTime;
-}
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('name', function () {
- return this.type + "/" + this.libraryID;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('Type', function () {
- return this.type[0].toUpperCase() + this.type.substr(1);
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('running', function () this._running);
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('stopping', function () this._stopping);
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finished', function () this._finished);
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('unfinishedRequests', function () {
- return this.totalRequests - this.finishedRequests;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finishedRequests', function () {
- return this._finishedReqs;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineSetter__('finishedRequests', function (val) {
- Zotero.debug("Finished requests: " + val);
- Zotero.debug("Total requests: " + this.totalRequests);
-
- this._finishedReqs = val;
-
- if (val == 0) {
- return;
- }
-
- // Last request
- if (val == this.totalRequests) {
- Zotero.debug(this.Type + " queue is done for library " + this.libraryID);
-
- // DEBUG info
- Zotero.debug("Active requests: " + this.activeRequests);
-
- if (this.activeRequests) {
- throw new Error(this.Type + " queue for library " + this.libraryID
- + " can't be done if there are active requests");
- }
-
- this._running = false;
- this._stopping = false;
- this._finished = true;
- this._requests = {};
- this._highPriority = [];
-
- var localChanges = this._localChanges;
- var remoteChanges = this._remoteChanges;
- var conflicts = this._conflicts.concat();
- var deferred = this._deferred;
- this._localChanges = false;
- this._remoteChanges = false;
- this._conflicts = [];
- this._deferred = null;
-
- if (!this._error) {
- Zotero.debug("Resolving promise for queue " + this.name);
- Zotero.debug(this._localChanges);
- Zotero.debug(this._remoteChanges);
- Zotero.debug(this._conflicts);
-
- deferred.resolve({
- libraryID: this.libraryID,
- type: this.type,
- localChanges: localChanges,
- remoteChanges: remoteChanges,
- conflicts: conflicts
- });
- }
- else {
- Zotero.debug("Rejecting promise for queue " + this.name);
- var e = this._error;
- this._error = false;
- e.libraryID = this.libraryID;
- e.type = this.type;
- deferred.reject(e);
- }
-
- return;
- }
-
- if (this._stopping) {
- return;
- }
- this.advance();
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('queuedRequests', function () {
- return this.unfinishedRequests - this.activeRequests;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('remaining', function () {
- var remaining = 0;
- for each(var request in this._requests) {
- remaining += request.remaining;
- }
- return remaining;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('percentage', function () {
- if (this.totalRequests == 0) {
- return 0;
- }
- if (this._finished) {
- return 100;
- }
-
- // Cache percentage for a second
- if (this._cachedPercentage && (new Date() - this._cachedPercentageTime) < 1000) {
- return this._cachedPercentage;
- }
-
- var completedRequests = 0;
- for each(var request in this._requests) {
- completedRequests += request.percentage / 100;
- }
- this._cachedPercentage = Math.round((completedRequests / this.totalRequests) * 100);
- this._cachedPercentageTime = new Date();
- return this._cachedPercentage;
-});
-
-
-Zotero.Sync.Storage.Queue.prototype.isRunning = function () {
- return this._running;
-}
-
-Zotero.Sync.Storage.Queue.prototype.isStopping = function () {
- return this._stopping;
-}
-
-
-/**
- * Add a request to this queue
- *
- * @param {Zotero.Sync.Storage.Request} request
- * @param {Boolean} highPriority Add or move request to high priority queue
- */
-Zotero.Sync.Storage.Queue.prototype.addRequest = function (request, highPriority) {
- if (this._finished) {
- this.reset();
- }
-
- request.queue = this;
- var name = request.name;
- Zotero.debug("Queuing " + this.type + " request '" + name + "' for library " + this.libraryID);
-
- if (this._requests[name]) {
- if (highPriority) {
- Zotero.debug("Moving " + name + " to high-priority queue");
- this._requests[name].importCallbacks(request);
- this._highPriority.push(name);
- return;
- }
-
- Zotero.debug("Request '" + name + "' already exists");
- return;
- }
-
- this._requests[name] = request;
- this.totalRequests++;
-
- if (highPriority) {
- this._highPriority.push(name);
- }
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.start = function () {
- if (!this._deferred || this._deferred.promise.isFulfilled()) {
- Zotero.debug("Creating deferred for queue " + this.name);
- this._deferred = Zotero.Promise.defer();
- }
- // The queue manager needs to know what queues were running in the
- // current session
- Zotero.Sync.Storage.QueueManager.addCurrentQueue(this);
-
- var self = this;
- setTimeout(function () {
- self.advance();
- }, 0);
-
- return this._deferred.promise;
-}
-
-
-
-/**
- * Start another request in this queue if there's an available slot
- */
-Zotero.Sync.Storage.Queue.prototype.advance = function () {
- this._running = true;
- this._finished = false;
-
- if (this._stopping) {
- Zotero.debug(this.Type + " queue for library " + this.libraryID
- + "is being stopped in Zotero.Sync.Storage.Queue.advance()", 2);
- return;
- }
-
- if (!this.queuedRequests) {
- Zotero.debug("No remaining requests in " + this.type
- + " queue for library " + this.libraryID + " ("
- + this.activeRequests + " active, "
- + this.finishedRequests + " finished)");
- return;
- }
-
- if (this.activeRequests >= this.maxConcurrentRequests) {
- Zotero.debug(this.Type + " queue for library " + this.libraryID
- + " is busy (" + this.activeRequests + "/"
- + this.maxConcurrentRequests + ")");
- return;
- }
-
-
-
- // Start the first unprocessed request
-
- // Try the high-priority queue first
- var self = this;
- var request, name;
- while (name = this._highPriority.shift()) {
- request = this._requests[name];
- if (request.isRunning() || request.isFinished()) {
- continue;
- }
-
- let requestName = name;
-
- Zotero.Promise.try(function () {
- var promise = request.start();
- self.advance();
- return promise;
- })
- .then(function (result) {
- if (result.localChanges) {
- self._localChanges = true;
- }
- if (result.remoteChanges) {
- self._remoteChanges = true;
- }
- if (result.conflict) {
- self.addConflict(
- requestName,
- result.conflict.local,
- result.conflict.remote
- );
- }
- })
- .catch(function (e) {
- self.error(e);
- });
-
- return;
- }
-
- // And then others
- for each(var request in this._requests) {
- if (request.isRunning() || request.isFinished()) {
- continue;
- }
-
- let requestName = request.name;
-
- // This isn't in a Zotero.Promise.try() because the request needs to get marked
- // as running immediately so that it doesn't get run again by a
- // subsequent advance() call.
- try {
- var promise = request.start();
- self.advance();
- }
- catch (e) {
- self.error(e);
- }
-
- promise.then(function (result) {
- if (result.localChanges) {
- self._localChanges = true;
- }
- if (result.remoteChanges) {
- self._remoteChanges = true;
- }
- if (result.conflict) {
- self.addConflict(
- requestName,
- result.conflict.local,
- result.conflict.remote
- );
- }
- })
- .catch(function (e) {
- self.error(e);
- });
-
- return;
- }
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.updateProgress = function () {
- Zotero.Sync.Storage.QueueManager.updateProgress();
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.addConflict = function (requestName, localData, remoteData) {
- Zotero.debug('===========');
- Zotero.debug(localData);
- Zotero.debug(remoteData);
-
- this._conflicts.push({
- name: requestName,
- localData: localData,
- remoteData: remoteData
- });
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.error = function (e) {
- if (!this._error) {
- if (this.isRunning()) {
- this._error = e;
- }
- else {
- Zotero.debug("Queue " + this.name + " was no longer running -- not assigning error", 2);
- }
- }
- Zotero.debug(e, 1);
- this.stop();
-}
-
-
-/**
- * Stops all requests in this queue
- */
-Zotero.Sync.Storage.Queue.prototype.stop = function () {
- if (!this._running) {
- Zotero.debug(this.Type + " queue for library " + this.libraryID
- + " is not running");
- return;
- }
- if (this._stopping) {
- Zotero.debug("Already stopping " + this.type + " queue for library "
- + this.libraryID);
- return;
- }
-
- Zotero.debug("Stopping " + this.type + " queue for library " + this.libraryID);
-
- // If no requests, finish manually
- /*if (this.activeRequests == 0) {
- this._finishedRequests = this._finishedRequests;
- return;
- }*/
-
- this._stopping = true;
- for each(var request in this._requests) {
- if (!request.isFinished()) {
- request.stop(true);
- }
- }
-
- Zotero.debug("Queue is stopped");
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.reset = function () {
- this._finished = false;
- this._finishedReqs = 0;
- this.totalRequests = 0;
-}
diff --git a/chrome/content/zotero/xpcom/storage/queueManager.js b/chrome/content/zotero/xpcom/storage/queueManager.js
@@ -1,370 +0,0 @@
-/*
- ***** BEGIN LICENSE BLOCK *****
-
- Copyright © 2009 Center for History and New Media
- George Mason University, Fairfax, Virginia, USA
- http://zotero.org
-
- This file is part of Zotero.
-
- Zotero is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Zotero is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
- ***** END LICENSE BLOCK *****
-*/
-
-
-Zotero.Sync.Storage.QueueManager = new function () {
- var _queues = {};
- var _currentQueues = [];
-
- this.start = Zotero.Promise.coroutine(function* (libraryID) {
- if (libraryID) {
- var queues = this.getAll(libraryID);
- var suffix = " for library " + libraryID;
- }
- else {
- var queues = this.getAll();
- var suffix = "";
- }
-
- Zotero.debug("Starting file sync queues" + suffix);
-
- var promises = [];
- for each(var queue in queues) {
- if (!queue.unfinishedRequests) {
- continue;
- }
- Zotero.debug("Starting queue " + queue.name);
- promises.push(queue.start());
- }
-
- if (!promises.length) {
- Zotero.debug("No files to sync" + suffix);
- }
-
- var results = yield Zotero.Promise.allSettled(promises);
- Zotero.debug("All storage queues are finished" + suffix);
-
- for (let i = 0; i < results.length; i++) {
- let result = results[i];
- // Check for conflicts to resolve
- if (result.state == "fulfilled") {
- result = result.value;
- if (result.conflicts.length) {
- Zotero.debug("Reconciling conflicts for library " + result.libraryID);
- Zotero.debug(result.conflicts);
- var data = yield _reconcileConflicts(result.conflicts);
- if (data) {
- _processMergeData(data);
- }
- }
- }
- }
-
- return promises;
- });
-
- this.stop = function (libraryID) {
- if (libraryID) {
- var queues = this.getAll(libraryID);
- }
- else {
- var queues = this.getAll();
- }
- for (var queue in queues) {
- queue.stop();
- }
- };
-
-
- /**
- * Retrieving a queue, creating a new one if necessary
- *
- * @param {String} queueName
- */
- this.get = function (queueName, libraryID, noInit) {
- if (typeof libraryID == 'undefined') {
- throw new Error("libraryID not specified");
- }
-
- var hash = queueName + "/" + libraryID;
-
- // Initialize the queue if it doesn't exist yet
- if (!_queues[hash]) {
- if (noInit) {
- return false;
- }
- var queue = new Zotero.Sync.Storage.Queue(queueName, libraryID);
- switch (queueName) {
- case 'download':
- queue.maxConcurrentRequests =
- Zotero.Prefs.get('sync.storage.maxDownloads')
- break;
-
- case 'upload':
- queue.maxConcurrentRequests =
- Zotero.Prefs.get('sync.storage.maxUploads')
- break;
-
- default:
- throw ("Invalid queue '" + queueName + "' in Zotero.Sync.Storage.QueueManager.get()");
- }
- _queues[hash] = queue;
- }
-
- return _queues[hash];
- };
-
-
- this.getAll = function (libraryID) {
- if (typeof libraryID == 'string') {
- throw new Error("libraryID must be a number or undefined");
- }
-
- var queues = [];
- for each(var queue in _queues) {
- if (typeof libraryID == 'undefined' || queue.libraryID === libraryID) {
- queues.push(queue);
- }
- }
- return queues;
- };
-
-
- this.addCurrentQueue = function (queue) {
- if (!this.hasCurrentQueue(queue)) {
- _currentQueues.push(queue.name);
- }
- }
-
-
- this.hasCurrentQueue = function (queue) {
- return _currentQueues.indexOf(queue.name) != -1;
- }
-
-
- /**
- * Stop all queues
- *
- * @param {Boolean} [skipStorageFinish=false] Don't call Zotero.Sync.Storage.finish()
- * when done (used when we stopped because of
- * an error)
- */
- this.cancel = function (skipStorageFinish) {
- Zotero.debug("Stopping all storage queues");
- for each(var queue in _queues) {
- if (queue.isRunning() && !queue.isStopping()) {
- queue.stop();
- }
- }
- }
-
-
- this.finish = function () {
- Zotero.debug("All storage queues are finished");
- _currentQueues = [];
- }
-
-
- /**
- * Calculate the current progress values and trigger a display update
- *
- * Also detects when all queues have finished and ends sync progress
- */
- this.updateProgress = function () {
- var activeRequests = 0;
- var allFinished = true;
- for each(var queue in _queues) {
- // Finished or never started
- if (!queue.isRunning() && !queue.isStopping()) {
- continue;
- }
- allFinished = false;
- activeRequests += queue.activeRequests;
- }
- if (activeRequests == 0) {
- _updateProgressMeters(0);
- if (allFinished) {
- this.finish();
- }
- return;
- }
-
- var status = {};
- for each(var queue in _queues) {
- if (!this.hasCurrentQueue(queue)) {
- continue;
- }
-
- if (!status[queue.libraryID]) {
- status[queue.libraryID] = {};
- }
- if (!status[queue.libraryID][queue.type]) {
- status[queue.libraryID][queue.type] = {};
- }
- status[queue.libraryID][queue.type].statusString = _getQueueStatus(queue);
- status[queue.libraryID][queue.type].percentage = queue.percentage;
- status[queue.libraryID][queue.type].totalRequests = queue.totalRequests;
- status[queue.libraryID][queue.type].finished = queue.finished;
- }
-
- _updateProgressMeters(activeRequests, status);
- }
-
-
- /**
- * Get a status string for a queue
- *
- * @param {Zotero.Sync.Storage.Queue} queue
- * @return {String}
- */
- function _getQueueStatus(queue) {
- var remaining = queue.remaining;
- var unfinishedRequests = queue.unfinishedRequests;
-
- if (!unfinishedRequests) {
- return Zotero.getString('sync.storage.none');
- }
-
- if (remaining > 1000) {
- var bytesRemaining = Zotero.getString(
- 'sync.storage.mbRemaining',
- Zotero.Utilities.numberFormat(remaining / 1000 / 1000, 1)
- );
- }
- else {
- var bytesRemaining = Zotero.getString(
- 'sync.storage.kbRemaining',
- Zotero.Utilities.numberFormat(remaining / 1000, 0)
- );
- }
- var totalRequests = queue.totalRequests;
- var filesRemaining = Zotero.getString(
- 'sync.storage.filesRemaining',
- [totalRequests - unfinishedRequests, totalRequests]
- );
- return bytesRemaining + ' (' + filesRemaining + ')';
- }
-
- /**
- * Cycle through windows, updating progress meters with new values
- */
- function _updateProgressMeters(activeRequests, status) {
- // Get overall percentage across queues
- var sum = 0, num = 0, percentage, total;
- for each(var libraryStatus in status) {
- for each(var queueStatus in libraryStatus) {
- percentage = queueStatus.percentage;
- total = queueStatus.totalRequests;
- sum += total * percentage;
- num += total;
- }
- }
- var percentage = Math.round(sum / num);
-
- var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
- .getService(Components.interfaces.nsIWindowMediator);
- var enumerator = wm.getEnumerator("navigator:browser");
- while (enumerator.hasMoreElements()) {
- var win = enumerator.getNext();
- if (!win.ZoteroPane) continue;
- var doc = win.ZoteroPane.document;
-
- var box = doc.getElementById("zotero-tb-sync-progress-box");
- var meter = doc.getElementById("zotero-tb-sync-progress");
-
- if (activeRequests == 0) {
- box.hidden = true;
- continue;
- }
-
- meter.setAttribute("value", percentage);
- box.hidden = false;
-
- var percentageLabel = doc.getElementById('zotero-tb-sync-progress-tooltip-progress');
- percentageLabel.lastChild.setAttribute('value', percentage + "%");
-
- var statusBox = doc.getElementById('zotero-tb-sync-progress-status');
- statusBox.data = status;
- }
- }
-
-
- var _reconcileConflicts = Zotero.Promise.coroutine(function* (conflicts) {
- var objectPairs = [];
- for each(var conflict in conflicts) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name);
- var item1 = yield item.clone(false, false, true);
- item1.setField('dateModified',
- Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true));
- var item2 = yield item.clone(false, false, true);
- item2.setField('dateModified',
- Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true));
- objectPairs.push([item1, item2]);
- }
-
- var io = {
- dataIn: {
- type: 'storagefile',
- captions: [
- Zotero.getString('sync.storage.localFile'),
- Zotero.getString('sync.storage.remoteFile'),
- Zotero.getString('sync.storage.savedFile')
- ],
- objects: objectPairs
- }
- };
-
- var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
- .getService(Components.interfaces.nsIWindowMediator);
- var lastWin = wm.getMostRecentWindow("navigator:browser");
- lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io);
-
- if (!io.dataOut) {
- return false;
- }
-
- // Since we're only putting cloned items into the merge window,
- // we have to manually set the ids
- for (var i=0; i<conflicts.length; i++) {
- io.dataOut[i].id = Zotero.Sync.Storage.getItemFromRequestName(conflicts[i].name).id;
- }
-
- return io.dataOut;
- });
-
-
- function _processMergeData(data) {
- if (!data.length) {
- return false;
- }
-
- for each(var mergeItem in data) {
- var itemID = mergeItem.id;
- var dateModified = mergeItem.ref.getField('dateModified');
- // Local
- if (dateModified == mergeItem.left.getField('dateModified')) {
- Zotero.Sync.Storage.setSyncState(
- itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
- );
- }
- // Remote
- else {
- Zotero.Sync.Storage.setSyncState(
- itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
- );
- }
- }
- }
-}
diff --git a/chrome/content/zotero/xpcom/storage/request.js b/chrome/content/zotero/xpcom/storage/request.js
@@ -1,368 +0,0 @@
-/*
- ***** BEGIN LICENSE BLOCK *****
-
- Copyright © 2009 Center for History and New Media
- George Mason University, Fairfax, Virginia, USA
- http://zotero.org
-
- This file is part of Zotero.
-
- Zotero is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Zotero is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
- ***** END LICENSE BLOCK *****
-*/
-
-
-/**
- * Transfer request for storage sync
- *
- * @param {String} name Identifier for request (e.g., "[libraryID]/[key]")
- * @param {Function} onStart Callback to run when request starts
- */
-Zotero.Sync.Storage.Request = function (name, callbacks) {
- Zotero.debug("Initializing request '" + name + "'");
-
- this.callbacks = ['onStart', 'onProgress'];
-
- this.name = name;
- this.channel = null;
- this.queue = null;
- this.progress = 0;
- this.progressMax = 0;
-
- this._deferred = Zotero.Promise.defer();
- this._running = false;
- this._stopping = false;
- this._percentage = 0;
- this._remaining = null;
- this._maxSize = null;
- this._finished = false;
- this._forceFinish = false;
- this._changesMade = false;
-
- for (var func in callbacks) {
- if (this.callbacks.indexOf(func) !== -1) {
- // Stuff all single functions into arrays
- this['_' + func] = typeof callbacks[func] === 'function' ? [callbacks[func]] : callbacks[func];
- }
- else {
- throw new Error("Invalid handler '" + func + "'");
- }
- }
-}
-
-
-Zotero.Sync.Storage.Request.prototype.setMaxSize = function (size) {
- this._maxSize = size;
-};
-
-
-/**
- * Add callbacks from another request to this request
- */
-Zotero.Sync.Storage.Request.prototype.importCallbacks = function (request) {
- for each(var name in this.callbacks) {
- name = '_' + name;
- if (request[name]) {
- // If no handlers for this event, add them all
- if (!this[name]) {
- this[name] = request[name];
- continue;
- }
- // Otherwise add functions that don't already exist
- var add = true;
- for each(var newFunc in request[name]) {
- for each(var currentFunc in this[name]) {
- if (newFunc.toString() === currentFunc.toString()) {
- Zotero.debug("Callback already exists in request -- not importing");
- add = false;
- break;
- }
- }
- if (add) {
- this[name].push(newFunc);
- }
- }
- }
- }
-}
-
-
-Zotero.Sync.Storage.Request.prototype.__defineGetter__('promise', function () {
- return this._deferred.promise;
-});
-
-
-Zotero.Sync.Storage.Request.prototype.__defineGetter__('percentage', function () {
- if (this._finished) {
- return 100;
- }
-
- if (this.progressMax == 0) {
- return 0;
- }
-
- var percentage = Math.round((this.progress / this.progressMax) * 100);
- if (percentage < this._percentage) {
- Zotero.debug(percentage + " is less than last percentage of "
- + this._percentage + " for request " + this.name, 2);
- Zotero.debug(this.progress);
- Zotero.debug(this.progressMax);
- percentage = this._percentage;
- }
- else if (percentage > 100) {
- Zotero.debug(percentage + " is greater than 100 for "
- + "request " + this.name, 2);
- Zotero.debug(this.progress);
- Zotero.debug(this.progressMax);
- percentage = 100;
- }
- else {
- this._percentage = percentage;
- }
- //Zotero.debug("Request '" + this.name + "' percentage is " + percentage);
- return percentage;
-});
-
-
-Zotero.Sync.Storage.Request.prototype.__defineGetter__('remaining', function () {
- if (this._finished) {
- return 0;
- }
-
- if (!this.progressMax) {
- if (this.queue.type == 'upload' && this._maxSize) {
- return Math.round(Zotero.Sync.Storage.compressionTracker.ratio * this._maxSize);
- }
-
- //Zotero.debug("Remaining not yet available for request '" + this.name + "'");
- return 0;
- }
-
- var remaining = this.progressMax - this.progress;
- if (this._remaining === null) {
- this._remaining = remaining;
- }
- else if (remaining > this._remaining) {
- Zotero.debug(remaining + " is greater than the last remaining amount of "
- + this._remaining + " for request " + this.name);
- remaining = this._remaining;
- }
- else if (remaining < 0) {
- Zotero.debug(remaining + " is less than 0 for request " + this.name);
- }
- else {
- this._remaining = remaining;
- }
- //Zotero.debug("Request '" + this.name + "' remaining is " + remaining);
- return remaining;
-});
-
-
-Zotero.Sync.Storage.Request.prototype.setChannel = function (channel) {
- this.channel = channel;
-}
-
-
-Zotero.Sync.Storage.Request.prototype.start = function () {
- if (!this.queue) {
- throw ("Request " + this.name + " must be added to a queue before starting");
- }
-
- Zotero.debug("Starting " + this.queue.name + " request " + this.name);
-
- if (this._running) {
- throw new Error("Request " + this.name + " already running");
- }
-
- this._running = true;
- this.queue.activeRequests++;
-
- if (this.queue.type == 'download') {
- Zotero.Sync.Storage.setItemDownloadPercentage(this.name, 0);
- }
-
- var self = this;
-
- // this._onStart is an array of promises returning changesMade.
- //
- // The main sync logic is triggered here.
-
- Zotero.Promise.all([f(this) for each(f in this._onStart)])
- .then(function (results) {
- return {
- localChanges: results.some(function (val) val && val.localChanges == true),
- remoteChanges: results.some(function (val) val && val.remoteChanges == true),
- conflict: results.reduce(function (prev, cur) {
- return prev.conflict ? prev : cur;
- }).conflict
- };
- })
- .then(function (results) {
- Zotero.debug(results);
-
- if (results.localChanges) {
- Zotero.debug("Changes were made by " + self.queue.name
- + " request " + self.name);
- }
- else {
- Zotero.debug("No changes were made by " + self.queue.name
- + " request " + self.name);
- }
-
- // This promise updates localChanges/remoteChanges on the queue
- self._deferred.resolve(results);
- })
- .catch(function (e) {
- if (self._stopping) {
- Zotero.debug("Skipping error for stopping request " + self.name);
- return;
- }
- Zotero.debug(self.queue.Type + " request " + self.name + " failed");
- self._deferred.reject(e);
- })
- // Finish the request (and in turn the queue, if this is the last request)
- .finally(function () {
- if (!self._finished) {
- self._finish();
- }
- });
-
- return this._deferred.promise;
-}
-
-
-Zotero.Sync.Storage.Request.prototype.isRunning = function () {
- return this._running;
-}
-
-
-Zotero.Sync.Storage.Request.prototype.isFinished = function () {
- return this._finished;
-}
-
-
-/**
- * Update counters for given request
- *
- * Also updates progress meter
- *
- * @param {Integer} progress Progress so far
- * (usually bytes transferred)
- * @param {Integer} progressMax Max progress value for this request
- * (usually total bytes)
- */
-Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, progressMax) {
- //Zotero.debug(progress + "/" + progressMax + " for request " + this.name);
-
- if (!this._running) {
- Zotero.debug("Trying to update finished request " + this.name + " in "
- + "Zotero.Sync.Storage.Request.onProgress() "
- + "(" + progress + "/" + progressMax + ")", 2);
- return;
- }
-
- if (!this.channel) {
- this.channel = channel;
- }
-
- // Workaround for invalid progress values (possibly related to
- // https://bugzilla.mozilla.org/show_bug.cgi?id=451991 and fixed in 3.1)
- if (progress < this.progress) {
- Zotero.debug("Invalid progress for request '"
- + this.name + "' (" + progress + " < " + this.progress + ")");
- return;
- }
-
- if (progressMax != this.progressMax) {
- Zotero.debug("progressMax has changed from " + this.progressMax
- + " to " + progressMax + " for request '" + this.name + "'", 2);
- }
-
- this.progress = progress;
- this.progressMax = progressMax;
- this.queue.updateProgress();
-
- if (this.queue.type == 'download') {
- Zotero.Sync.Storage.setItemDownloadPercentage(this.name, this.percentage);
- }
-
- if (this.onProgress) {
- for each(var f in this._onProgress) {
- f(progress, progressMax);
- }
- }
-}
-
-
-/**
- * Stop the request's underlying network request, if there is one
- */
-Zotero.Sync.Storage.Request.prototype.stop = function (force) {
- if (force) {
- this._forceFinish = true;
- }
-
- if (this.channel && this.channel.isPending()) {
- this._stopping = true;
-
- try {
- Zotero.debug("Stopping request '" + this.name + "'");
- this.channel.cancel(0x804b0002); // NS_BINDING_ABORTED
- }
- catch (e) {
- Zotero.debug(e);
- }
- }
- else {
- this._finish();
- }
-}
-
-
-/**
- * Mark request as finished and notify queue that it's done
- */
-Zotero.Sync.Storage.Request.prototype._finish = function () {
- // If an error occurred, we wait to finish the request, since doing
- // so might end the queue before the error flag has been set on the queue.
- // When the queue's error handler stops the queue, it stops the request
- // with stop(true) to force the finish to occur, allowing the queue's
- // promise to be rejected with the error.
- if (!this._forceFinish && this._deferred.promise.isRejected()) {
- return;
- }
-
- Zotero.debug("Finishing " + this.queue.name + " request '" + this.name + "'");
- this._finished = true;
- var active = this._running;
- this._running = false;
-
- Zotero.Sync.Storage.setItemDownloadPercentage(this.name, false);
-
- if (active) {
- this.queue.activeRequests--;
- }
- // TEMP: mechanism for failures?
- try {
- this.queue.finishedRequests++;
- this.queue.updateProgress();
- }
- catch (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- this._deferred.reject(e);
- throw e;
- }
-}
diff --git a/chrome/content/zotero/xpcom/storage/storageEngine.js b/chrome/content/zotero/xpcom/storage/storageEngine.js
@@ -0,0 +1,307 @@
+/*
+ ***** BEGIN LICENSE BLOCK *****
+
+ Copyright © 2015 Center for History and New Media
+ George Mason University, Fairfax, Virginia, USA
+ http://zotero.org
+
+ This file is part of Zotero.
+
+ Zotero is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Zotero is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Zotero. If not, see <http://www.gnu.org/licenses/>.
+
+ ***** END LICENSE BLOCK *****
+*/
+
+
+if (!Zotero.Sync.Storage) {
+ Zotero.Sync.Storage = {};
+}
+
+/**
+ * An Engine manages file sync processes for a given library
+ *
+ * @param {Object} options
+ * @param {Zotero.Sync.APIClient} options.apiClient
+ * @param {Integer} options.libraryID
+ * @param {Function} [onError] - Function to run on error
+ * @param {Boolean} [stopOnError]
+ */
+Zotero.Sync.Storage.Engine = function (options) {
+ if (options.apiClient == undefined) {
+ throw new Error("options.apiClient not set");
+ }
+ if (options.libraryID == undefined) {
+ throw new Error("options.libraryID not set");
+ }
+
+ this.apiClient = options.apiClient;
+ this.background = options.background;
+ this.firstInSession = options.firstInSession;
+ this.lastFullFileCheck = options.lastFullFileCheck;
+ this.libraryID = options.libraryID;
+ this.library = Zotero.Libraries.get(options.libraryID);
+
+ this.local = Zotero.Sync.Storage.Local;
+ this.utils = Zotero.Sync.Storage.Utilities;
+ this.mode = this.local.getModeForLibrary(this.libraryID);
+ var modeClass = this.utils.getClassForMode(this.mode);
+ this.controller = new modeClass(options);
+ this.setStatus = options.setStatus || function () {};
+ this.onError = options.onError || function (e) {};
+ this.stopOnError = options.stopOnError || false;
+
+ this.queues = [];
+ ['download', 'upload'].forEach(function (type) {
+ this.queues[type] = new ConcurrentCaller({
+ id: `${this.libraryID}/${type}`,
+ numConcurrent: Zotero.Prefs.get(
+ 'sync.storage.max' + Zotero.Utilities.capitalize(type) + 's'
+ ),
+ onError: this.onError,
+ stopOnError: this.stopOnError,
+ logger: Zotero.debug
+ });
+ }.bind(this))
+
+ this.maxCheckAge = 10800; // maximum age in seconds for upload modification check (3 hours)
+}
+
+Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function* () {
+ var libraryID = this.libraryID;
+ if (!Zotero.Prefs.get("sync.storage.enabled")) {
+ Zotero.debug("File sync is not enabled for " + this.library.name);
+ return false;
+ }
+
+ Zotero.debug("Starting file sync for " + this.library.name);
+
+ if (!this.controller.verified) {
+ Zotero.debug(`${this.mode} file sync is not active`);
+
+ throw new Error("Storage mode verification not implemented");
+
+ // TODO: Check server
+ }
+ if (this.controller.cacheCredentials) {
+ yield this.controller.cacheCredentials();
+ }
+
+ // Get library last-sync time for download-on-sync libraries.
+ var lastSyncTime = null;
+ var downloadAll = this.local.downloadOnSync(libraryID);
+ if (downloadAll) {
+ lastSyncTime = yield this.controller.getLastSyncTime(libraryID);
+ }
+
+ // Check for updated files to upload
+ if (!Zotero.Libraries.isFilesEditable(libraryID)) {
+ Zotero.debug("No file editing access -- skipping file modification check for "
+ + this.library.name);
+ }
+ // If this is a background sync, it's not the first sync of the session, the library has had
+ // at least one full check this session, and it's been less than maxCheckAge since the last
+ // full check of this library, check only files that were previously modified or opened
+ // recently
+ else if (this.background
+ && !this.firstInSession
+ && this.local.lastFullFileCheck[libraryID]
+ && (this.local.lastFullFileCheck[libraryID]
+ + (this.maxCheckAge * 1000)) > new Date().getTime()) {
+ let itemIDs = this.local.getFilesToCheck(libraryID, this.maxCheckAge);
+ yield this.local.checkForUpdatedFiles(libraryID, itemIDs);
+ }
+ // Otherwise check all files in library
+ else {
+ this.local.lastFullFileCheck[libraryID] = new Date().getTime();
+ yield this.local.checkForUpdatedFiles(libraryID);
+ }
+
+ yield this.local.resolveConflicts(libraryID);
+
+ var downloadForced = yield this.local.checkForForcedDownloads(libraryID);
+
+ // If we don't have any forced downloads, we can skip downloads if the last sync time hasn't
+ // changed or doesn't exist on the server (meaning there are no files)
+ if (downloadAll && !downloadForced) {
+ if (lastSyncTime) {
+ if (this.library.lastStorageSync == lastSyncTime) {
+ Zotero.debug("Last " + this.mode.toUpperCase() + " sync id hasn't changed for "
+ + this.library.name + " -- skipping file downloads");
+ downloadAll = false;
+ }
+ }
+ else {
+ Zotero.debug(`No last ${this.mode} sync time for ${this.library.name}`
+ + " -- skipping file downloads");
+ downloadAll = false;
+ }
+ }
+
+ // Get files to download
+ if (downloadAll || downloadForced) {
+ let itemIDs = yield this.local.getFilesToDownload(libraryID, !downloadAll);
+ if (itemIDs.length) {
+ Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ + "download for " + this.library.name);
+ for (let itemID of itemIDs) {
+ let item = yield Zotero.Items.getAsync(itemID);
+ yield this.queueItem(item);
+ }
+ }
+ else {
+ Zotero.debug("No files to download for " + this.library.name);
+ }
+ }
+
+ // Get files to upload
+ if (Zotero.Libraries.isFilesEditable(libraryID)) {
+ let itemIDs = yield this.local.getFilesToUpload(libraryID);
+ if (itemIDs.length) {
+ Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ + "upload for " + this.library.name);
+ for (let itemID of itemIDs) {
+ let item = yield Zotero.Items.getAsync(itemID, { noCache: true });
+ yield this.queueItem(item);
+ }
+ }
+ else {
+ Zotero.debug("No files to upload for " + this.library.name);
+ }
+ }
+ else {
+ Zotero.debug("No file editing access -- skipping file uploads for " + this.library.name);
+ }
+
+ var promises = {
+ download: this.queues.download.runAll(),
+ upload: this.queues.upload.runAll()
+ }
+
+ // Process the results
+ var changes = new Zotero.Sync.Storage.Result;
+ for (let type of ['download', 'upload']) {
+ let results = yield promises[type];
+
+ if (this.stopOnError) {
+ for (let p of results) {
+ if (p.isRejected()) {
+ let e = p.reason();
+ Zotero.debug(`File ${type} sync failed for ${this.library.name}`);
+ throw e;
+ }
+ }
+ }
+
+ Zotero.debug(`File ${type} sync finished for ${this.library.name}`);
+
+ changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value()));
+ }
+
+ // If files were uploaded, update the remote last-sync time
+ if (changes.remoteChanges) {
+ lastSyncTime = yield this.controller.setLastSyncTime(libraryID);
+ if (!lastSyncTime) {
+ throw new Error("Last sync time not set after sync");
+ }
+ }
+
+ // If there's a remote last-sync time from either the check before downloads or when it
+ // was changed after uploads, store that locally so we know we can skip download checks
+ // next time
+ if (lastSyncTime) {
+ this.library.lastStorageSync = lastSyncTime;
+ yield this.library.saveTx();
+ }
+
+ // If WebDAV sync, purge deleted and orphaned files
+ if (this.mode == 'webdav') {
+ try {
+ yield this.controller.purgeDeletedStorageFiles(libraryID);
+ yield this.controller.purgeOrphanedStorageFiles(libraryID);
+ }
+ catch (e) {
+ Zotero.logError(e);
+ }
+ }
+
+ if (!changes.localChanges) {
+ Zotero.debug("No local changes made during file sync");
+ }
+
+ Zotero.debug("Done with file sync for " + this.library.name);
+
+ return changes;
+})
+
+
+Zotero.Sync.Storage.Engine.prototype.stop = function () {
+ for (let type in this.queues) {
+ this.queues[type].stop();
+ }
+}
+
+Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) {
+ switch (yield this.local.getSyncState(item.id)) {
+ case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
+ var type = 'download';
+ var onStart = Zotero.Promise.method(function (request) {
+ return this.controller.downloadFile(request);
+ }.bind(this));
+ break;
+
+ case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
+ var type = 'upload';
+ var onStart = Zotero.Promise.method(function (request) {
+ return this.controller.uploadFile(request);
+ }.bind(this));
+ break;
+
+ case false:
+ Zotero.debug("Sync state for item " + item.id + " not found", 2);
+ return;
+
+ default:
+ throw new Error("Invalid sync state " + (yield this.local.getSyncState(item.id)));
+ }
+
+ var request = new Zotero.Sync.Storage.Request({
+ type,
+ libraryID: this.libraryID,
+ name: item.libraryKey,
+ onStart,
+ onProgress: this.onProgress
+ });
+ if (type == 'upload') {
+ try {
+ request.setMaxSize(yield Zotero.Attachments.getTotalFileSize(item));
+ }
+ // If this fails, ignore it, though we might fail later
+ catch (e) {
+ // But if the file doesn't exist yet, don't try to upload it
+ //
+ // This isn't a perfect test, because the file could still be in the process of being
+ // downloaded (e.g., from the web). It'd be better to download files to a temp
+ // directory and move them into place.
+ if (!(yield item.getFilePathAsync())) {
+ Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping");
+ return;
+ }
+
+ Zotero.logError(e);
+ }
+ }
+ this.queues[type].add(request.start.bind(request));
+})
diff --git a/chrome/content/zotero/xpcom/storage/storageLocal.js b/chrome/content/zotero/xpcom/storage/storageLocal.js
@@ -0,0 +1,1088 @@
+Zotero.Sync.Storage.Local = {
+ lastFullFileCheck: {},
+ uploadCheckFiles: [],
+
+ getClassForLibrary: function (libraryID) {
+ return Zotero.Sync.Storage.Utilities.getClassForMode(this.getModeForLibrary(libraryID));
+ },
+
+ getModeForLibrary: function (libraryID) {
+ var libraryType = Zotero.Libraries.getType(libraryID);
+ switch (libraryType) {
+ case 'user':
+ case 'publications':
+ return Zotero.Prefs.get("sync.storage.protocol") == 'webdav' ? 'webdav' : 'zfs';
+
+ case 'group':
+ return 'zfs';
+
+ default:
+ throw new Error(`Unexpected library type '${libraryType}'`);
+ }
+ },
+
+ setModeForLibrary: function (libraryID, mode) {
+ var libraryType = Zotero.Libraries.getType(libraryID);
+
+ if (libraryType != 'user') {
+ throw new Error(`Cannot set storage mode for ${libraryType} library`);
+ }
+
+ switch (mode) {
+ case 'webdav':
+ case 'zfs':
+ Zotero.Prefs.set("sync.storage.protocol", mode);
+ break;
+
+ default:
+ throw new Error(`Unexpected storage mode '${mode}'`);
+ }
+ },
+
+ /**
+ * Check or enable download-as-needed mode
+ *
+ * @param {Integer} [libraryID]
+ * @param {Boolean} [enable] - If true, enable download-as-needed mode for the given library
+ * @return {Boolean|undefined} - If 'enable' isn't set to true, return true if
+ * download-as-needed mode enabled and false if not
+ */
+ downloadAsNeeded: function (libraryID, enable) {
+ var pref = this._getDownloadPrefFromLibrary(libraryID);
+ var val = 'on-demand';
+ if (enable) {
+ Zotero.Prefs.set(pref, val);
+ return;
+ }
+ return Zotero.Prefs.get(pref) == val;
+ },
+
+ /**
+ * Check or enable download-on-sync mode
+ *
+ * @param {Integer} [libraryID]
+ * @param {Boolean} [enable] - If true, enable download-on-demand mode for the given library
+ * @return {Boolean|undefined} - If 'enable' isn't set to true, return true if
+ * download-as-needed mode enabled and false if not
+ */
+ downloadOnSync: function (libraryID, enable) {
+ var pref = this._getDownloadPrefFromLibrary(libraryID);
+ var val = 'on-demand';
+ if (enable) {
+ Zotero.Prefs.set(pref, val);
+ return;
+ }
+ return Zotero.Prefs.get(pref) == val;
+ },
+
+ _getDownloadPrefFromLibrary: function (libraryID) {
+ if (libraryID == Zotero.Libraries.userLibraryID) {
+ return 'sync.storage.downloadMode.personal';
+ }
+ // TODO: Library-specific settings
+
+ // Group library
+ return 'sync.storage.downloadMode.groups';
+ },
+
+ /**
+ * Get files to check for local modifications for uploading
+ *
+ * This includes files previously modified or opened externally via Zotero within maxCheckAge
+ */
+ getFilesToCheck: Zotero.Promise.coroutine(function* (libraryID, maxCheckAge) {
+ var minTime = new Date().getTime() - (maxCheckAge * 1000);
+
+ // Get files modified and synced since maxCheckAge
+ var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND "
+ + "storageModTime>=?";
+ var params = [
+ libraryID,
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL,
+ Zotero.Sync.Storage.SYNC_STATE_IN_SYNC,
+ minTime
+ ];
+ var itemIDs = yield Zotero.DB.columnQueryAsync(sql, params);
+
+ // Get files opened since maxCheckAge
+ itemIDs = itemIDs.concat(
+ this.uploadCheckFiles.filter(x => x.timestamp >= minTime).map(x => x.itemID)
+ );
+
+ return Zotero.Utilities.arrayUnique(itemIDs);
+ }),
+
+
+ /**
+ * Scans local files and marks any that have changed for uploading
+ * and any that are missing for downloading
+ *
+ * @param {Integer} libraryID
+ * @param {Integer[]} [itemIDs]
+ * @param {Object} [itemModTimes] Item mod times indexed by item ids;
+ * items with stored mod times
+ * that differ from the provided
+ * time but file mod times
+ * matching the stored time will
+ * be marked for download
+ * @return {Promise} Promise resolving to TRUE if any items changed state,
+ * FALSE otherwise
+ */
+ checkForUpdatedFiles: Zotero.Promise.coroutine(function* (libraryID, itemIDs, itemModTimes) {
+ var libraryName = Zotero.Libraries.getName(libraryID);
+ var msg = "Checking for locally changed attachment files in " + libraryName;
+
+ var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"]
+ .getService(Components.interfaces.nsIMemoryReporterManager);
+ memmgr.init();
+ //Zotero.debug("Memory usage: " + memmgr.resident);
+
+ if (itemIDs) {
+ if (!itemIDs.length) {
+ Zotero.debug("No files to check for local changes");
+ return false;
+ }
+ }
+ if (itemModTimes) {
+ if (!Object.keys(itemModTimes).length) {
+ return false;
+ }
+ msg += " in download-marking mode";
+ }
+
+ Zotero.debug(msg);
+
+ var changed = false;
+
+ if (!itemIDs) {
+ itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
+ }
+
+ // Can only handle a certain number of bound parameters at a time
+ var numIDs = itemIDs.length;
+ var maxIDs = Zotero.DB.MAX_BOUND_PARAMETERS - 10;
+ var done = 0;
+ var rows = [];
+
+ do {
+ let chunk = itemIDs.splice(0, maxIDs);
+ let sql = "SELECT itemID, linkMode, path, storageModTime, storageHash, syncState "
+ + "FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE linkMode IN (?,?) AND syncState IN (?,?)";
+ let params = [
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL,
+ Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
+ Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ ];
+ if (libraryID !== false) {
+ sql += " AND libraryID=?";
+ params.push(libraryID);
+ }
+ if (chunk.length) {
+ sql += " AND itemID IN (" + chunk.map(() => '?').join() + ")";
+ params = params.concat(chunk);
+ }
+ let chunkRows = yield Zotero.DB.queryAsync(sql, params);
+ if (chunkRows) {
+ rows = rows.concat(chunkRows);
+ }
+ done += chunk.length;
+ }
+ while (done < numIDs);
+
+ // If no files, or everything is already marked for download,
+ // we don't need to do anything
+ if (!rows.length) {
+ Zotero.debug("No in-sync or to-upload files found in " + libraryName);
+ return false;
+ }
+
+ // Index attachment data by item id
+ itemIDs = [];
+ var attachmentData = {};
+ for (let row of rows) {
+ var id = row.itemID;
+ itemIDs.push(id);
+ attachmentData[id] = {
+ linkMode: row.linkMode,
+ path: row.path,
+ mtime: row.storageModTime,
+ hash: row.storageHash,
+ state: row.syncState
+ };
+ }
+ rows = null;
+
+ var t = new Date();
+ var items = yield Zotero.Items.getAsync(itemIDs, { noCache: true });
+ var numItems = items.length;
+ var updatedStates = {};
+
+ //Zotero.debug("Memory usage: " + memmgr.resident);
+
+ var changed = false;
+ for (let i = 0; i < items.length; i++) {
+ let item = items[i];
+ // TODO: Catch error?
+ let state = yield this._checkForUpdatedFile(item, attachmentData[item.id]);
+ if (state !== false) {
+ yield Zotero.Sync.Storage.Local.setSyncState(item.id, state);
+ changed = true;
+ }
+ }
+
+ if (!items.length) {
+ Zotero.debug("No synced files have changed locally");
+ }
+
+ Zotero.debug(`Checked ${numItems} files in ${libraryName} in ` + (new Date() - t) + " ms");
+
+ return changed;
+ }),
+
+
+ _checkForUpdatedFile: Zotero.Promise.coroutine(function* (item, attachmentData, remoteModTime) {
+ var lk = item.libraryKey;
+ Zotero.debug("Checking attachment file for item " + lk, 4);
+
+ var path = item.getFilePath();
+ if (!path) {
+ Zotero.debug("Marking pathless attachment " + lk + " as in-sync");
+ return Zotero.Sync.Storage.SYNC_STATE_IN_SYNC;
+ }
+ var fileName = OS.Path.basename(path);
+ var file;
+
+ try {
+ file = yield OS.File.open(path);
+ let info = yield file.stat();
+ //Zotero.debug("Memory usage: " + memmgr.resident);
+
+ let fmtime = info.lastModificationDate.getTime();
+ //Zotero.debug("File modification time for item " + lk + " is " + fmtime);
+
+ if (fmtime < 0) {
+ Zotero.debug("File mod time " + fmtime + " is less than 0 -- interpreting as 0", 2);
+ fmtime = 0;
+ }
+
+ // If file is already marked for upload, skip check. Even if the file was changed
+ // both locally and remotely, conflicts are checked at upload time, so we don't need
+ // to worry about it here.
+ if ((yield this.getSyncState(item.id)) == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) {
+ Zotero.debug("File is already marked for upload");
+ return false;
+ }
+
+ //Zotero.debug("Stored mtime is " + attachmentData.mtime);
+ //Zotero.debug("File mtime is " + fmtime);
+
+ //BAIL AFTER DOWNLOAD MARKING MODE, OR CHECK LOCAL?
+ let mtime = attachmentData ? attachmentData.mtime : false;
+
+ // Download-marking mode
+ if (remoteModTime) {
+ Zotero.debug(`Remote mod time for item ${lk} is ${remoteModTime}`);
+
+ // Ignore attachments whose stored mod times haven't changed
+ mtime = mtime !== false ? mtime : (yield this.getSyncedModificationTime(item.id));
+ if (mtime == remoteModTime) {
+ Zotero.debug(`Synced mod time (${mtime}) hasn't changed for item ${lk}`);
+ return false;
+ }
+
+ Zotero.debug(`Marking attachment ${lk} for download (stored mtime: ${mtime})`);
+ // DEBUG: Always set here, or allow further steps?
+ return Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
+ }
+
+ var same = !this.checkFileModTime(item, fmtime, mtime);
+ if (same) {
+ Zotero.debug("File has not changed");
+ return false;
+ }
+
+ // If file hash matches stored hash, only the mod time changed, so skip
+ let fileHash = yield Zotero.Utilities.Internal.md5Async(file);
+
+ var hash = attachmentData ? attachmentData.hash : (yield this.getSyncedHash(item.id));
+ if (hash && hash == fileHash) {
+ // We have to close the file before modifying it from the main
+ // thread (at least on Windows, where assigning lastModifiedTime
+ // throws an NS_ERROR_FILE_IS_LOCKED otherwise)
+ yield file.close();
+
+ Zotero.debug("Mod time didn't match (" + fmtime + " != " + mtime + ") "
+ + "but hash did for " + fileName + " for item " + lk
+ + " -- updating file mod time");
+ try {
+ yield OS.File.setDates(path, null, mtime);
+ }
+ catch (e) {
+ Zotero.File.checkPathAccessError(e, path, 'update');
+ }
+ return false;
+ }
+
+ // Mark file for upload
+ Zotero.debug("Marking attachment " + lk + " as changed "
+ + "(" + mtime + " != " + fmtime + ")");
+ return Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
+ }
+ catch (e) {
+ if (e instanceof OS.File.Error &&
+ (e.becauseNoSuchFile
+ // This can happen if a path is too long on Windows,
+ // e.g. a file is being accessed on a VM through a share
+ // (and probably in other cases).
+ || (e.winLastError && e.winLastError == 3)
+ // Handle long filenames on OS X/Linux
+ || (e.unixErrno && e.unixErrno == 63))) {
+ Zotero.debug("Marking attachment " + lk + " as missing");
+ return Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD;
+ }
+
+ if (e instanceof OS.File.Error) {
+ if (e.becauseClosed) {
+ Zotero.debug("File was closed", 2);
+ }
+ Zotero.debug(e);
+ Zotero.debug(e.toString());
+ throw new Error(`Error for operation '${e.operation}' for ${path}`);
+ }
+
+ throw e;
+ }
+ finally {
+ if (file) {
+ //Zotero.debug("Closing file for item " + lk);
+ file.close();
+ }
+ }
+ }),
+
+ /**
+ *
+ * @param {Zotero.Item} item
+ * @param {Integer} fmtime - File modification time in milliseconds
+ * @param {Integer} mtime - Remote modification time in milliseconds
+ * @return {Boolean} - True if file modification time differs from remote mod time,
+ * false otherwise
+ */
+ checkFileModTime(item, fmtime, mtime) {
+ var libraryKey = item.libraryKey;
+
+ if (fmtime == mtime) {
+ Zotero.debug(`Mod time for ${libraryKey} matches remote file -- skipping`);
+ }
+ // Compare floored timestamps for filesystems that don't support millisecond
+ // precision (e.g., HFS+)
+ else if (Math.floor(mtime / 1000) * 1000 == fmtime
+ || Math.floor(fmtime / 1000) * 1000 == mtime) {
+ Zotero.debug(`File mod times for ${libraryKey} are within one-second precision `
+ + "(" + fmtime + " ≅ " + mtime + ") -- skipping");
+ }
+ // Allow timestamp to be exactly one hour off to get around time zone issues
+ // -- there may be a proper way to fix this
+ else if (Math.abs(fmtime - mtime) == 3600000
+ // And check with one-second precision as well
+ || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+ || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+ Zotero.debug(`File mod time (${fmtime}) for {$libraryKey} is exactly one hour off `
+ + `remote file (${mtime}) -- assuming time zone issue and skipping`);
+ }
+ else {
+ return true;
+ }
+
+ return false;
+ },
+
+ checkForForcedDownloads: Zotero.Promise.coroutine(function* (libraryID) {
+ // Forced downloads happen even in on-demand mode
+ var sql = "SELECT COUNT(*) FROM items JOIN itemAttachments USING (itemID) "
+ + "WHERE libraryID=? AND syncState=?";
+ return !!(yield Zotero.DB.valueQueryAsync(
+ sql, [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]
+ ));
+ }),
+
+
+ /**
+ * Get files marked as ready to download
+ *
+ * @param {Integer} libraryID
+ * @return {Promise<Number[]>} - Promise for an array of attachment itemIDs
+ */
+ getFilesToDownload: function (libraryID, forcedOnly) {
+ var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE libraryID=? AND syncState IN (?";
+ var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD];
+ if (!forcedOnly) {
+ sql += ",?";
+ params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD);
+ }
+ sql += ") "
+ // Skip attachments with empty path, which can't be saved, and files with .zotero*
+ // paths, which have somehow ended up in some users' libraries
+ + "AND path!='' AND path NOT LIKE 'storage:.zotero%'";
+ return Zotero.DB.columnQueryAsync(sql, params);
+ },
+
+
+ /**
+ * Get files marked as ready to upload
+ *
+ * @param {Integer} libraryID
+ * @return {Promise<Number[]>} - Promise for an array of attachment itemIDs
+ */
+ getFilesToUpload: function (libraryID) {
+ var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE libraryID=? AND syncState IN (?,?) AND linkMode IN (?,?)";
+ var params = [
+ libraryID,
+ Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
+ Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD,
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL
+ ];
+ return Zotero.DB.columnQueryAsync(sql, params);
+ },
+
+
+ /**
+ * @param {Integer} libraryID
+ * @return {Promise<String[]>} - Promise for an array of item keys
+ */
+ getDeletedFiles: function (libraryID) {
+ var sql = "SELECT key FROM storageDeleteLog WHERE libraryID=?";
+ return Zotero.DB.columnQueryAsync(sql, libraryID);
+ },
+
+
+ /**
+ * @param {Integer} itemID
+ */
+ getSyncState: function (itemID) {
+ var sql = "SELECT syncState FROM itemAttachments WHERE itemID=?";
+ return Zotero.DB.valueQueryAsync(sql, itemID);
+ },
+
+
+ /**
+ * @param {Integer} itemID
+ * @param {Integer} syncState Constant from Zotero.Sync.Storage
+ */
+ setSyncState: Zotero.Promise.method(function (itemID, syncState) {
+ switch (syncState) {
+ case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_IN_SYNC:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT:
+ break;
+
+ default:
+ throw new Error("Invalid sync state " + syncState);
+ }
+
+ var sql = "UPDATE itemAttachments SET syncState=? WHERE itemID=?";
+ return Zotero.DB.valueQueryAsync(sql, [syncState, itemID]);
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @return {Integer|NULL} Mod time as timestamp in ms,
+ * or NULL if never synced
+ */
+ getSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID) {
+ var sql = "SELECT storageModTime FROM itemAttachments WHERE itemID=?";
+ var mtime = yield Zotero.DB.valueQueryAsync(sql, itemID);
+ if (mtime === false) {
+ throw new Error("Item " + itemID + " not found")
+ }
+ return mtime;
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @param {Integer} mtime - File modification time as timestamp in ms
+ * @param {Boolean} [updateItem=FALSE] - Update clientDateModified field of attachment item
+ */
+ setSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID, mtime, updateItem) {
+ if (mtime < 0) {
+ Components.utils.reportError("Invalid file mod time " + mtime
+ + " in Zotero.Storage.setSyncedModificationTime()");
+ mtime = 0;
+ }
+
+ Zotero.DB.requireTransaction();
+
+ var sql = "UPDATE itemAttachments SET storageModTime=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [mtime, itemID]);
+
+ if (updateItem) {
+ // Update item date modified so the new mod time will be synced
+ let sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
+ }
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @return {Promise<String|null|false>} - File hash, null if never synced, if false if
+ * file doesn't exist
+ */
+ getSyncedHash: Zotero.Promise.coroutine(function* (itemID) {
+ var sql = "SELECT storageHash FROM itemAttachments WHERE itemID=?";
+ var hash = yield Zotero.DB.valueQueryAsync(sql, itemID);
+ if (hash === false) {
+ throw new Error("Item " + itemID + " not found");
+ }
+ return hash;
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @param {String} hash File hash
+ * @param {Boolean} [updateItem=FALSE] Update dateModified field of
+ * attachment item
+ */
+ setSyncedHash: Zotero.Promise.coroutine(function* (itemID, hash, updateItem) {
+ if (hash !== null && hash.length != 32) {
+ throw new Error("Invalid file hash '" + hash + "'");
+ }
+
+ Zotero.DB.requireTransaction();
+
+ var sql = "UPDATE itemAttachments SET storageHash=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [hash, itemID]);
+
+ if (updateItem) {
+ // Update item date modified so the new mod time will be synced
+ var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
+ }
+ }),
+
+
+ /**
+ * Extract a downloaded file and update the database metadata
+ *
+ * @param {Zotero.Item} data.item
+ * @param {Integer} data.mtime
+ * @param {String} data.md5
+ * @param {Boolean} data.compressed
+ * @return {Promise}
+ */
+ processDownload: Zotero.Promise.coroutine(function* (data) {
+ if (!data) {
+ throw new Error("'data' not set");
+ }
+ if (!data.item) {
+ throw new Error("'data.item' not set");
+ }
+ if (!data.mtime) {
+ throw new Error("'data.mtime' not set");
+ }
+ if (data.mtime != parseInt(data.mtime)) {
+ throw new Error("Invalid mod time '" + data.mtime + "'");
+ }
+ if (!data.compressed && !data.md5) {
+ throw new Error("'data.md5' is required if 'data.compressed'");
+ }
+
+ var item = data.item;
+ var mtime = parseInt(data.mtime);
+ var md5 = data.md5;
+
+ // TODO: Test file hash
+
+ if (data.compressed) {
+ var newPath = yield this._processZipDownload(item);
+ }
+ else {
+ var newPath = yield this._processSingleFileDownload(item);
+ }
+
+ // If newPath is set, the file was renamed, so set item filename to that
+ // and mark for updated
+ var path = yield item.getFilePathAsync();
+ if (newPath && path != newPath) {
+ // If library isn't editable but filename was changed, update
+ // database without updating the item's mod time, which would result
+ // in a library access error
+ if (!Zotero.Items.isEditable(item)) {
+ Zotero.debug("File renamed without library access -- "
+ + "updating itemAttachments path", 3);
+ yield item.relinkAttachmentFile(newPath, true);
+ }
+ else {
+ yield item.relinkAttachmentFile(newPath);
+ }
+
+ path = newPath;
+ }
+
+ if (!path) {
+ // This can happen if an HTML snapshot filename was changed and synced
+ // elsewhere but the renamed file wasn't synced, so the ZIP doesn't
+ // contain a file with the known name
+ Components.utils.reportError("File '" + item.attachmentFilename
+ + "' not found after processing download " + item.libraryKey);
+ return new Zotero.Sync.Storage.Result({
+ localChanges: false
+ });
+ }
+
+ try {
+ // If hash not provided (e.g., WebDAV), calculate it now
+ if (!md5) {
+ md5 = yield item.attachmentHash;
+ }
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, path, 'update');
+ }
+
+ // Set the file mtime to the time from the server
+ yield OS.File.setDates(path, null, new Date(parseInt(mtime)));
+
+ yield Zotero.DB.executeTransaction(function* () {
+ yield this.setSyncedHash(item.id, md5);
+ yield this.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ yield this.setSyncedModificationTime(item.id, mtime);
+ }.bind(this));
+
+ return new Zotero.Sync.Storage.Result({
+ localChanges: true
+ });
+ }),
+
+
+ _processSingleFileDownload: Zotero.Promise.coroutine(function* (item) {
+ var tempFilePath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
+
+ if (!(yield OS.File.exists(tempFilePath))) {
+ Zotero.debug(tempFilePath, 1);
+ throw new Error("Downloaded file not found");
+ }
+
+ var parentDirPath = Zotero.Attachments.getStorageDirectory(item).path;
+ if (!(yield OS.File.exists(parentDirPath))) {
+ yield Zotero.Attachments.createDirectoryForItem(item);
+ }
+
+ yield this._deleteExistingAttachmentFiles(item);
+
+ var path = item.getFilePath();
+ if (!path) {
+ throw new Error("Empty path for item " + item.key);
+ }
+ // Don't save Windows aliases
+ if (path.endsWith('.lnk')) {
+ return false;
+ }
+
+ var fileName = OS.Path.basename(path);
+ var renamed = false;
+
+ // Make sure the new filename is valid, in case an invalid character made it over
+ // (e.g., from before we checked for them)
+ var filteredName = Zotero.File.getValidFileName(fileName);
+ if (filteredName != fileName) {
+ Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
+ fileName = filteredName;
+ path = OS.Path.dirname(path, fileName);
+ renamed = true;
+ }
+
+ Zotero.debug("Moving download file " + OS.Path.basename(tempFilePath)
+ + " into attachment directory as '" + fileName + "'");
+ try {
+ var finalFileName = Zotero.File.createShortened(
+ path, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644
+ );
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, path, 'create');
+ }
+
+ if (finalFileName != fileName) {
+ Zotero.debug("Changed filename '" + fileName + "' to '" + finalFileName + "'");
+
+ fileName = finalFileName;
+ path = OS.Path.dirname(path, fileName);
+
+ // Abort if Windows path limitation would cause filenames to be overly truncated
+ if (Zotero.isWin && fileName.length < 40) {
+ try {
+ yield OS.File.remove(path);
+ }
+ catch (e) {}
+ // TODO: localize
+ var msg = "Due to a Windows path length limitation, your Zotero data directory "
+ + "is too deep in the filesystem for syncing to work reliably. "
+ + "Please relocate your Zotero data to a higher directory.";
+ Zotero.debug(msg, 1);
+ throw new Error(msg);
+ }
+
+ renamed = true;
+ }
+
+ try {
+ yield OS.File.move(tempFilePath, path);
+ }
+ catch (e) {
+ try {
+ yield OS.File.remove(tempFilePath);
+ }
+ catch (e) {}
+
+ Zotero.File.checkFileAccessError(e, path, 'create');
+ }
+
+ // processDownload() needs to know that we're renaming the file
+ return renamed ? path : null;
+ }),
+
+
+ _processZipDownload: Zotero.Promise.coroutine(function* (item) {
+ var zipFile = Zotero.getTempDirectory();
+ zipFile.append(item.key + '.tmp');
+
+ if (!zipFile.exists()) {
+ Zotero.debug(zipFile.path);
+ throw new Error(`Downloaded ZIP file not found for item ${item.libraryKey}`);
+ }
+
+ var zipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"].
+ createInstance(Components.interfaces.nsIZipReader);
+ try {
+ zipReader.open(zipFile);
+ zipReader.test(null);
+
+ Zotero.debug("ZIP file is OK");
+ }
+ catch (e) {
+ Zotero.debug(zipFile.leafName + " is not a valid ZIP file", 2);
+ zipReader.close();
+
+ try {
+ zipFile.remove(false);
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, zipFile, 'delete');
+ }
+
+ // TODO: Remove prop file to trigger reuploading, in case it was an upload error?
+
+ return false;
+ }
+
+ var parentDir = Zotero.Attachments.getStorageDirectory(item);
+ if (!parentDir.exists()) {
+ yield Zotero.Attachments.createDirectoryForItem(item);
+ }
+
+ try {
+ yield this._deleteExistingAttachmentFiles(item);
+ }
+ catch (e) {
+ zipReader.close();
+ throw (e);
+ }
+
+ var returnFile = null;
+ var count = 0;
+
+ var itemFileName = item.attachmentFilename;
+
+ var entries = zipReader.findEntries(null);
+ while (entries.hasMore()) {
+ count++;
+ var entryName = entries.getNext();
+ var b64re = /%ZB64$/;
+ if (entryName.match(b64re)) {
+ var fileName = Zotero.Utilities.Internal.Base64.decode(
+ entryName.replace(b64re, '')
+ );
+ }
+ else {
+ var fileName = entryName;
+ }
+
+ if (fileName.startsWith('.zotero')) {
+ Zotero.debug("Skipping " + fileName);
+ continue;
+ }
+
+ Zotero.debug("Extracting " + fileName);
+
+ var primaryFile = false;
+ var filtered = false;
+ var renamed = false;
+
+ // Make sure the new filename is valid, in case an invalid character
+ // somehow make it into the ZIP (e.g., from before we checked for them)
+ //
+ // Do this before trying to use the relative descriptor, since otherwise
+ // it might fail silently and select the parent directory
+ var filteredName = Zotero.File.getValidFileName(fileName);
+ if (filteredName != fileName) {
+ Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
+ fileName = filteredName;
+ filtered = true;
+ }
+
+ // Name in ZIP is a relative descriptor, so file has to be reconstructed
+ // using setRelativeDescriptor()
+ var destFile = parentDir.clone();
+ destFile.QueryInterface(Components.interfaces.nsILocalFile);
+ destFile.setRelativeDescriptor(parentDir, fileName);
+
+ fileName = destFile.leafName;
+
+ // If only one file in zip and it doesn't match the known filename,
+ // take our chances and use that name
+ if (count == 1 && !entries.hasMore() && itemFileName) {
+ // May not be necessary, but let's be safe
+ itemFileName = Zotero.File.getValidFileName(itemFileName);
+ if (itemFileName != fileName) {
+ Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2);
+ Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
+ fileName = itemFileName;
+ destFile.leafName = fileName;
+ renamed = true;
+ }
+ }
+
+ var primaryFile = itemFileName == fileName;
+ if (primaryFile && filtered) {
+ renamed = true;
+ }
+
+ if (destFile.exists()) {
+ var msg = "ZIP entry '" + fileName + "' " + "already exists";
+ Zotero.debug(msg, 2);
+ Components.utils.reportError(msg + " in " + funcName);
+ Zotero.debug(destFile.path);
+ continue;
+ }
+
+ try {
+ Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
+ }
+ catch (e) {
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+
+ zipReader.close();
+
+ Zotero.File.checkFileAccessError(e, destFile, 'create');
+ }
+
+ if (destFile.leafName != fileName) {
+ Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
+
+ // Abort if Windows path limitation would cause filenames to be overly truncated
+ if (Zotero.isWin && destFile.leafName.length < 40) {
+ try {
+ destFile.remove(false);
+ }
+ catch (e) {}
+ zipReader.close();
+ // TODO: localize
+ var msg = "Due to a Windows path length limitation, your Zotero data directory "
+ + "is too deep in the filesystem for syncing to work reliably. "
+ + "Please relocate your Zotero data to a higher directory.";
+ Zotero.debug(msg, 1);
+ throw new Error(msg);
+ }
+
+ if (primaryFile) {
+ renamed = true;
+ }
+ }
+
+ try {
+ zipReader.extract(entryName, destFile);
+ }
+ catch (e) {
+ try {
+ destFile.remove(false);
+ }
+ catch (e) {}
+
+ // For advertising junk files, ignore a bug on Windows where
+ // destFile.create() works but zipReader.extract() doesn't
+ // when the path length is close to 255.
+ if (destFile.leafName.match(/[a-zA-Z0-9+=]{130,}/)) {
+ var msg = "Ignoring error extracting '" + destFile.path + "'";
+ Zotero.debug(msg, 2);
+ Zotero.debug(e, 2);
+ Components.utils.reportError(msg + " in " + funcName);
+ continue;
+ }
+
+ zipReader.close();
+
+ Zotero.File.checkFileAccessError(e, destFile, 'create');
+ }
+
+ destFile.permissions = 0644;
+
+ // If we're renaming the main file, processDownload() needs to know
+ if (renamed) {
+ returnFile = destFile.path;
+ }
+ }
+ zipReader.close();
+ zipFile.remove(false);
+
+ return returnFile;
+ }),
+
+
+ _deleteExistingAttachmentFiles: Zotero.Promise.coroutine(function* (item) {
+ var parentDir = Zotero.Attachments.getStorageDirectory(item).path;
+ return this._deleteExistingFilesInDirectory(parentDir);
+ }),
+
+
+ _deleteExistingFilesInDirectory: Zotero.Promise.coroutine(function* (dir) {
+ var dirsToDelete = [];
+ var iterator = new OS.File.DirectoryIterator(dir);
+ try {
+ yield iterator.forEach(function (entry) {
+ return Zotero.Promise.coroutine(function* () {
+ if (entry.isDir) {
+ dirsToDelete.push(entry.path);
+ }
+ else {
+ try {
+ yield OS.File.remove(entry.path);
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, entry.path, 'delete');
+ }
+ }
+ })();
+ });
+ }
+ finally {
+ iterator.close();
+ }
+ for (let path of dirsToDelete) {
+ yield this._deleteExistingFilesInDirectory(path);
+ }
+ }),
+
+
+ /**
+ * @return {Promise<Object[]>} - A promise for an array of conflict objects
+ */
+ getConflicts: Zotero.Promise.coroutine(function* (libraryID) {
+ var sql = "SELECT itemID, version FROM items JOIN itemAttachments USING (itemID) "
+ + "WHERE libraryID=? AND syncState=?";
+ var rows = yield Zotero.DB.queryAsync(
+ sql,
+ [
+ { int: libraryID },
+ Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ ]
+ );
+ var keyVersionPairs = rows.map(function (row) {
+ var { libraryID, key } = Zotero.Items.getLibraryAndKeyFromID(row.itemID);
+ return [key, row.version];
+ });
+ var cacheObjects = yield Zotero.Sync.Data.Local.getCacheObjects(
+ 'item', libraryID, keyVersionPairs
+ );
+ if (!cacheObjects.length) return [];
+
+ var cacheObjectsByKey = {};
+ cacheObjects.forEach(obj => cacheObjectsByKey[obj.key] = obj);
+
+ var items = [];
+ var localItems = yield Zotero.Items.getAsync(rows.map(row => row.itemID));
+ for (let localItem of localItems) {
+ // Use the mtime for the dateModified field, since that's all that's shown in the
+ // CR window at the moment
+ let localItemJSON = yield localItem.toJSON();
+ localItemJSON.dateModified = Zotero.Date.dateToISO(
+ new Date(yield localItem.attachmentModificationTime)
+ );
+
+ let remoteItemJSON = cacheObjectsByKey[localItem.key];
+ if (!remoteItemJSON) {
+ Zotero.logError("Cached object not found for item " + localItem.libraryKey);
+ continue;
+ }
+ remoteItemJSON = remoteItemJSON.data;
+ remoteItemJSON.dateModified = Zotero.Date.dateToISO(new Date(remoteItemJSON.mtime));
+ items.push({
+ left: localItemJSON,
+ right: remoteItemJSON,
+ changes: [],
+ conflicts: []
+ })
+ }
+ return items;
+ }),
+
+
+ resolveConflicts: Zotero.Promise.coroutine(function* (libraryID) {
+ var conflicts = yield this.getConflicts(libraryID);
+ if (!conflicts.length) return false;
+
+ Zotero.debug("Reconciling conflicts for " + Zotero.Libraries.get(libraryID).name);
+
+ var io = {
+ dataIn: {
+ type: 'file',
+ captions: [
+ Zotero.getString('sync.storage.localFile'),
+ Zotero.getString('sync.storage.remoteFile'),
+ Zotero.getString('sync.storage.savedFile')
+ ],
+ conflicts
+ }
+ };
+
+ var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
+ .getService(Components.interfaces.nsIWindowMediator);
+ var lastWin = wm.getMostRecentWindow("navigator:browser");
+ lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io);
+
+ if (!io.dataOut) {
+ return false;
+ }
+ yield Zotero.DB.executeTransaction(function* () {
+ for (let i = 0; i < conflicts.length; i++) {
+ let conflict = conflicts[i];
+ let mtime = io.dataOut[i].dateModified;
+ // Local
+ if (mtime == conflict.left.dateModified) {
+ syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD;
+ }
+ // Remote
+ else {
+ syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
+ }
+ let itemID = Zotero.Items.getIDFromLibraryAndKey(libraryID, conflict.left.key);
+ yield Zotero.Sync.Storage.Local.setSyncState(itemID, syncState);
+ }
+ });
+ return true;
+ })
+}
diff --git a/chrome/content/zotero/xpcom/storage/storageRequest.js b/chrome/content/zotero/xpcom/storage/storageRequest.js
@@ -0,0 +1,292 @@
+/*
+ ***** BEGIN LICENSE BLOCK *****
+
+ Copyright © 2009 Center for History and New Media
+ George Mason University, Fairfax, Virginia, USA
+ http://zotero.org
+
+ This file is part of Zotero.
+
+ Zotero is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Zotero is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Zotero. If not, see <http://www.gnu.org/licenses/>.
+
+ ***** END LICENSE BLOCK *****
+*/
+
+
+/**
+ * Transfer request for storage sync
+ *
+ * @param {Object} options
+ * @param {String} options.type
+ * @param {Integer} options.libraryID
+ * @param {String} options.name - Identifier for request (e.g., "[libraryID]/[key]")
+ * @param {Function|Function[]} [options.onStart]
+ * @param {Function|Function[]} [options.onProgress]
+ * @param {Function|Function[]} [options.onStop]
+ */
+Zotero.Sync.Storage.Request = function (options) {
+ if (!options.type) throw new Error("type must be provided");
+ if (!options.libraryID) throw new Error("libraryID must be provided");
+ if (!options.name) throw new Error("name must be provided");
+ ['type', 'libraryID', 'name'].forEach(x => this[x] = options[x]);
+
+ Zotero.debug(`Initializing ${this.type} request ${this.name}`);
+
+ this.callbacks = ['onStart', 'onProgress', 'onStop'];
+
+ this.Type = Zotero.Utilities.capitalize(this.type);
+ this.channel = null;
+ this.queue = null;
+ this.progress = 0;
+ this.progressMax = 0;
+
+ this._deferred = Zotero.Promise.defer();
+ this._running = false;
+ this._stopping = false;
+ this._percentage = 0;
+ this._remaining = null;
+ this._maxSize = null;
+ this._finished = false;
+
+ for (let name of this.callbacks) {
+ if (!options[name]) continue;
+ this['_' + name] = Array.isArray(options[name]) ? options[name] : [options[name]];
+ }
+}
+
+
+Zotero.Sync.Storage.Request.prototype.setMaxSize = function (size) {
+ this._maxSize = size;
+};
+
+
+/**
+ * Add callbacks from another request to this request
+ */
+Zotero.Sync.Storage.Request.prototype.importCallbacks = function (request) {
+ for each(var name in this.callbacks) {
+ name = '_' + name;
+ if (request[name]) {
+ // If no handlers for this event, add them all
+ if (!this[name]) {
+ this[name] = request[name];
+ continue;
+ }
+ // Otherwise add functions that don't already exist
+ var add = true;
+ for each(var newFunc in request[name]) {
+ for each(var currentFunc in this[name]) {
+ if (newFunc.toString() === currentFunc.toString()) {
+ Zotero.debug("Callback already exists in request -- not importing");
+ add = false;
+ break;
+ }
+ }
+ if (add) {
+ this[name].push(newFunc);
+ }
+ }
+ }
+ }
+}
+
+
+Zotero.Sync.Storage.Request.prototype.__defineGetter__('percentage', function () {
+ if (this._finished) {
+ return 100;
+ }
+
+ if (this.progressMax == 0) {
+ return 0;
+ }
+
+ var percentage = Math.round((this.progress / this.progressMax) * 100);
+ if (percentage < this._percentage) {
+ Zotero.debug(percentage + " is less than last percentage of "
+ + this._percentage + " for request " + this.name, 2);
+ Zotero.debug(this.progress);
+ Zotero.debug(this.progressMax);
+ percentage = this._percentage;
+ }
+ else if (percentage > 100) {
+ Zotero.debug(percentage + " is greater than 100 for "
+ + "request " + this.name, 2);
+ Zotero.debug(this.progress);
+ Zotero.debug(this.progressMax);
+ percentage = 100;
+ }
+ else {
+ this._percentage = percentage;
+ }
+ //Zotero.debug("Request '" + this.name + "' percentage is " + percentage);
+ return percentage;
+});
+
+
+Zotero.Sync.Storage.Request.prototype.__defineGetter__('remaining', function () {
+ if (this._finished) {
+ return 0;
+ }
+
+ if (!this.progressMax) {
+ if (this.type == 'upload' && this._maxSize) {
+ return Math.round(Zotero.Sync.Storage.compressionTracker.ratio * this._maxSize);
+ }
+
+ //Zotero.debug("Remaining not yet available for request '" + this.name + "'");
+ return 0;
+ }
+
+ var remaining = this.progressMax - this.progress;
+ if (this._remaining === null) {
+ this._remaining = remaining;
+ }
+ else if (remaining > this._remaining) {
+ Zotero.debug(remaining + " is greater than the last remaining amount of "
+ + this._remaining + " for request " + this.name);
+ remaining = this._remaining;
+ }
+ else if (remaining < 0) {
+ Zotero.debug(remaining + " is less than 0 for request " + this.name);
+ }
+ else {
+ this._remaining = remaining;
+ }
+ //Zotero.debug("Request '" + this.name + "' remaining is " + remaining);
+ return remaining;
+});
+
+
+Zotero.Sync.Storage.Request.prototype.setChannel = function (channel) {
+ this.channel = channel;
+}
+
+
+Zotero.Sync.Storage.Request.prototype.start = Zotero.Promise.coroutine(function* () {
+ Zotero.debug("Starting " + this.type + " request " + this.name);
+
+ if (this._running) {
+ throw new Error(this.type + " request " + this.name + " already running");
+ }
+
+ if (!this._onStart) {
+ throw new Error("onStart not provided -- nothing to do!");
+ }
+
+ this._running = true;
+
+ // this._onStart is an array of promises for objects of result flags, which are combined
+ // into a single object here
+ //
+ // The main sync logic is triggered here.
+ try {
+ var results = yield Zotero.Promise.all(this._onStart.map(f => f(this)));
+
+ var result = new Zotero.Sync.Storage.Result;
+ result.updateFromResults(results);
+
+ Zotero.debug(this.Type + " request " + this.name + " finished");
+ Zotero.debug(result + "");
+
+ return result;
+ }
+ catch (e) {
+ Zotero.logError(this.Type + " request " + this.name + " failed");
+ throw e;
+ }
+ finally {
+ this._finished = true;
+ this._running = false;
+
+ if (this._onStop) {
+ this._onStop.forEach(x => x());
+ }
+ }
+});
+
+
+Zotero.Sync.Storage.Request.prototype.isRunning = function () {
+ return this._running;
+}
+
+
+Zotero.Sync.Storage.Request.prototype.isFinished = function () {
+ return this._finished;
+}
+
+
+/**
+ * Update counters for given request
+ *
+ * Also updates progress meter
+ *
+ * @param {Integer} progress Progress so far
+ * (usually bytes transferred)
+ * @param {Integer} progressMax Max progress value for this request
+ * (usually total bytes)
+ */
+Zotero.Sync.Storage.Request.prototype.onProgress = function (progress, progressMax) {
+ //Zotero.debug(progress + "/" + progressMax + " for request " + this.name);
+
+ if (!this._running) {
+ Zotero.debug("Trying to update finished request " + this.name + " in "
+ + "Zotero.Sync.Storage.Request.onProgress() "
+ + "(" + progress + "/" + progressMax + ")", 2);
+ return;
+ }
+
+ // Workaround for invalid progress values (possibly related to
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=451991 and fixed in 3.1)
+ if (progress < this.progress) {
+ Zotero.debug("Invalid progress for request '"
+ + this.name + "' (" + progress + " < " + this.progress + ")");
+ return;
+ }
+
+ if (progressMax != this.progressMax) {
+ Zotero.debug("progressMax has changed from " + this.progressMax
+ + " to " + progressMax + " for request '" + this.name + "'", 2);
+ }
+
+ this.progress = progress;
+ this.progressMax = progressMax;
+
+ if (this.type == 'download') {
+ Zotero.Sync.Storage.setItemDownloadPercentage(this.name, this.percentage);
+ }
+
+ if (this.onProgress) {
+ for each(var f in this._onProgress) {
+ f(progress, progressMax);
+ }
+ }
+}
+
+
+/**
+ * Stop the request's underlying network request, if there is one
+ */
+Zotero.Sync.Storage.Request.prototype.stop = function (force) {
+ if (this.channel && this.channel.isPending()) {
+ this._stopping = true;
+
+ try {
+ Zotero.debug(`Stopping ${this.type} request '${this.name} '`);
+ this.channel.cancel(0x804b0002); // NS_BINDING_ABORTED
+ }
+ catch (e) {
+ Zotero.debug(e, 1);
+ }
+ }
+}
diff --git a/chrome/content/zotero/xpcom/storage/storageResult.js b/chrome/content/zotero/xpcom/storage/storageResult.js
@@ -0,0 +1,47 @@
+"use strict";
+
+/**
+ * @property {Boolean} localChanges - Changes were made locally. For logging purposes only.
+ * @property {Boolean} remoteChanges - Changes were made on the server. This causes the
+ * last-sync time to be updated on the server (WebDAV) or retrieved (ZFS) and stored locally
+ * to skip additional file syncs until further server changes are made.
+ * @property {Boolean} syncRequired - A data sync is required to upload local changes
+ * @propretty {Boolean} fileSyncRequired - Another file sync is required to handle files left in
+ * conflict
+ */
+Zotero.Sync.Storage.Result = function (options = {}) {
+ this._props = ['localChanges', 'remoteChanges', 'syncRequired', 'fileSyncRequired'];
+ for (let prop of this._props) {
+ this[prop] = options[prop] || false;
+ }
+}
+
+/**
+ * Update the properties on this object from multiple Result objects
+ *
+ * @param {Zotero.Sync.Storage.Result[]} results
+ */
+Zotero.Sync.Storage.Result.prototype.updateFromResults = function (results) {
+ for (let prop of this._props) {
+ if (!this[prop]) {
+ for (let result of results) {
+ if (!(result instanceof Zotero.Sync.Storage.Result)) {
+ Zotero.debug(result, 1);
+ throw new Error("'result' is not a storage result");
+ }
+ if (result[prop]) {
+ this[prop] = true;
+ }
+ }
+ }
+ }
+}
+
+
+Zotero.Sync.Storage.Result.prototype.toString = function () {
+ var obj = {};
+ for (let prop of this._props) {
+ obj[prop] = this[prop] || false;
+ }
+ return JSON.stringify(obj, null, " ");
+}
diff --git a/chrome/content/zotero/xpcom/storage/storageUtilities.js b/chrome/content/zotero/xpcom/storage/storageUtilities.js
@@ -0,0 +1,67 @@
+Zotero.Sync.Storage.Utilities = {
+ getClassForMode: function (mode) {
+ switch (mode) {
+ case 'zfs':
+ return Zotero.Sync.Storage.ZFS_Module;
+
+ case 'webdav':
+ return Zotero.Sync.Storage.WebDAV_Module;
+
+ default:
+ throw new Error("Invalid storage mode '" + mode + "'");
+ }
+ },
+
+ getItemFromRequest: function (request) {
+ var [libraryID, key] = request.name.split('/');
+ return Zotero.Items.getByLibraryAndKey(libraryID, key);
+ },
+
+
+ /**
+ * Create zip file of attachment directory in the temp directory
+ *
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise<Boolean>} - True if the zip file was created, false otherwise
+ */
+ createUploadFile: Zotero.Promise.coroutine(function* (request) {
+ var item = this.getItemFromRequest(request);
+ Zotero.debug("Creating ZIP file for item " + item.libraryKey);
+
+ switch (item.attachmentLinkMode) {
+ case Zotero.Attachments.LINK_MODE_LINKED_FILE:
+ case Zotero.Attachments.LINK_MODE_LINKED_URL:
+ throw new Error("Upload file must be an imported snapshot or file");
+ }
+
+ var zipFile = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
+
+ return Zotero.File.zipDirectory(
+ Zotero.Attachments.getStorageDirectory(item).path,
+ zipFile,
+ {
+ onStopRequest: function (req, context, status) {
+ var zipFileName = OS.Path.basename(zipFile);
+
+ var originalSize = 0;
+ for (let entry of context.entries) {
+ let zipEntry = context.zipWriter.getEntry(entry.name);
+ if (!zipEntry) {
+ Zotero.logError("ZIP entry '" + entry.name + "' not found for "
+ + "request '" + request.name + "'")
+ continue;
+ }
+ originalSize += zipEntry.realSize;
+ }
+
+ Zotero.debug("Zip of " + zipFileName + " finished with status " + status
+ + " (original " + Math.round(originalSize / 1024) + "KB, "
+ + "compressed " + Math.round(context.zipWriter.file.fileSize / 1024) + "KB, "
+ + Math.round(
+ ((originalSize - context.zipWriter.file.fileSize) / originalSize) * 100
+ ) + "% reduction)");
+ }
+ }
+ );
+ })
+}
diff --git a/chrome/content/zotero/xpcom/storage/streamListener.js b/chrome/content/zotero/xpcom/storage/streamListener.js
@@ -30,10 +30,9 @@
* Possible properties of data object:
* - onStart: f(request)
* - onProgress: f(request, progress, progressMax)
- * - onStop: f(request, status, response, data)
- * - onCancel: f(request, status, data)
+ * - onStop: f(request, status, response)
+ * - onCancel: f(request, status)
* - streams: array of streams to close on completion
- * - Other values to pass to onStop()
*/
Zotero.Sync.Storage.StreamListener = function (data) {
this._data = data;
@@ -110,17 +109,15 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
onStateChange: function (wp, request, stateFlags, status) {
- Zotero.debug("onStateChange");
- Zotero.debug(stateFlags);
- Zotero.debug(status);
+ Zotero.debug("onStateChange with " + stateFlags);
- if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START)
- && (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) {
- this._onStart(request);
- }
- else if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP)
- && (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) {
- this._onStop(request, status);
+ if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_REQUEST) {
+ if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) {
+ this._onStart(request);
+ }
+ else if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) {
+ this._onStop(request, status);
+ }
}
},
@@ -148,18 +145,38 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
// nsIChannelEventSink
- onChannelRedirect: function (oldChannel, newChannel, flags) {
+ //
+ // If this._data.onChannelRedirect exists, it should return a promise resolving to true to
+ // follow the redirect or false to cancel it
+ onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
Zotero.debug('onChannelRedirect');
+ if (this._data && this._data.onChannelRedirect) {
+ let result = yield this._data.onChannelRedirect(oldChannel, newChannel, flags);
+ if (!result) {
+ oldChannel.cancel(Components.results.NS_BINDING_ABORTED);
+ newChannel.cancel(Components.results.NS_BINDING_ABORTED);
+ return false;
+ }
+ }
+
// if redirecting, store the new channel
this._channel = newChannel;
- },
+ }),
asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) {
Zotero.debug('asyncOnRedirect');
- this.onChannelRedirect(oldChan, newChan, flags);
- redirectCallback.onRedirectVerifyCallback(0);
+ this.onChannelRedirect(oldChan, newChan, flags)
+ .then(function (result) {
+ redirectCallback.onRedirectVerifyCallback(
+ result ? Components.results.NS_SUCCEEDED : Components.results.NS_FAILED
+ );
+ })
+ .catch(function (e) {
+ Zotero.logError(e);
+ redirectCallback.onRedirectVerifyCallback(Components.results.NS_FAILED);
+ });
},
// nsIHttpEventSink
@@ -177,8 +194,7 @@ Zotero.Sync.Storage.StreamListener.prototype = {
_onStart: function (request) {
Zotero.debug('Starting request');
if (this._data && this._data.onStart) {
- var data = this._getPassData();
- this._data.onStart(request, data);
+ this._data.onStart(request);
}
},
@@ -189,7 +205,6 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
_onStop: function (request, status) {
- Zotero.debug('Request ended with status ' + status);
var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED
if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) {
@@ -201,9 +216,11 @@ Zotero.Sync.Storage.StreamListener.prototype = {
Zotero.debug("Request responseStatus not available", 1);
status = 0;
}
+ Zotero.debug('Request ended with status code ' + status);
request.QueryInterface(Components.interfaces.nsIRequest);
}
else {
+ Zotero.debug('Request ended with status ' + status);
status = 0;
}
@@ -213,38 +230,20 @@ Zotero.Sync.Storage.StreamListener.prototype = {
}
}
- var data = this._getPassData();
-
if (cancelled) {
if (this._data.onCancel) {
- this._data.onCancel(request, status, data);
+ this._data.onCancel(request, status);
}
}
else {
if (this._data.onStop) {
- this._data.onStop(request, status, this._response, data);
+ this._data.onStop(request, status, this._response);
}
}
this._channel = null;
},
- _getPassData: function () {
- // Make copy of data without callbacks to pass along
- var passData = {};
- for (var i in this._data) {
- switch (i) {
- case "onStart":
- case "onProgress":
- case "onStop":
- case "onCancel":
- continue;
- }
- passData[i] = this._data[i];
- }
- return passData;
- },
-
// nsIInterfaceRequestor
getInterface: function (iid) {
try {
diff --git a/chrome/content/zotero/xpcom/storage/webdav.js b/chrome/content/zotero/xpcom/storage/webdav.js
@@ -24,1725 +24,1678 @@
*/
-Zotero.Sync.Storage.WebDAV = (function () {
- var _initialized = false;
- var _parentURI;
- var _rootURI;
- var _cachedCredentials = false;
+Zotero.Sync.Storage.WebDAV_Module = {};
+Zotero.Sync.Storage.WebDAV_Module.prototype = {
+ name: "WebDAV",
+ get verified() {
+ return Zotero.Prefs.get("sync.storage.verified");
+ },
- var _loginManagerHost = 'chrome://zotero';
- var _loginManagerURL = 'Zotero Storage Server';
+ _initialized: false,
+ _parentURI: null,
+ _rootURI: null,
+ _cachedCredentials: false,
- var _lastSyncIDLength = 30;
+ _loginManagerHost: 'chrome://zotero',
+ _loginManagerURL: 'Zotero Storage Server',
- //
- // Private methods
- //
- /**
- * Get mod time of file on storage server
- *
- * @param {Zotero.Item} item
- * @param {Function} callback Callback f(item, mdate)
- */
- function getStorageModificationTime(item, request) {
- var uri = getItemPropertyURI(item);
+ _lastSyncIDLength: 30,
+
+
+ get defaultError() {
+ return Zotero.getString('sync.storage.error.webdav.default');
+ },
+
+ get defaultErrorRestart() {
+ return Zotero.getString('sync.storage.error.webdav.defaultRestart', Zotero.appName);
+ },
+
+ get _username() {
+ return Zotero.Prefs.get('sync.storage.username');
+ },
+
+ get _password() {
+ var username = this._username;
- return Zotero.HTTP.promise("GET", uri,
- {
- debug: true,
- successCodes: [200, 300, 404],
- requestObserver: function (xmlhttp) {
- request.setChannel(xmlhttp.channel);
- }
- })
- .then(function (req) {
- checkResponse(req);
-
- // mod_speling can return 300s for 404s with base name matches
- if (req.status == 404 || req.status == 300) {
- return false;
- }
-
- // No modification time set
- if (!req.responseText) {
- return false;
- }
-
- var seconds = false;
- var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
- .createInstance(Components.interfaces.nsIDOMParser);
- try {
- var xml = parser.parseFromString(req.responseText, "text/xml");
- var mtime = xml.getElementsByTagName('mtime')[0].textContent;
- }
- catch (e) {
- Zotero.debug(e);
- var mtime = false;
- }
-
- // TEMP
- if (!mtime) {
- mtime = req.responseText;
- seconds = true;
- }
-
- var invalid = false;
-
- // Unix timestamps need to be converted to ms-based timestamps
- if (seconds) {
- if (mtime.match(/^[0-9]{1,10}$/)) {
- Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
- mtime = mtime * 1000;
- }
- else {
- invalid = true;
- }
- }
- else if (!mtime.match(/^[0-9]{1,13}$/)) {
- invalid = true;
- }
-
- // Delete invalid .prop files
- if (invalid) {
- var msg = "Invalid mod date '" + Zotero.Utilities.ellipsize(mtime, 20)
- + "' for item " + Zotero.Items.getLibraryKeyHash(item);
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- return deleteStorageFiles([item.key + ".prop"])
- .then(function (results) {
- throw new Error(Zotero.Sync.Storage.WebDAV.defaultError);
- });
- }
-
- return new Date(parseInt(mtime));
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- throw new Error("HTTP " + e.status + " error from WebDAV "
- + "server for GET request");
- }
- throw e;
- });
- }
+ if (!username) {
+ Zotero.debug('Username not set before getting Zotero.Sync.Storage.WebDAV.password');
+ return '';
+ }
+
+ Zotero.debug('Getting WebDAV password');
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins({}, this._loginManagerHost, this._loginManagerURL, null);
+
+ // Find user from returned array of nsILoginInfo objects
+ for (var i = 0; i < logins.length; i++) {
+ if (logins[i].username == username) {
+ return logins[i].password;
+ }
+ }
+
+ return '';
+ },
+
+ set _password(password) {
+ var username = this._username;
+ if (!username) {
+ Zotero.debug('Username not set before setting Zotero.Sync.Server.Mode.WebDAV.password');
+ return;
+ }
+
+ _cachedCredentials = false;
+
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins({}, this._loginManagerHost, this._loginManagerURL, null);
+
+ for (var i = 0; i < logins.length; i++) {
+ Zotero.debug('Clearing WebDAV passwords');
+ loginManager.removeLogin(logins[i]);
+ break;
+ }
+
+ if (password) {
+ Zotero.debug(this._loginManagerURL);
+ var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
+ Components.interfaces.nsILoginInfo, "init");
+ var loginInfo = new nsLoginInfo(this._loginManagerHost, this._loginManagerURL,
+ null, username, password, "", "");
+ loginManager.addLogin(loginInfo);
+ }
+ },
+ get rootURI() {
+ if (!this._rootURI) {
+ this._init();
+ }
+ return this._rootURI.clone();
+ },
- /**
- * Set mod time of file on storage server
- *
- * @param {Zotero.Item} item
- */
- function setStorageModificationTime(item) {
- var uri = getItemPropertyURI(item);
+ get parentURI() {
+ if (!this._parentURI) {
+ this._init();
+ }
+ return this._parentURI.clone();
+ },
+
+ init: function () {
+ this._rootURI = false;
+ this._parentURI = false;
- var mtime = item.attachmentModificationTime;
- var hash = item.attachmentHash;
+ var scheme = Zotero.Prefs.get('sync.storage.scheme');
+ switch (scheme) {
+ case 'http':
+ case 'https':
+ break;
+
+ default:
+ throw new Error("Invalid WebDAV scheme '" + scheme + "'");
+ }
- var prop = '<properties version="1">'
- + '<mtime>' + mtime + '</mtime>'
- + '<hash>' + hash + '</hash>'
- + '</properties>';
+ var url = Zotero.Prefs.get('sync.storage.url');
+ if (!url) {
+ var msg = "WebDAV URL not provided";
+ Zotero.debug(msg);
+ throw ({
+ message: msg,
+ name: "Z_ERROR_NO_URL",
+ filename: "webdav.js",
+ toString: function () { return this.message; }
+ });
+ }
- return Zotero.HTTP.promise("PUT", uri,
- { body: prop, debug: true, successCodes: [200, 201, 204] })
- .then(function (req) {
- return { mtime: mtime, hash: hash };
- })
- .catch(function (e) {
- throw new Error("HTTP " + e.xmlhttp.status
- + " from WebDAV server for HTTP PUT");
+ url = scheme + '://' + url;
+ var dir = "zotero";
+ var username = this._username;
+ var password = this._password;
+
+ if (!username) {
+ var msg = "WebDAV username not provided";
+ Zotero.debug(msg);
+ throw ({
+ message: msg,
+ name: "Z_ERROR_NO_USERNAME",
+ filename: "webdav.js",
+ toString: function () { return this.message; }
+ });
+ }
+
+ if (!password) {
+ var msg = "WebDAV password not provided";
+ Zotero.debug(msg);
+ throw ({
+ message: msg,
+ name: "Z_ERROR_NO_PASSWORD",
+ filename: "webdav.js",
+ toString: function () { return this.message; }
});
- };
+ }
+
+ var ios = Components.classes["@mozilla.org/network/io-service;1"].
+ getService(Components.interfaces.nsIIOService);
+ var uri = ios.newURI(url, null, null);
+ uri.username = encodeURIComponent(username);
+ uri.password = encodeURIComponent(password);
+ if (!uri.spec.match(/\/$/)) {
+ uri.spec += "/";
+ }
+ this._parentURI = uri;
+
+ var uri = uri.clone();
+ uri.spec += "zotero/";
+ this._rootURI = uri;
+ },
+ cacheCredentials: Zotero.Promise.coroutine(function* () {
+ if (this._cachedCredentials) {
+ Zotero.debug("WebDAV credentials are already cached");
+ return;
+ }
+
+ try {
+ var req = Zotero.HTTP.request("OPTIONS", this.rootURI);
+ checkResponse(req);
+
+ Zotero.debug("Credentials are cached");
+ this._cachedCredentials = true;
+ }
+ catch (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ let msg = "HTTP " + e.status + " error from WebDAV server "
+ + "for OPTIONS request";
+ Zotero.debug(msg, 1);
+ Components.utils.reportError(msg);
+ throw new Error(Zotero.Sync.Storage.WebDAV.defaultErrorRestart);
+ }
+ throw e;
+ }
+ }),
+
+
+ clearCachedCredentials: function() {
+ this._rootURI = this._parentURI = undefined;
+ this._cachedCredentials = false;
+ },
/**
- * Upload the generated ZIP file to the server
+ * Begin download process for individual file
*
- * @param {Object} Object with 'request' property
- * @return {void}
+ * @param {Zotero.Sync.Storage.Request} [request]
*/
- function processUploadFile(data) {
- /*
- updateSizeMultiplier(
- (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
- );
- */
- var request = data.request;
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
+ downloadFile: function (request, requeueCallback) {
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
+ if (!item) {
+ throw new Error("Item '" + request.name + "' not found");
+ }
+ // Retrieve modification time from server to store locally afterwards
return getStorageModificationTime(item, request)
.then(function (mdate) {
if (!request.isRunning()) {
- Zotero.debug("Upload request '" + request.name
+ Zotero.debug("Download request '" + request.name
+ "' is no longer running after getting mod time");
return false;
}
- // Check for conflict
- if (Zotero.Sync.Storage.getSyncState(item.id)
- != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
- if (mdate) {
- // Remote prop time
- var mtime = mdate.getTime();
-
- // Local file time
- var fmtime = item.attachmentModificationTime;
-
- var same = false;
- if (fmtime == mtime) {
- same = true;
- Zotero.debug("File mod time matches remote file -- skipping upload");
- }
- // Allow floored timestamps for filesystems that don't support
- // millisecond precision (e.g., HFS+)
- else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
- same = true;
- Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
- + "-- skipping upload");
- }
- // Allow timestamp to be exactly one hour off to get around
- // time zone issues -- there may be a proper way to fix this
- else if (Math.abs(fmtime - mtime) == 3600000
- // And check with one-second precision as well
- || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
- || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
- same = true;
- Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
- + "-- assuming time zone issue and skipping upload");
- }
-
- if (same) {
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return true;
- }
-
- var smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id);
- if (smtime != mtime) {
- Zotero.debug("Conflict -- last synced file mod time "
- + "does not match time on storage server"
- + " (" + smtime + " != " + mtime + ")");
- return {
- localChanges: false,
- remoteChanges: false,
- conflict: {
- local: { modTime: fmtime },
- remote: { modTime: mtime }
- }
- };
- }
- }
- else {
- Zotero.debug("Remote file not found for item " + item.id);
- }
+ if (!mdate) {
+ Zotero.debug("Remote file not found for item " + Zotero.Items.getLibraryKeyHash(item));
+ return false;
}
- var file = Zotero.getTempDirectory();
- file.append(item.key + '.zip');
-
- var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]
- .createInstance(Components.interfaces.nsIFileInputStream);
- fis.init(file, 0x01, 0, 0);
-
- var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"]
- .createInstance(Components.interfaces.nsIBufferedInputStream)
- bis.init(fis, 64 * 1024);
-
- var uri = getItemURI(item);
-
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var channel = ios.newChannelFromURI(uri);
- channel.QueryInterface(Components.interfaces.nsIUploadChannel);
- channel.setUploadStream(bis, 'application/octet-stream', -1);
- channel.QueryInterface(Components.interfaces.nsIHttpChannel);
- channel.requestMethod = 'PUT';
- channel.allowPipelining = false;
-
- channel.setRequestHeader('Keep-Alive', '', false);
- channel.setRequestHeader('Connection', '', false);
-
- var deferred = Zotero.Promise.defer();
+ var syncModTime = mdate.getTime();
- var listener = new Zotero.Sync.Storage.StreamListener(
+ // Skip download if local file exists and matches mod time
+ var file = item.getFile();
+ if (file && file.exists() && syncModTime == file.lastModifiedTime) {
+ Zotero.debug("File mod time matches remote file -- skipping download");
+
+ Zotero.DB.beginTransaction();
+ var syncState = Zotero.Sync.Storage.getSyncState(item.id);
+ var updateItem = syncState != 1;
+ Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
+ Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ Zotero.DB.commitTransaction();
+ return {
+ localChanges: true, // ?
+ remoteChanges: false,
+ syncRequired: false
+ };
+ }
+
+ var uri = getItemURI(item);
+ var destFile = Zotero.getTempDirectory();
+ destFile.append(item.key + '.zip.tmp');
+ if (destFile.exists()) {
+ destFile.remove(false);
+ }
+
+ var deferred = Zotero.Promise.defer();
+
+ var listener = new Zotero.Sync.Storage.StreamListener(
{
+ onStart: function (request, data) {
+ if (data.request.isFinished()) {
+ Zotero.debug("Download request " + data.request.name
+ + " stopped before download started -- closing channel");
+ request.cancel(0x804b0002); // NS_BINDING_ABORTED
+ deferred.resolve(false);
+ }
+ },
onProgress: function (a, b, c) {
- request.onProgress(a, b, c);
+ request.onProgress(a, b, c)
},
- onStop: function (httpRequest, status, response, data) {
+ onStop: function (request, status, response, data) {
data.request.setChannel(false);
- deferred.resolve(
- Zotero.Promise.try(function () {
- return onUploadComplete(httpRequest, status, response, data);
+ if (status == 404) {
+ var msg = "Remote ZIP file not found for item " + item.key;
+ Zotero.debug(msg, 2);
+ Components.utils.reportError(msg);
+
+ // Delete the orphaned prop file
+ deleteStorageFiles([item.key + ".prop"])
+ .finally(function (results) {
+ deferred.resolve(false);
})
- );
+ .done();
+ return;
+ }
+ else if (status != 200) {
+ var msg = "HTTP " + status + " from WebDAV server "
+ + " while downloading file";
+ Zotero.debug(msg, 1);
+ Components.utils.reportError(msg);
+ deferred.reject(Zotero.Sync.Storage.WebDAV.defaultError);
+ return;
+ }
+
+ // Don't try to process if the request has been cancelled
+ if (data.request.isFinished()) {
+ Zotero.debug("Download request " + data.request.name
+ + " is no longer running after file download");
+ deferred.resolve(false);
+ return;
+ }
+
+ Zotero.debug("Finished download of " + destFile.path);
+
+ try {
+ deferred.resolve(
+ Zotero.Sync.Storage.processDownload(
+ data, requeueCallback
+ )
+ );
+ }
+ catch (e) {
+ deferred.reject(e);
+ }
},
- onCancel: function (httpRequest, status, data) {
- onUploadCancel(httpRequest, status, data);
+ onCancel: function (request, status, data) {
+ Zotero.debug("Request cancelled");
deferred.resolve(false);
},
request: request,
item: item,
- streams: [fis, bis]
+ compressed: true,
+ syncModTime: syncModTime
}
);
- channel.notificationCallbacks = listener;
- var dispURI = uri.clone();
- if (dispURI.password) {
- dispURI.password = '********';
+ // Don't display password in console
+ var disp = uri.clone();
+ if (disp.password) {
+ disp.password = '********';
}
- Zotero.debug("HTTP PUT of " + file.leafName + " to " + dispURI.spec);
-
- channel.asyncOpen(listener, null);
+ Zotero.debug('Saving ' + disp.spec + ' with saveURI()');
+ const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
+ var wbp = Components
+ .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
+ .createInstance(nsIWBP);
+ wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
+ wbp.progressListener = listener;
+ Zotero.Utilities.Internal.saveURI(wbp, uri, destFile);
return deferred.promise;
});
- }
+ },
- function onUploadComplete(httpRequest, status, response, data) {
- var request = data.request;
- var item = data.item;
- var url = httpRequest.name;
-
- Zotero.debug("Upload of attachment " + item.key
- + " finished with status code " + status);
-
- switch (status) {
- case 200:
- case 201:
- case 204:
- break;
-
- case 403:
- case 500:
- Zotero.debug(response);
- throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
- " " + Zotero.getString('sync.storage.error.checkFileSyncSettings'));
-
- case 507:
- Zotero.debug(response);
- throw Zotero.getString('sync.storage.error.webdav.insufficientSpace');
-
- default:
- Zotero.debug(response);
- throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
- " " + Zotero.getString('sync.storage.error.checkFileSyncSettings')
- + "\n\n" + "HTTP " + status);
+ uploadFile: function (request) {
+ var deferred = Zotero.Promise.defer();
+ var created = Zotero.Sync.Storage.createUploadFile(
+ request,
+ function (data) {
+ if (!data) {
+ deferred.resolve(false);
+ return;
+ }
+ deferred.resolve(
+ Zotero.Promise.try(function () {
+ return processUploadFile(data);
+ })
+ );
+ }
+ );
+ if (!created) {
+ return Zotero.Promise.resolve(false);
}
+ return deferred.promise;
+ },
+
+
+ getLastSyncTime: function () {
+ var lastSyncURI = this.rootURI;
+ lastSyncURI.spec += "lastsync.txt";
- return setStorageModificationTime(item)
- .then(function (props) {
- if (!request.isRunning()) {
- Zotero.debug("Upload request '" + request.name
- + "' is no longer running after getting mod time");
- return false;
- }
-
- Zotero.DB.beginTransaction();
-
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true);
- Zotero.Sync.Storage.setSyncedHash(item.id, props.hash);
+ // Cache the credentials at the root URI
+ var self = this;
+ return Zotero.Promise.try(function () {
+ return self._cacheCredentials();
+ })
+ .then(function () {
+ return Zotero.HTTP.promise("GET", lastSyncURI,
+ { debug: true, successCodes: [200, 300, 404] });
+ })
+ .then(function (req) {
+ // If lastsync exists but not lastsync.txt, some servers try to
+ // be helpful and return 300.
+ if (req.status == 300 || req.status == 404) {
+ Zotero.debug("No last WebDAV sync file");
- Zotero.DB.commitTransaction();
+ // If no lastsync.txt, check previously used 'lastsync',
+ // and then delete it
+ let lastSyncURI = self.rootURI;
+ lastSyncURI.spec += "lastsync";
+ return Zotero.HTTP.promise("GET", lastSyncURI,
+ { debug: true, successCodes: [200, 404] })
+ .then(function (req) {
+ if (req.status == 404) {
+ return null;
+ }
+
+ Zotero.HTTP.promise("DELETE", lastSyncURI,
+ { debug: true, successCodes: [200, 204, 404] })
+ .done();
+
+ var lastModified = req.getResponseHeader("Last-Modified");
+ var date = new Date(lastModified);
+ Zotero.debug("Last successful WebDAV sync was " + date);
+ return Zotero.Date.toUnixTimestamp(date);
+ });
+ }
+
+ var lastModified = req.getResponseHeader("Last-Modified");
+ var date = new Date(lastModified);
+ Zotero.debug("Last successful WebDAV sync was " + date);
+
+ var re = new RegExp("^[a-zA-Z0-9]{" + _lastSyncIDLength + "}$");
+ if (!re.test(req.responseText)) {
+ Zotero.HTTP.promise("DELETE", lastSyncURI,
+ { debug: true, successCodes: [200, 204, 404] })
+ .done();
- try {
- var file = Zotero.getTempDirectory();
- file.append(item.key + '.zip');
- file.remove(false);
+ throw new Error("Invalid last sync id '" + req.responseText+ "'")
+ }
+
+ return req.responseText;
+ })
+ .catch(function (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ if (e.status == 403) {
+ Zotero.debug("Clearing WebDAV authentication credentials", 2);
+ _cachedCredentials = false;
}
- catch (e) {
- Components.utils.reportError(e);
+ else {
+ throw("HTTP " + e.status + " error from WebDAV server "
+ + "for GET request");
}
- return {
- localChanges: true,
- remoteChanges: true
- };
- });
- }
+ return Zotero.Promise.reject(e);
+ }
+ // TODO: handle browser offline exception
+ else {
+ throw (e);
+ }
+ });
+ },
- function onUploadCancel(httpRequest, status, data) {
- var request = data.request;
- var item = data.item;
-
- Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status);
-
- try {
- var file = Zotero.getTempDirectory();
- file.append(item.key + '.zip');
- file.remove(false);
+ setLastSyncTime: function (libraryID, localLastSyncID) {
+ if (libraryID != Zotero.Libraries.userLibraryID) {
+ throw new Error("libraryID must be user library");
}
- catch (e) {
- Components.utils.reportError(e);
- }
- }
-
-
- /**
- * Create a Zotero directory on the storage server
- */
- function createServerDirectory(callback) {
- var uri = Zotero.Sync.Storage.WebDAV.rootURI;
- Zotero.HTTP.WebDAV.doMkCol(uri, function (req) {
- Zotero.debug(req.responseText);
- Zotero.debug(req.status);
-
- switch (req.status) {
- case 201:
- return [uri, Zotero.Sync.Storage.SUCCESS];
-
- case 401:
- return [uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED];
-
- case 403:
- return [uri, Zotero.Sync.Storage.ERROR_FORBIDDEN];
-
- case 405:
- return [uri, Zotero.Sync.Storage.ERROR_NOT_ALLOWED];
-
- case 500:
- return [uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR];
-
- default:
- return [uri, Zotero.Sync.Storage.ERROR_UNKNOWN];
- }
- });
- }
-
-
- /**
- * Get the storage URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of file on storage server
- */
- function getItemURI(item) {
- var uri = Zotero.Sync.Storage.WebDAV.rootURI;
- uri.spec = uri.spec + item.key + '.zip';
- return uri;
- }
-
-
- /**
- * Get the storage property file URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of property file on storage server
- */
- function getItemPropertyURI(item) {
- var uri = Zotero.Sync.Storage.WebDAV.rootURI;
- uri.spec = uri.spec + item.key + '.prop';
- return uri;
- }
-
-
- /**
- * Get the storage property file URI corresponding to a given item storage URI
- *
- * @param {nsIURI} Item storage URI
- * @return {nsIURI|FALSE} Property file URI, or FALSE if not an item storage URI
- */
- function getPropertyURIFromItemURI(uri) {
- if (!uri.spec.match(/\.zip$/)) {
- return false;
- }
- var propURI = uri.clone();
- propURI.QueryInterface(Components.interfaces.nsIURL);
- propURI.fileName = uri.fileName.replace(/\.zip$/, '.prop');
- propURI.QueryInterface(Components.interfaces.nsIURI);
- return propURI;
- }
-
-
- /**
- * @inner
- * @param {String[]} files Remote filenames to delete (e.g., ZIPs)
- * @param {Function} callback Passed object containing three arrays:
- * 'deleted', 'missing', and 'error',
- * each containing filenames
- */
- function deleteStorageFiles(files) {
- var results = {
- deleted: [],
- missing: [],
- error: []
- };
-
- if (files.length == 0) {
- return Zotero.Promise.resolve(results);
+
+ // DEBUG: is this necessary for WebDAV?
+ if (localLastSyncID) {
+ var sql = "REPLACE INTO version VALUES (?, ?)";
+ Zotero.DB.query(
+ sql, ['storage_webdav_' + libraryID, localLastSyncID]
+ );
+ return;
}
- let deleteURI = _rootURI.clone();
- // This should never happen, but let's be safe
- if (!deleteURI.spec.match(/\/$/)) {
- return Zotero.Promise.reject("Root URI does not end in slash in "
- + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()");
- }
+ var uri = this.rootURI;
+ var successFileURI = uri.clone();
+ successFileURI.spec += "lastsync.txt";
- var funcs = [];
- for (let i=0; i<files.length; i++) {
- let fileName = files[i];
- let baseName = fileName.match(/^([^\.]+)/)[1];
- funcs.push(function () {
- let deleteURI = _rootURI.clone();
- deleteURI.QueryInterface(Components.interfaces.nsIURL);
- deleteURI.fileName = fileName;
- deleteURI.QueryInterface(Components.interfaces.nsIURI);
- return Zotero.HTTP.promise("DELETE", deleteURI, { successCodes: [200, 204, 404] })
- .then(function (req) {
- switch (req.status) {
- case 204:
- // IIS 5.1 and Sakai return 200
- case 200:
- var fileDeleted = true;
- break;
-
- case 404:
- var fileDeleted = true;
- break;
- }
-
- // If an item file URI, get the property URI
- var deletePropURI = getPropertyURIFromItemURI(deleteURI);
-
- // If we already deleted the prop file, skip it
- if (!deletePropURI || results.deleted.indexOf(deletePropURI.fileName) != -1) {
- if (fileDeleted) {
- results.deleted.push(baseName);
- }
- else {
- results.missing.push(baseName);
- }
- return;
- }
-
- let propFileName = deletePropURI.fileName;
-
- // Delete property file
- return Zotero.HTTP.promise("DELETE", deletePropURI, { successCodes: [200, 204, 404] })
- .then(function (req) {
- switch (req.status) {
- case 204:
- // IIS 5.1 and Sakai return 200
- case 200:
- results.deleted.push(baseName);
- break;
-
- case 404:
- if (fileDeleted) {
- results.deleted.push(baseName);
- }
- else {
- results.missing.push(baseName);
- }
- break;
- }
- });
- })
- .catch(function (e) {
- results.error.push(baseName);
- throw e;
- });
- });
- }
+ // Generate a random id for the last-sync id
+ var id = Zotero.Utilities.randomString(_lastSyncIDLength);
- Components.utils.import("resource://zotero/concurrent-caller.js");
- var caller = new ConcurrentCaller(4);
- caller.stopOnError = true;
- caller.setLogger(function (msg) Zotero.debug(msg));
- caller.onError(function (e) Components.utils.reportError(e));
- return caller.fcall(funcs)
+ return Zotero.HTTP.promise("PUT", successFileURI,
+ { body: id, debug: true, successCodes: [200, 201, 204] })
.then(function () {
- return results;
+ var sql = "REPLACE INTO version VALUES (?, ?)";
+ Zotero.DB.query(
+ sql, ['storage_webdav_' + libraryID, id]
+ );
+ })
+ .catch(function (e) {
+ var msg = "HTTP " + e.status + " error from WebDAV server "
+ + "for PUT request";
+ Zotero.debug(msg, 2);
+ Components.utils.reportError(msg);
+ throw Zotero.Sync.Storage.WebDAV.defaultError;
});
- }
+ },
- /**
- * Checks for an invalid SSL certificate and throws a nice error
- */
- function checkResponse(req) {
- var channel = req.channel;
- if (!channel instanceof Ci.nsIChannel) {
- Zotero.Sync.Storage.EventManager.error('No HTTPS channel available');
- }
+
+ checkServer: function () {
+ var deferred = Zotero.Promise.defer();
- // Check if the error we encountered is really an SSL error
- // Logic borrowed from https://developer.mozilla.org/en-US/docs/How_to_check_the_security_state_of_an_XMLHTTPRequest_over_SSL
- // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/ssl/sslerr.h
- // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/util/secerr.h
- var secErrLimit = Ci.nsINSSErrorsService.NSS_SEC_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE;
- var secErr = Math.abs(Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE) - (channel.status & 0xffff);
- var sslErrLimit = Ci.nsINSSErrorsService.NSS_SSL_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE;
- var sslErr = Math.abs(Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE) - (channel.status & 0xffff);
- if( (secErr < 0 || secErr > secErrLimit) && (sslErr < 0 || sslErr > sslErrLimit) ) {
- return;
+ try {
+ // Clear URIs
+ this.init();
+
+ var parentURI = this.parentURI;
+ var uri = this.rootURI;
+ }
+ catch (e) {
+ switch (e.name) {
+ case 'Z_ERROR_NO_URL':
+ deferred.resolve([null, Zotero.Sync.Storage.ERROR_NO_URL]);
+
+ case 'Z_ERROR_NO_USERNAME':
+ deferred.resolve([null, Zotero.Sync.Storage.ERROR_NO_USERNAME]);
+
+ case 'Z_ERROR_NO_PASSWORD':
+ deferred.resolve([null, Zotero.Sync.Storage.ERROR_NO_PASSWORD]);
+
+ default:
+ Zotero.debug(e);
+ Components.utils.reportError(e);
+ deferred.resolve([null, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ }
+
+ return deferred.promise;
}
- var secInfo = channel.securityInfo;
- if (secInfo instanceof Ci.nsITransportSecurityInfo) {
- secInfo.QueryInterface(Ci.nsITransportSecurityInfo);
- if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_INSECURE) == Ci.nsIWebProgressListener.STATE_IS_INSECURE) {
- var host = 'host';
+ var xmlstr = "<propfind xmlns='DAV:'><prop>"
+ // IIS 5.1 requires at least one property in PROPFIND
+ + "<getcontentlength/>"
+ + "</prop></propfind>";
+
+ // Test whether URL is WebDAV-enabled
+ var request = Zotero.HTTP.doOptions(uri, function (req) {
+ // Timeout
+ if (req.status == 0) {
try {
- host = channel.URI.host;
+ checkResponse(req);
}
catch (e) {
- Zotero.debug(e);
+ deferred.reject(e);
}
- var msg = Zotero.getString('sync.storage.error.webdav.sslCertificateError', host);
- // In Standalone, provide cert_override.txt instructions and a
- // button to open the Zotero profile directory
- if (Zotero.isStandalone) {
- msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.seeCertOverrideDocumentation');
- var buttonText = Zotero.getString('general.openDocumentation');
- var func = function () {
- var zp = Zotero.getActiveZoteroPane();
- zp.loadURI("https://www.zotero.org/support/kb/cert_override", { shiftKey: true });
- };
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNREACHABLE]);
+ }
+
+ Zotero.debug(req.getAllResponseHeaders());
+ Zotero.debug(req.responseText);
+ Zotero.debug(req.status);
+
+ switch (req.status) {
+ case 400:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_BAD_REQUEST]);
+
+ case 401:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+
+ case 403:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
+
+ case 500:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
+ }
+
+ var dav = req.getResponseHeader("DAV");
+ if (dav == null) {
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_NOT_DAV]);
+ }
+
+ // Get the Authorization header used in case we need to do a request
+ // on the parent below
+ var channelAuthorization = Zotero.HTTP.getChannelAuthorization(req.channel);
+
+ var headers = { Depth: 0 };
+
+ // Test whether Zotero directory exists
+ Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (req) {
+ Zotero.debug(req.responseText);
+ Zotero.debug(req.status);
+
+ switch (req.status) {
+ case 207:
+ // Test if missing files return 404s
+ var missingFileURI = uri.clone();
+ missingFileURI.spec += "nonexistent.prop";
+ Zotero.HTTP.promise("GET", missingFileURI, { successCodes: [404], debug: true })
+ .then(function () {
+ // Test if Zotero directory is writable
+ var testFileURI = uri.clone();
+ testFileURI.spec += "zotero-test-file.prop";
+ Zotero.HTTP.WebDAV.doPut(testFileURI, " ", function (req) {
+ Zotero.debug(req.responseText);
+ Zotero.debug(req.status);
+
+ switch (req.status) {
+ case 200:
+ case 201:
+ case 204:
+ Zotero.HTTP.doGet(
+ testFileURI,
+ function (req) {
+ Zotero.debug(req.responseText);
+ Zotero.debug(req.status);
+
+ switch (req.status) {
+ case 200:
+ // Delete test file
+ Zotero.HTTP.WebDAV.doDelete(
+ testFileURI,
+ function (req) {
+ Zotero.debug(req.responseText);
+ Zotero.debug(req.status);
+
+ switch (req.status) {
+ case 200: // IIS 5.1 and Sakai return 200
+ case 204:
+ return deferred.resolve([uri, Zotero.Sync.Storage.SUCCESS]);
+
+ case 401:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+
+ case 403:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
+
+ default:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ }
+ }
+ );
+ return;
+
+ case 401:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+
+ case 403:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
+
+ // This can happen with cloud storage services
+ // backed by S3 or other eventually consistent
+ // data stores.
+ //
+ // This can also be from IIS 6+, which is configured
+ // not to serve .prop files.
+ // http://support.microsoft.com/kb/326965
+ case 404:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FILE_MISSING_AFTER_UPLOAD]);
+
+ case 500:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
+
+ default:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ }
+ }
+ );
+ return;
+
+ case 401:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+
+ case 403:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
+
+ case 500:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
+
+ default:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ }
+ });
+ })
+ .catch(function (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ if (e.status >= 200 && e.status < 300) {
+ deferred.resolve([uri, Zotero.Sync.Storage.ERROR_NONEXISTENT_FILE_NOT_MISSING]);
+ }
+ else if (e.status == 401) {
+ deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+ }
+ else if (e.status == 403) {
+ deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
+ }
+ else if (e.status == 500) {
+ deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
+ }
+ else {
+ deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ }
+ }
+ else {
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+ deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ }
+ })
+ .done();
+ return;
+
+ case 400:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_BAD_REQUEST]);
+
+ case 401:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+
+ case 403:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
+
+ case 404:
+ // Include Authorization header from /zotero request,
+ // since Firefox probably won't apply it to the parent request
+ var newHeaders = {};
+ for (var header in headers) {
+ newHeaders[header] = headers[header];
+ }
+ newHeaders["Authorization"] = channelAuthorization;
+
+ // Zotero directory wasn't found, so see if at least
+ // the parent directory exists
+ Zotero.HTTP.WebDAV.doProp("PROPFIND", parentURI, xmlstr,
+ function (req) {
+ Zotero.debug(req.responseText);
+ Zotero.debug(req.status);
+
+ switch (req.status) {
+ // Parent directory existed
+ case 207:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_ZOTERO_DIR_NOT_FOUND]);
+
+ case 400:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_BAD_REQUEST]);
+
+ case 401:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+
+ // Parent directory wasn't found either
+ case 404:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_PARENT_DIR_NOT_FOUND]);
+
+ default:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ }
+ }, newHeaders);
+ return;
+
+ case 500:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
+
+ default:
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
}
- // In Firefox display a button to load the WebDAV URL
- else {
- msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
- var buttonText = Zotero.getString('sync.storage.error.webdav.loadURL');
- var func = function () {
- var zp = Zotero.getActiveZoteroPane();
- zp.loadURI(channel.URI.spec, { shiftKey: true });
- };
+ }, headers);
+ });
+
+ if (!request) {
+ return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_OFFLINE]);
+ }
+
+ // Pass XMLHttpRequest to progress handler
+ setTimeout(function () {
+ var obj = {};
+ obj.xmlhttp = request;
+ deferred.notify(obj)
+ }, 0);
+
+ return deferred.promise;
+ },
+
+
+ /**
+ * Handles the result of WebDAV verification, displaying an alert if necessary.
+ *
+ * @return bool True if the verification succeeded, false otherwise
+ */
+ checkServerCallback: function (uri, status, window, skipSuccessMessage) {
+ var promptService =
+ Components.classes["@mozilla.org/embedcomp/prompt-service;1"].
+ createInstance(Components.interfaces.nsIPromptService);
+ if (uri) {
+ var spec = uri.scheme + '://' + uri.hostPort + uri.path;
+ }
+
+ switch (status) {
+ case Zotero.Sync.Storage.SUCCESS:
+ return true;
+
+ case Zotero.Sync.Storage.ERROR_NO_URL:
+ var errorMessage = Zotero.getString('sync.storage.error.webdav.enterURL');
+ break;
+
+ case Zotero.Sync.Storage.ERROR_NO_USERNAME:
+ var errorMessage = Zotero.getString('sync.error.usernameNotSet');
+ break;
+
+ case Zotero.Sync.Storage.ERROR_NO_PASSWORD:
+ var errorMessage = Zotero.getString('sync.error.enterPassword');
+ break;
+
+ case Zotero.Sync.Storage.ERROR_UNREACHABLE:
+ var errorMessage = Zotero.getString('sync.storage.error.serverCouldNotBeReached', uri.host);
+ break;
+
+ case Zotero.Sync.Storage.ERROR_NOT_DAV:
+ var errorMessage = Zotero.getString('sync.storage.error.webdav.invalidURL', spec);
+ break;
+
+ case Zotero.Sync.Storage.ERROR_AUTH_FAILED:
+ var errorTitle = Zotero.getString('general.permissionDenied');
+ var errorMessage = Zotero.localeJoin([
+ Zotero.getString('sync.storage.error.webdav.invalidLogin'),
+ Zotero.getString('sync.storage.error.checkFileSyncSettings')
+ ]);
+ break;
+
+ case Zotero.Sync.Storage.ERROR_FORBIDDEN:
+ var errorTitle = Zotero.getString('general.permissionDenied');
+ var errorMessage = Zotero.localeJoin([
+ Zotero.getString('sync.storage.error.webdav.permissionDenied', uri.path),
+ Zotero.getString('sync.storage.error.checkFileSyncSettings')
+ ]);
+ break;
+
+ case Zotero.Sync.Storage.ERROR_PARENT_DIR_NOT_FOUND:
+ var errorTitle = Zotero.getString('sync.storage.error.directoryNotFound');
+ var parentSpec = spec.replace(/\/zotero\/$/, "");
+ var errorMessage = Zotero.getString('sync.storage.error.doesNotExist', parentSpec);
+ break;
+
+ case Zotero.Sync.Storage.ERROR_ZOTERO_DIR_NOT_FOUND:
+ var create = promptService.confirmEx(
+ window,
+ Zotero.getString('sync.storage.error.directoryNotFound'),
+ Zotero.getString('sync.storage.error.doesNotExist', spec) + "\n\n"
+ + Zotero.getString('sync.storage.error.createNow'),
+ promptService.BUTTON_POS_0
+ * promptService.BUTTON_TITLE_IS_STRING
+ + promptService.BUTTON_POS_1
+ * promptService.BUTTON_TITLE_CANCEL,
+ Zotero.getString('general.create'),
+ null, null, null, {}
+ );
+
+ if (create != 0) {
+ return;
}
- var e = new Zotero.Error(
- msg,
- 0,
- {
- dialogText: msg,
- dialogButtonText: buttonText,
- dialogButtonCallback: func
+ createServerDirectory(function (uri, status) {
+ switch (status) {
+ case Zotero.Sync.Storage.SUCCESS:
+ if (!skipSuccessMessage) {
+ promptService.alert(
+ window,
+ Zotero.getString('sync.storage.serverConfigurationVerified'),
+ Zotero.getString('sync.storage.fileSyncSetUp')
+ );
+ }
+ Zotero.Prefs.set("sync.storage.verified", true);
+ return true;
+
+ case Zotero.Sync.Storage.ERROR_FORBIDDEN:
+ var errorTitle = Zotero.getString('general.permissionDenied');
+ var errorMessage = Zotero.getString('sync.storage.error.permissionDeniedAtAddress') + "\n\n"
+ + spec + "\n\n"
+ + Zotero.getString('sync.storage.error.checkFileSyncSettings');
+ break;
}
- );
- throw e;
- }
- else if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_BROKEN) == Ci.nsIWebProgressListener.STATE_IS_BROKEN) {
- var msg = Zotero.getString('sync.storage.error.webdav.sslConnectionError', host) +
- Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
- var e = new Zotero.Error(
- msg,
- 0,
- {
- dialogText: msg,
- dialogButtonText: Zotero.getString('sync.storage.error.webdav.loadURL'),
- dialogButtonCallback: function () {
- var zp = Zotero.getActiveZoteroPane();
- zp.loadURI(channel.URI.spec, { shiftKey: true });
- }
+
+ // TEMP
+ if (!errorMessage) {
+ var errorMessage = status;
}
- );
- throw e;
+ promptService.alert(window, errorTitle, errorMessage);
+ });
+
+ return false;
+
+ case Zotero.Sync.Storage.ERROR_FILE_MISSING_AFTER_UPLOAD:
+ var errorTitle = Zotero.getString("general.warning");
+ var errorMessage = Zotero.getString('sync.storage.error.webdav.fileMissingAfterUpload');
+ Zotero.Prefs.set("sync.storage.verified", true);
+ break;
+
+ case Zotero.Sync.Storage.ERROR_SERVER_ERROR:
+ var errorTitle = Zotero.getString('sync.storage.error.webdav.serverConfig.title');
+ var errorMessage = Zotero.getString('sync.storage.error.webdav.serverConfig')
+ + "\n\n" + Zotero.getString('sync.storage.error.checkFileSyncSettings');
+ break;
+
+ case Zotero.Sync.Storage.ERROR_UNKNOWN:
+ var errorMessage = Zotero.localeJoin([
+ Zotero.getString('general.unknownErrorOccurred'),
+ Zotero.getString('sync.storage.error.checkFileSyncSettings')
+ ]);
+ break;
+
+ case Zotero.Sync.Storage.ERROR_NONEXISTENT_FILE_NOT_MISSING:
+ var errorTitle = Zotero.getString('sync.storage.error.webdav.serverConfig.title');
+ var errorMessage = Zotero.getString('sync.storage.error.webdav.nonexistentFileNotMissing');
+ break;
+ }
+
+ if (!skipSuccessMessage) {
+ if (!errorTitle) {
+ var errorTitle = Zotero.getString("general.error");
+ }
+ // TEMP
+ if (!errorMessage) {
+ var errorMessage = status;
}
+ promptService.alert(window, errorTitle, errorMessage);
}
- }
-
-
- //
- // Public methods (called via Zotero.Sync.Storage.WebDAV)
- //
- var obj = new Zotero.Sync.Storage.Mode;
- obj.name = "WebDAV";
-
- Object.defineProperty(obj, "defaultError", {
- get: function () Zotero.getString('sync.storage.error.webdav.default')
- });
+ return false;
+ },
- Object.defineProperty(obj, "defaultErrorRestart", {
- get: function () Zotero.getString('sync.storage.error.webdav.defaultRestart', Zotero.appName)
- });
- Object.defineProperty(obj, "includeUserFiles", {
- get: function () {
- return Zotero.Prefs.get("sync.storage.enabled") && Zotero.Prefs.get("sync.storage.protocol") == 'webdav';
+ /**
+ * Remove files on storage server that were deleted locally
+ *
+ * @param {Integer} libraryID
+ */
+ purgeDeletedStorageFiles: Zotero.Promise.coroutine(function* (libraryID) {
+ Zotero.debug("Purging deleted storage files");
+ var files = yield Zotero.Sync.Storage.Local.getDeletedFiles(libraryID);
+ if (!files.length) {
+ Zotero.debug("No files to delete remotely");
+ return false;
}
- });
- obj.includeGroupItems = false;
-
- Object.defineProperty(obj, "_verified", {
- get: function () Zotero.Prefs.get("sync.storage.verified")
- });
+
+ // Add .zip extension
+ var files = files.map(file => file + ".zip");
+
+ var results = yield deleteStorageFiles(files)
+
+ // Remove deleted and nonexistent files from storage delete log
+ var toPurge = results.deleted.concat(results.missing);
+ if (toPurge.length > 0) {
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Utilities.Internal.forEachChunkAsync(
+ toPurge,
+ Zotero.DB.MAX_BOUND_PARAMETERS,
+ function (chunk) {
+ var sql = "DELETE FROM storageDeleteLog WHERE libraryID=? AND key IN ("
+ + chunk.map(() => '?').join() + ")";
+ return Zotero.DB.queryAsync(sql, [libraryID].concat(chunk));
+ }
+ );
+ });
+ }
+
+ Zotero.debug(results);
+
+ return results.deleted.length;
+ }),
- Object.defineProperty(obj, "_username", {
- get: function () Zotero.Prefs.get('sync.storage.username')
- });
- Object.defineProperty(obj, "_password", {
- get: function () {
- var username = this._username;
+ /**
+ * Delete orphaned storage files older than a day before last sync time
+ */
+ purgeOrphanedStorageFiles: function (libraryID) {
+ // Note: libraryID not currently used
+
+ return Zotero.Promise.try(function () {
+ const daysBeforeSyncTime = 1;
- if (!username) {
- Zotero.debug('Username not set before getting Zotero.Sync.Storage.WebDAV.password');
- return '';
+ // If recently purged, skip
+ var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge');
+ var days = 10;
+ if (lastpurge && new Date(lastpurge * 1000) > (new Date() - (1000 * 60 * 60 * 24 * days))) {
+ return false;
}
- Zotero.debug('Getting WebDAV password');
- var loginManager = Components.classes["@mozilla.org/login-manager;1"]
- .getService(Components.interfaces.nsILoginManager);
- var logins = loginManager.findLogins({}, _loginManagerHost, _loginManagerURL, null);
-
- // Find user from returned array of nsILoginInfo objects
- for (var i = 0; i < logins.length; i++) {
- if (logins[i].username == username) {
- return logins[i].password;
- }
- }
+ Zotero.debug("Purging orphaned storage files");
- return '';
- },
-
- set: function (password) {
- var username = this._username;
- if (!username) {
- Zotero.debug('Username not set before setting Zotero.Sync.Server.Mode.WebDAV.password');
- return;
- }
+ var uri = this.rootURI;
+ var path = uri.path;
- _cachedCredentials = false;
+ var xmlstr = "<propfind xmlns='DAV:'><prop>"
+ + "<getlastmodified/>"
+ + "</prop></propfind>";
- var loginManager = Components.classes["@mozilla.org/login-manager;1"]
- .getService(Components.interfaces.nsILoginManager);
- var logins = loginManager.findLogins({}, _loginManagerHost, _loginManagerURL, null);
+ var lastSyncDate = new Date(Zotero.Sync.Server.lastLocalSyncTime * 1000);
- for (var i = 0; i < logins.length; i++) {
- Zotero.debug('Clearing WebDAV passwords');
- loginManager.removeLogin(logins[i]);
- break;
- }
+ var deferred = Zotero.Promise.defer();
- if (password) {
- Zotero.debug(_loginManagerURL);
- var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
- Components.interfaces.nsILoginInfo, "init");
- var loginInfo = new nsLoginInfo(_loginManagerHost, _loginManagerURL,
- null, username, password, "", "");
- loginManager.addLogin(loginInfo);
- }
- }
- });
-
- Object.defineProperty(obj, "rootURI", {
- get: function () {
- if (!_rootURI) {
- this._init();
- }
- return _rootURI.clone();
- }
- });
-
- Object.defineProperty(obj, "parentURI", {
- get: function () {
- if (!_parentURI) {
- this._init();
- }
- return _parentURI.clone();
- }
- });
-
- obj._init = function () {
- _rootURI = false;
- _parentURI = false;
-
- var scheme = Zotero.Prefs.get('sync.storage.scheme');
- switch (scheme) {
- case 'http':
- case 'https':
- break;
+ Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (xmlhttp) {
+ Zotero.Promise.try(function () {
+ Zotero.debug(xmlhttp.responseText);
+
+ var funcName = "Zotero.Sync.Storage.purgeOrphanedStorageFiles()";
+
+ var responseNode = xmlhttp.responseXML.documentElement;
+ responseNode.xpath = function (path) {
+ return Zotero.Utilities.xpath(this, path, { D: 'DAV:' });
+ };
+
+ var deleteFiles = [];
+ var trailingSlash = !!path.match(/\/$/);
+ for each(var response in responseNode.xpath("D:response")) {
+ var href = Zotero.Utilities.xpathText(
+ response, "D:href", { D: 'DAV:' }
+ ) || "";
+ Zotero.debug(href);
+
+ // Strip trailing slash if there isn't one on the root path
+ if (!trailingSlash) {
+ href = href.replace(/\/$/, "");
+ }
+
+ // Absolute
+ if (href.match(/^https?:\/\//)) {
+ var ios = Components.classes["@mozilla.org/network/io-service;1"].
+ getService(Components.interfaces.nsIIOService);
+ var href = ios.newURI(href, null, null);
+ href = href.path;
+ }
+
+ // Skip root URI
+ if (href == path
+ // Some Apache servers respond with a "/zotero" href
+ // even for a "/zotero/" request
+ || (trailingSlash && href + '/' == path)
+ // Try URL-encoded as well, as above
+ || decodeURIComponent(href) == path) {
+ continue;
+ }
+
+ if (href.indexOf(path) == -1
+ // Try URL-encoded as well, in case there's a '~' or similar
+ // character in the URL and the server (e.g., Sakai) is
+ // encoding the value
+ && decodeURIComponent(href).indexOf(path) == -1) {
+ throw new Error(
+ "DAV:href '" + href + "' does not begin with path '"
+ + path + "' in " + funcName
+ );
+ }
+
+ var matches = href.match(/[^\/]+$/);
+ if (!matches) {
+ throw new Error(
+ "Unexpected href '" + href + "' in " + funcName
+ );
+ }
+ var file = matches[0];
+
+ if (file.indexOf('.') == 0) {
+ Zotero.debug("Skipping hidden file " + file);
+ continue;
+ }
+ if (!file.match(/\.zip$/) && !file.match(/\.prop$/)) {
+ Zotero.debug("Skipping file " + file);
+ continue;
+ }
+
+ var key = file.replace(/\.(zip|prop)$/, '');
+ var item = Zotero.Items.getByLibraryAndKey(null, key);
+ if (item) {
+ Zotero.debug("Skipping existing file " + file);
+ continue;
+ }
+
+ Zotero.debug("Checking orphaned file " + file);
+
+ // TODO: Parse HTTP date properly
+ Zotero.debug(response.innerHTML);
+ var lastModified = Zotero.Utilities.xpathText(
+ response, ".//D:getlastmodified", { D: 'DAV:' }
+ );
+ lastModified = Zotero.Date.strToISO(lastModified);
+ lastModified = Zotero.Date.sqlToDate(lastModified);
+
+ // Delete files older than a day before last sync time
+ var days = (lastSyncDate - lastModified) / 1000 / 60 / 60 / 24;
+
+ if (days > daysBeforeSyncTime) {
+ deleteFiles.push(file);
+ }
+ }
+
+ return deleteStorageFiles(deleteFiles)
+ .then(function (results) {
+ Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000))
+ Zotero.debug(results);
+ });
+ })
+ .catch(function (e) {
+ deferred.reject(e);
+ })
+ .then(function () {
+ deferred.resolve();
+ });
+ }, { Depth: 1 });
- default:
- throw new Error("Invalid WebDAV scheme '" + scheme + "'");
- }
-
- var url = Zotero.Prefs.get('sync.storage.url');
- if (!url) {
- var msg = "WebDAV URL not provided";
- Zotero.debug(msg);
- throw ({
- message: msg,
- name: "Z_ERROR_NO_URL",
- filename: "webdav.js",
- toString: function () { return this.message; }
- });
- }
-
- url = scheme + '://' + url;
- var dir = "zotero";
- var username = this._username;
- var password = this._password;
-
- if (!username) {
- var msg = "WebDAV username not provided";
- Zotero.debug(msg);
- throw ({
- message: msg,
- name: "Z_ERROR_NO_USERNAME",
- filename: "webdav.js",
- toString: function () { return this.message; }
- });
- }
-
- if (!password) {
- var msg = "WebDAV password not provided";
- Zotero.debug(msg);
- throw ({
- message: msg,
- name: "Z_ERROR_NO_PASSWORD",
- filename: "webdav.js",
- toString: function () { return this.message; }
- });
- }
-
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var uri = ios.newURI(url, null, null);
- uri.username = encodeURIComponent(username);
- uri.password = encodeURIComponent(password);
- if (!uri.spec.match(/\/$/)) {
- uri.spec += "/";
- }
- _parentURI = uri;
-
- var uri = uri.clone();
- uri.spec += "zotero/";
- _rootURI = uri;
- };
+ return deferred.promise;
+ }.bind(this));
+ },
-
- obj.clearCachedCredentials = function() {
- _rootURI = _parentURI = undefined;
- _cachedCredentials = false;
- };
+ //
+ // Private methods
+ //
/**
- * Begin download process for individual file
+ * Get mod time of file on storage server
*
- * @param {Zotero.Sync.Storage.Request} [request]
+ * @param {Zotero.Item} item
+ * @param {Function} callback Callback f(item, mdate)
*/
- obj._downloadFile = function (request) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- if (!item) {
- throw new Error("Item '" + request.name + "' not found");
- }
+ _getStorageModificationTime: function (item, request) {
+ var uri = getItemPropertyURI(item);
- // Retrieve modification time from server to store locally afterwards
- return getStorageModificationTime(item, request)
- .then(function (mdate) {
- if (!request.isRunning()) {
- Zotero.debug("Download request '" + request.name
- + "' is no longer running after getting mod time");
- return false;
+ return Zotero.HTTP.promise("GET", uri,
+ {
+ debug: true,
+ successCodes: [200, 300, 404],
+ requestObserver: function (xmlhttp) {
+ request.setChannel(xmlhttp.channel);
}
+ })
+ .then(function (req) {
+ checkResponse(req);
- if (!mdate) {
- Zotero.debug("Remote file not found for item " + Zotero.Items.getLibraryKeyHash(item));
+ // mod_speling can return 300s for 404s with base name matches
+ if (req.status == 404 || req.status == 300) {
return false;
}
- var syncModTime = mdate.getTime();
+ // No modification time set
+ if (!req.responseText) {
+ return false;
+ }
- // Skip download if local file exists and matches mod time
- var file = item.getFile();
- if (file && file.exists() && syncModTime == file.lastModifiedTime) {
- Zotero.debug("File mod time matches remote file -- skipping download");
-
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- var updateItem = syncState != 1;
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return {
- localChanges: true
- };
+ var seconds = false;
+ var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
+ .createInstance(Components.interfaces.nsIDOMParser);
+ try {
+ var xml = parser.parseFromString(req.responseText, "text/xml");
+ var mtime = xml.getElementsByTagName('mtime')[0].textContent;
+ }
+ catch (e) {
+ Zotero.debug(e);
+ var mtime = false;
}
- var uri = getItemURI(item);
- var destFile = Zotero.getTempDirectory();
- destFile.append(item.key + '.zip.tmp');
- if (destFile.exists()) {
- destFile.remove(false);
+ // TEMP
+ if (!mtime) {
+ mtime = req.responseText;
+ seconds = true;
}
- var deferred = Zotero.Promise.defer();
+ var invalid = false;
- var listener = new Zotero.Sync.Storage.StreamListener(
- {
- onStart: function (request, data) {
- if (data.request.isFinished()) {
- Zotero.debug("Download request " + data.request.name
- + " stopped before download started -- closing channel");
- request.cancel(0x804b0002); // NS_BINDING_ABORTED
- deferred.resolve(false);
- }
- },
- onProgress: function (a, b, c) {
- request.onProgress(a, b, c)
- },
- onStop: function (request, status, response, data) {
- data.request.setChannel(false);
-
- if (status == 404) {
- var msg = "Remote ZIP file not found for item " + item.key;
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg);
-
- // Delete the orphaned prop file
- deleteStorageFiles([item.key + ".prop"])
- .finally(function (results) {
- deferred.resolve(false);
- })
- .done();
- return;
- }
- else if (status != 200) {
- var msg = "HTTP " + status + " from WebDAV server "
- + " while downloading file";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- deferred.reject(Zotero.Sync.Storage.WebDAV.defaultError);
- return;
- }
-
- // Don't try to process if the request has been cancelled
- if (data.request.isFinished()) {
- Zotero.debug("Download request " + data.request.name
- + " is no longer running after file download");
- deferred.resolve(false);
- return;
- }
-
- Zotero.debug("Finished download of " + destFile.path);
-
- try {
- deferred.resolve(Zotero.Sync.Storage.processDownload(data));
- }
- catch (e) {
- deferred.reject(e);
- }
- },
- onCancel: function (request, status, data) {
- Zotero.debug("Request cancelled");
- deferred.resolve(false);
- },
- request: request,
- item: item,
- compressed: true,
- syncModTime: syncModTime
+ // Unix timestamps need to be converted to ms-based timestamps
+ if (seconds) {
+ if (mtime.match(/^[0-9]{1,10}$/)) {
+ Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
+ mtime = mtime * 1000;
}
- );
+ else {
+ invalid = true;
+ }
+ }
+ else if (!mtime.match(/^[0-9]{1,13}$/)) {
+ invalid = true;
+ }
- // Don't display password in console
- var disp = uri.clone();
- if (disp.password) {
- disp.password = '********';
+ // Delete invalid .prop files
+ if (invalid) {
+ var msg = "Invalid mod date '" + Zotero.Utilities.ellipsize(mtime, 20)
+ + "' for item " + Zotero.Items.getLibraryKeyHash(item);
+ Zotero.debug(msg, 1);
+ Components.utils.reportError(msg);
+ return deleteStorageFiles([item.key + ".prop"])
+ .then(function (results) {
+ throw new Error(Zotero.Sync.Storage.WebDAV.defaultError);
+ });
}
- Zotero.debug('Saving ' + disp.spec + ' with saveURI()');
- const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
- var wbp = Components
- .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
- .createInstance(nsIWBP);
- wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
- wbp.progressListener = listener;
- Zotero.Utilities.Internal.saveURI(wbp, uri, destFile);
- return deferred.promise;
+ return new Date(parseInt(mtime));
+ })
+ .catch(function (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ throw new Error("HTTP " + e.status + " error from WebDAV "
+ + "server for GET request");
+ }
+ throw e;
});
- };
+ },
- obj._uploadFile = function (request) {
- var deferred = Zotero.Promise.defer();
- var created = Zotero.Sync.Storage.createUploadFile(
- request,
- function (data) {
- if (!data) {
- deferred.resolve(false);
- return;
- }
- deferred.resolve(
- Zotero.Promise.try(function () {
- return processUploadFile(data);
- })
- );
- }
- );
- if (!created) {
- return Zotero.Promise.resolve(false);
- }
- return deferred.promise;
- };
+ /**
+ * Set mod time of file on storage server
+ *
+ * @param {Zotero.Item} item
+ */
+ _setStorageModificationTime: Zotero.Promise.coroutine(function* (item) {
+ var uri = getItemPropertyURI(item);
+
+ var mtime = item.attachmentModificationTime;
+ var hash = yield item.attachmentHash;
+
+ var prop = '<properties version="1">'
+ + '<mtime>' + mtime + '</mtime>'
+ + '<hash>' + hash + '</hash>'
+ + '</properties>';
+
+ return Zotero.HTTP.promise("PUT", uri,
+ { body: prop, debug: true, successCodes: [200, 201, 204] })
+ .then(function (req) {
+ return { mtime: mtime, hash: hash };
+ })
+ .catch(function (e) {
+ throw new Error("HTTP " + e.xmlhttp.status
+ + " from WebDAV server for HTTP PUT");
+ })
+ }),
- obj._getLastSyncTime = function () {
- var lastSyncURI = this.rootURI;
- lastSyncURI.spec += "lastsync.txt";
+
+ /**
+ * Upload the generated ZIP file to the server
+ *
+ * @param {Object} Object with 'request' property
+ * @return {void}
+ */
+ _processUploadFile: Zotero.Promise.coroutine(function* (data) {
+ /*
+ updateSizeMultiplier(
+ (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
+ );
+ */
+ var request = data.request;
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
- // Cache the credentials at the root URI
- var self = this;
- return Zotero.Promise.try(function () {
- return self._cacheCredentials();
- })
- .then(function () {
- return Zotero.HTTP.promise("GET", lastSyncURI,
- { debug: true, successCodes: [200, 300, 404] });
- })
- .then(function (req) {
- // If lastsync exists but not lastsync.txt, some servers try to
- // be helpful and return 300.
- if (req.status == 300 || req.status == 404) {
- Zotero.debug("No last WebDAV sync file");
-
- // If no lastsync.txt, check previously used 'lastsync',
- // and then delete it
- let lastSyncURI = self.rootURI;
- lastSyncURI.spec += "lastsync";
- return Zotero.HTTP.promise("GET", lastSyncURI,
- { debug: true, successCodes: [200, 404] })
- .then(function (req) {
- if (req.status == 404) {
- return null;
- }
-
- Zotero.HTTP.promise("DELETE", lastSyncURI,
- { debug: true, successCodes: [200, 204, 404] })
- .done();
-
- var lastModified = req.getResponseHeader("Last-Modified");
- var date = new Date(lastModified);
- Zotero.debug("Last successful WebDAV sync was " + date);
- return Zotero.Date.toUnixTimestamp(date);
- });
- }
-
- var lastModified = req.getResponseHeader("Last-Modified");
- var date = new Date(lastModified);
- Zotero.debug("Last successful WebDAV sync was " + date);
-
- var re = new RegExp("^[a-zA-Z0-9]{" + _lastSyncIDLength + "}$");
- if (!re.test(req.responseText)) {
- Zotero.HTTP.promise("DELETE", lastSyncURI,
- { debug: true, successCodes: [200, 204, 404] })
- .done();
+ var mdate = getStorageModificationTime(item, request);
+
+ if (!request.isRunning()) {
+ Zotero.debug("Upload request '" + request.name
+ + "' is no longer running after getting mod time");
+ return false;
+ }
+
+ // Check for conflict
+ if (Zotero.Sync.Storage.getSyncState(item.id)
+ != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
+ if (mdate) {
+ // Local file time
+ var fmtime = yield item.attachmentModificationTime;
+ // Remote prop time
+ var mtime = mdate.getTime();
- throw new Error("Invalid last sync id '" + req.responseText+ "'")
- }
-
- return req.responseText;
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status == 403) {
- Zotero.debug("Clearing WebDAV authentication credentials", 2);
- _cachedCredentials = false;
- }
- else {
- throw("HTTP " + e.status + " error from WebDAV server "
- + "for GET request");
+ var same = !(yield Zotero.Sync.Storage.checkFileModTime(item, fmtime, mtime));
+ if (same) {
+ Zotero.DB.beginTransaction();
+ var syncState = Zotero.Sync.Storage.getSyncState(item.id);
+ Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
+ Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ Zotero.DB.commitTransaction();
+ return true;
}
- return Zotero.Promise.reject(e);
+ let smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (smtime != mtime) {
+ Zotero.debug("Conflict -- last synced file mod time "
+ + "does not match time on storage server"
+ + " (" + smtime + " != " + mtime + ")");
+ return {
+ localChanges: false,
+ remoteChanges: false,
+ syncRequired: false,
+ conflict: {
+ local: { modTime: fmtime },
+ remote: { modTime: mtime }
+ }
+ };
+ }
}
- // TODO: handle browser offline exception
else {
- throw (e);
+ Zotero.debug("Remote file not found for item " + item.id);
}
- });
- };
-
-
- obj._setLastSyncTime = function (libraryID, localLastSyncID) {
- if (libraryID != Zotero.Libraries.userLibraryID) {
- throw new Error("libraryID must be user library");
}
- // DEBUG: is this necessary for WebDAV?
- if (localLastSyncID) {
- var sql = "REPLACE INTO version VALUES (?, ?)";
- Zotero.DB.query(
- sql, ['storage_webdav_' + libraryID, localLastSyncID]
- );
- return;
- }
+ var file = Zotero.getTempDirectory();
+ file.append(item.key + '.zip');
- var uri = this.rootURI;
- var successFileURI = uri.clone();
- successFileURI.spec += "lastsync.txt";
+ var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]
+ .createInstance(Components.interfaces.nsIFileInputStream);
+ fis.init(file, 0x01, 0, 0);
- // Generate a random id for the last-sync id
- var id = Zotero.Utilities.randomString(_lastSyncIDLength);
+ var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"]
+ .createInstance(Components.interfaces.nsIBufferedInputStream)
+ bis.init(fis, 64 * 1024);
- return Zotero.HTTP.promise("PUT", successFileURI,
- { body: id, debug: true, successCodes: [200, 201, 204] })
- .then(function () {
- var sql = "REPLACE INTO version VALUES (?, ?)";
- Zotero.DB.query(
- sql, ['storage_webdav_' + libraryID, id]
- );
- })
- .catch(function (e) {
- var msg = "HTTP " + e.status + " error from WebDAV server "
- + "for PUT request";
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg);
- throw Zotero.Sync.Storage.WebDAV.defaultError;
- });
- };
-
-
- obj._cacheCredentials = function () {
- if (_cachedCredentials) {
- Zotero.debug("WebDAV credentials are already cached");
- return;
- }
+ var uri = getItemURI(item);
+
+ var ios = Components.classes["@mozilla.org/network/io-service;1"].
+ getService(Components.interfaces.nsIIOService);
+ var channel = ios.newChannelFromURI(uri);
+ channel.QueryInterface(Components.interfaces.nsIUploadChannel);
+ channel.setUploadStream(bis, 'application/octet-stream', -1);
+ channel.QueryInterface(Components.interfaces.nsIHttpChannel);
+ channel.requestMethod = 'PUT';
+ channel.allowPipelining = false;
+
+ channel.setRequestHeader('Keep-Alive', '', false);
+ channel.setRequestHeader('Connection', '', false);
- return Zotero.HTTP.promise("OPTIONS", this.rootURI)
- .then(function (req) {
- checkResponse(req);
-
- Zotero.debug("Credentials are cached");
- _cachedCredentials = true;
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- var msg = "HTTP " + e.status + " error from WebDAV server "
- + "for OPTIONS request";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- throw new Error(Zotero.Sync.Storage.WebDAV.defaultErrorRestart);
- }
- throw e;
- });
- };
-
-
- obj._checkServer = function () {
var deferred = Zotero.Promise.defer();
- try {
- // Clear URIs
- this.init();
-
- var parentURI = this.parentURI;
- var uri = this.rootURI;
- }
- catch (e) {
- switch (e.name) {
- case 'Z_ERROR_NO_URL':
- deferred.resolve([null, Zotero.Sync.Storage.ERROR_NO_URL]);
-
- case 'Z_ERROR_NO_USERNAME':
- deferred.resolve([null, Zotero.Sync.Storage.ERROR_NO_USERNAME]);
-
- case 'Z_ERROR_NO_PASSWORD':
- deferred.resolve([null, Zotero.Sync.Storage.ERROR_NO_PASSWORD]);
+ var listener = new Zotero.Sync.Storage.StreamListener(
+ {
+ onProgress: function (a, b, c) {
+ request.onProgress(a, b, c);
+ },
+ onStop: function (httpRequest, status, response, data) {
+ data.request.setChannel(false);
- default:
- Zotero.debug(e);
- Components.utils.reportError(e);
- deferred.resolve([null, Zotero.Sync.Storage.ERROR_UNKNOWN]);
+ deferred.resolve(
+ Zotero.Promise.try(function () {
+ return onUploadComplete(httpRequest, status, response, data);
+ })
+ );
+ },
+ onCancel: function (httpRequest, status, data) {
+ onUploadCancel(httpRequest, status, data);
+ deferred.resolve(false);
+ },
+ request: request,
+ item: item,
+ streams: [fis, bis]
}
-
- return deferred.promise;
+ );
+ channel.notificationCallbacks = listener;
+
+ var dispURI = uri.clone();
+ if (dispURI.password) {
+ dispURI.password = '********';
}
+ Zotero.debug("HTTP PUT of " + file.leafName + " to " + dispURI.spec);
- var xmlstr = "<propfind xmlns='DAV:'><prop>"
- // IIS 5.1 requires at least one property in PROPFIND
- + "<getcontentlength/>"
- + "</prop></propfind>";
+ channel.asyncOpen(listener, null);
+
+ return deferred.promise;
+ }),
+
+
+ _onUploadComplete: function (httpRequest, status, response, data) {
+ var request = data.request;
+ var item = data.item;
+ var url = httpRequest.name;
+
+ Zotero.debug("Upload of attachment " + item.key
+ + " finished with status code " + status);
+
+ switch (status) {
+ case 200:
+ case 201:
+ case 204:
+ break;
+
+ case 403:
+ case 500:
+ Zotero.debug(response);
+ throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
+ " " + Zotero.getString('sync.storage.error.checkFileSyncSettings'));
+
+ case 507:
+ Zotero.debug(response);
+ throw Zotero.getString('sync.storage.error.webdav.insufficientSpace');
+
+ default:
+ Zotero.debug(response);
+ throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
+ " " + Zotero.getString('sync.storage.error.checkFileSyncSettings')
+ + "\n\n" + "HTTP " + status);
+ }
- // Test whether URL is WebDAV-enabled
- var request = Zotero.HTTP.doOptions(uri, function (req) {
- // Timeout
- if (req.status == 0) {
+ return setStorageModificationTime(item)
+ .then(function (props) {
+ if (!request.isRunning()) {
+ Zotero.debug("Upload request '" + request.name
+ + "' is no longer running after getting mod time");
+ return false;
+ }
+
+ Zotero.DB.beginTransaction();
+
+ Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true);
+ Zotero.Sync.Storage.setSyncedHash(item.id, props.hash);
+
+ Zotero.DB.commitTransaction();
+
try {
- checkResponse(req);
+ var file = Zotero.getTempDirectory();
+ file.append(item.key + '.zip');
+ file.remove(false);
}
catch (e) {
- deferred.reject(e);
+ Components.utils.reportError(e);
}
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNREACHABLE]);
- }
-
- Zotero.debug(req.getAllResponseHeaders());
+ return {
+ localChanges: true,
+ remoteChanges: true,
+ syncRequired: true
+ };
+ });
+ },
+
+
+ _onUploadCancel: function (httpRequest, status, data) {
+ var request = data.request;
+ var item = data.item;
+
+ Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status);
+
+ try {
+ var file = Zotero.getTempDirectory();
+ file.append(item.key + '.zip');
+ file.remove(false);
+ }
+ catch (e) {
+ Components.utils.reportError(e);
+ }
+ },
+
+
+ /**
+ * Create a Zotero directory on the storage server
+ */
+ _createServerDirectory: function (callback) {
+ var uri = Zotero.Sync.Storage.WebDAV.rootURI;
+ Zotero.HTTP.WebDAV.doMkCol(uri, function (req) {
Zotero.debug(req.responseText);
Zotero.debug(req.status);
switch (req.status) {
- case 400:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_BAD_REQUEST]);
+ case 201:
+ return [uri, Zotero.Sync.Storage.SUCCESS];
case 401:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
+ return [uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED];
case 403:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
-
- case 500:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
- }
-
- var dav = req.getResponseHeader("DAV");
- if (dav == null) {
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_NOT_DAV]);
- }
-
- // Get the Authorization header used in case we need to do a request
- // on the parent below
- var channelAuthorization = Zotero.HTTP.getChannelAuthorization(req.channel);
-
- var headers = { Depth: 0 };
-
- // Test whether Zotero directory exists
- Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (req) {
- Zotero.debug(req.responseText);
- Zotero.debug(req.status);
+ return [uri, Zotero.Sync.Storage.ERROR_FORBIDDEN];
- switch (req.status) {
- case 207:
- // Test if missing files return 404s
- var missingFileURI = uri.clone();
- missingFileURI.spec += "nonexistent.prop";
- Zotero.HTTP.promise("GET", missingFileURI, { successCodes: [404], debug: true })
- .then(function () {
- // Test if Zotero directory is writable
- var testFileURI = uri.clone();
- testFileURI.spec += "zotero-test-file.prop";
- Zotero.HTTP.WebDAV.doPut(testFileURI, " ", function (req) {
- Zotero.debug(req.responseText);
- Zotero.debug(req.status);
-
- switch (req.status) {
- case 200:
- case 201:
- case 204:
- Zotero.HTTP.doGet(
- testFileURI,
- function (req) {
- Zotero.debug(req.responseText);
- Zotero.debug(req.status);
-
- switch (req.status) {
- case 200:
- // Delete test file
- Zotero.HTTP.WebDAV.doDelete(
- testFileURI,
- function (req) {
- Zotero.debug(req.responseText);
- Zotero.debug(req.status);
-
- switch (req.status) {
- case 200: // IIS 5.1 and Sakai return 200
- case 204:
- return deferred.resolve([uri, Zotero.Sync.Storage.SUCCESS]);
-
- case 401:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
-
- case 403:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
-
- default:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
- }
- }
- );
- return;
-
- case 401:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
-
- case 403:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
-
- // This can happen with cloud storage services
- // backed by S3 or other eventually consistent
- // data stores.
- //
- // This can also be from IIS 6+, which is configured
- // not to serve .prop files.
- // http://support.microsoft.com/kb/326965
- case 404:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FILE_MISSING_AFTER_UPLOAD]);
-
- case 500:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
-
- default:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
- }
- }
- );
- return;
-
- case 401:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
-
- case 403:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
-
- case 500:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
-
- default:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
- }
- });
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status >= 200 && e.status < 300) {
- deferred.resolve([uri, Zotero.Sync.Storage.ERROR_NONEXISTENT_FILE_NOT_MISSING]);
- }
- else if (e.status == 401) {
- deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
- }
- else if (e.status == 403) {
- deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
- }
- else if (e.status == 500) {
- deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
- }
- else {
- deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
- }
- }
- else {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
- }
- })
- .done();
- return;
-
- case 400:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_BAD_REQUEST]);
-
- case 401:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
-
- case 403:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]);
-
- case 404:
- // Include Authorization header from /zotero request,
- // since Firefox probably won't apply it to the parent request
- var newHeaders = {};
- for (var header in headers) {
- newHeaders[header] = headers[header];
- }
- newHeaders["Authorization"] = channelAuthorization;
-
- // Zotero directory wasn't found, so see if at least
- // the parent directory exists
- Zotero.HTTP.WebDAV.doProp("PROPFIND", parentURI, xmlstr,
- function (req) {
- Zotero.debug(req.responseText);
- Zotero.debug(req.status);
-
- switch (req.status) {
- // Parent directory existed
- case 207:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_ZOTERO_DIR_NOT_FOUND]);
-
- case 400:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_BAD_REQUEST]);
-
- case 401:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]);
-
- // Parent directory wasn't found either
- case 404:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_PARENT_DIR_NOT_FOUND]);
-
- default:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
- }
- }, newHeaders);
- return;
-
- case 500:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]);
-
- default:
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_UNKNOWN]);
- }
- }, headers);
+ case 405:
+ return [uri, Zotero.Sync.Storage.ERROR_NOT_ALLOWED];
+
+ case 500:
+ return [uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR];
+
+ default:
+ return [uri, Zotero.Sync.Storage.ERROR_UNKNOWN];
+ }
});
-
- if (!request) {
- return deferred.resolve([uri, Zotero.Sync.Storage.ERROR_OFFLINE]);
- }
-
- // Pass XMLHttpRequest to progress handler
- setTimeout(function () {
- var obj = {};
- obj.xmlhttp = request;
- deferred.notify(obj)
- }, 0);
-
- return deferred.promise;
- };
+ },
/**
- * Handles the result of WebDAV verification, displaying an alert if necessary.
+ * Get the storage URI for an item
*
- * @return bool True if the verification succeeded, false otherwise
+ * @inner
+ * @param {Zotero.Item}
+ * @return {nsIURI} URI of file on storage server
*/
- obj._checkServerCallback = function (uri, status, window, skipSuccessMessage) {
- var promptService =
- Components.classes["@mozilla.org/embedcomp/prompt-service;1"].
- createInstance(Components.interfaces.nsIPromptService);
- if (uri) {
- var spec = uri.scheme + '://' + uri.hostPort + uri.path;
+ _getItemURI: function (item) {
+ var uri = Zotero.Sync.Storage.WebDAV.rootURI;
+ uri.spec = uri.spec + item.key + '.zip';
+ return uri;
+ },
+
+
+ /**
+ * Get the storage property file URI for an item
+ *
+ * @inner
+ * @param {Zotero.Item}
+ * @return {nsIURI} URI of property file on storage server
+ */
+ _getItemPropertyURI: function (item) {
+ var uri = Zotero.Sync.Storage.WebDAV.rootURI;
+ uri.spec = uri.spec + item.key + '.prop';
+ return uri;
+ },
+
+
+ /**
+ * Get the storage property file URI corresponding to a given item storage URI
+ *
+ * @param {nsIURI} Item storage URI
+ * @return {nsIURI|FALSE} Property file URI, or FALSE if not an item storage URI
+ */
+ _getPropertyURIFromItemURI: function (uri) {
+ if (!uri.spec.match(/\.zip$/)) {
+ return false;
}
+ var propURI = uri.clone();
+ propURI.QueryInterface(Components.interfaces.nsIURL);
+ propURI.fileName = uri.fileName.replace(/\.zip$/, '.prop');
+ propURI.QueryInterface(Components.interfaces.nsIURI);
+ return propURI;
+ },
+
+
+ /**
+ * @inner
+ * @param {String[]} files Remote filenames to delete (e.g., ZIPs)
+ * @param {Function} callback Passed object containing three arrays:
+ * 'deleted', 'missing', and 'error',
+ * each containing filenames
+ */
+ _deleteStorageFiles: function (files) {
+ var results = {
+ deleted: [],
+ missing: [],
+ error: []
+ };
- switch (status) {
- case Zotero.Sync.Storage.SUCCESS:
- return true;
-
- case Zotero.Sync.Storage.ERROR_NO_URL:
- var errorMessage = Zotero.getString('sync.storage.error.webdav.enterURL');
- break;
-
- case Zotero.Sync.Storage.ERROR_NO_USERNAME:
- var errorMessage = Zotero.getString('sync.error.usernameNotSet');
- break;
-
- case Zotero.Sync.Storage.ERROR_NO_PASSWORD:
- var errorMessage = Zotero.getString('sync.error.enterPassword');
- break;
-
- case Zotero.Sync.Storage.ERROR_UNREACHABLE:
- var errorMessage = Zotero.getString('sync.storage.error.serverCouldNotBeReached', uri.host);
- break;
-
- case Zotero.Sync.Storage.ERROR_NOT_DAV:
- var errorMessage = Zotero.getString('sync.storage.error.webdav.invalidURL', spec);
- break;
-
- case Zotero.Sync.Storage.ERROR_AUTH_FAILED:
- var errorTitle = Zotero.getString('general.permissionDenied');
- var errorMessage = Zotero.localeJoin([
- Zotero.getString('sync.storage.error.webdav.invalidLogin'),
- Zotero.getString('sync.storage.error.checkFileSyncSettings')
- ]);
- break;
-
- case Zotero.Sync.Storage.ERROR_FORBIDDEN:
- var errorTitle = Zotero.getString('general.permissionDenied');
- var errorMessage = Zotero.localeJoin([
- Zotero.getString('sync.storage.error.webdav.permissionDenied', uri.path),
- Zotero.getString('sync.storage.error.checkFileSyncSettings')
- ]);
- break;
-
- case Zotero.Sync.Storage.ERROR_PARENT_DIR_NOT_FOUND:
- var errorTitle = Zotero.getString('sync.storage.error.directoryNotFound');
- var parentSpec = spec.replace(/\/zotero\/$/, "");
- var errorMessage = Zotero.getString('sync.storage.error.doesNotExist', parentSpec);
- break;
-
- case Zotero.Sync.Storage.ERROR_ZOTERO_DIR_NOT_FOUND:
- var create = promptService.confirmEx(
- window,
- Zotero.getString('sync.storage.error.directoryNotFound'),
- Zotero.getString('sync.storage.error.doesNotExist', spec) + "\n\n"
- + Zotero.getString('sync.storage.error.createNow'),
- promptService.BUTTON_POS_0
- * promptService.BUTTON_TITLE_IS_STRING
- + promptService.BUTTON_POS_1
- * promptService.BUTTON_TITLE_CANCEL,
- Zotero.getString('general.create'),
- null, null, null, {}
- );
-
- if (create != 0) {
- return;
- }
-
- createServerDirectory(function (uri, status) {
- switch (status) {
- case Zotero.Sync.Storage.SUCCESS:
- if (!skipSuccessMessage) {
- promptService.alert(
- window,
- Zotero.getString('sync.storage.serverConfigurationVerified'),
- Zotero.getString('sync.storage.fileSyncSetUp')
- );
- }
- Zotero.Prefs.set("sync.storage.verified", true);
- return true;
+ if (files.length == 0) {
+ return Zotero.Promise.resolve(results);
+ }
+
+ let deleteURI = _rootURI.clone();
+ // This should never happen, but let's be safe
+ if (!deleteURI.spec.match(/\/$/)) {
+ return Zotero.Promise.reject("Root URI does not end in slash in "
+ + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()");
+ }
+
+ var funcs = [];
+ for (let i=0; i<files.length; i++) {
+ let fileName = files[i];
+ let baseName = fileName.match(/^([^\.]+)/)[1];
+ funcs.push(function () {
+ let deleteURI = _rootURI.clone();
+ deleteURI.QueryInterface(Components.interfaces.nsIURL);
+ deleteURI.fileName = fileName;
+ deleteURI.QueryInterface(Components.interfaces.nsIURI);
+ return Zotero.HTTP.promise("DELETE", deleteURI, { successCodes: [200, 204, 404] })
+ .then(function (req) {
+ switch (req.status) {
+ case 204:
+ // IIS 5.1 and Sakai return 200
+ case 200:
+ var fileDeleted = true;
+ break;
- case Zotero.Sync.Storage.ERROR_FORBIDDEN:
- var errorTitle = Zotero.getString('general.permissionDenied');
- var errorMessage = Zotero.getString('sync.storage.error.permissionDeniedAtAddress') + "\n\n"
- + spec + "\n\n"
- + Zotero.getString('sync.storage.error.checkFileSyncSettings');
+ case 404:
+ var fileDeleted = true;
break;
}
- // TEMP
- if (!errorMessage) {
- var errorMessage = status;
+ // If an item file URI, get the property URI
+ var deletePropURI = getPropertyURIFromItemURI(deleteURI);
+
+ // If we already deleted the prop file, skip it
+ if (!deletePropURI || results.deleted.indexOf(deletePropURI.fileName) != -1) {
+ if (fileDeleted) {
+ results.deleted.push(baseName);
+ }
+ else {
+ results.missing.push(baseName);
+ }
+ return;
}
- promptService.alert(window, errorTitle, errorMessage);
- });
-
- return false;
-
- case Zotero.Sync.Storage.ERROR_FILE_MISSING_AFTER_UPLOAD:
- var errorTitle = Zotero.getString("general.warning");
- var errorMessage = Zotero.getString('sync.storage.error.webdav.fileMissingAfterUpload');
- Zotero.Prefs.set("sync.storage.verified", true);
- break;
-
- case Zotero.Sync.Storage.ERROR_SERVER_ERROR:
- var errorTitle = Zotero.getString('sync.storage.error.webdav.serverConfig.title');
- var errorMessage = Zotero.getString('sync.storage.error.webdav.serverConfig')
- + "\n\n" + Zotero.getString('sync.storage.error.checkFileSyncSettings');
- break;
-
- case Zotero.Sync.Storage.ERROR_UNKNOWN:
- var errorMessage = Zotero.localeJoin([
- Zotero.getString('general.unknownErrorOccurred'),
- Zotero.getString('sync.storage.error.checkFileSyncSettings')
- ]);
- break;
-
- case Zotero.Sync.Storage.ERROR_NONEXISTENT_FILE_NOT_MISSING:
- var errorTitle = Zotero.getString('sync.storage.error.webdav.serverConfig.title');
- var errorMessage = Zotero.getString('sync.storage.error.webdav.nonexistentFileNotMissing');
- break;
+
+ let propFileName = deletePropURI.fileName;
+
+ // Delete property file
+ return Zotero.HTTP.promise("DELETE", deletePropURI, { successCodes: [200, 204, 404] })
+ .then(function (req) {
+ switch (req.status) {
+ case 204:
+ // IIS 5.1 and Sakai return 200
+ case 200:
+ results.deleted.push(baseName);
+ break;
+
+ case 404:
+ if (fileDeleted) {
+ results.deleted.push(baseName);
+ }
+ else {
+ results.missing.push(baseName);
+ }
+ break;
+ }
+ });
+ })
+ .catch(function (e) {
+ results.error.push(baseName);
+ throw e;
+ });
+ });
}
- if (!skipSuccessMessage) {
- if (!errorTitle) {
- var errorTitle = Zotero.getString("general.error");
- }
- // TEMP
- if (!errorMessage) {
- var errorMessage = status;
- }
- promptService.alert(window, errorTitle, errorMessage);
- }
- return false;
- };
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ var caller = new ConcurrentCaller(4);
+ caller.stopOnError = true;
+ caller.setLogger(function (msg) Zotero.debug(msg));
+ caller.onError(function (e) Components.utils.reportError(e));
+ return caller.fcall(funcs)
+ .then(function () {
+ return results;
+ });
+ },
/**
- * Remove files on storage server that were deleted locally
- *
- * @param {Function} callback Passed number of files deleted
+ * Checks for an invalid SSL certificate and throws a nice error
*/
- obj._purgeDeletedStorageFiles = function () {
- return Zotero.Promise.try(function () {
- if (!this.includeUserFiles) {
- return false;
- }
-
- Zotero.debug("Purging deleted storage files");
- var files = Zotero.Sync.Storage.getDeletedFiles();
- if (!files) {
- Zotero.debug("No files to delete remotely");
- return false;
- }
-
- // Add .zip extension
- var files = files.map(function (file) file + ".zip");
-
- return deleteStorageFiles(files)
- .then(function (results) {
- // Remove deleted and nonexistent files from storage delete log
- var toPurge = results.deleted.concat(results.missing);
- if (toPurge.length > 0) {
- var done = 0;
- var maxFiles = 999;
- var numFiles = toPurge.length;
-
- Zotero.DB.beginTransaction();
-
- do {
- var chunk = toPurge.splice(0, maxFiles);
- var sql = "DELETE FROM storageDeleteLog WHERE key IN ("
- + chunk.map(function () '?').join() + ")";
- Zotero.DB.query(sql, chunk);
- done += chunk.length;
- }
- while (done < numFiles);
-
- Zotero.DB.commitTransaction();
+ _checkResponse: function (req) {
+ var channel = req.channel;
+ if (!channel instanceof Ci.nsIChannel) {
+ Zotero.Sync.Storage.EventManager.error('No HTTPS channel available');
+ }
+
+ // Check if the error we encountered is really an SSL error
+ // Logic borrowed from https://developer.mozilla.org/en-US/docs/How_to_check_the_security_state_of_an_XMLHTTPRequest_over_SSL
+ // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/ssl/sslerr.h
+ // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/util/secerr.h
+ var secErrLimit = Ci.nsINSSErrorsService.NSS_SEC_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE;
+ var secErr = Math.abs(Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE) - (channel.status & 0xffff);
+ var sslErrLimit = Ci.nsINSSErrorsService.NSS_SSL_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE;
+ var sslErr = Math.abs(Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE) - (channel.status & 0xffff);
+ if( (secErr < 0 || secErr > secErrLimit) && (sslErr < 0 || sslErr > sslErrLimit) ) {
+ return;
+ }
+
+ var secInfo = channel.securityInfo;
+ if (secInfo instanceof Ci.nsITransportSecurityInfo) {
+ secInfo.QueryInterface(Ci.nsITransportSecurityInfo);
+ if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_INSECURE) == Ci.nsIWebProgressListener.STATE_IS_INSECURE) {
+ var host = 'host';
+ try {
+ host = channel.URI.host;
+ }
+ catch (e) {
+ Zotero.debug(e);
}
- Zotero.debug(results);
+ var msg = Zotero.getString('sync.storage.error.webdav.sslCertificateError', host);
+ // In Standalone, provide cert_override.txt instructions and a
+ // button to open the Zotero profile directory
+ if (Zotero.isStandalone) {
+ msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.seeCertOverrideDocumentation');
+ var buttonText = Zotero.getString('general.openDocumentation');
+ var func = function () {
+ var zp = Zotero.getActiveZoteroPane();
+ zp.loadURI("https://www.zotero.org/support/kb/cert_override", { shiftKey: true });
+ };
+ }
+ // In Firefox display a button to load the WebDAV URL
+ else {
+ msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
+ var buttonText = Zotero.getString('sync.storage.error.webdav.loadURL');
+ var func = function () {
+ var zp = Zotero.getActiveZoteroPane();
+ zp.loadURI(channel.URI.spec, { shiftKey: true });
+ };
+ }
- return results.deleted.length;
- });
- }.bind(this));
- };
-
-
- /**
- * Delete orphaned storage files older than a day before last sync time
- */
- obj._purgeOrphanedStorageFiles = function () {
- return Zotero.Promise.try(function () {
- const daysBeforeSyncTime = 1;
-
- if (!this.includeUserFiles) {
- return false;
- }
-
- // If recently purged, skip
- var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge');
- var days = 10;
- if (lastpurge && new Date(lastpurge * 1000) > (new Date() - (1000 * 60 * 60 * 24 * days))) {
- return false;
+ var e = new Zotero.Error(
+ msg,
+ 0,
+ {
+ dialogText: msg,
+ dialogButtonText: buttonText,
+ dialogButtonCallback: func
+ }
+ );
+ throw e;
}
-
- Zotero.debug("Purging orphaned storage files");
-
- var uri = this.rootURI;
- var path = uri.path;
-
- var xmlstr = "<propfind xmlns='DAV:'><prop>"
- + "<getlastmodified/>"
- + "</prop></propfind>";
-
- var lastSyncDate = new Date(Zotero.Sync.Server.lastLocalSyncTime * 1000);
-
- var deferred = Zotero.Promise.defer();
-
- Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (xmlhttp) {
- Zotero.Promise.try(function () {
- Zotero.debug(xmlhttp.responseText);
-
- var funcName = "Zotero.Sync.Storage.purgeOrphanedStorageFiles()";
-
- var responseNode = xmlhttp.responseXML.documentElement;
- responseNode.xpath = function (path) {
- return Zotero.Utilities.xpath(this, path, { D: 'DAV:' });
- };
-
- var deleteFiles = [];
- var trailingSlash = !!path.match(/\/$/);
- for each(var response in responseNode.xpath("D:response")) {
- var href = Zotero.Utilities.xpathText(
- response, "D:href", { D: 'DAV:' }
- ) || "";
- Zotero.debug(href);
-
- // Strip trailing slash if there isn't one on the root path
- if (!trailingSlash) {
- href = href.replace(/\/$/, "");
- }
-
- // Absolute
- if (href.match(/^https?:\/\//)) {
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var href = ios.newURI(href, null, null);
- href = href.path;
- }
-
- // Skip root URI
- if (href == path
- // Some Apache servers respond with a "/zotero" href
- // even for a "/zotero/" request
- || (trailingSlash && href + '/' == path)
- // Try URL-encoded as well, as above
- || decodeURIComponent(href) == path) {
- continue;
- }
-
- if (href.indexOf(path) == -1
- // Try URL-encoded as well, in case there's a '~' or similar
- // character in the URL and the server (e.g., Sakai) is
- // encoding the value
- && decodeURIComponent(href).indexOf(path) == -1) {
- throw new Error(
- "DAV:href '" + href + "' does not begin with path '"
- + path + "' in " + funcName
- );
- }
-
- var matches = href.match(/[^\/]+$/);
- if (!matches) {
- throw new Error(
- "Unexpected href '" + href + "' in " + funcName
- );
- }
- var file = matches[0];
-
- if (file.indexOf('.') == 0) {
- Zotero.debug("Skipping hidden file " + file);
- continue;
- }
- if (!file.match(/\.zip$/) && !file.match(/\.prop$/)) {
- Zotero.debug("Skipping file " + file);
- continue;
- }
-
- var key = file.replace(/\.(zip|prop)$/, '');
- var item = Zotero.Items.getByLibraryAndKey(null, key);
- if (item) {
- Zotero.debug("Skipping existing file " + file);
- continue;
- }
-
- Zotero.debug("Checking orphaned file " + file);
-
- // TODO: Parse HTTP date properly
- Zotero.debug(response.innerHTML);
- var lastModified = Zotero.Utilities.xpathText(
- response, ".//D:getlastmodified", { D: 'DAV:' }
- );
- lastModified = Zotero.Date.strToISO(lastModified);
- lastModified = Zotero.Date.sqlToDate(lastModified);
-
- // Delete files older than a day before last sync time
- var days = (lastSyncDate - lastModified) / 1000 / 60 / 60 / 24;
-
- if (days > daysBeforeSyncTime) {
- deleteFiles.push(file);
+ else if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_BROKEN) == Ci.nsIWebProgressListener.STATE_IS_BROKEN) {
+ var msg = Zotero.getString('sync.storage.error.webdav.sslConnectionError', host) +
+ Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
+ var e = new Zotero.Error(
+ msg,
+ 0,
+ {
+ dialogText: msg,
+ dialogButtonText: Zotero.getString('sync.storage.error.webdav.loadURL'),
+ dialogButtonCallback: function () {
+ var zp = Zotero.getActiveZoteroPane();
+ zp.loadURI(channel.URI.spec, { shiftKey: true });
}
}
-
- return deleteStorageFiles(deleteFiles)
- .then(function (results) {
- Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000))
- Zotero.debug(results);
- });
- })
- .catch(function (e) {
- deferred.reject(e);
- })
- .then(function () {
- deferred.resolve();
- });
- }, { Depth: 1 });
-
- return deferred.promise;
- }.bind(this));
- };
-
- return obj;
-}());
+ );
+ throw e;
+ }
+ }
+ }
+}
diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js
@@ -24,605 +24,874 @@
*/
-Zotero.Sync.Storage.ZFS = (function () {
- var _rootURI;
- var _userURI;
- var _headers = {
- "Zotero-API-Version" : ZOTERO_CONFIG.API_VERSION
- };
- var _cachedCredentials = false;
- var _s3Backoff = 1;
- var _s3ConsecutiveFailures = 0;
- var _maxS3Backoff = 60;
- var _maxS3ConsecutiveFailures = 5;
+Zotero.Sync.Storage.ZFS_Module = function (options) {
+ this.options = options;
+ this.apiClient = options.apiClient;
+
+ this._s3Backoff = 1;
+ this._s3ConsecutiveFailures = 0;
+ this._maxS3Backoff = 60;
+ this._maxS3ConsecutiveFailures = 5;
+};
+Zotero.Sync.Storage.ZFS_Module.prototype = {
+ name: "ZFS",
+ verified: true,
/**
- * Get file metadata on storage server
+ * @return {Promise} A promise for the last sync time
+ */
+ getLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
+ var params = this._getRequestParams(libraryID, "laststoragesync");
+ var uri = this.apiClient.buildRequestURI(params);
+
+ try {
+ let req = yield this.apiClient.makeRequest(
+ "GET", uri, { successCodes: [200, 404], debug: true }
+ );
+
+ // Not yet synced
+ if (req.status == 404) {
+ Zotero.debug("No last sync time for library " + libraryID);
+ return null;
+ }
+
+ let ts = req.responseText;
+ let date = new Date(ts * 1000);
+ Zotero.debug("Last successful ZFS sync for library " + libraryID + " was " + date);
+ return ts;
+ }
+ catch (e) {
+ Zotero.logError(e);
+ throw e;
+ }
+ }),
+
+
+ setLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
+ var params = this._getRequestParams(libraryID, "laststoragesync");
+ var uri = this.apiClient.buildRequestURI(params);
+
+ try {
+ var req = yield this.apiClient.makeRequest(
+ "POST", uri, { successCodes: [200, 404], debug: true }
+ );
+ }
+ catch (e) {
+ var msg = "Unexpected status code " + e.xmlhttp.status + " setting last file sync time";
+ Zotero.logError(e);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+
+ // Not yet synced
+ //
+ // TODO: Don't call this at all if no files uploaded
+ if (req.status == 404) {
+ return;
+ }
+
+ var time = req.responseText;
+ if (parseInt(time) != time) {
+ Zotero.logError(`Unexpected response ${time} setting last file sync time`);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ return parseInt(time);
+ }),
+
+
+ /**
+ * Begin download process for individual file
*
- * @param {Zotero.Item} item
- * @param {Function} callback Callback f(item, etag)
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise<Boolean>} - True if file download, false if not
*/
- function getStorageFileInfo(item, request) {
- var funcName = "Zotero.Sync.Storage.ZFS.getStorageFileInfo()";
+ downloadFile: Zotero.Promise.coroutine(function* (request) {
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
+ if (!item) {
+ throw new Error("Item '" + request.name + "' not found");
+ }
- return Zotero.HTTP.promise("GET", getItemInfoURI(item),
+ var path = item.getFilePath();
+ if (!path) {
+ Zotero.debug(`Cannot download file for attachment ${item.libraryKey} with no path`);
+ return new Zotero.Sync.Storage.Result;
+ }
+
+ var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
+
+ // saveURI() below appears not to create empty files for Content-Length: 0,
+ // so we create one here just in case, which also lets us check file access
+ try {
+ let file = yield OS.File.open(destPath, {
+ truncate: true
+ });
+ file.close();
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, destPath, 'create');
+ }
+
+ var deferred = Zotero.Promise.defer();
+ var requestData = {item};
+
+ var listener = new Zotero.Sync.Storage.StreamListener(
{
- successCodes: [200, 404],
- headers: _headers,
- requestObserver: function (xmlhttp) {
- request.setChannel(xmlhttp.channel);
- }
- })
- .then(function (req) {
- if (req.status == 404) {
- return false;
- }
-
- var info = {};
- info.hash = req.getResponseHeader('ETag');
- if (!info.hash) {
- var msg = "Hash not found in info response in " + funcName
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Zotero.debug(req.status);
- Zotero.debug(req.responseText);
- Components.utils.reportError(msg);
+ onStart: function (req) {
+ if (request.isFinished()) {
+ Zotero.debug("Download request " + request.name
+ + " stopped before download started -- closing channel");
+ req.cancel(Components.results.NS_BINDING_ABORTED);
+ deferred.resolve(false);
+ }
+ },
+ onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
+ // These will be used in processDownload() if the download succeeds
+ oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel);
+
+ Zotero.debug("CHANNEL HERE FOR " + item.libraryKey + " WITH " + oldChannel.status);
+ Zotero.debug(oldChannel.URI.spec);
+ Zotero.debug(newChannel.URI.spec);
+
+ var header;
try {
- Zotero.debug(req.getAllResponseHeaders());
+ header = "Zotero-File-Modification-Time";
+ requestData.mtime = oldChannel.getResponseHeader(header);
+ header = "Zotero-File-MD5";
+ requestData.md5 = oldChannel.getResponseHeader(header);
+ header = "Zotero-File-Compressed";
+ requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes';
}
catch (e) {
- Zotero.debug("Response headers unavailable");
+ deferred.reject(new Error(`${header} header not set in file request for ${item.libraryKey}`));
+ return false;
}
- var msg = Zotero.getString('sync.storage.error.zfs.restart', Zotero.appName);
- throw msg;
- }
- info.filename = req.getResponseHeader('X-Zotero-Filename');
- var mtime = req.getResponseHeader('X-Zotero-Modification-Time');
- info.mtime = parseInt(mtime);
- info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes';
- Zotero.debug(info);
-
- return info;
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.xmlhttp.status == 0) {
- var msg = "Request cancelled getting storage file info";
+
+ if (!(yield OS.File.exists(path))) {
+ return true;
+ }
+
+ var updateHash = false;
+ var fileModTime = yield item.attachmentModificationTime;
+ if (requestData.mtime == fileModTime) {
+ Zotero.debug("File mod time matches remote file -- skipping download of "
+ + item.libraryKey);
+ }
+ // If not compressed, check hash, in case only timestamp changed
+ else if (!requestData.compressed && (yield item.attachmentHash) == requestData.md5) {
+ Zotero.debug("File hash matches remote file -- skipping download of "
+ + item.libraryKey);
+ updateHash = true;
}
else {
- var msg = "Unexpected status code " + e.xmlhttp.status
- + " getting storage file info for item " + item.libraryKey;
+ return true;
}
- Zotero.debug(msg, 1);
- Zotero.debug(e.xmlhttp.responseText);
- Components.utils.reportError(msg);
- throw new Error(Zotero.Sync.Storage.defaultError);
- }
-
- throw e;
- });
- }
-
-
- /**
- * Upload the file to the server
- *
- * @param {Object} Object with 'request' property
- * @return {void}
- */
- function processUploadFile(data) {
- /*
- updateSizeMultiplier(
- (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
- );
- */
-
- var request = data.request;
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- return getStorageFileInfo(item, request)
- .then(function (info) {
- if (request.isFinished()) {
- Zotero.debug("Upload request '" + request.name
- + "' is no longer running after getting file info");
- return false;
- }
-
- // Check for conflict
- if (Zotero.Sync.Storage.getSyncState(item.id)
- != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
- if (info) {
- // Remote mod time
- var mtime = info.mtime;
- // Local file time
- var fmtime = item.attachmentModificationTime;
-
- var same = false;
- var useLocal = false;
- if (fmtime == mtime) {
- same = true;
- Zotero.debug("File mod time matches remote file -- skipping upload");
- }
- // Allow floored timestamps for filesystems that don't support
- // millisecond precision (e.g., HFS+)
- else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
- same = true;
- Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
- + "-- skipping upload");
- }
- // Allow timestamp to be exactly one hour off to get around
- // time zone issues -- there may be a proper way to fix this
- else if (Math.abs(fmtime - mtime) == 3600000
- // And check with one-second precision as well
- || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
- || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
- same = true;
- Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
- + "-- assuming time zone issue and skipping upload");
+
+ // Update local metadata and stop request, skipping file download
+ yield Zotero.DB.executeTransaction(function* () {
+ if (updateHash) {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, requestData.md5);
}
- // Ignore maxed-out 32-bit ints, from brief problem after switch to 32-bit servers
- else if (mtime == 2147483647) {
- Zotero.debug("Remote mod time is invalid -- uploading local file version");
- useLocal = true;
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, requestData.mtime
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+ return false;
+ }),
+ onProgress: function (a, b, c) {
+ request.onProgress(a, b, c)
+ },
+ onStop: function (req, status, res) {
+ request.setChannel(false);
+
+ if (status != 200) {
+ if (status == 404) {
+ Zotero.debug("Remote file not found for item " + item.libraryKey);
+ deferred.resolve(new Zotero.Sync.Storage.Result);
+ return;
}
- if (same) {
- Zotero.debug(Zotero.Sync.Storage.getSyncedModificationTime(item.id));
+ // If S3 connection is interrupted, delay and retry, or bail if too many
+ // consecutive failures
+ if (status == 0) {
+ if (this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) {
+ let libraryKey = item.libraryKey;
+ let msg = "S3 returned 0 for " + libraryKey + " -- retrying download"
+ Components.utils.reportError(msg);
+ Zotero.debug(msg, 1);
+ if (this._s3Backoff < this._maxS3Backoff) {
+ this._s3Backoff *= 2;
+ }
+ this._s3ConsecutiveFailures++;
+ Zotero.debug("Delaying " + libraryKey + " download for "
+ + this._s3Backoff + " seconds", 2);
+ Zotero.Promise.delay(this._s3Backoff * 1000)
+ .then(function () {
+ deferred.resolve(this._downloadFile(request));
+ });
+ return;
+ }
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return {
- localChanges: true,
- remoteChanges: false
- };
+ Zotero.debug(this._s3ConsecutiveFailures
+ + " consecutive S3 failures -- aborting", 1);
+ this._s3ConsecutiveFailures = 0;
}
- var smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id);
- if (!useLocal && smtime != mtime) {
- Zotero.debug("Conflict -- last synced file mod time "
- + "does not match time on storage server"
- + " (" + smtime + " != " + mtime + ")");
- return {
- localChanges: false,
- remoteChanges: false,
- conflict: {
- local: { modTime: fmtime },
- remote: { modTime: mtime }
- }
- };
+ var msg = "Unexpected status code " + status + " for GET " + uri;
+ Zotero.debug(msg, 1);
+ Components.utils.reportError(msg);
+ // Output saved content, in case an error was captured
+ try {
+ let sample = Zotero.File.getContents(destPath, null, 4096);
+ if (sample) {
+ Zotero.debug(sample, 1);
+ }
+ }
+ catch (e) {
+ Zotero.debug(e, 1);
}
+ deferred.reject(new Error(Zotero.Sync.Storage.defaultError));
+ return;
}
- else {
- Zotero.debug("Remote file not found for item " + item.libraryID + "/" + item.key);
+
+ // Don't try to process if the request has been cancelled
+ if (request.isFinished()) {
+ Zotero.debug("Download request " + request.name
+ + " is no longer running after file download", 2);
+ deferred.resolve(false);
+ return;
+ }
+
+ Zotero.debug("Finished download of " + destPath);
+
+ try {
+ deferred.resolve(
+ Zotero.Sync.Storage.Local.processDownload(requestData)
+ );
+ }
+ catch (e) {
+ Zotero.debug("REJECTING");
+ deferred.reject(e);
+ }
+ }.bind(this),
+ onCancel: function (req, status) {
+ Zotero.debug("Request cancelled");
+ if (deferred.promise.isPending()) {
+ deferred.resolve(false);
}
}
+ }
+ );
+
+ var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(params);
+ var headers = this.apiClient.getHeaders();
+
+ Zotero.debug('Saving ' + uri);
+ const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
+ var wbp = Components
+ .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
+ .createInstance(nsIWBP);
+ wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
+ wbp.progressListener = listener;
+ Zotero.Utilities.Internal.saveURI(wbp, uri, destPath, headers);
+
+ return deferred.promise;
+ }),
+
+
+ uploadFile: Zotero.Promise.coroutine(function* (request) {
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
+ let created = yield Zotero.Sync.Storage.Utilities.createUploadFile(request);
+ if (!created) {
+ return new Zotero.Sync.Storage.Result;
+ }
+ return this._processUploadFile(request);
+ }
+ return this._processUploadFile(request);
+ }),
+
+
+ /**
+ * Remove all synced files from the server
+ */
+ purgeDeletedStorageFiles: Zotero.Promise.coroutine(function* () {
+ var sql = "SELECT value FROM settings WHERE setting=? AND key=?";
+ var values = yield Zotero.DB.columnQueryAsync(sql, ['storage', 'zfsPurge']);
+ if (!values) {
+ return false;
+ }
+
+ Zotero.debug("Unlinking synced files on ZFS");
+
+ var uri = this.userURI;
+ uri.spec += "removestoragefiles?";
+ // Unused
+ for each(var value in values) {
+ switch (value) {
+ case 'user':
+ uri.spec += "user=1&";
+ break;
- return getFileUploadParameters(
- item,
- function (item, target, uploadKey, params) {
- return postFile(request, item, target, uploadKey, params);
- },
- function () {
- updateItemFileInfo(item);
- return {
- localChanges: true,
- remoteChanges: false
- };
- }
- );
- });
- }
+ case 'group':
+ uri.spec += "group=1&";
+ break;
+
+ default:
+ throw new Error("Invalid zfsPurge value '" + value + "'");
+ }
+ }
+ uri.spec = uri.spec.substr(0, uri.spec.length - 1);
+
+ yield Zotero.HTTP.request("POST", uri, "");
+
+ var sql = "DELETE FROM settings WHERE setting=? AND key=?";
+ yield Zotero.DB.queryAsync(sql, ['storage', 'zfsPurge']);
+ }),
+
+
+ //
+ // Private methods
+ //
+ _getRequestParams: function (libraryID, target) {
+ var library = Zotero.Libraries.get(libraryID);
+ return {
+ libraryType: library.libraryType,
+ libraryTypeID: library.libraryTypeID,
+ target
+ };
+ },
/**
- * Get mod time of file on storage server
+ * Get authorization from API for uploading file
*
- * @param {Zotero.Item} item
- * @param {Function} uploadCallback Callback f(request, item, target, params)
- * @param {Function} existsCallback Callback f() to call when file already exists
- * on server and uploading isn't necessary
+ * @param {Zotero.Item} item
+ * @return {Object|String} - Object with upload params or 'exists'
*/
- function getFileUploadParameters(item, uploadCallback, existsCallback) {
- var funcName = "Zotero.Sync.Storage.ZFS.getFileUploadParameters()";
-
- var uri = getItemURI(item);
+ _getFileUploadParameters: Zotero.Promise.coroutine(function* (item) {
+ var funcName = "Zotero.Sync.Storage.ZFS._getFileUploadParameters()";
- if (Zotero.Attachments.getNumFiles(item) > 1) {
- var file = Zotero.getTempDirectory();
- var filename = item.key + '.zip';
- file.append(filename);
- uri.spec = uri.spec;
- var zip = true;
+ var path = item.getFilePath();
+ var filename = OS.Path.basename(path);
+ var zip = yield Zotero.Attachments.hasMultipleFiles(item);
+ if (zip) {
+ var uploadPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
}
else {
- var file = item.getFile();
- var filename = file.leafName;
- var zip = false;
+ var uploadPath = path;
}
- var mtime = item.attachmentModificationTime;
+ var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(params);
+
+ // TODO: One-step uploads
+ /*var headers = {
+ "Content-Type": "application/json"
+ };
+ var storedHash = yield Zotero.Sync.Storage.Local.getSyncedHash(item.id);
+ //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (storedHash) {
+ headers["If-Match"] = storedHash;
+ }
+ else {
+ headers["If-None-Match"] = "*";
+ }
+ var mtime = yield item.attachmentModificationTime;
var hash = Zotero.Utilities.Internal.md5(file);
+ var json = {
+ md5: hash,
+ mtime,
+ filename,
+ size: file.fileSize
+ };
+ var charset = item.attachmentCharset;
+ var contentType = item.attachmentContentType;
+ if (charset) {
+ json.charset = charset;
+ }
+ if (contentType) {
+ json.contentType = contentType;
+ }
+ if (zip) {
+ json.zip = true;
+ }
+
+ try {
+ var req = yield this.apiClient.makeRequest(
+ "POST", uri, { body: JSON.stringify(json), headers, debug: true }
+ );
+ }*/
+
+ var headers = {
+ "Content-Type": "application/x-www-form-urlencoded"
+ };
+ var storedHash = yield Zotero.Sync.Storage.Local.getSyncedHash(item.id);
+ //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (storedHash) {
+ headers["If-Match"] = storedHash;
+ }
+ else {
+ headers["If-None-Match"] = "*";
+ }
- var body = "md5=" + hash + "&filename=" + encodeURIComponent(filename)
- + "&filesize=" + file.fileSize + "&mtime=" + mtime;
+ // Build POST body
+ var mtime = yield item.attachmentModificationTime;
+ var params = {
+ md5: yield item.attachmentHash,
+ mtime,
+ filename,
+ filesize: (yield OS.File.stat(uploadPath)).size
+ };
+ var charset = item.attachmentCharset;
+ var contentType = item.attachmentContentType;
+ if (charset) {
+ params.charset = charset;
+ }
+ if (contentType) {
+ params.contentType = contentType;
+ }
if (zip) {
- body += "&zip=1";
+ params.zipMD5 = yield Zotero.Utilities.Internal.md5Async(uploadPath);
+ params.zipFilename = OS.Path.basename(uploadPath);
+ }
+ var body = [];
+ for (let i in params) {
+ body.push(i + "=" + encodeURIComponent(params[i]));
}
+ body = body.join('&');
- return Zotero.HTTP.promise("POST", uri, { body: body, headers: _headers, debug: true })
- .then(function (req) {
- if (!req.responseXML) {
- throw new Error("Invalid response retrieving file upload parameters");
- }
-
- var rootTag = req.responseXML.documentElement.tagName;
-
- if (rootTag != 'upload' && rootTag != 'exists') {
- throw new Error("Invalid response retrieving file upload parameters");
- }
-
- // File was already available, so uploading isn't required
- if (rootTag == 'exists') {
- return existsCallback();
+ try {
+ var req = yield this.apiClient.makeRequest(
+ "POST",
+ uri,
+ {
+ body,
+ headers,
+ // This should include all errors in _handleUploadAuthorizationFailure()
+ successCodes: [200, 201, 204, 403, 404, 412, 413],
+ debug: true
}
-
- var url = req.responseXML.getElementsByTagName('url')[0].textContent;
- var uploadKey = req.responseXML.getElementsByTagName('key')[0].textContent;
- var params = {}, p = '';
- var paramNodes = req.responseXML.getElementsByTagName('params')[0].childNodes;
- for (var i = 0; i < paramNodes.length; i++) {
- params[paramNodes[i].tagName] = paramNodes[i].textContent;
+ );
+ }
+ catch (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ let msg = "Unexpected status code " + e.status + " in " + funcName
+ + " (" + item.libraryKey + ")";
+ Zotero.logError(msg);
+ Zotero.debug(e.xmlhttp.getAllResponseHeaders());
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ throw e;
+ }
+
+ var result = yield this._handleUploadAuthorizationFailure(req, item);
+ if (result instanceof Zotero.Sync.Storage.Result) {
+ return result;
+ }
+
+ try {
+ var json = JSON.parse(req.responseText);
+ }
+ catch (e) {
+ Zotero.logError(e);
+ Zotero.debug(req.responseText, 1);
+ }
+ if (!json) {
+ throw new Error("Invalid response retrieving file upload parameters");
+ }
+
+ if (!json.uploadKey && !json.exists) {
+ throw new Error("Invalid response retrieving file upload parameters");
+ }
+
+ if (json.exists) {
+ let version = req.getResponseHeader('Last-Modified-Version');
+ if (!version) {
+ throw new Error("Last-Modified-Version not provided");
+ }
+ json.version = version;
+ }
+
+ Zotero.debug('=-=-=--=');
+ Zotero.debug(json);
+
+ // TEMP
+ //
+ // Passed through to _updateItemFileInfo()
+ json.mtime = mtime;
+ json.md5 = params.md5;
+ if (storedHash) {
+ json.storedHash = storedHash;
+ }
+
+ return json;
+ }),
+
+
+ /**
+ * Handle known errors from upload authorization request
+ *
+ * These must be included in successCodes in _getFileUploadParameters()
+ */
+ _handleUploadAuthorizationFailure: Zotero.Promise.coroutine(function* (req, item) {
+ //
+ // These must be included in successCodes above.
+ // TODO: 429?
+ if (req.status == 403) {
+ let groupID = Zotero.Groups.getGroupIDFromLibraryID(item.libraryID);
+ let e = new Zotero.Error(
+ "File editing denied for group",
+ "ZFS_FILE_EDITING_DENIED",
+ {
+ groupID: groupID
}
- return uploadCallback(item, url, uploadKey, params);
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status == 413) {
- var retry = e.xmlhttp.getResponseHeader('Retry-After');
- if (retry) {
- var minutes = Math.round(retry / 60);
- var e = new Zotero.Error(
- Zotero.getString('sync.storage.error.zfs.tooManyQueuedUploads', minutes),
- "ZFS_UPLOAD_QUEUE_LIMIT"
- );
- throw e;
- }
-
- var text, buttonText = null, buttonCallback;
-
- // Group file
- if (item.libraryID) {
- var group = Zotero.Groups.getByLibraryID(item.libraryID);
- text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n"
- + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2');
- }
- // Personal file
- else {
- text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n"
- + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2');
- buttonText = Zotero.getString('sync.storage.openAccountSettings');
- buttonCallback = function () {
- var url = "https://www.zotero.org/settings/storage";
-
- var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
- .getService(Components.interfaces.nsIWindowMediator);
- var win = wm.getMostRecentWindow("navigator:browser");
- win.ZoteroPane.loadURI(url);
- }
- }
-
- text += "\n\n" + filename + " (" + Math.round(file.fileSize / 1024) + "KB)";
-
- var e = new Zotero.Error(
- Zotero.getString('sync.storage.error.zfs.fileWouldExceedQuota', filename),
- "ZFS_OVER_QUOTA",
- {
- dialogText: text,
- dialogButtonText: buttonText,
- dialogButtonCallback: buttonCallback
- }
- );
- e.errorType = 'warning';
- Zotero.debug(e, 2);
- Components.utils.reportError(e);
- throw e;
- }
- else if (e.status == 403) {
- var groupID = Zotero.Groups.getGroupIDFromLibraryID(item.libraryID);
- var e = new Zotero.Error(
- "File editing denied for group",
- "ZFS_FILE_EDITING_DENIED",
- {
- groupID: groupID
- }
- );
- throw e;
- }
- else if (e.status == 404) {
- Components.utils.reportError("Unexpected status code 404 in " + funcName
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")");
- if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) {
- Components.utils.reportError("Skipping automatic client reset due to debug pref");
- return;
- }
- if (!Zotero.Sync.Server.canAutoResetClient) {
- Components.utils.reportError("Client has already been auto-reset -- manual sync required");
- return;
- }
- Zotero.Sync.Server.resetClient();
- Zotero.Sync.Server.canAutoResetClient = false;
- throw new Error(Zotero.Sync.Storage.defaultError);
- }
+ );
+ throw e;
+ }
+ else if (req.status == 404) {
+ Components.utils.reportError("Unexpected status code 404 in upload authorization "
+ + "request (" + item.libraryKey + ")");
+ if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) {
+ Components.utils.reportError("Skipping automatic client reset due to debug pref");
+ }
+ if (!Zotero.Sync.Server.canAutoResetClient) {
+ Components.utils.reportError("Client has already been auto-reset -- manual sync required");
+ }
+
+ // TODO: Make an API request to fix this
+
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ else if (req.status == 412) {
+ Zotero.debug("412 BUT WE'RE COOL");
+ let version = req.getResponseHeader('Last-Modified-Version');
+ if (!version) {
+ throw new Error("Last-Modified-Version header not provided");
+ }
+ if (version > item.version) {
+ return new Zotero.Sync.Storage.Result({
+ syncRequired: true
+ });
+ }
+ if (version < item.version) {
+ throw new Error("Last-Modified-Version is lower than item version "
+ + `(${version} < ${item.version})`);
+ }
+
+ // Get updated item metadata
+ let library = Zotero.Libraries.get(item.libraryID);
+ let json = yield this.apiClient.downloadObjects(
+ library.libraryType,
+ library.libraryTypeID,
+ 'item',
+ [item.key]
+ )[0];
+ if (!Array.isArray(json)) {
+ Zotero.logError(json);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ if (json.length > 1) {
+ throw new Error("More than one result for item lookup");
+ }
+
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', item.libraryID, json);
+ json = json[0];
+
+ if (json.data.version > item.version) {
+ return new Zotero.Sync.Storage.Result({
+ syncRequired: true
+ });
+ }
+
+ let fileHash = yield item.attachmentHash;
+ let fileModTime = yield item.attachmentModificationTime;
+
+ Zotero.debug("MD5");
+ Zotero.debug(json.data.md5);
+ Zotero.debug(fileHash);
+
+ if (json.data.md5 == fileHash) {
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, fileModTime
+ );
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, fileHash);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+ return new Zotero.Sync.Storage.Result;
+ }
+
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ return new Zotero.Sync.Storage.Result({
+ fileSyncRequired: true
+ });
+ }
+ else if (req.status == 413) {
+ let retry = req.getResponseHeader('Retry-After');
+ if (retry) {
+ let minutes = Math.round(retry / 60);
+ throw new Zotero.Error(
+ Zotero.getString('sync.storage.error.zfs.tooManyQueuedUploads', minutes),
+ "ZFS_UPLOAD_QUEUE_LIMIT"
+ );
+ }
+
+ let text, buttonText = null, buttonCallback;
+
+ // Group file
+ if (item.libraryID) {
+ var group = Zotero.Groups.getByLibraryID(item.libraryID);
+ text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n"
+ + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2');
+ }
+ // Personal file
+ else {
+ text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n"
+ + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2');
+ buttonText = Zotero.getString('sync.storage.openAccountSettings');
+ buttonCallback = function () {
+ var url = "https://www.zotero.org/settings/storage";
- var msg = "Unexpected status code " + e.status + " in " + funcName
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Zotero.debug(e.xmlhttp.getAllResponseHeaders());
- Components.utils.reportError(msg);
- throw new Error(Zotero.Sync.Storage.defaultError);
+ var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
+ .getService(Components.interfaces.nsIWindowMediator);
+ var win = wm.getMostRecentWindow("navigator:browser");
+ win.ZoteroPane.loadURI(url);
}
-
- throw e;
- });
- }
+ }
+
+ text += "\n\n" + filename + " (" + Math.round(file.fileSize / 1024) + "KB)";
+
+ let e = new Zotero.Error(
+ Zotero.getString('sync.storage.error.zfs.fileWouldExceedQuota', filename),
+ "ZFS_OVER_QUOTA",
+ {
+ dialogText: text,
+ dialogButtonText: buttonText,
+ dialogButtonCallback: buttonCallback
+ }
+ );
+ e.errorType = 'warning';
+ Zotero.debug(e, 2);
+ Components.utils.reportError(e);
+ throw e;
+ }
+ }),
- function postFile(request, item, url, uploadKey, params) {
+ /**
+ * Given parameters from authorization, upload file to S3
+ */
+ _uploadFile: Zotero.Promise.coroutine(function* (request, item, params) {
if (request.isFinished()) {
- Zotero.debug("Upload request " + request.name + " is no longer running after getting upload parameters");
- return false;
+ Zotero.debug("Upload request " + request.name + " is no longer running after getting "
+ + "upload parameters");
+ return new Zotero.Sync.Storage.Result;
}
- var file = getUploadFile(item);
+ var file = yield this._getUploadFile(item);
- // TODO: make sure this doesn't appear in file
- var boundary = "---------------------------" + Math.random().toString().substr(2);
+ Components.utils.importGlobalProperties(["File"]);
+ file = File(file);
- var mis = Components.classes["@mozilla.org/io/multiplex-input-stream;1"]
- .createInstance(Components.interfaces.nsIMultiplexInputStream);
+ var blob = new Blob([params.prefix, file, params.suffix]);
- // Add parameters
- for (var key in params) {
- var storage = Components.classes["@mozilla.org/storagestream;1"]
- .createInstance(Components.interfaces.nsIStorageStream);
- storage.init(4096, 4294967295, null); // PR_UINT32_MAX
- var out = storage.getOutputStream(0);
-
- var conv = Components.classes["@mozilla.org/intl/converter-output-stream;1"]
- .createInstance(Components.interfaces.nsIConverterOutputStream);
- conv.init(out, null, 4096, "?");
-
- var str = "--" + boundary + '\r\nContent-Disposition: form-data; name="' + key + '"'
- + '\r\n\r\n' + params[key] + '\r\n';
- conv.writeString(str);
- conv.close();
+ try {
+ var req = yield Zotero.HTTP.request(
+ "POST",
+ params.url,
+ {
+ headers: {
+ "Content-Type": params.contentType
+ },
+ body: blob,
+ requestObserver: function (req) {
+ request.setChannel(req.channel);
+ req.upload.addEventListener("progress", function (event) {
+ if (event.lengthComputable) {
+ request.onProgress(event.loaded, event.total);
+ }
+ });
+ },
+ debug: true,
+ successCodes: [201]
+ }
+ );
+ }
+ catch (e) {
+ // For timeouts and failures from S3, which happen intermittently,
+ // wait a little and try again
+ let timeoutMessage = "Your socket connection to the server was not read from or "
+ + "written to within the timeout period.";
+ if (e.status == 0
+ || (e.status == 400 && e.xmlhttp.responseText.indexOf(timeoutMessage) != -1)) {
+ if (this._s3ConsecutiveFailures >= this._maxS3ConsecutiveFailures) {
+ Zotero.debug(this._s3ConsecutiveFailures
+ + " consecutive S3 failures -- aborting", 1);
+ this._s3ConsecutiveFailures = 0;
+ }
+ else {
+ let msg = "S3 returned " + e.status + " (" + item.libraryKey + ") "
+ + "-- retrying upload"
+ Zotero.logError(msg);
+ Zotero.debug(e.xmlhttp.responseText, 1);
+ if (this._s3Backoff < this._maxS3Backoff) {
+ this._s3Backoff *= 2;
+ }
+ this._s3ConsecutiveFailures++;
+ Zotero.debug("Delaying " + item.libraryKey + " upload for "
+ + this._s3Backoff + " seconds", 2);
+ yield Zotero.Promise.delay(this._s3Backoff * 1000);
+ return this._uploadFile(request, item, params);
+ }
+ }
+ else if (e.status == 500) {
+ // TODO: localize
+ throw new Error("File upload failed. Please try again.");
+ }
+ else {
+ Zotero.logError(`Unexpected file upload status ${e.status} (${item.libraryKey})`);
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e.xmlhttp.responseText);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
- var instr = storage.newInputStream(0);
- mis.appendStream(instr);
+ // TODO: Detect cancel?
+ //onUploadCancel(httpRequest, status, data)
+ //deferred.resolve(false);
}
- // Add file
- var sis = Components.classes["@mozilla.org/io/string-input-stream;1"]
- .createInstance(Components.interfaces.nsIStringInputStream);
- var str = "--" + boundary + '\r\nContent-Disposition: form-data; name="file"\r\n\r\n';
- sis.setData(str, -1);
- mis.appendStream(sis);
-
- var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]
- .createInstance(Components.interfaces.nsIFileInputStream);
- fis.init(file, 0x01, 0, Components.interfaces.nsIFileInputStream.CLOSE_ON_EOF
- | Components.interfaces.nsIFileInputStream.REOPEN_ON_REWIND);
-
- var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"]
- .createInstance(Components.interfaces.nsIBufferedInputStream)
- bis.init(fis, 64 * 1024);
- mis.appendStream(bis);
-
- // End request
- var sis = Components.classes["@mozilla.org/io/string-input-stream;1"]
- .createInstance(Components.interfaces.nsIStringInputStream);
- var str = "\r\n--" + boundary + "--";
- sis.setData(str, -1);
- mis.appendStream(sis);
-
+ request.setChannel(false);
+ return this._onUploadComplete(req, request, item, params);
+ }),
+
+
+ /**
+ * Post-upload file registration with API
+ */
+ _onUploadComplete: Zotero.Promise.coroutine(function* (req, request, item, params) {
+ var uploadKey = params.uploadKey;
- /* var cstream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
- createInstance(Components.interfaces.nsIConverterInputStream);
- cstream.init(mis, "UTF-8", 0, 0); // you can use another encoding here if you wish
+ Zotero.debug("Upload of attachment " + item.key + " finished with status code " + req.status);
+ Zotero.debug(req.responseText);
- let (str = {}) {
- cstream.readString(-1, str); // read the whole file and put it in str.value
- data = str.value;
+ // Decrease backoff delay on successful upload
+ if (this._s3Backoff > 1) {
+ this._s3Backoff /= 2;
}
- cstream.close(); // this closes fstream
- alert(data);
- */
-
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var uri = ios.newURI(url, null, null);
- var channel = ios.newChannelFromURI(uri);
-
- channel.QueryInterface(Components.interfaces.nsIUploadChannel);
- channel.setUploadStream(mis, "multipart/form-data", -1);
- channel.QueryInterface(Components.interfaces.nsIHttpChannel);
- channel.requestMethod = 'POST';
- channel.allowPipelining = false;
- channel.setRequestHeader('Keep-Alive', '', false);
- channel.setRequestHeader('Connection', '', false);
- channel.setRequestHeader("Content-Type", "multipart/form-data; boundary=" + boundary, false);
- //channel.setRequestHeader('Date', date, false);
-
- request.setChannel(channel);
+ // And reset consecutive failures
+ this._s3ConsecutiveFailures = 0;
- var deferred = Zotero.Promise.defer();
+ var requestParams = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(requestParams);
+ var headers = {
+ "Content-Type": "application/x-www-form-urlencoded"
+ };
+ if (params.storedHash) {
+ headers["If-Match"] = params.storedHash;
+ }
+ else {
+ headers["If-None-Match"] = "*";
+ }
+ var body = "upload=" + uploadKey;
- var listener = new Zotero.Sync.Storage.StreamListener(
- {
- onProgress: function (a, b, c) {
- request.onProgress(a, b, c);
- },
- onStop: function (httpRequest, status, response, data) {
- data.request.setChannel(false);
-
- // For timeouts and failures from S3, which happen intermittently,
- // wait a little and try again
- let timeoutMessage = "Your socket connection to the server was not read from or "
- + "written to within the timeout period.";
- if (status == 0 || (status == 400 && response.indexOf(timeoutMessage) != -1)) {
- if (_s3ConsecutiveFailures >= _maxS3ConsecutiveFailures) {
- Zotero.debug(_s3ConsecutiveFailures
- + " consecutive S3 failures -- aborting", 1);
- _s3ConsecutiveFailures = 0;
- }
- else {
- let libraryKey = Zotero.Items.getLibraryKeyHash(item);
- let msg = "S3 returned " + status
- + " (" + libraryKey + ") -- retrying upload"
- Components.utils.reportError(msg);
- Zotero.debug(msg, 1);
- Zotero.debug(response, 1);
- if (_s3Backoff < _maxS3Backoff) {
- _s3Backoff *= 2;
- }
- _s3ConsecutiveFailures++;
- Zotero.debug("Delaying " + libraryKey + " upload for "
- + _s3Backoff + " seconds", 2);
- Q.delay(_s3Backoff * 1000)
- .then(function () {
- deferred.resolve(postFile(request, item, url, uploadKey, params));
- });
- return;
- }
+ // Register upload on server
+ try {
+ req = yield this.apiClient.makeRequest(
+ "POST",
+ uri,
+ {
+ body,
+ headers,
+ successCodes: [204],
+ requestObserver: function (xmlhttp) {
+ request.setChannel(xmlhttp.channel);
}
-
- deferred.resolve(onUploadComplete(httpRequest, status, response, data));
- },
- onCancel: function (httpRequest, status, data) {
- onUploadCancel(httpRequest, status, data)
- deferred.resolve(false);
- },
- request: request,
- item: item,
- uploadKey: uploadKey,
- streams: [mis]
- }
- );
- channel.notificationCallbacks = listener;
+ }
+ );
+ }
+ catch (e) {
+ let msg = `Unexpected file registration status ${e.status} (${item.libraryKey})`;
+ Zotero.logError(msg);
+ Zotero.logError(e.xmlhttp.responseText);
+ Zotero.debug(e.xmlhttp.getAllResponseHeaders());
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
- var dispURI = uri.clone();
- if (dispURI.password) {
- dispURI.password = '********';
+ var version = req.getResponseHeader('Last-Modified-Version');
+ if (!version) {
+ throw new Error("Last-Modified-Version not provided");
}
- Zotero.debug("HTTP POST of " + file.leafName + " to " + dispURI.spec);
+ params.version = version;
- channel.asyncOpen(listener, null);
+ yield this._updateItemFileInfo(item, params);
- return deferred.promise;
- }
+ return new Zotero.Sync.Storage.Result({
+ localChanges: true,
+ remoteChanges: true
+ });
+ }),
- function onUploadComplete(httpRequest, status, response, data) {
- return Q.try(function () {
- var request = data.request;
- var item = data.item;
- var uploadKey = data.uploadKey;
-
- Zotero.debug("Upload of attachment " + item.key
- + " finished with status code " + status);
-
- Zotero.debug(response);
+ /**
+ * Update the local attachment item with the mtime and hash of the uploaded file and the
+ * library version returned by the upload request, and save a modified version of the item
+ * to the sync cache
+ */
+ _updateItemFileInfo: Zotero.Promise.coroutine(function* (item, params) {
+ // Mark as in-sync
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
- switch (status) {
- case 201:
- // Decrease backoff delay on successful upload
- if (_s3Backoff > 1) {
- _s3Backoff /= 2;
- }
- // And reset consecutive failures
- _s3ConsecutiveFailures = 0;
- break;
-
- case 500:
- throw new Error("File upload failed. Please try again.");
-
- default:
- var msg = "Unexpected file upload status " + status
- + " in Zotero.Sync.Storage.ZFS.onUploadComplete()"
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- Components.utils.reportError(response);
- throw new Error(Zotero.Sync.Storage.defaultError);
+ // Store file mod time and hash
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, params.mtime);
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, params.md5);
+ // Update sync cache with new file metadata and version from server
+ var json = yield Zotero.Sync.Data.Local.getCacheObject(
+ 'item', item.libraryID, item.key, item.version
+ );
+ if (json) {
+ json.version = params.version;
+ json.data.version = params.version;
+ json.data.mtime = params.mtime;
+ json.data.md5 = params.md5;
+ yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
}
+ // Update item with new version from server
+ yield Zotero.Items.updateVersion([item.id], params.version);
- var uri = getItemURI(item);
- var body = "update=" + uploadKey + "&mtime=" + item.attachmentModificationTime;
-
- // Register upload on server
- return Zotero.HTTP.promise("POST", uri, { body: body, headers: _headers, successCodes: [204] })
- .then(function (req) {
- updateItemFileInfo(item);
- return {
- localChanges: true,
- remoteChanges: true
- };
- })
- .catch(function (e) {
- var msg = "Unexpected file registration status " + e.status
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Zotero.debug(e.xmlhttp.responseText);
- Zotero.debug(e.xmlhttp.getAllResponseHeaders());
- Components.utils.reportError(msg);
- Components.utils.reportError(e.xmlhttp.responseText);
- throw new Error(Zotero.Sync.Storage.defaultError);
- });
+ // TODO: Can filename, contentType, and charset change the attachment item?
});
- }
-
-
- function updateItemFileInfo(item) {
- // Mark as changed locally
- Zotero.DB.beginTransaction();
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
-
- // Store file mod time
- var mtime = item.attachmentModificationTime;
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, mtime, true);
-
- // Store file hash of individual files
- if (Zotero.Attachments.getNumFiles(item) == 1) {
- var hash = item.attachmentHash;
- Zotero.Sync.Storage.setSyncedHash(item.id, hash);
- }
-
- Zotero.DB.commitTransaction();
try {
- if (Zotero.Attachments.getNumFiles(item) > 1) {
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
var file = Zotero.getTempDirectory();
file.append(item.key + '.zip');
- file.remove(false);
+ yield OS.File.remove(file.path);
}
}
catch (e) {
Components.utils.reportError(e);
}
- }
+ }),
- function onUploadCancel(httpRequest, status, data) {
+ _onUploadCancel: Zotero.Promise.coroutine(function* (httpRequest, status, data) {
var request = data.request;
var item = data.item;
Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status);
try {
- if (Zotero.Attachments.getNumFiles(item) > 1) {
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
var file = Zotero.getTempDirectory();
file.append(item.key + '.zip');
file.remove(false);
@@ -631,40 +900,11 @@ Zotero.Sync.Storage.ZFS = (function () {
catch (e) {
Components.utils.reportError(e);
}
- }
-
-
- /**
- * Get the storage URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of file on storage server
- */
- function getItemURI(item) {
- var uri = Zotero.Sync.Storage.ZFS.rootURI;
- // Be sure to mirror parameter changes to getItemInfoURI() below
- uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1';
- return uri;
- }
-
-
- /**
- * Get the storage info URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of file on storage server with info flag
- */
- function getItemInfoURI(item) {
- var uri = Zotero.Sync.Storage.ZFS.rootURI;
- uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1&info=1';
- return uri;
- }
+ }),
- function getUploadFile(item) {
- if (Zotero.Attachments.getNumFiles(item) > 1) {
+ _getUploadFile: Zotero.Promise.coroutine(function* (item) {
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
var file = Zotero.getTempDirectory();
var filename = item.key + '.zip';
file.append(filename);
@@ -673,500 +913,169 @@ Zotero.Sync.Storage.ZFS = (function () {
var file = item.getFile();
}
return file;
- }
-
-
- //
- // Public methods (called via Zotero.Sync.Storage.ZFS)
- //
- var obj = new Zotero.Sync.Storage.Mode;
- obj.name = "ZFS";
-
- Object.defineProperty(obj, "includeUserFiles", {
- get: function () {
- return Zotero.Prefs.get("sync.storage.enabled") && Zotero.Prefs.get("sync.storage.protocol") == 'zotero';
- }
- });
-
- Object.defineProperty(obj, "includeGroupFiles", {
- get: function () {
- return Zotero.Prefs.get("sync.storage.groups.enabled");
- }
- });
-
- obj._verified = true;
-
- Object.defineProperty(obj, "rootURI", {
- get: function () {
- if (!_rootURI) {
- this._init();
- }
- return _rootURI.clone();
- }
- });
-
- Object.defineProperty(obj, "userURI", {
- get: function () {
- if (!_userURI) {
- this._init();
- }
- return _userURI.clone();
- }
- });
-
-
- obj._init = function () {
- _rootURI = false;
- _userURI = false;
-
- var url = ZOTERO_CONFIG.API_URL;
- var username = Zotero.Sync.Server.username;
- var password = Zotero.Sync.Server.password;
-
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var uri = ios.newURI(url, null, null);
- uri.username = encodeURIComponent(username);
- uri.password = encodeURIComponent(password);
- _rootURI = uri;
-
- uri = uri.clone();
- uri.spec += 'users/' + Zotero.Users.getCurrentUserID() + '/';
- _userURI = uri;
- };
+ }),
- obj.clearCachedCredentials = function() {
- _rootURI = _userURI = undefined;
- _cachedCredentials = false;
- };
/**
- * Begin download process for individual file
+ * Get attachment item metadata on storage server
*
- * @param {Zotero.Sync.Storage.Request} [request]
+ * @param {Zotero.Item} item
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise<Object>|false} - Promise for object with 'hash', 'filename', 'mtime',
+ * 'compressed', or false if item not found
*/
- obj._downloadFile = function (request) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- if (!item) {
- throw new Error("Item '" + request.name + "' not found");
- }
+ _getStorageFileInfo: Zotero.Promise.coroutine(function* (item, request) {
+ var funcName = "Zotero.Sync.Storage.ZFS._getStorageFileInfo()";
- var self = this;
+ var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(params);
- // Retrieve file info from server to store locally afterwards
- return getStorageFileInfo(item, request)
- .then(function (info) {
- if (!request.isRunning()) {
- Zotero.debug("Download request '" + request.name
- + "' is no longer running after getting remote file info");
- return false;
- }
-
- if (!info) {
- Zotero.debug("Remote file not found for item " + item.libraryID + "/" + item.key);
- return false;
- }
-
- var syncModTime = info.mtime;
- var syncHash = info.hash;
-
- var file = item.getFile();
- // Skip download if local file exists and matches mod time
- if (file && file.exists()) {
- if (syncModTime == file.lastModifiedTime) {
- Zotero.debug("File mod time matches remote file -- skipping download");
-
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //var updateItem = syncState != 1;
- var updateItem = false;
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return {
- localChanges: true,
- remoteChanges: false
- };
- }
- // If not compressed, check hash, in case only timestamp changed
- else if (!info.compressed && item.attachmentHash == syncHash) {
- Zotero.debug("File hash matches remote file -- skipping download");
-
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //var updateItem = syncState != 1;
- var updateItem = false;
- if (!info.compressed) {
- Zotero.Sync.Storage.setSyncedHash(item.id, syncHash, false);
- }
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return {
- localChanges: true,
- remoteChanges: false
- };
- }
- }
-
- var destFile = Zotero.getTempDirectory();
- if (info.compressed) {
- destFile.append(item.key + '.zip.tmp');
- }
- else {
- destFile.append(item.key + '.tmp');
- }
-
- if (destFile.exists()) {
- try {
- destFile.remove(false);
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, destFile, 'delete');
+ try {
+ let req = yield this.apiClient.makeRequest(
+ "GET",
+ uri,
+ {
+ successCodes: [200, 404],
+ requestObserver: function (xmlhttp) {
+ request.setChannel(xmlhttp.channel);
}
}
-
- // saveURI() below appears not to create empty files for Content-Length: 0,
- // so we create one here just in case
+ );
+ if (req.status == 404) {
+ return new Zotero.Sync.Storage.Result;
+ }
+
+ let info = {};
+ info.hash = req.getResponseHeader('ETag');
+ if (!info.hash) {
+ let msg = `Hash not found in info response in ${funcName} (${item.libraryKey})`;
+ Zotero.debug(msg, 1);
+ Zotero.debug(req.status);
+ Zotero.debug(req.responseText);
+ Components.utils.reportError(msg);
try {
- destFile.create(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
+ Zotero.debug(req.getAllResponseHeaders());
}
catch (e) {
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- var deferred = Zotero.Promise.defer();
-
- var listener = new Zotero.Sync.Storage.StreamListener(
- {
- onStart: function (request, data) {
- if (data.request.isFinished()) {
- Zotero.debug("Download request " + data.request.name
- + " stopped before download started -- closing channel");
- request.cancel(0x804b0002); // NS_BINDING_ABORTED
- deferred.resolve(false);
- }
- },
- onProgress: function (a, b, c) {
- request.onProgress(a, b, c)
- },
- onStop: function (request, status, response, data) {
- data.request.setChannel(false);
-
- if (status != 200) {
- if (status == 404) {
- deferred.resolve(false);
- return;
- }
-
- if (status == 0) {
- if (_s3ConsecutiveFailures >= _maxS3ConsecutiveFailures) {
- Zotero.debug(_s3ConsecutiveFailures
- + " consecutive S3 failures -- aborting", 1);
- _s3ConsecutiveFailures = 0;
- }
- else {
- let libraryKey = Zotero.Items.getLibraryKeyHash(item);
- let msg = "S3 returned " + status
- + " (" + libraryKey + ") -- retrying download"
- Components.utils.reportError(msg);
- Zotero.debug(msg, 1);
- if (_s3Backoff < _maxS3Backoff) {
- _s3Backoff *= 2;
- }
- _s3ConsecutiveFailures++;
- Zotero.debug("Delaying " + libraryKey + " download for "
- + _s3Backoff + " seconds", 2);
- Q.delay(_s3Backoff * 1000)
- .then(function () {
- deferred.resolve(self._downloadFile(data.request));
- });
- return;
- }
- }
-
- var msg = "Unexpected status code " + status
- + " for request " + data.request.name
- + " in Zotero.Sync.Storage.ZFS.downloadFile()";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- // Ignore files not found in S3
- try {
- Zotero.debug(Zotero.File.getContents(destFile, null, 4096), 1);
- }
- catch (e) {
- Zotero.debug(e, 1);
- }
- deferred.reject(Zotero.Sync.Storage.defaultError);
- return;
- }
-
- // Don't try to process if the request has been cancelled
- if (data.request.isFinished()) {
- Zotero.debug("Download request " + data.request.name
- + " is no longer running after file download", 2);
- deferred.resolve(false);
- return;
- }
-
- Zotero.debug("Finished download of " + destFile.path);
-
- try {
- deferred.resolve(Zotero.Sync.Storage.processDownload(data));
- }
- catch (e) {
- deferred.reject(e);
- }
- },
- onCancel: function (request, status, data) {
- Zotero.debug("Request cancelled");
- deferred.resolve(false);
- },
- request: request,
- item: item,
- compressed: info.compressed,
- syncModTime: syncModTime,
- syncHash: syncHash
- }
- );
-
- var uri = getItemURI(item);
-
- // Don't display password in console
- var disp = uri.clone();
- if (disp.password) {
- disp.password = "********";
- }
- Zotero.debug('Saving ' + disp.spec + ' with saveURI()');
- const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
- var wbp = Components
- .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
- .createInstance(nsIWBP);
- wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
- wbp.progressListener = listener;
- Zotero.Utilities.Internal.saveURI(wbp, uri, destFile);
-
- return deferred.promise;
- });
- };
-
-
- obj._uploadFile = function (request) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- if (Zotero.Attachments.getNumFiles(item) > 1) {
- var deferred = Zotero.Promise.defer();
- var created = Zotero.Sync.Storage.createUploadFile(
- request,
- function (data) {
- if (!data) {
- deferred.resolve(false);
- return;
- }
- deferred.resolve(processUploadFile(data));
+ Zotero.debug("Response headers unavailable");
}
- );
- if (!created) {
- return Zotero.Promise.resolve(false);
- }
- return deferred.promise;
- }
- else {
- return processUploadFile({ request: request });
- }
- };
-
-
- /**
- * @return {Promise} A promise for the last sync time
- */
- obj._getLastSyncTime = function (libraryID) {
- var lastSyncURI = this._getLastSyncURI(libraryID);
-
- var self = this;
- return Zotero.Promise.try(function () {
- // Cache the credentials at the root
- return self._cacheCredentials();
- })
- .then(function () {
- return Zotero.HTTP.promise("GET", lastSyncURI,
- { headers: _headers, successCodes: [200, 404], debug: true });
- })
- .then(function (req) {
- // Not yet synced
- if (req.status == 404) {
- Zotero.debug("No last sync time for library " + libraryID);
- return null;
+ let e = Zotero.getString('sync.storage.error.zfs.restart', Zotero.appName);
+ throw new Error(e);
}
+ info.filename = req.getResponseHeader('X-Zotero-Filename');
+ let mtime = req.getResponseHeader('X-Zotero-Modification-Time');
+ info.mtime = parseInt(mtime);
+ info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes';
+ Zotero.debug(info);
- var ts = req.responseText;
- var date = new Date(ts * 1000);
- Zotero.debug("Last successful ZFS sync for library "
- + libraryID + " was " + date);
- return ts;
- })
- .catch(function (e) {
+ return info;
+ }
+ catch (e) {
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status == 401 || e.status == 403) {
- Zotero.debug("Clearing ZFS authentication credentials", 2);
- _cachedCredentials = false;
+ if (e.xmlhttp.status == 0) {
+ var msg = "Request cancelled getting storage file info";
}
-
- return Zotero.Promise.reject(e);
- }
- // TODO: handle browser offline exception
- else {
- throw e;
- }
- });
- };
-
-
- obj._setLastSyncTime = function (libraryID, localLastSyncTime) {
- if (localLastSyncTime) {
- var sql = "REPLACE INTO version VALUES (?, ?)";
- Zotero.DB.query(
- sql, ['storage_zfs_' + libraryID, { int: localLastSyncTime }]
- );
- return;
- }
-
- var lastSyncURI = this._getLastSyncURI(libraryID);
-
- return Zotero.HTTP.promise("POST", lastSyncURI, { headers: _headers, successCodes: [200, 404], debug: true })
- .then(function (req) {
- // Not yet synced
- //
- // TODO: Don't call this at all if no files uploaded
- if (req.status == 404) {
- return;
+ else {
+ var msg = "Unexpected status code " + e.xmlhttp.status
+ + " getting storage file info for item " + item.libraryKey;
}
-
- var ts = req.responseText;
-
- var sql = "REPLACE INTO version VALUES (?, ?)";
- Zotero.DB.query(
- sql, ['storage_zfs_' + libraryID, { int: ts }]
- );
- })
- .catch(function (e) {
- var msg = "Unexpected status code " + e.xmlhttp.status
- + " setting last file sync time";
Zotero.debug(msg, 1);
+ Zotero.debug(e.xmlhttp.responseText);
Components.utils.reportError(msg);
throw new Error(Zotero.Sync.Storage.defaultError);
- });
- };
-
-
- obj._getLastSyncURI = function (libraryID) {
- if (libraryID === Zotero.Libraries.userLibraryID) {
- var lastSyncURI = this.userURI;
- }
- else if (libraryID) {
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var uri = ios.newURI(Zotero.URI.getLibraryURI(libraryID), null, null);
- var path = uri.path;
- // We don't want the user URI, but it already has the right domain
- // and credentials, so just start with that and replace the path
- var lastSyncURI = this.userURI;
- lastSyncURI.path = path + "/";
- }
- else {
- throw new Error("libraryID not specified");
+ }
+
+ throw e;
}
- lastSyncURI.spec += "laststoragesync";
- return lastSyncURI;
- }
+ }),
- obj._cacheCredentials = function () {
- if (_cachedCredentials) {
- Zotero.debug("ZFS credentials are already cached");
- return Zotero.Promise.resolve();
- }
+ /**
+ * Upload the file to the server
+ *
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise}
+ */
+ _processUploadFile: Zotero.Promise.coroutine(function* (request) {
+ /*
+ updateSizeMultiplier(
+ (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
+ );
+ */
- var uri = this.rootURI;
- // TODO: move to root uri
- uri.spec += "?auth=1";
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
- return Zotero.HTTP.promise("GET", uri, { headers: _headers }).
- then(function (req) {
- Zotero.debug("Credentials are cached");
- _cachedCredentials = true;
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status == 401) {
- var msg = "File sync login failed\n\n"
- + "Check your username and password in the Sync "
- + "pane of the Zotero preferences.";
- throw (msg);
- }
-
- var msg = "Unexpected status code " + e.status + " "
- + "caching ZFS credentials";
- Zotero.debug(msg, 1);
- throw (msg);
+
+ /*var info = yield this._getStorageFileInfo(item, request);
+
+ if (request.isFinished()) {
+ Zotero.debug("Upload request '" + request.name
+ + "' is no longer running after getting file info");
+ return false;
+ }
+
+ // Check for conflict
+ if ((yield Zotero.Sync.Storage.Local.getSyncState(item.id))
+ != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
+ if (info) {
+ // Local file time
+ var fmtime = yield item.attachmentModificationTime;
+ // Remote mod time
+ var mtime = info.mtime;
+
+ var useLocal = false;
+ var same = !(yield Zotero.Sync.Storage.checkFileModTime(item, fmtime, mtime));
+
+ // Ignore maxed-out 32-bit ints, from brief problem after switch to 32-bit servers
+ if (!same && mtime == 2147483647) {
+ Zotero.debug("Remote mod time is invalid -- uploading local file version");
+ useLocal = true;
}
- else {
- throw (e);
+
+ if (same) {
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime);
+ yield Zotero.Sync.Storage.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+ return {
+ localChanges: true,
+ remoteChanges: false
+ };
}
- });
- };
-
-
- /**
- * Remove all synced files from the server
- */
- obj._purgeDeletedStorageFiles = function () {
- return Zotero.Promise.try(function () {
- // Cache the credentials at the root
- return this._cacheCredentials();
- }.bind(this))
- then(function () {
- // If we don't have a user id we've never synced and don't need to bother
- if (!Zotero.Users.getCurrentUserID()) {
- return false;
- }
-
- var sql = "SELECT value FROM settings WHERE setting=? AND key=?";
- var values = Zotero.DB.columnQuery(sql, ['storage', 'zfsPurge']);
- if (!values) {
- return false;
- }
-
- // TODO: promisify
-
- Zotero.debug("Unlinking synced files on ZFS");
-
- var uri = this.userURI;
- uri.spec += "removestoragefiles?";
- // Unused
- for each(var value in values) {
- switch (value) {
- case 'user':
- uri.spec += "user=1&";
- break;
-
- case 'group':
- uri.spec += "group=1&";
- break;
-
- default:
- throw "Invalid zfsPurge value '" + value
- + "' in ZFS purgeDeletedStorageFiles()";
+
+ let smtime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (!useLocal && smtime != mtime) {
+ Zotero.debug("Conflict -- last synced file mod time "
+ + "does not match time on storage server"
+ + " (" + smtime + " != " + mtime + ")");
+ return {
+ localChanges: false,
+ remoteChanges: false,
+ conflict: {
+ local: { modTime: fmtime },
+ remote: { modTime: mtime }
+ }
+ };
}
}
- uri.spec = uri.spec.substr(0, uri.spec.length - 1);
-
- return Zotero.HTTP.promise("POST", uri, "")
- .then(function (req) {
- var sql = "DELETE FROM settings WHERE setting=? AND key=?";
- Zotero.DB.query(sql, ['storage', 'zfsPurge']);
+ else {
+ Zotero.debug("Remote file not found for item " + item.libraryKey);
+ }
+ }*/
+
+ var result = yield this._getFileUploadParameters(item);
+ if (result.exists) {
+ yield this._updateItemFileInfo(item, result);
+ return new Zotero.Sync.Storage.Result({
+ localChanges: true,
+ remoteChanges: true
});
- }.bind(this));
- };
-
- return obj;
-}());
+ }
+ else if (result instanceof Zotero.Sync.Storage.Result) {
+ return result;
+ }
+ return this._uploadFile(request, item, result);
+ })
+}
diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js
@@ -1488,43 +1488,6 @@ Zotero.Sync.Server = new function () {
-Zotero.BufferedInputListener = function (callback) {
- this._callback = callback;
-}
-
-Zotero.BufferedInputListener.prototype = {
- binaryInputStream: null,
- size: 0,
- data: '',
-
- onStartRequest: function(request, context) {},
-
- onStopRequest: function(request, context, status) {
- this.binaryInputStream.close();
- delete this.binaryInputStream;
-
- this._callback(this.data);
- },
-
- onDataAvailable: function(request, context, inputStream, offset, count) {
- this.size += count;
-
- this.binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"]
- .createInstance(Components.interfaces.nsIBinaryInputStream)
- this.binaryInputStream.setInputStream(inputStream);
- this.data += this.binaryInputStream.readBytes(this.binaryInputStream.available());
- },
-
- QueryInterface: function (iid) {
- if (iid.equals(Components.interfaces.nsISupports)
- || iid.equals(Components.interfaces.nsIStreamListener)) {
- return this;
- }
- throw Components.results.NS_ERROR_NO_INTERFACE;
- }
-}
-
-
Zotero.Sync.Server.Data = new function() {
var _noMergeTypes = ['search'];
diff --git a/chrome/content/zotero/xpcom/sync/syncAPIClient.js b/chrome/content/zotero/xpcom/sync/syncAPIClient.js
@@ -28,14 +28,15 @@ if (!Zotero.Sync) {
}
Zotero.Sync.APIClient = function (options) {
- this.baseURL = options.baseURL;
- this.apiKey = options.apiKey;
- this.concurrentCaller = options.concurrentCaller;
+ if (!options.baseURL) throw new Error("baseURL not set");
+ if (!options.apiVersion) throw new Error("apiVersion not set");
+ if (!options.apiKey) throw new Error("apiKey not set");
+ if (!options.caller) throw new Error("caller not set");
- if (options.apiVersion == undefined) {
- throw new Error("options.apiVersion not set");
- }
+ this.baseURL = options.baseURL;
this.apiVersion = options.apiVersion;
+ this.apiKey = options.apiKey;
+ this.caller = options.caller;
}
Zotero.Sync.APIClient.prototype = {
@@ -44,7 +45,7 @@ Zotero.Sync.APIClient.prototype = {
getKeyInfo: Zotero.Promise.coroutine(function* () {
var uri = this.baseURL + "keys/" + this.apiKey;
- var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] });
+ var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) {
return false;
}
@@ -63,7 +64,7 @@ Zotero.Sync.APIClient.prototype = {
if (!userID) throw new Error("User ID not provided");
var uri = this.baseURL + "users/" + userID + "/groups?format=versions";
- var xmlhttp = yield this._makeRequest("GET", uri);
+ var xmlhttp = yield this.makeRequest("GET", uri);
return this._parseJSON(xmlhttp.responseText);
}),
@@ -76,7 +77,7 @@ Zotero.Sync.APIClient.prototype = {
if (!groupID) throw new Error("Group ID not provided");
var uri = this.baseURL + "groups/" + groupID;
- var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] });
+ var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) {
return false;
}
@@ -93,7 +94,7 @@ Zotero.Sync.APIClient.prototype = {
if (since) {
params.since = since;
}
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
var options = {
successCodes: [200, 304]
};
@@ -102,7 +103,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": since
};
}
- var xmlhttp = yield this._makeRequest("GET", uri, options);
+ var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) {
return false;
}
@@ -128,8 +129,8 @@ Zotero.Sync.APIClient.prototype = {
libraryTypeID: libraryTypeID,
since: since || 0
};
- var uri = this._buildRequestURI(params);
- var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 409] });
+ var uri = this.buildRequestURI(params);
+ var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 409] });
if (xmlhttp.status == 409) {
Zotero.debug(`'since' value '${since}' is earlier than the beginning of the delete log`);
return false;
@@ -154,7 +155,7 @@ Zotero.Sync.APIClient.prototype = {
* @param {String} libraryType 'user' or 'group'
* @param {Integer} libraryTypeID userID or groupID
* @param {String} objectType 'item', 'collection', 'search'
- * @param {Object} queryParams Query parameters (see _buildRequestURI())
+ * @param {Object} queryParams Query parameters (see buildRequestURI())
* @return {Promise<Object>|FALSE} Object with 'libraryVersion' and 'results'
*/
getVersions: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, queryParams, libraryVersion) {
@@ -176,7 +177,7 @@ Zotero.Sync.APIClient.prototype = {
}
// TODO: Use pagination
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
var options = {
successCodes: [200, 304]
@@ -186,7 +187,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": libraryVersion
};
}
- var xmlhttp = yield this._makeRequest("GET", uri, options);
+ var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) {
return false;
}
@@ -256,10 +257,10 @@ Zotero.Sync.APIClient.prototype = {
if (objectType == 'item') {
params.includeTrashed = 1;
}
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
return [
- this._makeRequest("GET", uri)
+ this.makeRequest("GET", uri)
.then(function (xmlhttp) {
return this._parseJSON(xmlhttp.responseText)
}.bind(this))
@@ -294,9 +295,9 @@ Zotero.Sync.APIClient.prototype = {
libraryType: libraryType,
libraryTypeID: libraryTypeID
};
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
- var xmlhttp = yield this._makeRequest(method, uri, {
+ var xmlhttp = yield this.makeRequest(method, uri, {
headers: {
"If-Unmodified-Since-Version": version
},
@@ -319,7 +320,7 @@ Zotero.Sync.APIClient.prototype = {
}),
- _buildRequestURI: function (params) {
+ buildRequestURI: function (params) {
var uri = this.baseURL;
switch (params.libraryType) {
@@ -332,6 +333,10 @@ Zotero.Sync.APIClient.prototype = {
break;
}
+ if (params.target === undefined) {
+ throw new Error("'target' not provided");
+ }
+
uri += "/" + params.target;
if (params.objectKey) {
@@ -382,30 +387,33 @@ Zotero.Sync.APIClient.prototype = {
},
- _makeRequest: function (method, uri, options) {
- if (!options) {
- options = {};
- }
- if (!options.headers) {
- options.headers = {};
+ getHeaders: function (headers = {}) {
+ headers["Zotero-API-Version"] = this.apiVersion;
+ if (this.apiKey) {
+ headers["Zotero-API-Key"] = this.apiKey;
}
- options.headers["Zotero-API-Version"] = this.apiVersion;
+ return headers;
+ },
+
+
+ makeRequest: function (method, uri, options = {}) {
+ options.headers = this.getHeaders(options.headers);
options.dontCache = true;
options.foreground = !options.background;
options.responseType = options.responseType || 'text';
- if (this.apiKey) {
- options.headers.Authorization = "Bearer " + this.apiKey;
- }
- var self = this;
- return this.concurrentCaller.fcall(function () {
- return Zotero.HTTP.request(method, uri, options)
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- self._checkResponse(e.xmlhttp);
- }
+ return this.caller.start(Zotero.Promise.coroutine(function* () {
+ try {
+ var xmlhttp = yield Zotero.HTTP.request(method, uri, options);
+ this._checkBackoff(xmlhttp);
+ return xmlhttp;
+ }
+ catch (e) {
+ /*if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ this._checkRetry(e.xmlhttp);
+ }*/
throw e;
- });
- });
+ }
+ }.bind(this)));
},
@@ -422,21 +430,6 @@ Zotero.Sync.APIClient.prototype = {
},
- _checkResponse: function (xmlhttp) {
- this._checkBackoff(xmlhttp);
- this._checkAuth(xmlhttp);
- },
-
-
- _checkAuth: function (xmlhttp) {
- if (xmlhttp.status == 403) {
- var e = new Zotero.Error(Zotero.getString('sync.error.invalidLogin'), "INVALID_SYNC_LOGIN");
- e.fatal = true;
- throw e;
- }
- },
-
-
_checkBackoff: function (xmlhttp) {
var backoff = xmlhttp.getResponseHeader("Backoff");
if (backoff) {
@@ -444,7 +437,7 @@ Zotero.Sync.APIClient.prototype = {
if (backoff > 3600) {
// TODO: Update status?
- this.concurrentCaller.pause(backoff * 1000);
+ this.caller.pause(backoff * 1000);
}
}
}
diff --git a/chrome/content/zotero/xpcom/sync/syncEngine.js b/chrome/content/zotero/xpcom/sync/syncEngine.js
@@ -76,7 +76,13 @@ Zotero.Sync.Data.Engine = function (options) {
onError: this.onError
}
- this.syncCachePromise = Zotero.Promise.resolve().bind(this);
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ this.syncCacheProcessor = new ConcurrentCaller({
+ id: "Sync Cache Processor",
+ numConcurrent: 1,
+ logger: Zotero.debug,
+ stopOnError: this.stopOnError
+ });
};
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1;
@@ -167,12 +173,8 @@ Zotero.Sync.Data.Engine.prototype.start = Zotero.Promise.coroutine(function* ()
}
}
- // TEMP: make more reliable
- while (this.syncCachePromise.isPending()) {
- Zotero.debug("Waiting for sync cache to be processed");
- yield this.syncCachePromise;
- yield Zotero.Promise.delay(50);
- }
+ Zotero.debug("Waiting for sync cache to be processed");
+ yield this.syncCacheProcessor.wait();
yield Zotero.Libraries.updateLastSyncTime(this.libraryID);
@@ -286,12 +288,7 @@ Zotero.Sync.Data.Engine.prototype._startDownload = Zotero.Promise.coroutine(func
}
// Wait for sync process to clear
- // TEMP: make more reliable
- while (this.syncCachePromise.isPending()) {
- Zotero.debug("Waiting for sync cache to be processed");
- yield this.syncCachePromise;
- yield Zotero.Promise.delay(50);
- }
+ yield this.syncCacheProcessor.wait();
//
// Get deleted objects
@@ -671,7 +668,8 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
if (state == 'successful') {
// Update local object with saved data if necessary
- yield obj.fromJSON(current.data);
+ yield obj.loadAllData();
+ obj.fromJSON(current.data);
toSave.push(obj);
toCache.push(current);
}
@@ -701,8 +699,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
// Handle failed objects
for (let index in json.results.failed) {
- let e = json.results.failed[index];
- Zotero.logError(e.message);
+ let { code, message } = json.results.failed[index];
+ e = new Error(message);
+ e.name = "ZoteroUploadObjectError";
+ e.code = code;
+ Zotero.logError(e);
// This shouldn't happen, because the upload request includes a library
// version and should prevent an outdated upload before the object version is
@@ -711,12 +712,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
return this.UPLOAD_RESULT_OBJECT_CONFLICT;
}
- if (this.stopOnError) {
- Zotero.debug("WE FAILED!!!");
- throw new Error(e.message);
- }
if (this.onError) {
- this.onError(e.message);
+ this.onError(e);
+ }
+ if (this.stopOnError) {
+ throw new Error(e);
}
batch[index].tries++;
// Mark 400 errors as permanently failed
@@ -990,7 +990,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
this._failedCheck();
let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
- let ObjectType = objectType[0].toUpperCase() + objectType.substr(1);
+ let ObjectType = Zotero.Utilities.capitalize(objectType);
// TODO: localize
this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName);
@@ -1037,8 +1037,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
let cacheVersions = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersions(
this.libraryID, objectType
);
- // Queue objects that are out of date or don't exist locally and aren't up-to-date
- // in the cache
+ // Queue objects that are out of date or don't exist locally
for (let key in results.versions) {
let version = results.versions[key];
let obj = yield objectsClass.getByLibraryAndKeyAsync(this.libraryID, key, {
@@ -1060,12 +1059,12 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
}
if (obj) {
- Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " + obj.libraryKey
+ Zotero.debug(ObjectType + " " + obj.libraryKey
+ " is older than version in sync cache");
}
else {
- Zotero.debug(Zotero.Utilities.capitalize(objectType) + " "
- + this.libraryID + "/" + key + " in sync cache not found locally");
+ Zotero.debug(ObjectType + " " + this.libraryID + "/" + key
+ + " in sync cache not found locally");
}
toDownload.push(key);
@@ -1127,7 +1126,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
break;
}
- yield this.syncCachePromise;
+ yield this.syncCacheProcessor.wait();
yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion);
@@ -1145,20 +1144,19 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
* @param {String} objectType
*/
Zotero.Sync.Data.Engine.prototype._processCache = function (objectType) {
- var self = this;
- this.syncCachePromise = this.syncCachePromise.then(function () {
- self._failedCheck();
+ this.syncCacheProcessor.start(function () {
+ this._failedCheck();
return Zotero.Sync.Data.Local.processSyncCacheForObjectType(
- self.libraryID, objectType, self.options
+ this.libraryID, objectType, this.options
)
.catch(function (e) {
Zotero.logError(e);
- if (self.stopOnError) {
+ if (this.stopOnError) {
Zotero.debug("WE FAILED!!!");
- self.failed = e;
+ this.failed = e;
}
- });
- })
+ }.bind(this));
+ }.bind(this))
}
diff --git a/chrome/content/zotero/xpcom/sync/syncEventListeners.js b/chrome/content/zotero/xpcom/sync/syncEventListeners.js
@@ -39,10 +39,9 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
var syncSQL = "REPLACE INTO syncDeleteLog (syncObjectTypeID, libraryID, key, synced) "
+ "VALUES (?, ?, ?, 0)";
+ var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
- if (type == 'item' && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
- }
+ var storageForLibrary = {};
return Zotero.DB.executeTransaction(function* () {
for (let i = 0; i < ids.length; i++) {
@@ -74,18 +73,25 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
key
]
);
- if (storageSQL && oldItem.itemType == 'attachment' &&
- [
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL
- ].indexOf(oldItem.linkMode) != -1) {
- yield Zotero.DB.queryAsync(
- storageSQL,
- [
- libraryID,
- key
- ]
- );
+
+ if (type == 'item') {
+ if (storageForLibrary[libraryID] === undefined) {
+ storageForLibrary[libraryID] =
+ Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
+ }
+ if (storageForLibrary[libraryID] && oldItem.itemType == 'attachment' &&
+ [
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL
+ ].indexOf(oldItem.linkMode) != -1) {
+ yield Zotero.DB.queryAsync(
+ storageSQL,
+ [
+ libraryID,
+ key
+ ]
+ );
+ }
}
}
});
@@ -215,3 +221,23 @@ Zotero.Sync.EventListeners.progressListener = {
}
};
+
+
+Zotero.Sync.EventListeners.StorageFileOpenListener = {
+ init: function () {
+ Zotero.Notifier.registerObserver(this, ['file'], 'storageFileOpen');
+ },
+
+ notify: function (event, type, ids, extraData) {
+ if (event == 'open' && type == 'file') {
+ let timestamp = new Date().getTime();
+
+ for (let i = 0; i < ids.length; i++) {
+ Zotero.Sync.Storage.Local.uploadCheckFiles.push({
+ itemID: ids[i],
+ timestamp: timestamp
+ });
+ }
+ }
+ }
+}
diff --git a/chrome/content/zotero/xpcom/sync/syncLocal.js b/chrome/content/zotero/xpcom/sync/syncLocal.js
@@ -28,6 +28,8 @@ if (!Zotero.Sync.Data) {
}
Zotero.Sync.Data.Local = {
+ _loginManagerHost: 'https://api.zotero.org',
+ _loginManagerRealm: 'Zotero Web API',
_lastSyncTime: null,
_lastClassicSyncTime: null,
@@ -39,6 +41,71 @@ Zotero.Sync.Data.Local = {
}),
+ getAPIKey: function () {
+ var apiKey = Zotero.Prefs.get('devAPIKey');
+ if (apiKey) {
+ return apiKey;
+ }
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins(
+ {}, this._loginManagerHost, null, this._loginManagerRealm
+ );
+ // Get API from returned array of nsILoginInfo objects
+ if (logins.length) {
+ return logins[0].password;
+ }
+ if (!apiKey) {
+ let username = Zotero.Prefs.get('sync.server.username');
+ if (username) {
+ let password = Zotero.Sync.Data.Local.getLegacyPassword(username);
+ if (!password) {
+ return false;
+ }
+ throw new Error("Unimplemented");
+ // Get API key from server
+
+ // Store API key
+
+ // Remove old logins and username pref
+ }
+ }
+ return apiKey;
+ },
+
+
+ setAPIKey: function (apiKey) {
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
+ Components.interfaces.nsILoginInfo, "init");
+ var loginInfo = new nsLoginInfo(
+ this._loginManagerHost,
+ null,
+ this._loginManagerRealm,
+ 'API Key',
+ apiKey,
+ "",
+ ""
+ );
+ loginManager.addLogin(loginInfo);
+ },
+
+
+ getLegacyPassword: function (username) {
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins({}, "chrome://zotero", "Zotero Storage Server", null);
+ // Find user from returned array of nsILoginInfo objects
+ for (let login of logins) {
+ if (login.username == username) {
+ return login.password;
+ }
+ }
+ return false;
+ },
+
+
getLastSyncTime: function () {
if (_lastSyncTime === null) {
throw new Error("Last sync time not yet loaded");
@@ -86,7 +153,7 @@ Zotero.Sync.Data.Local = {
var sql = "SELECT " + objectsClass.idColumn + " FROM " + objectsClass.table
+ " WHERE libraryID=? AND synced=0";
- // RETRIEVE PARENT DOWN? EVEN POSSIBLE?
+ // TODO: RETRIEVE PARENT DOWN? EVEN POSSIBLE?
// items via parent
// collections via getDescendents?
@@ -154,6 +221,35 @@ Zotero.Sync.Data.Local = {
}),
+ getCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, keyVersionPairs) {
+ if (!keyVersionPairs.length) return [];
+ var sql = "SELECT data FROM syncCache SC JOIN (SELECT "
+ + keyVersionPairs.map(function (pair) {
+ Zotero.DataObjectUtilities.checkKey(pair[0]);
+ return "'" + pair[0] + "' AS key, " + parseInt(pair[1]) + " AS version";
+ }).join(" UNION SELECT ")
+ + ") AS pairs ON (pairs.key=SC.key AND pairs.version=SC.version) "
+ + "WHERE libraryID=? AND "
+ + "syncObjectTypeID IN (SELECT syncObjectTypeID FROM syncObjectTypes WHERE name=?)";
+ var rows = yield Zotero.DB.columnQueryAsync(sql, [libraryID, objectType]);
+ return rows.map(row => JSON.parse(row));
+ }),
+
+
+ saveCacheObject: Zotero.Promise.coroutine(function* (objectType, libraryID, json) {
+ json = this._checkCacheJSON(json);
+
+ Zotero.debug("Saving to sync cache:");
+ Zotero.debug(json);
+
+ var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType);
+ var sql = "INSERT OR REPLACE INTO syncCache "
+ + "(libraryID, key, syncObjectTypeID, version, data) VALUES (?, ?, ?, ?, ?)";
+ var params = [libraryID, json.key, syncObjectTypeID, json.version, JSON.stringify(json)];
+ return Zotero.DB.queryAsync(sql, params);
+ }),
+
+
saveCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, jsonArray) {
if (!Array.isArray(jsonArray)) {
throw new Error("'json' must be an array");
@@ -165,20 +261,7 @@ Zotero.Sync.Data.Local = {
return;
}
- jsonArray = jsonArray.map(o => {
- if (o.key === undefined) {
- throw new Error("Missing 'key' property in JSON");
- }
- if (o.version === undefined) {
- throw new Error("Missing 'version' property in JSON");
- }
- // If direct data object passed, wrap in fake response object
- return o.data === undefined ? {
- key: o.key,
- version: o.version,
- data: o
- } : o;
- });
+ jsonArray = jsonArray.map(json => this._checkCacheJSON(json));
Zotero.debug("Saving to sync cache:");
Zotero.debug(jsonArray);
@@ -206,6 +289,22 @@ Zotero.Sync.Data.Local = {
}),
+ _checkCacheJSON: function (json) {
+ if (json.key === undefined) {
+ throw new Error("Missing 'key' property in JSON");
+ }
+ if (json.version === undefined) {
+ throw new Error("Missing 'version' property in JSON");
+ }
+ // If direct data object passed, wrap in fake response object
+ return json.data === undefined ? {
+ key: json.key,
+ version: json.version,
+ data: json
+ } : json;
+ },
+
+
processSyncCache: Zotero.Promise.coroutine(function* (libraryID, options) {
for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(libraryID)) {
yield this.processSyncCacheForObjectType(libraryID, objectType, options);
@@ -213,8 +312,7 @@ Zotero.Sync.Data.Local = {
}),
- processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options) {
- options = options || {};
+ processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options = {}) {
var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
var ObjectType = Zotero.Utilities.capitalize(objectType);
@@ -227,7 +325,6 @@ Zotero.Sync.Data.Local = {
var numSkipped = 0;
var data = yield this._getUnwrittenData(libraryID, objectType);
-
if (!data.length) {
Zotero.debug("No unwritten " + objectTypePlural + " in sync cache");
return;
@@ -260,9 +357,9 @@ Zotero.Sync.Data.Local = {
for (let i = 0; i < chunk.length; i++) {
let json = chunk[i];
let jsonData = json.data;
- let isNewObject;
let objectKey = json.key;
+ Zotero.debug(`Processing ${objectType} ${libraryID}/${objectKey}`);
Zotero.debug(json);
if (!jsonData) {
@@ -302,26 +399,22 @@ Zotero.Sync.Data.Local = {
}*/
}
+ let isNewObject = false;
+ let skipCache = false;
let obj = yield objectsClass.getByLibraryAndKeyAsync(
libraryID, objectKey, { noCache: true }
);
if (obj) {
Zotero.debug("Matching local " + objectType + " exists", 4);
- isNewObject = false;
- // Local object has not been modified since last sync
- if (obj.synced) {
- // Overwrite local below
- }
- else {
+ // Local object has been modified since last sync
+ if (!obj.synced) {
Zotero.debug("Local " + objectType + " " + obj.libraryKey
+ " has been modified since last sync", 4);
let cachedJSON = yield this.getCacheObject(
objectType, obj.libraryID, obj.key, obj.version
);
- Zotero.debug("GOT CACHED");
- Zotero.debug(cachedJSON);
let jsonDataLocal = yield obj.toJSON();
@@ -333,42 +426,51 @@ Zotero.Sync.Data.Local = {
['dateAdded', 'dateModified']
);
- // If no changes, update local version and keep as unsynced
+ // If no changes, update local version number and mark as synced
if (!result.changes.length && !result.conflicts.length) {
- Zotero.debug("No remote changes to apply to local " + objectType
- + " " + obj.libraryKey);
- yield obj.updateVersion(json.version);
+ Zotero.debug("No remote changes to apply to local "
+ + objectType + " " + obj.libraryKey);
+ obj.version = json.version;
+ obj.synced = true;
+ yield obj.save();
continue;
}
- // If no conflicts, apply remote changes automatically
- if (!result.conflicts.length) {
- Zotero.DataObjectUtilities.applyChanges(
- jsonData, result.changes
- );
- let saved = yield this._saveObjectFromJSON(obj, jsonData, options);
- if (saved) numSaved++;
+ if (result.conflicts.length) {
+ if (objectType != 'item') {
+ throw new Error(`Unexpected conflict on ${objectType} object`);
+ }
+ Zotero.debug("Conflict!");
+ conflicts.push({
+ left: jsonDataLocal,
+ right: jsonData,
+ changes: result.changes,
+ conflicts: result.conflicts
+ });
continue;
}
- if (objectType != 'item') {
- throw new Error(`Unexpected conflict on ${objectType} object`);
- }
-
- conflicts.push({
- left: jsonDataLocal,
- right: jsonData,
- changes: result.changes,
- conflicts: result.conflicts
- });
- continue;
+ // If no conflicts, apply remote changes automatically
+ Zotero.debug(`Applying remote changes to ${objectType} `
+ + obj.libraryKey);
+ Zotero.debug(result.changes);
+ Zotero.DataObjectUtilities.applyChanges(
+ jsonDataLocal, result.changes
+ );
+ // Transfer properties that aren't in the changeset
+ ['version', 'dateAdded', 'dateModified'].forEach(x => {
+ if (jsonDataLocal[x] !== jsonData[x]) {
+ Zotero.debug(`Applying remote '${x}' value`);
+ }
+ jsonDataLocal[x] = jsonData[x];
+ })
+ jsonData = jsonDataLocal;
}
-
- let saved = yield this._saveObjectFromJSON(obj, jsonData, options);
- if (saved) numSaved++;
}
// Object doesn't exist locally
else {
+ Zotero.debug(ObjectType + " doesn't exist locally");
+
isNewObject = true;
// Check if object has been deleted locally
@@ -376,6 +478,8 @@ Zotero.Sync.Data.Local = {
objectType, libraryID, objectKey
);
if (dateDeleted) {
+ Zotero.debug(ObjectType + " was deleted locally");
+
switch (objectType) {
case 'item':
conflicts.push({
@@ -410,24 +514,30 @@ Zotero.Sync.Data.Local = {
obj.key = objectKey;
yield obj.loadPrimaryData();
- let saved = yield this._saveObjectFromJSON(obj, jsonData, options, {
- // Don't cache new items immediately, which skips reloading after save
- skipCache: true
- });
- if (saved) numSaved++;
+ // Don't cache new items immediately, which skips reloading after save
+ skipCache = true;
+ }
+
+ let saved = yield this._saveObjectFromJSON(
+ obj, jsonData, options, { skipCache }
+ );
+ // Mark updated attachments for download
+ if (saved && objectType == 'item' && obj.isImportedAttachment()) {
+ yield this._checkAttachmentForDownload(
+ obj, jsonData.mtime, isNewObject
+ );
+ }
+
+ if (saved) {
+ numSaved++;
}
}
}.bind(this));
}.bind(this)
);
- // Keep retrying if we skipped any, as long as we're still making progress
- if (numSkipped && numSaved != 0) {
- Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
- yield this.processSyncCacheForObjectType(libraryID, objectType, options);
- }
-
if (conflicts.length) {
+ // Sort conflicts by local Date Modified/Deleted
conflicts.sort(function (a, b) {
var d1 = a.left.dateDeleted || a.left.dateModified;
var d2 = b.left.dateDeleted || b.left.dateModified;
@@ -442,6 +552,7 @@ Zotero.Sync.Data.Local = {
var mergeData = this.resolveConflicts(conflicts);
if (mergeData) {
+ Zotero.debug("Processing resolved conflicts");
let mergeOptions = {};
Object.assign(mergeOptions, options);
// Tell _saveObjectFromJSON not to save with 'synced' set to true
@@ -484,11 +595,55 @@ Zotero.Sync.Data.Local = {
}
}
+ // Keep retrying if we skipped any, as long as we're still making progress
+ if (numSkipped && numSaved != 0) {
+ Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
+ return this.processSyncCacheForObjectType(libraryID, objectType, options);
+ }
+
data = yield this._getUnwrittenData(libraryID, objectType);
- Zotero.debug("Skipping " + data.length + " "
- + (data.length == 1 ? objectType : objectTypePlural)
- + " in sync cache");
- return data;
+ if (data.length) {
+ Zotero.debug(`Skipping ${data.length} `
+ + (data.length == 1 ? objectType : objectTypePlural)
+ + " in sync cache");
+ }
+ }),
+
+
+ _checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) {
+ var markToDownload = false;
+ if (!isNewObject) {
+ // Convert previously used Unix timestamps to ms-based timestamps
+ if (mtime < 10000000000) {
+ Zotero.debug("Converting Unix timestamp '" + mtime + "' to ms");
+ mtime = mtime * 1000;
+ }
+ var fmtime = null;
+ try {
+ fmtime = yield item.attachmentModificationTime;
+ }
+ catch (e) {
+ // This will probably fail later too, but ignore it for now
+ Zotero.logError(e);
+ }
+ if (fmtime) {
+ let state = Zotero.Sync.Storage.Local.checkFileModTime(item, fmtime, mtime);
+ if (state !== false) {
+ markToDownload = true;
+ }
+ }
+ else {
+ markToDownload = true;
+ }
+ }
+ else {
+ markToDownload = true;
+ }
+ if (markToDownload) {
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ }
}),
@@ -501,6 +656,8 @@ Zotero.Sync.Data.Local = {
resolveConflicts: function (conflicts) {
+ Zotero.debug("Showing conflict resolution window");
+
var io = {
dataIn: {
captions: [
@@ -511,9 +668,7 @@ Zotero.Sync.Data.Local = {
conflicts
}
};
-
var url = 'chrome://zotero/content/merge.xul';
-
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
@@ -553,7 +708,8 @@ Zotero.Sync.Data.Local = {
_saveObjectFromJSON: Zotero.Promise.coroutine(function* (obj, json, options) {
try {
- yield obj.fromJSON(json);
+ yield obj.loadAllData();
+ obj.fromJSON(json);
if (!options.saveAsChanged) {
obj.version = json.version;
obj.synced = true;
@@ -611,6 +767,11 @@ Zotero.Sync.Data.Local = {
var changeset1 = Zotero.DataObjectUtilities.diff(originalJSON, currentJSON, ignoreFields);
var changeset2 = Zotero.DataObjectUtilities.diff(originalJSON, newJSON, ignoreFields);
+ Zotero.debug("CHANGESET1");
+ Zotero.debug(changeset1);
+ Zotero.debug("CHANGESET2");
+ Zotero.debug(changeset2);
+
var conflicts = [];
for (let i = 0; i < changeset1.length; i++) {
@@ -725,27 +886,43 @@ Zotero.Sync.Data.Local = {
var conflicts = [];
for (let i = 0; i < changeset.length; i++) {
- let c = changeset[i];
+ let c2 = changeset[i];
// Member changes are additive only, so ignore removals
- if (c.op.endsWith('-remove')) {
+ if (c2.op.endsWith('-remove')) {
continue;
}
// Record member changes
- if (c.op.startsWith('member-') || c.op.startsWith('property-member-')) {
- changes.push(c);
+ if (c2.op.startsWith('member-') || c2.op.startsWith('property-member-')) {
+ changes.push(c2);
continue;
}
// Automatically apply remote changes for non-items, even if in conflict
if (objectType != 'item') {
- changes.push(c);
+ changes.push(c2);
continue;
}
// Field changes are conflicts
- conflicts.push(c);
+ //
+ // Since we don't know what changed, use only 'add' and 'delete'
+ if (c2.op == 'modify') {
+ c2.op = 'add';
+ }
+ let val = currentJSON[c2.field];
+ let c1 = {
+ field: c2.field,
+ op: val !== undefined ? 'add' : 'delete'
+ };
+ if (val !== undefined) {
+ c1.value = val;
+ }
+ if (c2.op == 'modify') {
+ c2.op = 'add';
+ }
+ conflicts.push([c1, c2]);
}
return { changes, conflicts };
diff --git a/chrome/content/zotero/xpcom/sync/syncRunner.js b/chrome/content/zotero/xpcom/sync/syncRunner.js
@@ -29,34 +29,62 @@ if (!Zotero.Sync) {
Zotero.Sync = {};
}
-Zotero.Sync.Runner_Module = function () {
+// Initialized as Zotero.Sync.Runner in zotero.js
+Zotero.Sync.Runner_Module = function (options = {}) {
+ const stopOnError = true;
+
Zotero.defineProperty(this, 'background', { get: () => _background });
Zotero.defineProperty(this, 'lastSyncStatus', { get: () => _lastSyncStatus });
- const stopOnError = true;
+ this.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
+ this.apiVersion = options.apiVersion || ZOTERO_CONFIG.API_VERSION;
+ this.apiKey = options.apiKey || Zotero.Sync.Data.Local.getAPIKey();
+
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ this.caller = new ConcurrentCaller(4);
+ this.caller.setLogger(msg => Zotero.debug(msg));
+ this.caller.stopOnError = stopOnError;
+ this.caller.onError = function (e) {
+ this.addError(e);
+ if (e.fatal) {
+ this.caller.stop();
+ throw e;
+ }
+ }.bind(this);
var _autoSyncTimer;
var _background;
var _firstInSession = true;
var _syncInProgress = false;
+ var _syncEngines = [];
+ var _storageEngines = [];
+
var _lastSyncStatus;
var _currentSyncStatusLabel;
var _currentLastSyncLabel;
var _errors = [];
+ this.getAPIClient = function () {
+ return new Zotero.Sync.APIClient({
+ baseURL: this.baseURL,
+ apiVersion: this.apiVersion,
+ apiKey: this.apiKey,
+ caller: this.caller
+ });
+ }
+
+
/**
* Begin a sync session
*
- * @param {Object} [options]
- * @param {String} [apiKey]
- * @param {Boolean} [background=false] - Whether this is a background request, which prevents
- * some alerts from being shown
- * @param {String} [baseURL]
- * @param {Integer[]} [libraries] - IDs of libraries to sync
- * @param {Function} [onError] - Function to pass errors to instead of handling internally
- * (used for testing)
+ * @param {Object} [options]
+ * @param {Boolean} [options.background=false] Whether this is a background request, which
+ * prevents some alerts from being shown
+ * @param {Integer[]} [options.libraries] IDs of libraries to sync
+ * @param {Function} [options.onError] Function to pass errors to instead of
+ * handling internally (used for testing)
*/
this.sync = Zotero.Promise.coroutine(function* (options = {}) {
// Clear message list
@@ -84,14 +112,13 @@ Zotero.Sync.Runner_Module = function () {
// Purge deleted objects so they don't cause sync errors (e.g., long tags)
yield Zotero.purgeDataObjects(true);
- options.apiKey = options.apiKey || Zotero.Prefs.get('devAPIKey');
- if (!options.apiKey) {
- let msg = "API key not provided";
+ if (!this.apiKey) {
+ let msg = "API key not set";
let e = new Zotero.Error(msg, 0, { dialogButtonText: null })
this.updateIcons(e);
+ _syncInProgress = false;
return false;
}
- options.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
if (_firstInSession) {
options.firstInSession = true;
_firstInSession = false;
@@ -102,66 +129,45 @@ Zotero.Sync.Runner_Module = function () {
this.updateIcons('animate');
try {
- Components.utils.import("resource://zotero/concurrent-caller.js");
- var caller = new ConcurrentCaller(4); // TEMP: one for now
- caller.setLogger(msg => Zotero.debug(msg));
- caller.stopOnError = stopOnError;
- caller.onError = function (e) {
- this.addError(e);
- if (e.fatal) {
- caller.stop();
- throw e;
- }
- }.bind(this);
-
- // TODO: Use a single client for all operations?
- var client = new Zotero.Sync.APIClient({
- baseURL: options.baseURL,
- apiVersion: ZOTERO_CONFIG.API_VERSION,
- apiKey: options.apiKey,
- concurrentCaller: caller,
- background: options.background
- });
+ let client = this.getAPIClient();
- var keyInfo = yield this.checkAccess(client, options);
+ let keyInfo = yield this.checkAccess(client, options);
if (!keyInfo) {
- this.stop();
+ this.end();
Zotero.debug("Syncing cancelled");
return false;
}
- var libraries = yield this.checkLibraries(client, options, keyInfo, libraries);
-
- for (let libraryID of libraries) {
- try {
- let engine = new Zotero.Sync.Data.Engine({
- libraryID: libraryID,
- apiClient: client,
- setStatus: this.setSyncStatus.bind(this),
- stopOnError: stopOnError,
- onError: this.addError.bind(this)
- });
- yield engine.start();
- }
- catch (e) {
- Zotero.debug("Sync failed for library " + libraryID);
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- this.checkError(e);
+ let engineOptions = {
+ apiClient: client,
+ caller: this.caller,
+ setStatus: this.setSyncStatus.bind(this),
+ stopOnError,
+ onError: function (e) {
if (options.onError) {
options.onError(e);
}
else {
- this.addError(e);
- }
- if (stopOnError || e.fatal) {
- caller.stop();
- break;
+ this.addError.bind(this);
}
+ }.bind(this),
+ background: _background,
+ firstInSession: _firstInSession
+ };
+
+ let nextLibraries = yield this.checkLibraries(
+ client, options, keyInfo, options.libraries
+ );
+ // Sync data, files, and then any data that needs to be uploaded
+ let attempt = 1;
+ while (nextLibraries.length) {
+ if (attempt > 3) {
+ throw new Error("Too many sync attempts -- stopping");
}
+ nextLibraries = yield _doDataSync(nextLibraries, engineOptions);
+ nextLibraries = yield _doFileSync(nextLibraries, engineOptions);
+ attempt++;
}
-
- yield Zotero.Sync.Data.Local.updateLastSyncTime();
}
catch (e) {
if (options.onError) {
@@ -171,62 +177,19 @@ Zotero.Sync.Runner_Module = function () {
this.addError(e);
}
}
-
- this.stop();
+ finally {
+ this.end();
+ }
Zotero.debug("Done syncing");
- return;
-
- var storageSync = function () {
- Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
-
- Zotero.Sync.Storage.sync(options)
- .then(function (results) {
- Zotero.debug("File sync is finished");
-
- if (results.errors.length) {
- Zotero.debug(results.errors, 1);
- for each(var e in results.errors) {
- Components.utils.reportError(e);
- }
- Zotero.Sync.Runner.setErrors(results.errors);
- return;
- }
-
- if (results.changesMade) {
- Zotero.debug("Changes made during file sync "
- + "-- performing additional data sync");
- Zotero.Sync.Server.sync(finalCallbacks);
- }
- else {
- Zotero.Sync.Runner.stop();
- }
- })
- .catch(function (e) {
- Zotero.debug("File sync failed", 1);
- Zotero.Sync.Runner.error(e);
- })
- .done();
- };
+ /*if (results.changesMade) {
+ Zotero.debug("Changes made during file sync "
+ + "-- performing additional data sync");
+ this.sync(options);
+ }*/
- Zotero.Sync.Server.sync({
- // Sync 1 success
- onSuccess: storageSync,
-
- // Sync 1 skip
- onSkip: storageSync,
-
- // Sync 1 stop
- onStop: function () {
- Zotero.Sync.Runner.stop();
- },
-
- // Sync 1 error
- onError: function (e) {
- Zotero.Sync.Runner.error(e);
- }
- });
+ return;
});
@@ -242,8 +205,9 @@ Zotero.Sync.Runner_Module = function () {
}
// Sanity check
- if (!json.userID) throw new Error("userID not found in response");
- if (!json.username) throw new Error("username not found in response");
+ if (!json.userID) throw new Error("userID not found in key response");
+ if (!json.username) throw new Error("username not found in key response");
+ if (!json.access) throw new Error("'access' not found in key response");
// Make sure user hasn't changed, and prompt to update database if so
if (!(yield this.checkUser(json.userID, json.username))) {
@@ -446,8 +410,6 @@ Zotero.Sync.Runner_Module = function () {
*
* @param {Integer} userID New userID
* @param {Integer} libraryID New libraryID
- * @param {Integer} noServerData The server account is empty — this is
- * the account after a server clear
* @return {Boolean} - True to continue, false to cancel
*/
this.checkUser = Zotero.Promise.coroutine(function* (userID, username) {
@@ -544,7 +506,154 @@ Zotero.Sync.Runner_Module = function () {
});
+ var _doDataSync = Zotero.Promise.coroutine(function* (libraries, options, skipUpdateLastSyncTime) {
+ var successfulLibraries = [];
+ for (let libraryID of libraries) {
+ try {
+ let opts = {};
+ Object.assign(opts, options);
+ opts.libraryID = libraryID;
+
+ let engine = new Zotero.Sync.Data.Engine(opts);
+ yield engine.start();
+ successfulLibraries.push(libraryID);
+ }
+ catch (e) {
+ Zotero.debug("Sync failed for library " + libraryID);
+ Zotero.logError(e);
+ this.checkError(e);
+ if (options.onError) {
+ options.onError(e);
+ }
+ else {
+ this.addError(e);
+ }
+ if (stopOnError || e.fatal) {
+ Zotero.debug("Stopping on error", 1);
+ options.caller.stop();
+ break;
+ }
+ }
+ }
+ // Update last-sync time if any libraries synced
+ // TEMP: Do we want to show updated time if some libraries haven't synced?
+ if (!libraries.length || successfulLibraries.length) {
+ yield Zotero.Sync.Data.Local.updateLastSyncTime();
+ }
+ return successfulLibraries;
+ }.bind(this));
+
+
+ var _doFileSync = Zotero.Promise.coroutine(function* (libraries, options) {
+ Zotero.debug("Starting file syncing");
+ this.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
+ let librariesToSync = [];
+ for (let libraryID of libraries) {
+ try {
+ let opts = {};
+ Object.assign(opts, options);
+ opts.libraryID = libraryID;
+
+ let tries = 3;
+ while (true) {
+ if (tries == 0) {
+ throw new Error("Too many file sync attempts for library " + libraryID);
+ }
+ tries--;
+ let engine = new Zotero.Sync.Storage.Engine(opts);
+ let results = yield engine.start();
+ if (results.syncRequired) {
+ librariesToSync.push(libraryID);
+ }
+ else if (results.fileSyncRequired) {
+ Zotero.debug("Another file sync required -- restarting");
+ continue;
+ }
+ break;
+ }
+ }
+ catch (e) {
+ Zotero.debug("File sync failed for library " + libraryID);
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+ this.checkError(e);
+ if (options.onError) {
+ options.onError(e);
+ }
+ else {
+ this.addError(e);
+ }
+ if (stopOnError || e.fatal) {
+ options.caller.stop();
+ break;
+ }
+ }
+ }
+ Zotero.debug("Done with file syncing");
+ return librariesToSync;
+ }.bind(this));
+
+
+ /**
+ * Download a single file on demand (not within a sync process)
+ */
+ this.downloadFile = Zotero.Promise.coroutine(function* (item, requestCallbacks) {
+ if (Zotero.HTTP.browserIsOffline()) {
+ Zotero.debug("Browser is offline", 2);
+ return false;
+ }
+
+ // TEMP
+ var options = {};
+
+ var itemID = item.id;
+ var modeClass = Zotero.Sync.Storage.Local.getClassForLibrary(item.libraryID);
+ var controller = new modeClass({
+ apiClient: this.getAPIClient()
+ });
+
+ // TODO: verify WebDAV on-demand?
+ if (!controller.verified) {
+ Zotero.debug("File syncing is not active for item's library -- skipping download");
+ return false;
+ }
+
+ if (!item.isImportedAttachment()) {
+ throw new Error("Not an imported attachment");
+ }
+
+ if (yield item.getFilePathAsync()) {
+ Zotero.debug("File already exists -- replacing");
+ }
+
+ // TODO: start sync icon?
+ // TODO: create queue for cancelling
+
+ if (!requestCallbacks) {
+ requestCallbacks = {};
+ }
+ var onStart = function (request) {
+ return controller.downloadFile(request);
+ };
+ var request = new Zotero.Sync.Storage.Request({
+ type: 'download',
+ libraryID: item.libraryID,
+ name: item.libraryKey,
+ onStart: requestCallbacks.onStart
+ ? [onStart, requestCallbacks.onStart]
+ : [onStart]
+ });
+ return request.start();
+ });
+
+
this.stop = function () {
+ _syncEngines.forEach(engine => engine.stop());
+ _storageEngines.forEach(engine => engine.stop());
+ }
+
+
+ this.end = function () {
this.updateIcons(_errors);
_errors = [];
_syncInProgress = false;
@@ -669,7 +778,6 @@ Zotero.Sync.Runner_Module = function () {
if (libraryID) {
e.libraryID = libraryID;
}
- Zotero.logError(e);
_errors.push(this.parseError(e));
}
@@ -1027,7 +1135,8 @@ Zotero.Sync.Runner_Module = function () {
var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
if (!lastSyncTime) {
try {
- lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime()
+ lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime();
+ Zotero.debug(lastSyncTime);
}
catch (e) {
Zotero.debug(e, 2);
@@ -1052,5 +1161,3 @@ Zotero.Sync.Runner_Module = function () {
_currentLastSyncLabel.hidden = false;
}
}
-
-Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;
diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js
@@ -607,6 +607,7 @@ Components.utils.import("resource://gre/modules/osfile.jsm");
yield Zotero.Sync.Data.Local.init();
yield Zotero.Sync.Data.Utilities.init();
Zotero.Sync.EventListeners.init();
+ Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;
Zotero.MIMETypeHandler.init();
yield Zotero.Proxies.init();
@@ -2706,6 +2707,9 @@ Zotero.Browser = new function() {
if(!win) {
var win = Services.ww.activeWindow;
}
+ if (!win) {
+ throw new Error("Parent window not available for hidden browser");
+ }
}
// Create a hidden browser
diff --git a/chrome/content/zotero/zoteroPane.js b/chrome/content/zotero/zoteroPane.js
@@ -1325,6 +1325,7 @@ var ZoteroPane = new function()
else if (item.isAttachment()) {
var attachmentBox = document.getElementById('zotero-attachment-box');
attachmentBox.mode = this.collectionsView.editable ? 'edit' : 'view';
+ yield item.loadNote();
attachmentBox.item = item;
document.getElementById('zotero-item-pane-content').selectedIndex = 3;
@@ -3692,38 +3693,41 @@ var ZoteroPane = new function()
}
}
else {
- if (!item.isImportedAttachment() || !Zotero.Sync.Storage.downloadAsNeeded(item.libraryID)) {
+ if (!item.isImportedAttachment()
+ || !Zotero.Sync.Storage.Local.downloadAsNeeded(item.libraryID)) {
this.showAttachmentNotFoundDialog(itemID, noLocateOnMissing);
return;
}
let downloadedItem = item;
- yield Zotero.Sync.Storage.downloadFile(
- downloadedItem,
- {
- onProgress: function (progress, progressMax) {}
- }
- )
- .then(function () {
- if (!downloadedItem.getFile()) {
- ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing);
- return;
- }
-
- // check if unchanged?
- // maybe not necessary, since we'll get an error if there's an error
-
-
- Zotero.Notifier.trigger('redraw', 'item', []);
- Zotero.debug('downloaded');
- Zotero.debug(downloadedItem.id);
- return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
- })
- .catch(function (e) {
+ try {
+ yield Zotero.Sync.Runner.downloadFile(
+ downloadedItem,
+ {
+ onProgress: function (progress, progressMax) {}
+ }
+ );
+ }
+ catch (e) {
// TODO: show error somewhere else
Zotero.debug(e, 1);
ZoteroPane_Local.syncAlert(e);
- });
+ return;
+ }
+
+ if (!(yield downloadedItem.getFilePathAsync())) {
+ ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing);
+ return;
+ }
+
+ // check if unchanged?
+ // maybe not necessary, since we'll get an error if there's an error
+
+
+ Zotero.Notifier.trigger('redraw', 'item', []);
+ Zotero.debug('downloaded');
+ Zotero.debug(downloadedItem.id);
+ return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
}
}
});
@@ -3962,7 +3966,7 @@ var ZoteroPane = new function()
this.syncAlert = function (e) {
- e = Zotero.Sync.Runner.parseSyncError(e);
+ e = Zotero.Sync.Runner.parseError(e);
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);
diff --git a/chrome/content/zotero/zoteroPane.xul b/chrome/content/zotero/zoteroPane.xul
@@ -195,9 +195,10 @@
</hbox>
<hbox align="center" pack="end">
<hbox id="zotero-tb-sync-progress-box" hidden="true" align="center">
+ <!-- TODO: localize -->
<toolbarbutton id="zotero-tb-sync-storage-cancel"
- tooltiptext="Cancel Storage Sync"
- oncommand="Zotero.Sync.Storage.QueueManager.cancel()"/>
+ tooltiptext="Stop sync"
+ oncommand="Zotero.Sync.Runner.stop()"/>
<progressmeter id="zotero-tb-sync-progress" mode="determined"
value="0" tooltip="zotero-tb-sync-progress-tooltip">
</progressmeter>
diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties
@@ -961,12 +961,12 @@ rtfScan.saveTitle = Select a location in which to save the formatted file
rtfScan.scannedFileSuffix = (Scanned)
-file.accessError.theFile = The file '%S'
-file.accessError.aFile = A file
-file.accessError.cannotBe = cannot be
-file.accessError.created = created
-file.accessError.updated = updated
-file.accessError.deleted = deleted
+file.accessError.theFileCannotBeCreated = The file '%S' cannot be created.
+file.accessError.theFileCannotBeUpdated = The file '%S' cannot be updated.
+file.accessError.theFileCannotBeDeleted = The file '%S' cannot be deleted.
+file.accessError.aFileCannotBeCreated = A file cannot be created.
+file.accessError.aFileCannotBeUpdated = A file cannot be updated.
+file.accessError.aFileCannotBeDeleted = A file cannot be deleted.
file.accessError.message.windows = Check that the file is not currently in use, that its permissions allow write access, and that it has a valid filename.
file.accessError.message.other = Check that the file is not currently in use and that its permissions allow write access.
file.accessError.restart = Restarting your computer or disabling security software may also help.
diff --git a/components/zotero-service.js b/components/zotero-service.js
@@ -107,11 +107,12 @@ const xpcomFilesLocal = [
'sync/syncRunner',
'sync/syncUtilities',
'storage',
+ 'storage/storageEngine',
+ 'storage/storageLocal',
+ 'storage/storageRequest',
+ 'storage/storageResult',
+ 'storage/storageUtilities',
'storage/streamListener',
- 'storage/queueManager',
- 'storage/queue',
- 'storage/request',
- 'storage/mode',
'storage/zfs',
'storage/webdav',
'syncedSettings',
diff --git a/test/resource/httpd.js b/test/resource/httpd.js
@@ -0,0 +1,5356 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * An implementation of an HTTP server both as a loadable script and as an XPCOM
+ * component. See the accompanying README file for user documentation on
+ * httpd.js.
+ */
+
+this.EXPORTED_SYMBOLS = [
+ "HTTP_400",
+ "HTTP_401",
+ "HTTP_402",
+ "HTTP_403",
+ "HTTP_404",
+ "HTTP_405",
+ "HTTP_406",
+ "HTTP_407",
+ "HTTP_408",
+ "HTTP_409",
+ "HTTP_410",
+ "HTTP_411",
+ "HTTP_412",
+ "HTTP_413",
+ "HTTP_414",
+ "HTTP_415",
+ "HTTP_417",
+ "HTTP_500",
+ "HTTP_501",
+ "HTTP_502",
+ "HTTP_503",
+ "HTTP_504",
+ "HTTP_505",
+ "HttpError",
+ "HttpServer",
+];
+
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+const CC = Components.Constructor;
+
+const PR_UINT32_MAX = Math.pow(2, 32) - 1;
+
+/** True if debugging output is enabled, false otherwise. */
+var DEBUG = false; // non-const *only* so tweakable in server tests
+
+/** True if debugging output should be timestamped. */
+var DEBUG_TIMESTAMP = false; // non-const so tweakable in server tests
+
+var gGlobalObject = this;
+
+/**
+ * Asserts that the given condition holds. If it doesn't, the given message is
+ * dumped, a stack trace is printed, and an exception is thrown to attempt to
+ * stop execution (which unfortunately must rely upon the exception not being
+ * accidentally swallowed by the code that uses it).
+ */
+function NS_ASSERT(cond, msg)
+{
+ if (DEBUG && !cond)
+ {
+ dumpn("###!!!");
+ dumpn("###!!! ASSERTION" + (msg ? ": " + msg : "!"));
+ dumpn("###!!! Stack follows:");
+
+ var stack = new Error().stack.split(/\n/);
+ dumpn(stack.map(function(val) { return "###!!! " + val; }).join("\n"));
+
+ throw Cr.NS_ERROR_ABORT;
+ }
+}
+
+/** Constructs an HTTP error object. */
+this.HttpError = function HttpError(code, description)
+{
+ this.code = code;
+ this.description = description;
+}
+HttpError.prototype =
+{
+ toString: function()
+ {
+ return this.code + " " + this.description;
+ }
+};
+
+/**
+ * Errors thrown to trigger specific HTTP server responses.
+ */
+this.HTTP_400 = new HttpError(400, "Bad Request");
+this.HTTP_401 = new HttpError(401, "Unauthorized");
+this.HTTP_402 = new HttpError(402, "Payment Required");
+this.HTTP_403 = new HttpError(403, "Forbidden");
+this.HTTP_404 = new HttpError(404, "Not Found");
+this.HTTP_405 = new HttpError(405, "Method Not Allowed");
+this.HTTP_406 = new HttpError(406, "Not Acceptable");
+this.HTTP_407 = new HttpError(407, "Proxy Authentication Required");
+this.HTTP_408 = new HttpError(408, "Request Timeout");
+this.HTTP_409 = new HttpError(409, "Conflict");
+this.HTTP_410 = new HttpError(410, "Gone");
+this.HTTP_411 = new HttpError(411, "Length Required");
+this.HTTP_412 = new HttpError(412, "Precondition Failed");
+this.HTTP_413 = new HttpError(413, "Request Entity Too Large");
+this.HTTP_414 = new HttpError(414, "Request-URI Too Long");
+this.HTTP_415 = new HttpError(415, "Unsupported Media Type");
+this.HTTP_417 = new HttpError(417, "Expectation Failed");
+
+this.HTTP_500 = new HttpError(500, "Internal Server Error");
+this.HTTP_501 = new HttpError(501, "Not Implemented");
+this.HTTP_502 = new HttpError(502, "Bad Gateway");
+this.HTTP_503 = new HttpError(503, "Service Unavailable");
+this.HTTP_504 = new HttpError(504, "Gateway Timeout");
+this.HTTP_505 = new HttpError(505, "HTTP Version Not Supported");
+
+/** Creates a hash with fields corresponding to the values in arr. */
+function array2obj(arr)
+{
+ var obj = {};
+ for (var i = 0; i < arr.length; i++)
+ obj[arr[i]] = arr[i];
+ return obj;
+}
+
+/** Returns an array of the integers x through y, inclusive. */
+function range(x, y)
+{
+ var arr = [];
+ for (var i = x; i <= y; i++)
+ arr.push(i);
+ return arr;
+}
+
+/** An object (hash) whose fields are the numbers of all HTTP error codes. */
+const HTTP_ERROR_CODES = array2obj(range(400, 417).concat(range(500, 505)));
+
+
+/**
+ * The character used to distinguish hidden files from non-hidden files, a la
+ * the leading dot in Apache. Since that mechanism also hides files from
+ * easy display in LXR, ls output, etc. however, we choose instead to use a
+ * suffix character. If a requested file ends with it, we append another
+ * when getting the file on the server. If it doesn't, we just look up that
+ * file. Therefore, any file whose name ends with exactly one of the character
+ * is "hidden" and available for use by the server.
+ */
+const HIDDEN_CHAR = "^";
+
+/**
+ * The file name suffix indicating the file containing overridden headers for
+ * a requested file.
+ */
+const HEADERS_SUFFIX = HIDDEN_CHAR + "headers" + HIDDEN_CHAR;
+
+/** Type used to denote SJS scripts for CGI-like functionality. */
+const SJS_TYPE = "sjs";
+
+/** Base for relative timestamps produced by dumpn(). */
+var firstStamp = 0;
+
+/** dump(str) with a trailing "\n" -- only outputs if DEBUG. */
+function dumpn(str)
+{
+ if (DEBUG)
+ {
+ var prefix = "HTTPD-INFO | ";
+ if (DEBUG_TIMESTAMP)
+ {
+ if (firstStamp === 0)
+ firstStamp = Date.now();
+
+ var elapsed = Date.now() - firstStamp; // milliseconds
+ var min = Math.floor(elapsed / 60000);
+ var sec = (elapsed % 60000) / 1000;
+
+ if (sec < 10)
+ prefix += min + ":0" + sec.toFixed(3) + " | ";
+ else
+ prefix += min + ":" + sec.toFixed(3) + " | ";
+ }
+
+ dump(prefix + str + "\n");
+ }
+}
+
+/** Dumps the current JS stack if DEBUG. */
+function dumpStack()
+{
+ // peel off the frames for dumpStack() and Error()
+ var stack = new Error().stack.split(/\n/).slice(2);
+ stack.forEach(dumpn);
+}
+
+
+/** The XPCOM thread manager. */
+var gThreadManager = null;
+
+/** The XPCOM prefs service. */
+var gRootPrefBranch = null;
+function getRootPrefBranch()
+{
+ if (!gRootPrefBranch)
+ {
+ gRootPrefBranch = Cc["@mozilla.org/preferences-service;1"]
+ .getService(Ci.nsIPrefBranch);
+ }
+ return gRootPrefBranch;
+}
+
+/**
+ * JavaScript constructors for commonly-used classes; precreating these is a
+ * speedup over doing the same from base principles. See the docs at
+ * http://developer.mozilla.org/en/docs/Components.Constructor for details.
+ */
+const ServerSocket = CC("@mozilla.org/network/server-socket;1",
+ "nsIServerSocket",
+ "init");
+const ScriptableInputStream = CC("@mozilla.org/scriptableinputstream;1",
+ "nsIScriptableInputStream",
+ "init");
+const Pipe = CC("@mozilla.org/pipe;1",
+ "nsIPipe",
+ "init");
+const FileInputStream = CC("@mozilla.org/network/file-input-stream;1",
+ "nsIFileInputStream",
+ "init");
+const ConverterInputStream = CC("@mozilla.org/intl/converter-input-stream;1",
+ "nsIConverterInputStream",
+ "init");
+const WritablePropertyBag = CC("@mozilla.org/hash-property-bag;1",
+ "nsIWritablePropertyBag2");
+const SupportsString = CC("@mozilla.org/supports-string;1",
+ "nsISupportsString");
+
+/* These two are non-const only so a test can overwrite them. */
+var BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+ "nsIBinaryInputStream",
+ "setInputStream");
+var BinaryOutputStream = CC("@mozilla.org/binaryoutputstream;1",
+ "nsIBinaryOutputStream",
+ "setOutputStream");
+
+/**
+ * Returns the RFC 822/1123 representation of a date.
+ *
+ * @param date : Number
+ * the date, in milliseconds from midnight (00:00:00), January 1, 1970 GMT
+ * @returns string
+ * the representation of the given date
+ */
+function toDateString(date)
+{
+ //
+ // rfc1123-date = wkday "," SP date1 SP time SP "GMT"
+ // date1 = 2DIGIT SP month SP 4DIGIT
+ // ; day month year (e.g., 02 Jun 1982)
+ // time = 2DIGIT ":" 2DIGIT ":" 2DIGIT
+ // ; 00:00:00 - 23:59:59
+ // wkday = "Mon" | "Tue" | "Wed"
+ // | "Thu" | "Fri" | "Sat" | "Sun"
+ // month = "Jan" | "Feb" | "Mar" | "Apr"
+ // | "May" | "Jun" | "Jul" | "Aug"
+ // | "Sep" | "Oct" | "Nov" | "Dec"
+ //
+
+ const wkdayStrings = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
+ const monthStrings = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+
+ /**
+ * Processes a date and returns the encoded UTC time as a string according to
+ * the format specified in RFC 2616.
+ *
+ * @param date : Date
+ * the date to process
+ * @returns string
+ * a string of the form "HH:MM:SS", ranging from "00:00:00" to "23:59:59"
+ */
+ function toTime(date)
+ {
+ var hrs = date.getUTCHours();
+ var rv = (hrs < 10) ? "0" + hrs : hrs;
+
+ var mins = date.getUTCMinutes();
+ rv += ":";
+ rv += (mins < 10) ? "0" + mins : mins;
+
+ var secs = date.getUTCSeconds();
+ rv += ":";
+ rv += (secs < 10) ? "0" + secs : secs;
+
+ return rv;
+ }
+
+ /**
+ * Processes a date and returns the encoded UTC date as a string according to
+ * the date1 format specified in RFC 2616.
+ *
+ * @param date : Date
+ * the date to process
+ * @returns string
+ * a string of the form "HH:MM:SS", ranging from "00:00:00" to "23:59:59"
+ */
+ function toDate1(date)
+ {
+ var day = date.getUTCDate();
+ var month = date.getUTCMonth();
+ var year = date.getUTCFullYear();
+
+ var rv = (day < 10) ? "0" + day : day;
+ rv += " " + monthStrings[month];
+ rv += " " + year;
+
+ return rv;
+ }
+
+ date = new Date(date);
+
+ const fmtString = "%wkday%, %date1% %time% GMT";
+ var rv = fmtString.replace("%wkday%", wkdayStrings[date.getUTCDay()]);
+ rv = rv.replace("%time%", toTime(date));
+ return rv.replace("%date1%", toDate1(date));
+}
+
+/**
+ * Prints out a human-readable representation of the object o and its fields,
+ * omitting those whose names begin with "_" if showMembers != true (to ignore
+ * "private" properties exposed via getters/setters).
+ */
+function printObj(o, showMembers)
+{
+ var s = "******************************\n";
+ s += "o = {\n";
+ for (var i in o)
+ {
+ if (typeof(i) != "string" ||
+ (showMembers || (i.length > 0 && i[0] != "_")))
+ s+= " " + i + ": " + o[i] + ",\n";
+ }
+ s += " };\n";
+ s += "******************************";
+ dumpn(s);
+}
+
+/**
+ * Instantiates a new HTTP server.
+ */
+function nsHttpServer()
+{
+ if (!gThreadManager)
+ gThreadManager = Cc["@mozilla.org/thread-manager;1"].getService();
+
+ /** The port on which this server listens. */
+ this._port = undefined;
+
+ /** The socket associated with this. */
+ this._socket = null;
+
+ /** The handler used to process requests to this server. */
+ this._handler = new ServerHandler(this);
+
+ /** Naming information for this server. */
+ this._identity = new ServerIdentity();
+
+ /**
+ * Indicates when the server is to be shut down at the end of the request.
+ */
+ this._doQuit = false;
+
+ /**
+ * True if the socket in this is closed (and closure notifications have been
+ * sent and processed if the socket was ever opened), false otherwise.
+ */
+ this._socketClosed = true;
+
+ /**
+ * Used for tracking existing connections and ensuring that all connections
+ * are properly cleaned up before server shutdown; increases by 1 for every
+ * new incoming connection.
+ */
+ this._connectionGen = 0;
+
+ /**
+ * Hash of all open connections, indexed by connection number at time of
+ * creation.
+ */
+ this._connections = {};
+}
+nsHttpServer.prototype =
+{
+ classID: Components.ID("{54ef6f81-30af-4b1d-ac55-8ba811293e41}"),
+
+ // NSISERVERSOCKETLISTENER
+
+ /**
+ * Processes an incoming request coming in on the given socket and contained
+ * in the given transport.
+ *
+ * @param socket : nsIServerSocket
+ * the socket through which the request was served
+ * @param trans : nsISocketTransport
+ * the transport for the request/response
+ * @see nsIServerSocketListener.onSocketAccepted
+ */
+ onSocketAccepted: function(socket, trans)
+ {
+ dumpn("*** onSocketAccepted(socket=" + socket + ", trans=" + trans + ")");
+
+ dumpn(">>> new connection on " + trans.host + ":" + trans.port);
+
+ const SEGMENT_SIZE = 8192;
+ const SEGMENT_COUNT = 1024;
+ try
+ {
+ var input = trans.openInputStream(0, SEGMENT_SIZE, SEGMENT_COUNT)
+ .QueryInterface(Ci.nsIAsyncInputStream);
+ var output = trans.openOutputStream(0, 0, 0);
+ }
+ catch (e)
+ {
+ dumpn("*** error opening transport streams: " + e);
+ trans.close(Cr.NS_BINDING_ABORTED);
+ return;
+ }
+
+ var connectionNumber = ++this._connectionGen;
+
+ try
+ {
+ var conn = new Connection(input, output, this, socket.port, trans.port,
+ connectionNumber);
+ var reader = new RequestReader(conn);
+
+ // XXX add request timeout functionality here!
+
+ // Note: must use main thread here, or we might get a GC that will cause
+ // threadsafety assertions. We really need to fix XPConnect so that
+ // you can actually do things in multi-threaded JS. :-(
+ input.asyncWait(reader, 0, 0, gThreadManager.mainThread);
+ }
+ catch (e)
+ {
+ // Assume this connection can't be salvaged and bail on it completely;
+ // don't attempt to close it so that we can assert that any connection
+ // being closed is in this._connections.
+ dumpn("*** error in initial request-processing stages: " + e);
+ trans.close(Cr.NS_BINDING_ABORTED);
+ return;
+ }
+
+ this._connections[connectionNumber] = conn;
+ dumpn("*** starting connection " + connectionNumber);
+ },
+
+ /**
+ * Called when the socket associated with this is closed.
+ *
+ * @param socket : nsIServerSocket
+ * the socket being closed
+ * @param status : nsresult
+ * the reason the socket stopped listening (NS_BINDING_ABORTED if the server
+ * was stopped using nsIHttpServer.stop)
+ * @see nsIServerSocketListener.onStopListening
+ */
+ onStopListening: function(socket, status)
+ {
+ dumpn(">>> shutting down server on port " + socket.port);
+ for (var n in this._connections) {
+ if (!this._connections[n]._requestStarted) {
+ this._connections[n].close();
+ }
+ }
+ this._socketClosed = true;
+ if (this._hasOpenConnections()) {
+ dumpn("*** open connections!!!");
+ }
+ if (!this._hasOpenConnections())
+ {
+ dumpn("*** no open connections, notifying async from onStopListening");
+
+ // Notify asynchronously so that any pending teardown in stop() has a
+ // chance to run first.
+ var self = this;
+ var stopEvent =
+ {
+ run: function()
+ {
+ dumpn("*** _notifyStopped async callback");
+ self._notifyStopped();
+ }
+ };
+ gThreadManager.currentThread
+ .dispatch(stopEvent, Ci.nsIThread.DISPATCH_NORMAL);
+ }
+ },
+
+ // NSIHTTPSERVER
+
+ //
+ // see nsIHttpServer.start
+ //
+ start: function(port)
+ {
+ this._start(port, "localhost")
+ },
+
+ _start: function(port, host)
+ {
+ if (this._socket)
+ throw Cr.NS_ERROR_ALREADY_INITIALIZED;
+
+ this._port = port;
+ this._doQuit = this._socketClosed = false;
+
+ this._host = host;
+
+ // The listen queue needs to be long enough to handle
+ // network.http.max-persistent-connections-per-server or
+ // network.http.max-persistent-connections-per-proxy concurrent
+ // connections, plus a safety margin in case some other process is
+ // talking to the server as well.
+ var prefs = getRootPrefBranch();
+ var maxConnections = 5 + Math.max(
+ prefs.getIntPref("network.http.max-persistent-connections-per-server"),
+ prefs.getIntPref("network.http.max-persistent-connections-per-proxy"));
+
+ try
+ {
+ var loopback = true;
+ if (this._host != "127.0.0.1" && this._host != "localhost") {
+ var loopback = false;
+ }
+
+ // When automatically selecting a port, sometimes the chosen port is
+ // "blocked" from clients. We don't want to use these ports because
+ // tests will intermittently fail. So, we simply keep trying to to
+ // get a server socket until a valid port is obtained. We limit
+ // ourselves to finite attempts just so we don't loop forever.
+ var ios = Cc["@mozilla.org/network/io-service;1"]
+ .getService(Ci.nsIIOService);
+ var socket;
+ for (var i = 100; i; i--)
+ {
+ var temp = new ServerSocket(this._port,
+ loopback, // true = localhost, false = everybody
+ maxConnections);
+
+ var allowed = ios.allowPort(temp.port, "http");
+ if (!allowed)
+ {
+ dumpn(">>>Warning: obtained ServerSocket listens on a blocked " +
+ "port: " + temp.port);
+ }
+
+ if (!allowed && this._port == -1)
+ {
+ dumpn(">>>Throwing away ServerSocket with bad port.");
+ temp.close();
+ continue;
+ }
+
+ socket = temp;
+ break;
+ }
+
+ if (!socket) {
+ throw new Error("No socket server available. Are there no available ports?");
+ }
+
+ dumpn(">>> listening on port " + socket.port + ", " + maxConnections +
+ " pending connections");
+ socket.asyncListen(this);
+ this._port = socket.port;
+ this._identity._initialize(socket.port, host, true);
+ this._socket = socket;
+ }
+ catch (e)
+ {
+ dump("\n!!! could not start server on port " + port + ": " + e + "\n\n");
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ }
+ },
+
+ //
+ // see nsIHttpServer.stop
+ //
+ stop: function(callback)
+ {
+ if (!callback)
+ throw Cr.NS_ERROR_NULL_POINTER;
+ if (!this._socket)
+ throw Cr.NS_ERROR_UNEXPECTED;
+
+ this._stopCallback = typeof callback === "function"
+ ? callback
+ : function() { callback.onStopped(); };
+
+ dumpn(">>> stopping listening on port " + this._socket.port);
+ this._socket.close();
+ this._socket = null;
+
+ // We can't have this identity any more, and the port on which we're running
+ // this server now could be meaningless the next time around.
+ this._identity._teardown();
+
+ this._doQuit = false;
+
+ // socket-close notification and pending request completion happen async
+ },
+
+ //
+ // see nsIHttpServer.registerFile
+ //
+ registerFile: function(path, file)
+ {
+ if (file && (!file.exists() || file.isDirectory()))
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ this._handler.registerFile(path, file);
+ },
+
+ //
+ // see nsIHttpServer.registerDirectory
+ //
+ registerDirectory: function(path, directory)
+ {
+ // XXX true path validation!
+ if (path.charAt(0) != "/" ||
+ path.charAt(path.length - 1) != "/" ||
+ (directory &&
+ (!directory.exists() || !directory.isDirectory())))
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ // XXX determine behavior of nonexistent /foo/bar when a /foo/bar/ mapping
+ // exists!
+
+ this._handler.registerDirectory(path, directory);
+ },
+
+ //
+ // see nsIHttpServer.registerPathHandler
+ //
+ registerPathHandler: function(path, handler)
+ {
+ this._handler.registerPathHandler(path, handler);
+ },
+
+ //
+ // see nsIHttpServer.registerPrefixHandler
+ //
+ registerPrefixHandler: function(prefix, handler)
+ {
+ this._handler.registerPrefixHandler(prefix, handler);
+ },
+
+ //
+ // see nsIHttpServer.registerErrorHandler
+ //
+ registerErrorHandler: function(code, handler)
+ {
+ this._handler.registerErrorHandler(code, handler);
+ },
+
+ //
+ // see nsIHttpServer.setIndexHandler
+ //
+ setIndexHandler: function(handler)
+ {
+ this._handler.setIndexHandler(handler);
+ },
+
+ //
+ // see nsIHttpServer.registerContentType
+ //
+ registerContentType: function(ext, type)
+ {
+ this._handler.registerContentType(ext, type);
+ },
+
+ //
+ // see nsIHttpServer.serverIdentity
+ //
+ get identity()
+ {
+ return this._identity;
+ },
+
+ //
+ // see nsIHttpServer.getState
+ //
+ getState: function(path, k)
+ {
+ return this._handler._getState(path, k);
+ },
+
+ //
+ // see nsIHttpServer.setState
+ //
+ setState: function(path, k, v)
+ {
+ return this._handler._setState(path, k, v);
+ },
+
+ //
+ // see nsIHttpServer.getSharedState
+ //
+ getSharedState: function(k)
+ {
+ return this._handler._getSharedState(k);
+ },
+
+ //
+ // see nsIHttpServer.setSharedState
+ //
+ setSharedState: function(k, v)
+ {
+ return this._handler._setSharedState(k, v);
+ },
+
+ //
+ // see nsIHttpServer.getObjectState
+ //
+ getObjectState: function(k)
+ {
+ return this._handler._getObjectState(k);
+ },
+
+ //
+ // see nsIHttpServer.setObjectState
+ //
+ setObjectState: function(k, v)
+ {
+ return this._handler._setObjectState(k, v);
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpServer) ||
+ iid.equals(Ci.nsIServerSocketListener) ||
+ iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // NON-XPCOM PUBLIC API
+
+ /**
+ * Returns true iff this server is not running (and is not in the process of
+ * serving any requests still to be processed when the server was last
+ * stopped after being run).
+ */
+ isStopped: function()
+ {
+ return this._socketClosed && !this._hasOpenConnections();
+ },
+
+ // PRIVATE IMPLEMENTATION
+
+ /** True if this server has any open connections to it, false otherwise. */
+ _hasOpenConnections: function()
+ {
+ //
+ // If we have any open connections, they're tracked as numeric properties on
+ // |this._connections|. The non-standard __count__ property could be used
+ // to check whether there are any properties, but standard-wise, even
+ // looking forward to ES5, there's no less ugly yet still O(1) way to do
+ // this.
+ //
+ for (var n in this._connections)
+ return true;
+ return false;
+ },
+
+ /** Calls the server-stopped callback provided when stop() was called. */
+ _notifyStopped: function()
+ {
+ NS_ASSERT(this._stopCallback !== null, "double-notifying?");
+ NS_ASSERT(!this._hasOpenConnections(), "should be done serving by now");
+
+ //
+ // NB: We have to grab this now, null out the member, *then* call the
+ // callback here, or otherwise the callback could (indirectly) futz with
+ // this._stopCallback by starting and immediately stopping this, at
+ // which point we'd be nulling out a field we no longer have a right to
+ // modify.
+ //
+ var callback = this._stopCallback;
+ this._stopCallback = null;
+ try
+ {
+ callback();
+ }
+ catch (e)
+ {
+ // not throwing because this is specified as being usually (but not
+ // always) asynchronous
+ dump("!!! error running onStopped callback: " + e + "\n");
+ }
+ },
+
+ /**
+ * Notifies this server that the given connection has been closed.
+ *
+ * @param connection : Connection
+ * the connection that was closed
+ */
+ _connectionClosed: function(connection)
+ {
+ NS_ASSERT(connection.number in this._connections,
+ "closing a connection " + this + " that we never added to the " +
+ "set of open connections?");
+ NS_ASSERT(this._connections[connection.number] === connection,
+ "connection number mismatch? " +
+ this._connections[connection.number]);
+ delete this._connections[connection.number];
+
+ // Fire a pending server-stopped notification if it's our responsibility.
+ if (!this._hasOpenConnections() && this._socketClosed)
+ this._notifyStopped();
+ // Bug 508125: Add a GC here else we'll use gigabytes of memory running
+ // mochitests. We can't rely on xpcshell doing an automated GC, as that
+ // would interfere with testing GC stuff...
+ Components.utils.forceGC();
+ },
+
+ /**
+ * Requests that the server be shut down when possible.
+ */
+ _requestQuit: function()
+ {
+ dumpn(">>> requesting a quit");
+ dumpStack();
+ this._doQuit = true;
+ }
+};
+
+this.HttpServer = nsHttpServer;
+
+//
+// RFC 2396 section 3.2.2:
+//
+// host = hostname | IPv4address
+// hostname = *( domainlabel "." ) toplabel [ "." ]
+// domainlabel = alphanum | alphanum *( alphanum | "-" ) alphanum
+// toplabel = alpha | alpha *( alphanum | "-" ) alphanum
+// IPv4address = 1*digit "." 1*digit "." 1*digit "." 1*digit
+//
+
+const HOST_REGEX =
+ new RegExp("^(?:" +
+ // *( domainlabel "." )
+ "(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)*" +
+ // toplabel
+ "[a-z](?:[a-z0-9-]*[a-z0-9])?" +
+ "|" +
+ // IPv4 address
+ "\\d+\\.\\d+\\.\\d+\\.\\d+" +
+ ")$",
+ "i");
+
+
+/**
+ * Represents the identity of a server. An identity consists of a set of
+ * (scheme, host, port) tuples denoted as locations (allowing a single server to
+ * serve multiple sites or to be used behind both HTTP and HTTPS proxies for any
+ * host/port). Any incoming request must be to one of these locations, or it
+ * will be rejected with an HTTP 400 error. One location, denoted as the
+ * primary location, is the location assigned in contexts where a location
+ * cannot otherwise be endogenously derived, such as for HTTP/1.0 requests.
+ *
+ * A single identity may contain at most one location per unique host/port pair;
+ * other than that, no restrictions are placed upon what locations may
+ * constitute an identity.
+ */
+function ServerIdentity()
+{
+ /** The scheme of the primary location. */
+ this._primaryScheme = "http";
+
+ /** The hostname of the primary location. */
+ this._primaryHost = "127.0.0.1"
+
+ /** The port number of the primary location. */
+ this._primaryPort = -1;
+
+ /**
+ * The current port number for the corresponding server, stored so that a new
+ * primary location can always be set if the current one is removed.
+ */
+ this._defaultPort = -1;
+
+ /**
+ * Maps hosts to maps of ports to schemes, e.g. the following would represent
+ * https://example.com:789/ and http://example.org/:
+ *
+ * {
+ * "xexample.com": { 789: "https" },
+ * "xexample.org": { 80: "http" }
+ * }
+ *
+ * Note the "x" prefix on hostnames, which prevents collisions with special
+ * JS names like "prototype".
+ */
+ this._locations = { "xlocalhost": {} };
+}
+ServerIdentity.prototype =
+{
+ // NSIHTTPSERVERIDENTITY
+
+ //
+ // see nsIHttpServerIdentity.primaryScheme
+ //
+ get primaryScheme()
+ {
+ if (this._primaryPort === -1)
+ throw Cr.NS_ERROR_NOT_INITIALIZED;
+ return this._primaryScheme;
+ },
+
+ //
+ // see nsIHttpServerIdentity.primaryHost
+ //
+ get primaryHost()
+ {
+ if (this._primaryPort === -1)
+ throw Cr.NS_ERROR_NOT_INITIALIZED;
+ return this._primaryHost;
+ },
+
+ //
+ // see nsIHttpServerIdentity.primaryPort
+ //
+ get primaryPort()
+ {
+ if (this._primaryPort === -1)
+ throw Cr.NS_ERROR_NOT_INITIALIZED;
+ return this._primaryPort;
+ },
+
+ //
+ // see nsIHttpServerIdentity.add
+ //
+ add: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ var entry = this._locations["x" + host];
+ if (!entry)
+ this._locations["x" + host] = entry = {};
+
+ entry[port] = scheme;
+ },
+
+ //
+ // see nsIHttpServerIdentity.remove
+ //
+ remove: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ var entry = this._locations["x" + host];
+ if (!entry)
+ return false;
+
+ var present = port in entry;
+ delete entry[port];
+
+ if (this._primaryScheme == scheme &&
+ this._primaryHost == host &&
+ this._primaryPort == port &&
+ this._defaultPort !== -1)
+ {
+ // Always keep at least one identity in existence at any time, unless
+ // we're in the process of shutting down (the last condition above).
+ this._primaryPort = -1;
+ this._initialize(this._defaultPort, host, false);
+ }
+
+ return present;
+ },
+
+ //
+ // see nsIHttpServerIdentity.has
+ //
+ has: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ return "x" + host in this._locations &&
+ scheme === this._locations["x" + host][port];
+ },
+
+ //
+ // see nsIHttpServerIdentity.has
+ //
+ getScheme: function(host, port)
+ {
+ this._validate("http", host, port);
+
+ var entry = this._locations["x" + host];
+ if (!entry)
+ return "";
+
+ return entry[port] || "";
+ },
+
+ //
+ // see nsIHttpServerIdentity.setPrimary
+ //
+ setPrimary: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ this.add(scheme, host, port);
+
+ this._primaryScheme = scheme;
+ this._primaryHost = host;
+ this._primaryPort = port;
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpServerIdentity) || iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // PRIVATE IMPLEMENTATION
+
+ /**
+ * Initializes the primary name for the corresponding server, based on the
+ * provided port number.
+ */
+ _initialize: function(port, host, addSecondaryDefault)
+ {
+ this._host = host;
+ if (this._primaryPort !== -1)
+ this.add("http", host, port);
+ else
+ this.setPrimary("http", "localhost", port);
+ this._defaultPort = port;
+
+ // Only add this if we're being called at server startup
+ if (addSecondaryDefault && host != "127.0.0.1")
+ this.add("http", "127.0.0.1", port);
+ },
+
+ /**
+ * Called at server shutdown time, unsets the primary location only if it was
+ * the default-assigned location and removes the default location from the
+ * set of locations used.
+ */
+ _teardown: function()
+ {
+ if (this._host != "127.0.0.1") {
+ // Not the default primary location, nothing special to do here
+ this.remove("http", "127.0.0.1", this._defaultPort);
+ }
+
+ // This is a *very* tricky bit of reasoning here; make absolutely sure the
+ // tests for this code pass before you commit changes to it.
+ if (this._primaryScheme == "http" &&
+ this._primaryHost == this._host &&
+ this._primaryPort == this._defaultPort)
+ {
+ // Make sure we don't trigger the readding logic in .remove(), then remove
+ // the default location.
+ var port = this._defaultPort;
+ this._defaultPort = -1;
+ this.remove("http", this._host, port);
+
+ // Ensure a server start triggers the setPrimary() path in ._initialize()
+ this._primaryPort = -1;
+ }
+ else
+ {
+ // No reason not to remove directly as it's not our primary location
+ this.remove("http", this._host, this._defaultPort);
+ }
+ },
+
+ /**
+ * Ensures scheme, host, and port are all valid with respect to RFC 2396.
+ *
+ * @throws NS_ERROR_ILLEGAL_VALUE
+ * if any argument doesn't match the corresponding production
+ */
+ _validate: function(scheme, host, port)
+ {
+ if (scheme !== "http" && scheme !== "https")
+ {
+ dumpn("*** server only supports http/https schemes: '" + scheme + "'");
+ dumpStack();
+ throw Cr.NS_ERROR_ILLEGAL_VALUE;
+ }
+ if (!HOST_REGEX.test(host))
+ {
+ dumpn("*** unexpected host: '" + host + "'");
+ throw Cr.NS_ERROR_ILLEGAL_VALUE;
+ }
+ if (port < 0 || port > 65535)
+ {
+ dumpn("*** unexpected port: '" + port + "'");
+ throw Cr.NS_ERROR_ILLEGAL_VALUE;
+ }
+ }
+};
+
+
+/**
+ * Represents a connection to the server (and possibly in the future the thread
+ * on which the connection is processed).
+ *
+ * @param input : nsIInputStream
+ * stream from which incoming data on the connection is read
+ * @param output : nsIOutputStream
+ * stream to write data out the connection
+ * @param server : nsHttpServer
+ * the server handling the connection
+ * @param port : int
+ * the port on which the server is running
+ * @param outgoingPort : int
+ * the outgoing port used by this connection
+ * @param number : uint
+ * a serial number used to uniquely identify this connection
+ */
+function Connection(input, output, server, port, outgoingPort, number)
+{
+ dumpn("*** opening new connection " + number + " on port " + outgoingPort);
+
+ /** Stream of incoming data. */
+ this.input = input;
+
+ /** Stream for outgoing data. */
+ this.output = output;
+
+ /** The server associated with this request. */
+ this.server = server;
+
+ /** The port on which the server is running. */
+ this.port = port;
+
+ /** The outgoing poort used by this connection. */
+ this._outgoingPort = outgoingPort;
+
+ /** The serial number of this connection. */
+ this.number = number;
+
+ /**
+ * The request for which a response is being generated, null if the
+ * incoming request has not been fully received or if it had errors.
+ */
+ this.request = null;
+
+ /** This allows a connection to disambiguate between a peer initiating a
+ * close and the socket being forced closed on shutdown.
+ */
+ this._closed = false;
+
+ /** State variable for debugging. */
+ this._processed = false;
+
+ /** whether or not 1st line of request has been received */
+ this._requestStarted = false;
+}
+Connection.prototype =
+{
+ /** Closes this connection's input/output streams. */
+ close: function()
+ {
+ if (this._closed)
+ return;
+
+ dumpn("*** closing connection " + this.number +
+ " on port " + this._outgoingPort);
+
+ this.input.close();
+ this.output.close();
+ this._closed = true;
+
+ var server = this.server;
+ server._connectionClosed(this);
+
+ // If an error triggered a server shutdown, act on it now
+ if (server._doQuit)
+ server.stop(function() { /* not like we can do anything better */ });
+ },
+
+ /**
+ * Initiates processing of this connection, using the data in the given
+ * request.
+ *
+ * @param request : Request
+ * the request which should be processed
+ */
+ process: function(request)
+ {
+ NS_ASSERT(!this._closed && !this._processed);
+
+ this._processed = true;
+
+ this.request = request;
+ this.server._handler.handleResponse(this);
+ },
+
+ /**
+ * Initiates processing of this connection, generating a response with the
+ * given HTTP error code.
+ *
+ * @param code : uint
+ * an HTTP code, so in the range [0, 1000)
+ * @param request : Request
+ * incomplete data about the incoming request (since there were errors
+ * during its processing
+ */
+ processError: function(code, request)
+ {
+ NS_ASSERT(!this._closed && !this._processed);
+
+ this._processed = true;
+ this.request = request;
+ this.server._handler.handleError(code, this);
+ },
+
+ /** Converts this to a string for debugging purposes. */
+ toString: function()
+ {
+ return "<Connection(" + this.number +
+ (this.request ? ", " + this.request.path : "") +"): " +
+ (this._closed ? "closed" : "open") + ">";
+ },
+
+ requestStarted: function()
+ {
+ this._requestStarted = true;
+ }
+};
+
+
+
+/** Returns an array of count bytes from the given input stream. */
+function readBytes(inputStream, count)
+{
+ return new BinaryInputStream(inputStream).readByteArray(count);
+}
+
+
+
+/** Request reader processing states; see RequestReader for details. */
+const READER_IN_REQUEST_LINE = 0;
+const READER_IN_HEADERS = 1;
+const READER_IN_BODY = 2;
+const READER_FINISHED = 3;
+
+
+/**
+ * Reads incoming request data asynchronously, does any necessary preprocessing,
+ * and forwards it to the request handler. Processing occurs in three states:
+ *
+ * READER_IN_REQUEST_LINE Reading the request's status line
+ * READER_IN_HEADERS Reading headers in the request
+ * READER_IN_BODY Reading the body of the request
+ * READER_FINISHED Entire request has been read and processed
+ *
+ * During the first two stages, initial metadata about the request is gathered
+ * into a Request object. Once the status line and headers have been processed,
+ * we start processing the body of the request into the Request. Finally, when
+ * the entire body has been read, we create a Response and hand it off to the
+ * ServerHandler to be given to the appropriate request handler.
+ *
+ * @param connection : Connection
+ * the connection for the request being read
+ */
+function RequestReader(connection)
+{
+ /** Connection metadata for this request. */
+ this._connection = connection;
+
+ /**
+ * A container providing line-by-line access to the raw bytes that make up the
+ * data which has been read from the connection but has not yet been acted
+ * upon (by passing it to the request handler or by extracting request
+ * metadata from it).
+ */
+ this._data = new LineData();
+
+ /**
+ * The amount of data remaining to be read from the body of this request.
+ * After all headers in the request have been read this is the value in the
+ * Content-Length header, but as the body is read its value decreases to zero.
+ */
+ this._contentLength = 0;
+
+ /** The current state of parsing the incoming request. */
+ this._state = READER_IN_REQUEST_LINE;
+
+ /** Metadata constructed from the incoming request for the request handler. */
+ this._metadata = new Request(connection.port);
+
+ /**
+ * Used to preserve state if we run out of line data midway through a
+ * multi-line header. _lastHeaderName stores the name of the header, while
+ * _lastHeaderValue stores the value we've seen so far for the header.
+ *
+ * These fields are always either both undefined or both strings.
+ */
+ this._lastHeaderName = this._lastHeaderValue = undefined;
+}
+RequestReader.prototype =
+{
+ // NSIINPUTSTREAMCALLBACK
+
+ /**
+ * Called when more data from the incoming request is available. This method
+ * then reads the available data from input and deals with that data as
+ * necessary, depending upon the syntax of already-downloaded data.
+ *
+ * @param input : nsIAsyncInputStream
+ * the stream of incoming data from the connection
+ */
+ onInputStreamReady: function(input)
+ {
+ dumpn("*** onInputStreamReady(input=" + input + ") on thread " +
+ gThreadManager.currentThread + " (main is " +
+ gThreadManager.mainThread + ")");
+ dumpn("*** this._state == " + this._state);
+
+ // Handle cases where we get more data after a request error has been
+ // discovered but *before* we can close the connection.
+ var data = this._data;
+ if (!data)
+ return;
+
+ try
+ {
+ data.appendBytes(readBytes(input, input.available()));
+ }
+ catch (e)
+ {
+ if (streamClosed(e))
+ {
+ dumpn("*** WARNING: unexpected error when reading from socket; will " +
+ "be treated as if the input stream had been closed");
+ dumpn("*** WARNING: actual error was: " + e);
+ }
+
+ // We've lost a race -- input has been closed, but we're still expecting
+ // to read more data. available() will throw in this case, and since
+ // we're dead in the water now, destroy the connection.
+ dumpn("*** onInputStreamReady called on a closed input, destroying " +
+ "connection");
+ this._connection.close();
+ return;
+ }
+
+ switch (this._state)
+ {
+ default:
+ NS_ASSERT(false, "invalid state: " + this._state);
+ break;
+
+ case READER_IN_REQUEST_LINE:
+ if (!this._processRequestLine())
+ break;
+ /* fall through */
+
+ case READER_IN_HEADERS:
+ if (!this._processHeaders())
+ break;
+ /* fall through */
+
+ case READER_IN_BODY:
+ this._processBody();
+ }
+
+ if (this._state != READER_FINISHED)
+ input.asyncWait(this, 0, 0, gThreadManager.currentThread);
+ },
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(aIID)
+ {
+ if (aIID.equals(Ci.nsIInputStreamCallback) ||
+ aIID.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // PRIVATE API
+
+ /**
+ * Processes unprocessed, downloaded data as a request line.
+ *
+ * @returns boolean
+ * true iff the request line has been fully processed
+ */
+ _processRequestLine: function()
+ {
+ NS_ASSERT(this._state == READER_IN_REQUEST_LINE);
+
+ // Servers SHOULD ignore any empty line(s) received where a Request-Line
+ // is expected (section 4.1).
+ var data = this._data;
+ var line = {};
+ var readSuccess;
+ while ((readSuccess = data.readLine(line)) && line.value == "")
+ dumpn("*** ignoring beginning blank line...");
+
+ // if we don't have a full line, wait until we do
+ if (!readSuccess)
+ return false;
+
+ // we have the first non-blank line
+ try
+ {
+ this._parseRequestLine(line.value);
+ this._state = READER_IN_HEADERS;
+ this._connection.requestStarted();
+ return true;
+ }
+ catch (e)
+ {
+ this._handleError(e);
+ return false;
+ }
+ },
+
+ /**
+ * Processes stored data, assuming it is either at the beginning or in
+ * the middle of processing request headers.
+ *
+ * @returns boolean
+ * true iff header data in the request has been fully processed
+ */
+ _processHeaders: function()
+ {
+ NS_ASSERT(this._state == READER_IN_HEADERS);
+
+ // XXX things to fix here:
+ //
+ // - need to support RFC 2047-encoded non-US-ASCII characters
+
+ try
+ {
+ var done = this._parseHeaders();
+ if (done)
+ {
+ var request = this._metadata;
+
+ // XXX this is wrong for requests with transfer-encodings applied to
+ // them, particularly chunked (which by its nature can have no
+ // meaningful Content-Length header)!
+ this._contentLength = request.hasHeader("Content-Length")
+ ? parseInt(request.getHeader("Content-Length"), 10)
+ : 0;
+ dumpn("_processHeaders, Content-length=" + this._contentLength);
+
+ this._state = READER_IN_BODY;
+ }
+ return done;
+ }
+ catch (e)
+ {
+ this._handleError(e);
+ return false;
+ }
+ },
+
+ /**
+ * Processes stored data, assuming it is either at the beginning or in
+ * the middle of processing the request body.
+ *
+ * @returns boolean
+ * true iff the request body has been fully processed
+ */
+ _processBody: function()
+ {
+ NS_ASSERT(this._state == READER_IN_BODY);
+
+ // XXX handle chunked transfer-coding request bodies!
+
+ try
+ {
+ if (this._contentLength > 0)
+ {
+ var data = this._data.purge();
+ var count = Math.min(data.length, this._contentLength);
+ dumpn("*** loading data=" + data + " len=" + data.length +
+ " excess=" + (data.length - count));
+
+ var bos = new BinaryOutputStream(this._metadata._bodyOutputStream);
+ bos.writeByteArray(data, count);
+ this._contentLength -= count;
+ }
+
+ dumpn("*** remaining body data len=" + this._contentLength);
+ if (this._contentLength == 0)
+ {
+ this._validateRequest();
+ this._state = READER_FINISHED;
+ this._handleResponse();
+ return true;
+ }
+
+ return false;
+ }
+ catch (e)
+ {
+ this._handleError(e);
+ return false;
+ }
+ },
+
+ /**
+ * Does various post-header checks on the data in this request.
+ *
+ * @throws : HttpError
+ * if the request was malformed in some way
+ */
+ _validateRequest: function()
+ {
+ NS_ASSERT(this._state == READER_IN_BODY);
+
+ dumpn("*** _validateRequest");
+
+ var metadata = this._metadata;
+ var headers = metadata._headers;
+
+ // 19.6.1.1 -- servers MUST report 400 to HTTP/1.1 requests w/o Host header
+ var identity = this._connection.server.identity;
+ if (metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1))
+ {
+ if (!headers.hasHeader("Host"))
+ {
+ dumpn("*** malformed HTTP/1.1 or greater request with no Host header!");
+ throw HTTP_400;
+ }
+
+ // If the Request-URI wasn't absolute, then we need to determine our host.
+ // We have to determine what scheme was used to access us based on the
+ // server identity data at this point, because the request just doesn't
+ // contain enough data on its own to do this, sadly.
+ if (!metadata._host)
+ {
+ var host, port;
+ var hostPort = headers.getHeader("Host");
+ var colon = hostPort.indexOf(":");
+ if (colon < 0)
+ {
+ host = hostPort;
+ port = "";
+ }
+ else
+ {
+ host = hostPort.substring(0, colon);
+ port = hostPort.substring(colon + 1);
+ }
+
+ // NB: We allow an empty port here because, oddly, a colon may be
+ // present even without a port number, e.g. "example.com:"; in this
+ // case the default port applies.
+ if (!HOST_REGEX.test(host) || !/^\d*$/.test(port))
+ {
+ dumpn("*** malformed hostname (" + hostPort + ") in Host " +
+ "header, 400 time");
+ throw HTTP_400;
+ }
+
+ // If we're not given a port, we're stuck, because we don't know what
+ // scheme to use to look up the correct port here, in general. Since
+ // the HTTPS case requires a tunnel/proxy and thus requires that the
+ // requested URI be absolute (and thus contain the necessary
+ // information), let's assume HTTP will prevail and use that.
+ port = +port || 80;
+
+ var scheme = identity.getScheme(host, port);
+ if (!scheme)
+ {
+ dumpn("*** unrecognized hostname (" + hostPort + ") in Host " +
+ "header, 400 time");
+ throw HTTP_400;
+ }
+
+ metadata._scheme = scheme;
+ metadata._host = host;
+ metadata._port = port;
+ }
+ }
+ else
+ {
+ NS_ASSERT(metadata._host === undefined,
+ "HTTP/1.0 doesn't allow absolute paths in the request line!");
+
+ metadata._scheme = identity.primaryScheme;
+ metadata._host = identity.primaryHost;
+ metadata._port = identity.primaryPort;
+ }
+
+ NS_ASSERT(identity.has(metadata._scheme, metadata._host, metadata._port),
+ "must have a location we recognize by now!");
+ },
+
+ /**
+ * Handles responses in case of error, either in the server or in the request.
+ *
+ * @param e
+ * the specific error encountered, which is an HttpError in the case where
+ * the request is in some way invalid or cannot be fulfilled; if this isn't
+ * an HttpError we're going to be paranoid and shut down, because that
+ * shouldn't happen, ever
+ */
+ _handleError: function(e)
+ {
+ // Don't fall back into normal processing!
+ this._state = READER_FINISHED;
+
+ var server = this._connection.server;
+ if (e instanceof HttpError)
+ {
+ var code = e.code;
+ }
+ else
+ {
+ dumpn("!!! UNEXPECTED ERROR: " + e +
+ (e.lineNumber ? ", line " + e.lineNumber : ""));
+
+ // no idea what happened -- be paranoid and shut down
+ code = 500;
+ server._requestQuit();
+ }
+
+ // make attempted reuse of data an error
+ this._data = null;
+
+ this._connection.processError(code, this._metadata);
+ },
+
+ /**
+ * Now that we've read the request line and headers, we can actually hand off
+ * the request to be handled.
+ *
+ * This method is called once per request, after the request line and all
+ * headers and the body, if any, have been received.
+ */
+ _handleResponse: function()
+ {
+ NS_ASSERT(this._state == READER_FINISHED);
+
+ // We don't need the line-based data any more, so make attempted reuse an
+ // error.
+ this._data = null;
+
+ this._connection.process(this._metadata);
+ },
+
+
+ // PARSING
+
+ /**
+ * Parses the request line for the HTTP request associated with this.
+ *
+ * @param line : string
+ * the request line
+ */
+ _parseRequestLine: function(line)
+ {
+ NS_ASSERT(this._state == READER_IN_REQUEST_LINE);
+
+ dumpn("*** _parseRequestLine('" + line + "')");
+
+ var metadata = this._metadata;
+
+ // clients and servers SHOULD accept any amount of SP or HT characters
+ // between fields, even though only a single SP is required (section 19.3)
+ var request = line.split(/[ \t]+/);
+ if (!request || request.length != 3)
+ {
+ dumpn("*** No request in line");
+ throw HTTP_400;
+ }
+
+ metadata._method = request[0];
+
+ // get the HTTP version
+ var ver = request[2];
+ var match = ver.match(/^HTTP\/(\d+\.\d+)$/);
+ if (!match)
+ {
+ dumpn("*** No HTTP version in line");
+ throw HTTP_400;
+ }
+
+ // determine HTTP version
+ try
+ {
+ metadata._httpVersion = new nsHttpVersion(match[1]);
+ if (!metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_0))
+ throw "unsupported HTTP version";
+ }
+ catch (e)
+ {
+ // we support HTTP/1.0 and HTTP/1.1 only
+ throw HTTP_501;
+ }
+
+
+ var fullPath = request[1];
+ var serverIdentity = this._connection.server.identity;
+
+ var scheme, host, port;
+
+ if (fullPath.charAt(0) != "/")
+ {
+ // No absolute paths in the request line in HTTP prior to 1.1
+ if (!metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1))
+ {
+ dumpn("*** Metadata version too low");
+ throw HTTP_400;
+ }
+
+ try
+ {
+ var uri = Cc["@mozilla.org/network/io-service;1"]
+ .getService(Ci.nsIIOService)
+ .newURI(fullPath, null, null);
+ fullPath = uri.path;
+ scheme = uri.scheme;
+ host = metadata._host = uri.asciiHost;
+ port = uri.port;
+ if (port === -1)
+ {
+ if (scheme === "http")
+ {
+ port = 80;
+ }
+ else if (scheme === "https")
+ {
+ port = 443;
+ }
+ else
+ {
+ dumpn("*** Unknown scheme: " + scheme);
+ throw HTTP_400;
+ }
+ }
+ }
+ catch (e)
+ {
+ // If the host is not a valid host on the server, the response MUST be a
+ // 400 (Bad Request) error message (section 5.2). Alternately, the URI
+ // is malformed.
+ dumpn("*** Threw when dealing with URI: " + e);
+ throw HTTP_400;
+ }
+
+ if (!serverIdentity.has(scheme, host, port) || fullPath.charAt(0) != "/")
+ {
+ dumpn("*** serverIdentity unknown or path does not start with '/'");
+ throw HTTP_400;
+ }
+ }
+
+ var splitter = fullPath.indexOf("?");
+ if (splitter < 0)
+ {
+ // _queryString already set in ctor
+ metadata._path = fullPath;
+ }
+ else
+ {
+ metadata._path = fullPath.substring(0, splitter);
+ metadata._queryString = fullPath.substring(splitter + 1);
+ }
+
+ metadata._scheme = scheme;
+ metadata._host = host;
+ metadata._port = port;
+ },
+
+ /**
+ * Parses all available HTTP headers in this until the header-ending CRLFCRLF,
+ * adding them to the store of headers in the request.
+ *
+ * @throws
+ * HTTP_400 if the headers are malformed
+ * @returns boolean
+ * true if all headers have now been processed, false otherwise
+ */
+ _parseHeaders: function()
+ {
+ NS_ASSERT(this._state == READER_IN_HEADERS);
+
+ dumpn("*** _parseHeaders");
+
+ var data = this._data;
+
+ var headers = this._metadata._headers;
+ var lastName = this._lastHeaderName;
+ var lastVal = this._lastHeaderValue;
+
+ var line = {};
+ while (true)
+ {
+ dumpn("*** Last name: '" + lastName + "'");
+ dumpn("*** Last val: '" + lastVal + "'");
+ NS_ASSERT(!((lastVal === undefined) ^ (lastName === undefined)),
+ lastName === undefined ?
+ "lastVal without lastName? lastVal: '" + lastVal + "'" :
+ "lastName without lastVal? lastName: '" + lastName + "'");
+
+ if (!data.readLine(line))
+ {
+ // save any data we have from the header we might still be processing
+ this._lastHeaderName = lastName;
+ this._lastHeaderValue = lastVal;
+ return false;
+ }
+
+ var lineText = line.value;
+ dumpn("*** Line text: '" + lineText + "'");
+ var firstChar = lineText.charAt(0);
+
+ // blank line means end of headers
+ if (lineText == "")
+ {
+ // we're finished with the previous header
+ if (lastName)
+ {
+ try
+ {
+ headers.setHeader(lastName, lastVal, true);
+ }
+ catch (e)
+ {
+ dumpn("*** setHeader threw on last header, e == " + e);
+ throw HTTP_400;
+ }
+ }
+ else
+ {
+ // no headers in request -- valid for HTTP/1.0 requests
+ }
+
+ // either way, we're done processing headers
+ this._state = READER_IN_BODY;
+ return true;
+ }
+ else if (firstChar == " " || firstChar == "\t")
+ {
+ // multi-line header if we've already seen a header line
+ if (!lastName)
+ {
+ dumpn("We don't have a header to continue!");
+ throw HTTP_400;
+ }
+
+ // append this line's text to the value; starts with SP/HT, so no need
+ // for separating whitespace
+ lastVal += lineText;
+ }
+ else
+ {
+ // we have a new header, so set the old one (if one existed)
+ if (lastName)
+ {
+ try
+ {
+ headers.setHeader(lastName, lastVal, true);
+ }
+ catch (e)
+ {
+ dumpn("*** setHeader threw on a header, e == " + e);
+ throw HTTP_400;
+ }
+ }
+
+ var colon = lineText.indexOf(":"); // first colon must be splitter
+ if (colon < 1)
+ {
+ dumpn("*** No colon or missing header field-name");
+ throw HTTP_400;
+ }
+
+ // set header name, value (to be set in the next loop, usually)
+ lastName = lineText.substring(0, colon);
+ lastVal = lineText.substring(colon + 1);
+ } // empty, continuation, start of header
+ } // while (true)
+ }
+};
+
+
+/** The character codes for CR and LF. */
+const CR = 0x0D, LF = 0x0A;
+
+/**
+ * Calculates the number of characters before the first CRLF pair in array, or
+ * -1 if the array contains no CRLF pair.
+ *
+ * @param array : Array
+ * an array of numbers in the range [0, 256), each representing a single
+ * character; the first CRLF is the lowest index i where
+ * |array[i] == "\r".charCodeAt(0)| and |array[i+1] == "\n".charCodeAt(0)|,
+ * if such an |i| exists, and -1 otherwise
+ * @param start : uint
+ * start index from which to begin searching in array
+ * @returns int
+ * the index of the first CRLF if any were present, -1 otherwise
+ */
+function findCRLF(array, start)
+{
+ for (var i = array.indexOf(CR, start); i >= 0; i = array.indexOf(CR, i + 1))
+ {
+ if (array[i + 1] == LF)
+ return i;
+ }
+ return -1;
+}
+
+
+/**
+ * A container which provides line-by-line access to the arrays of bytes with
+ * which it is seeded.
+ */
+function LineData()
+{
+ /** An array of queued bytes from which to get line-based characters. */
+ this._data = [];
+
+ /** Start index from which to search for CRLF. */
+ this._start = 0;
+}
+LineData.prototype =
+{
+ /**
+ * Appends the bytes in the given array to the internal data cache maintained
+ * by this.
+ */
+ appendBytes: function(bytes)
+ {
+ var count = bytes.length;
+ var quantum = 262144; // just above half SpiderMonkey's argument-count limit
+ if (count < quantum)
+ {
+ Array.prototype.push.apply(this._data, bytes);
+ return;
+ }
+
+ // Large numbers of bytes may cause Array.prototype.push to be called with
+ // more arguments than the JavaScript engine supports. In that case append
+ // bytes in fixed-size amounts until all bytes are appended.
+ for (var start = 0; start < count; start += quantum)
+ {
+ var slice = bytes.slice(start, Math.min(start + quantum, count));
+ Array.prototype.push.apply(this._data, slice);
+ }
+ },
+
+ /**
+ * Removes and returns a line of data, delimited by CRLF, from this.
+ *
+ * @param out
+ * an object whose "value" property will be set to the first line of text
+ * present in this, sans CRLF, if this contains a full CRLF-delimited line
+ * of text; if this doesn't contain enough data, the value of the property
+ * is undefined
+ * @returns boolean
+ * true if a full line of data could be read from the data in this, false
+ * otherwise
+ */
+ readLine: function(out)
+ {
+ var data = this._data;
+ var length = findCRLF(data, this._start);
+ if (length < 0)
+ {
+ this._start = data.length;
+
+ // But if our data ends in a CR, we have to back up one, because
+ // the first byte in the next packet might be an LF and if we
+ // start looking at data.length we won't find it.
+ if (data.length > 0 && data[data.length - 1] === CR)
+ --this._start;
+
+ return false;
+ }
+
+ // Reset for future lines.
+ this._start = 0;
+
+ //
+ // We have the index of the CR, so remove all the characters, including
+ // CRLF, from the array with splice, and convert the removed array
+ // (excluding the trailing CRLF characters) into the corresponding string.
+ //
+ var leading = data.splice(0, length + 2);
+ var quantum = 262144;
+ var line = "";
+ for (var start = 0; start < length; start += quantum)
+ {
+ var slice = leading.slice(start, Math.min(start + quantum, length));
+ line += String.fromCharCode.apply(null, slice);
+ }
+
+ out.value = line;
+ return true;
+ },
+
+ /**
+ * Removes the bytes currently within this and returns them in an array.
+ *
+ * @returns Array
+ * the bytes within this when this method is called
+ */
+ purge: function()
+ {
+ var data = this._data;
+ this._data = [];
+ return data;
+ }
+};
+
+
+
+/**
+ * Creates a request-handling function for an nsIHttpRequestHandler object.
+ */
+function createHandlerFunc(handler)
+{
+ return function(metadata, response) { handler.handle(metadata, response); };
+}
+
+
+/**
+ * The default handler for directories; writes an HTML response containing a
+ * slightly-formatted directory listing.
+ */
+function defaultIndexHandler(metadata, response)
+{
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var path = htmlEscape(decodeURI(metadata.path));
+
+ //
+ // Just do a very basic bit of directory listings -- no need for too much
+ // fanciness, especially since we don't have a style sheet in which we can
+ // stick rules (don't want to pollute the default path-space).
+ //
+
+ var body = '<html>\
+ <head>\
+ <title>' + path + '</title>\
+ </head>\
+ <body>\
+ <h1>' + path + '</h1>\
+ <ol style="list-style-type: none">';
+
+ var directory = metadata.getProperty("directory");
+ NS_ASSERT(directory && directory.isDirectory());
+
+ var fileList = [];
+ var files = directory.directoryEntries;
+ while (files.hasMoreElements())
+ {
+ var f = files.getNext().QueryInterface(Ci.nsIFile);
+ var name = f.leafName;
+ if (!f.isHidden() &&
+ (name.charAt(name.length - 1) != HIDDEN_CHAR ||
+ name.charAt(name.length - 2) == HIDDEN_CHAR))
+ fileList.push(f);
+ }
+
+ fileList.sort(fileSort);
+
+ for (var i = 0; i < fileList.length; i++)
+ {
+ var file = fileList[i];
+ try
+ {
+ var name = file.leafName;
+ if (name.charAt(name.length - 1) == HIDDEN_CHAR)
+ name = name.substring(0, name.length - 1);
+ var sep = file.isDirectory() ? "/" : "";
+
+ // Note: using " to delimit the attribute here because encodeURIComponent
+ // passes through '.
+ var item = '<li><a href="' + encodeURIComponent(name) + sep + '">' +
+ htmlEscape(name) + sep +
+ '</a></li>';
+
+ body += item;
+ }
+ catch (e) { /* some file system error, ignore the file */ }
+ }
+
+ body += ' </ol>\
+ </body>\
+ </html>';
+
+ response.bodyOutputStream.write(body, body.length);
+}
+
+/**
+ * Sorts a and b (nsIFile objects) into an aesthetically pleasing order.
+ */
+function fileSort(a, b)
+{
+ var dira = a.isDirectory(), dirb = b.isDirectory();
+
+ if (dira && !dirb)
+ return -1;
+ if (dirb && !dira)
+ return 1;
+
+ var namea = a.leafName.toLowerCase(), nameb = b.leafName.toLowerCase();
+ return nameb > namea ? -1 : 1;
+}
+
+
+/**
+ * Converts an externally-provided path into an internal path for use in
+ * determining file mappings.
+ *
+ * @param path
+ * the path to convert
+ * @param encoded
+ * true if the given path should be passed through decodeURI prior to
+ * conversion
+ * @throws URIError
+ * if path is incorrectly encoded
+ */
+function toInternalPath(path, encoded)
+{
+ if (encoded)
+ path = decodeURI(path);
+
+ var comps = path.split("/");
+ for (var i = 0, sz = comps.length; i < sz; i++)
+ {
+ var comp = comps[i];
+ if (comp.charAt(comp.length - 1) == HIDDEN_CHAR)
+ comps[i] = comp + HIDDEN_CHAR;
+ }
+ return comps.join("/");
+}
+
+const PERMS_READONLY = (4 << 6) | (4 << 3) | 4;
+
+/**
+ * Adds custom-specified headers for the given file to the given response, if
+ * any such headers are specified.
+ *
+ * @param file
+ * the file on the disk which is to be written
+ * @param metadata
+ * metadata about the incoming request
+ * @param response
+ * the Response to which any specified headers/data should be written
+ * @throws HTTP_500
+ * if an error occurred while processing custom-specified headers
+ */
+function maybeAddHeaders(file, metadata, response)
+{
+ var name = file.leafName;
+ if (name.charAt(name.length - 1) == HIDDEN_CHAR)
+ name = name.substring(0, name.length - 1);
+
+ var headerFile = file.parent;
+ headerFile.append(name + HEADERS_SUFFIX);
+
+ if (!headerFile.exists())
+ return;
+
+ const PR_RDONLY = 0x01;
+ var fis = new FileInputStream(headerFile, PR_RDONLY, PERMS_READONLY,
+ Ci.nsIFileInputStream.CLOSE_ON_EOF);
+
+ try
+ {
+ var lis = new ConverterInputStream(fis, "UTF-8", 1024, 0x0);
+ lis.QueryInterface(Ci.nsIUnicharLineInputStream);
+
+ var line = {value: ""};
+ var more = lis.readLine(line);
+
+ if (!more && line.value == "")
+ return;
+
+
+ // request line
+
+ var status = line.value;
+ if (status.indexOf("HTTP ") == 0)
+ {
+ status = status.substring(5);
+ var space = status.indexOf(" ");
+ var code, description;
+ if (space < 0)
+ {
+ code = status;
+ description = "";
+ }
+ else
+ {
+ code = status.substring(0, space);
+ description = status.substring(space + 1, status.length);
+ }
+
+ response.setStatusLine(metadata.httpVersion, parseInt(code, 10), description);
+
+ line.value = "";
+ more = lis.readLine(line);
+ }
+
+ // headers
+ while (more || line.value != "")
+ {
+ var header = line.value;
+ var colon = header.indexOf(":");
+
+ response.setHeader(header.substring(0, colon),
+ header.substring(colon + 1, header.length),
+ false); // allow overriding server-set headers
+
+ line.value = "";
+ more = lis.readLine(line);
+ }
+ }
+ catch (e)
+ {
+ dumpn("WARNING: error in headers for " + metadata.path + ": " + e);
+ throw HTTP_500;
+ }
+ finally
+ {
+ fis.close();
+ }
+}
+
+
+/**
+ * An object which handles requests for a server, executing default and
+ * overridden behaviors as instructed by the code which uses and manipulates it.
+ * Default behavior includes the paths / and /trace (diagnostics), with some
+ * support for HTTP error pages for various codes and fallback to HTTP 500 if
+ * those codes fail for any reason.
+ *
+ * @param server : nsHttpServer
+ * the server in which this handler is being used
+ */
+function ServerHandler(server)
+{
+ // FIELDS
+
+ /**
+ * The nsHttpServer instance associated with this handler.
+ */
+ this._server = server;
+
+ /**
+ * A FileMap object containing the set of path->nsILocalFile mappings for
+ * all directory mappings set in the server (e.g., "/" for /var/www/html/,
+ * "/foo/bar/" for /local/path/, and "/foo/bar/baz/" for /local/path2).
+ *
+ * Note carefully: the leading and trailing "/" in each path (not file) are
+ * removed before insertion to simplify the code which uses this. You have
+ * been warned!
+ */
+ this._pathDirectoryMap = new FileMap();
+
+ /**
+ * Custom request handlers for the server in which this resides. Path-handler
+ * pairs are stored as property-value pairs in this property.
+ *
+ * @see ServerHandler.prototype._defaultPaths
+ */
+ this._overridePaths = {};
+
+ /**
+ * Custom request handlers for the path prefixes on the server in which this
+ * resides. Path-handler pairs are stored as property-value pairs in this
+ * property.
+ *
+ * @see ServerHandler.prototype._defaultPaths
+ */
+ this._overridePrefixes = {};
+
+ /**
+ * Custom request handlers for the error handlers in the server in which this
+ * resides. Path-handler pairs are stored as property-value pairs in this
+ * property.
+ *
+ * @see ServerHandler.prototype._defaultErrors
+ */
+ this._overrideErrors = {};
+
+ /**
+ * Maps file extensions to their MIME types in the server, overriding any
+ * mapping that might or might not exist in the MIME service.
+ */
+ this._mimeMappings = {};
+
+ /**
+ * The default handler for requests for directories, used to serve directories
+ * when no index file is present.
+ */
+ this._indexHandler = defaultIndexHandler;
+
+ /** Per-path state storage for the server. */
+ this._state = {};
+
+ /** Entire-server state storage. */
+ this._sharedState = {};
+
+ /** Entire-server state storage for nsISupports values. */
+ this._objectState = {};
+}
+ServerHandler.prototype =
+{
+ // PUBLIC API
+
+ /**
+ * Handles a request to this server, responding to the request appropriately
+ * and initiating server shutdown if necessary.
+ *
+ * This method never throws an exception.
+ *
+ * @param connection : Connection
+ * the connection for this request
+ */
+ handleResponse: function(connection)
+ {
+ var request = connection.request;
+ var response = new Response(connection);
+
+ var path = request.path;
+ dumpn("*** path == " + path);
+
+ try
+ {
+ try
+ {
+ if (path in this._overridePaths)
+ {
+ // explicit paths first, then files based on existing directory mappings,
+ // then (if the file doesn't exist) built-in server default paths
+ dumpn("calling override for " + path);
+ this._overridePaths[path](request, response);
+ }
+ else
+ {
+ var longestPrefix = "";
+ for (let prefix in this._overridePrefixes) {
+ if (prefix.length > longestPrefix.length &&
+ path.substr(0, prefix.length) == prefix)
+ {
+ longestPrefix = prefix;
+ }
+ }
+ if (longestPrefix.length > 0)
+ {
+ dumpn("calling prefix override for " + longestPrefix);
+ this._overridePrefixes[longestPrefix](request, response);
+ }
+ else
+ {
+ this._handleDefault(request, response);
+ }
+ }
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort(e);
+ return;
+ }
+
+ if (!(e instanceof HttpError))
+ {
+ dumpn("*** unexpected error: e == " + e);
+ throw HTTP_500;
+ }
+ if (e.code !== 404)
+ throw e;
+
+ dumpn("*** default: " + (path in this._defaultPaths));
+
+ response = new Response(connection);
+ if (path in this._defaultPaths)
+ this._defaultPaths[path](request, response);
+ else
+ throw HTTP_404;
+ }
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort(e);
+ return;
+ }
+
+ var errorCode = "internal";
+
+ try
+ {
+ if (!(e instanceof HttpError))
+ throw e;
+
+ errorCode = e.code;
+ dumpn("*** errorCode == " + errorCode);
+
+ response = new Response(connection);
+ if (e.customErrorHandling)
+ e.customErrorHandling(response);
+ this._handleError(errorCode, request, response);
+ return;
+ }
+ catch (e2)
+ {
+ dumpn("*** error handling " + errorCode + " error: " +
+ "e2 == " + e2 + ", shutting down server");
+
+ connection.server._requestQuit();
+ response.abort(e2);
+ return;
+ }
+ }
+
+ response.complete();
+ },
+
+ //
+ // see nsIHttpServer.registerFile
+ //
+ registerFile: function(path, file)
+ {
+ if (!file)
+ {
+ dumpn("*** unregistering '" + path + "' mapping");
+ delete this._overridePaths[path];
+ return;
+ }
+
+ dumpn("*** registering '" + path + "' as mapping to " + file.path);
+ file = file.clone();
+
+ var self = this;
+ this._overridePaths[path] =
+ function(request, response)
+ {
+ if (!file.exists())
+ throw HTTP_404;
+
+ response.setStatusLine(request.httpVersion, 200, "OK");
+ self._writeFileResponse(request, file, response, 0, file.fileSize);
+ };
+ },
+
+ //
+ // see nsIHttpServer.registerPathHandler
+ //
+ registerPathHandler: function(path, handler)
+ {
+ // XXX true path validation!
+ if (path.charAt(0) != "/")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ this._handlerToField(handler, this._overridePaths, path);
+ },
+
+ //
+ // see nsIHttpServer.registerPrefixHandler
+ //
+ registerPrefixHandler: function(path, handler)
+ {
+ // XXX true path validation!
+ if (path.charAt(0) != "/" || path.charAt(path.length - 1) != "/")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ this._handlerToField(handler, this._overridePrefixes, path);
+ },
+
+ //
+ // see nsIHttpServer.registerDirectory
+ //
+ registerDirectory: function(path, directory)
+ {
+ // strip off leading and trailing '/' so that we can use lastIndexOf when
+ // determining exactly how a path maps onto a mapped directory --
+ // conditional is required here to deal with "/".substring(1, 0) being
+ // converted to "/".substring(0, 1) per the JS specification
+ var key = path.length == 1 ? "" : path.substring(1, path.length - 1);
+
+ // the path-to-directory mapping code requires that the first character not
+ // be "/", or it will go into an infinite loop
+ if (key.charAt(0) == "/")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ key = toInternalPath(key, false);
+
+ if (directory)
+ {
+ dumpn("*** mapping '" + path + "' to the location " + directory.path);
+ this._pathDirectoryMap.put(key, directory);
+ }
+ else
+ {
+ dumpn("*** removing mapping for '" + path + "'");
+ this._pathDirectoryMap.put(key, null);
+ }
+ },
+
+ //
+ // see nsIHttpServer.registerErrorHandler
+ //
+ registerErrorHandler: function(err, handler)
+ {
+ if (!(err in HTTP_ERROR_CODES))
+ dumpn("*** WARNING: registering non-HTTP/1.1 error code " +
+ "(" + err + ") handler -- was this intentional?");
+
+ this._handlerToField(handler, this._overrideErrors, err);
+ },
+
+ //
+ // see nsIHttpServer.setIndexHandler
+ //
+ setIndexHandler: function(handler)
+ {
+ if (!handler)
+ handler = defaultIndexHandler;
+ else if (typeof(handler) != "function")
+ handler = createHandlerFunc(handler);
+
+ this._indexHandler = handler;
+ },
+
+ //
+ // see nsIHttpServer.registerContentType
+ //
+ registerContentType: function(ext, type)
+ {
+ if (!type)
+ delete this._mimeMappings[ext];
+ else
+ this._mimeMappings[ext] = headerUtils.normalizeFieldValue(type);
+ },
+
+ // PRIVATE API
+
+ /**
+ * Sets or remove (if handler is null) a handler in an object with a key.
+ *
+ * @param handler
+ * a handler, either function or an nsIHttpRequestHandler
+ * @param dict
+ * The object to attach the handler to.
+ * @param key
+ * The field name of the handler.
+ */
+ _handlerToField: function(handler, dict, key)
+ {
+ // for convenience, handler can be a function if this is run from xpcshell
+ if (typeof(handler) == "function")
+ dict[key] = handler;
+ else if (handler)
+ dict[key] = createHandlerFunc(handler);
+ else
+ delete dict[key];
+ },
+
+ /**
+ * Handles a request which maps to a file in the local filesystem (if a base
+ * path has already been set; otherwise the 404 error is thrown).
+ *
+ * @param metadata : Request
+ * metadata for the incoming request
+ * @param response : Response
+ * an uninitialized Response to the given request, to be initialized by a
+ * request handler
+ * @throws HTTP_###
+ * if an HTTP error occurred (usually HTTP_404); note that in this case the
+ * calling code must handle post-processing of the response
+ */
+ _handleDefault: function(metadata, response)
+ {
+ dumpn("*** _handleDefault()");
+
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+
+ var path = metadata.path;
+ NS_ASSERT(path.charAt(0) == "/", "invalid path: <" + path + ">");
+
+ // determine the actual on-disk file; this requires finding the deepest
+ // path-to-directory mapping in the requested URL
+ var file = this._getFileForPath(path);
+
+ // the "file" might be a directory, in which case we either serve the
+ // contained index.html or make the index handler write the response
+ if (file.exists() && file.isDirectory())
+ {
+ file.append("index.html"); // make configurable?
+ if (!file.exists() || file.isDirectory())
+ {
+ metadata._ensurePropertyBag();
+ metadata._bag.setPropertyAsInterface("directory", file.parent);
+ this._indexHandler(metadata, response);
+ return;
+ }
+ }
+
+ // alternately, the file might not exist
+ if (!file.exists())
+ throw HTTP_404;
+
+ var start, end;
+ if (metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1) &&
+ metadata.hasHeader("Range") &&
+ this._getTypeFromFile(file) !== SJS_TYPE)
+ {
+ var rangeMatch = metadata.getHeader("Range").match(/^bytes=(\d+)?-(\d+)?$/);
+ if (!rangeMatch)
+ {
+ dumpn("*** Range header bogosity: '" + metadata.getHeader("Range") + "'");
+ throw HTTP_400;
+ }
+
+ if (rangeMatch[1] !== undefined)
+ start = parseInt(rangeMatch[1], 10);
+
+ if (rangeMatch[2] !== undefined)
+ end = parseInt(rangeMatch[2], 10);
+
+ if (start === undefined && end === undefined)
+ {
+ dumpn("*** More Range header bogosity: '" + metadata.getHeader("Range") + "'");
+ throw HTTP_400;
+ }
+
+ // No start given, so the end is really the count of bytes from the
+ // end of the file.
+ if (start === undefined)
+ {
+ start = Math.max(0, file.fileSize - end);
+ end = file.fileSize - 1;
+ }
+
+ // start and end are inclusive
+ if (end === undefined || end >= file.fileSize)
+ end = file.fileSize - 1;
+
+ if (start !== undefined && start >= file.fileSize) {
+ var HTTP_416 = new HttpError(416, "Requested Range Not Satisfiable");
+ HTTP_416.customErrorHandling = function(errorResponse)
+ {
+ maybeAddHeaders(file, metadata, errorResponse);
+ };
+ throw HTTP_416;
+ }
+
+ if (end < start)
+ {
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+ start = 0;
+ end = file.fileSize - 1;
+ }
+ else
+ {
+ response.setStatusLine(metadata.httpVersion, 206, "Partial Content");
+ var contentRange = "bytes " + start + "-" + end + "/" + file.fileSize;
+ response.setHeader("Content-Range", contentRange);
+ }
+ }
+ else
+ {
+ start = 0;
+ end = file.fileSize - 1;
+ }
+
+ // finally...
+ dumpn("*** handling '" + path + "' as mapping to " + file.path + " from " +
+ start + " to " + end + " inclusive");
+ this._writeFileResponse(metadata, file, response, start, end - start + 1);
+ },
+
+ /**
+ * Writes an HTTP response for the given file, including setting headers for
+ * file metadata.
+ *
+ * @param metadata : Request
+ * the Request for which a response is being generated
+ * @param file : nsILocalFile
+ * the file which is to be sent in the response
+ * @param response : Response
+ * the response to which the file should be written
+ * @param offset: uint
+ * the byte offset to skip to when writing
+ * @param count: uint
+ * the number of bytes to write
+ */
+ _writeFileResponse: function(metadata, file, response, offset, count)
+ {
+ const PR_RDONLY = 0x01;
+
+ var type = this._getTypeFromFile(file);
+ if (type === SJS_TYPE)
+ {
+ var fis = new FileInputStream(file, PR_RDONLY, PERMS_READONLY,
+ Ci.nsIFileInputStream.CLOSE_ON_EOF);
+
+ try
+ {
+ var sis = new ScriptableInputStream(fis);
+ var s = Cu.Sandbox(gGlobalObject);
+ s.importFunction(dump, "dump");
+ s.importFunction(atob, "atob");
+ s.importFunction(btoa, "btoa");
+
+ // Define a basic key-value state-preservation API across requests, with
+ // keys initially corresponding to the empty string.
+ var self = this;
+ var path = metadata.path;
+ s.importFunction(function getState(k)
+ {
+ return self._getState(path, k);
+ });
+ s.importFunction(function setState(k, v)
+ {
+ self._setState(path, k, v);
+ });
+ s.importFunction(function getSharedState(k)
+ {
+ return self._getSharedState(k);
+ });
+ s.importFunction(function setSharedState(k, v)
+ {
+ self._setSharedState(k, v);
+ });
+ s.importFunction(function getObjectState(k, callback)
+ {
+ callback(self._getObjectState(k));
+ });
+ s.importFunction(function setObjectState(k, v)
+ {
+ self._setObjectState(k, v);
+ });
+ s.importFunction(function registerPathHandler(p, h)
+ {
+ self.registerPathHandler(p, h);
+ });
+
+ // Make it possible for sjs files to access their location
+ this._setState(path, "__LOCATION__", file.path);
+
+ try
+ {
+ // Alas, the line number in errors dumped to console when calling the
+ // request handler is simply an offset from where we load the SJS file.
+ // Work around this in a reasonably non-fragile way by dynamically
+ // getting the line number where we evaluate the SJS file. Don't
+ // separate these two lines!
+ var line = new Error().lineNumber;
+ Cu.evalInSandbox(sis.read(file.fileSize), s, "latest");
+ }
+ catch (e)
+ {
+ dumpn("*** syntax error in SJS at " + file.path + ": " + e);
+ throw HTTP_500;
+ }
+
+ try
+ {
+ s.handleRequest(metadata, response);
+ }
+ catch (e)
+ {
+ dump("*** error running SJS at " + file.path + ": " +
+ e + " on line " +
+ (e instanceof Error
+ ? e.lineNumber + " in httpd.js"
+ : (e.lineNumber - line)) + "\n");
+ throw HTTP_500;
+ }
+ }
+ finally
+ {
+ fis.close();
+ }
+ }
+ else
+ {
+ try
+ {
+ response.setHeader("Last-Modified",
+ toDateString(file.lastModifiedTime),
+ false);
+ }
+ catch (e) { /* lastModifiedTime threw, ignore */ }
+
+ response.setHeader("Content-Type", type, false);
+ maybeAddHeaders(file, metadata, response);
+ response.setHeader("Content-Length", "" + count, false);
+
+ var fis = new FileInputStream(file, PR_RDONLY, PERMS_READONLY,
+ Ci.nsIFileInputStream.CLOSE_ON_EOF);
+
+ offset = offset || 0;
+ count = count || file.fileSize;
+ NS_ASSERT(offset === 0 || offset < file.fileSize, "bad offset");
+ NS_ASSERT(count >= 0, "bad count");
+ NS_ASSERT(offset + count <= file.fileSize, "bad total data size");
+
+ try
+ {
+ if (offset !== 0)
+ {
+ // Seek (or read, if seeking isn't supported) to the correct offset so
+ // the data sent to the client matches the requested range.
+ if (fis instanceof Ci.nsISeekableStream)
+ fis.seek(Ci.nsISeekableStream.NS_SEEK_SET, offset);
+ else
+ new ScriptableInputStream(fis).read(offset);
+ }
+ }
+ catch (e)
+ {
+ fis.close();
+ throw e;
+ }
+
+ let writeMore = function () {
+ gThreadManager.currentThread
+ .dispatch(writeData, Ci.nsIThread.DISPATCH_NORMAL);
+ }
+
+ var input = new BinaryInputStream(fis);
+ var output = new BinaryOutputStream(response.bodyOutputStream);
+ var writeData =
+ {
+ run: function()
+ {
+ var chunkSize = Math.min(65536, count);
+ count -= chunkSize;
+ NS_ASSERT(count >= 0, "underflow");
+
+ try
+ {
+ var data = input.readByteArray(chunkSize);
+ NS_ASSERT(data.length === chunkSize,
+ "incorrect data returned? got " + data.length +
+ ", expected " + chunkSize);
+ output.writeByteArray(data, data.length);
+ if (count === 0)
+ {
+ fis.close();
+ response.finish();
+ }
+ else
+ {
+ writeMore();
+ }
+ }
+ catch (e)
+ {
+ try
+ {
+ fis.close();
+ }
+ finally
+ {
+ response.finish();
+ }
+ throw e;
+ }
+ }
+ };
+
+ writeMore();
+
+ // Now that we know copying will start, flag the response as async.
+ response.processAsync();
+ }
+ },
+
+ /**
+ * Get the value corresponding to a given key for the given path for SJS state
+ * preservation across requests.
+ *
+ * @param path : string
+ * the path from which the given state is to be retrieved
+ * @param k : string
+ * the key whose corresponding value is to be returned
+ * @returns string
+ * the corresponding value, which is initially the empty string
+ */
+ _getState: function(path, k)
+ {
+ var state = this._state;
+ if (path in state && k in state[path])
+ return state[path][k];
+ return "";
+ },
+
+ /**
+ * Set the value corresponding to a given key for the given path for SJS state
+ * preservation across requests.
+ *
+ * @param path : string
+ * the path from which the given state is to be retrieved
+ * @param k : string
+ * the key whose corresponding value is to be set
+ * @param v : string
+ * the value to be set
+ */
+ _setState: function(path, k, v)
+ {
+ if (typeof v !== "string")
+ throw new Error("non-string value passed");
+ var state = this._state;
+ if (!(path in state))
+ state[path] = {};
+ state[path][k] = v;
+ },
+
+ /**
+ * Get the value corresponding to a given key for SJS state preservation
+ * across requests.
+ *
+ * @param k : string
+ * the key whose corresponding value is to be returned
+ * @returns string
+ * the corresponding value, which is initially the empty string
+ */
+ _getSharedState: function(k)
+ {
+ var state = this._sharedState;
+ if (k in state)
+ return state[k];
+ return "";
+ },
+
+ /**
+ * Set the value corresponding to a given key for SJS state preservation
+ * across requests.
+ *
+ * @param k : string
+ * the key whose corresponding value is to be set
+ * @param v : string
+ * the value to be set
+ */
+ _setSharedState: function(k, v)
+ {
+ if (typeof v !== "string")
+ throw new Error("non-string value passed");
+ this._sharedState[k] = v;
+ },
+
+ /**
+ * Returns the object associated with the given key in the server for SJS
+ * state preservation across requests.
+ *
+ * @param k : string
+ * the key whose corresponding object is to be returned
+ * @returns nsISupports
+ * the corresponding object, or null if none was present
+ */
+ _getObjectState: function(k)
+ {
+ if (typeof k !== "string")
+ throw new Error("non-string key passed");
+ return this._objectState[k] || null;
+ },
+
+ /**
+ * Sets the object associated with the given key in the server for SJS
+ * state preservation across requests.
+ *
+ * @param k : string
+ * the key whose corresponding object is to be set
+ * @param v : nsISupports
+ * the object to be associated with the given key; may be null
+ */
+ _setObjectState: function(k, v)
+ {
+ if (typeof k !== "string")
+ throw new Error("non-string key passed");
+ if (typeof v !== "object")
+ throw new Error("non-object value passed");
+ if (v && !("QueryInterface" in v))
+ {
+ throw new Error("must pass an nsISupports; use wrappedJSObject to ease " +
+ "pain when using the server from JS");
+ }
+
+ this._objectState[k] = v;
+ },
+
+ /**
+ * Gets a content-type for the given file, first by checking for any custom
+ * MIME-types registered with this handler for the file's extension, second by
+ * asking the global MIME service for a content-type, and finally by failing
+ * over to application/octet-stream.
+ *
+ * @param file : nsIFile
+ * the nsIFile for which to get a file type
+ * @returns string
+ * the best content-type which can be determined for the file
+ */
+ _getTypeFromFile: function(file)
+ {
+ try
+ {
+ var name = file.leafName;
+ var dot = name.lastIndexOf(".");
+ if (dot > 0)
+ {
+ var ext = name.slice(dot + 1);
+ if (ext in this._mimeMappings)
+ return this._mimeMappings[ext];
+ }
+ return Cc["@mozilla.org/uriloader/external-helper-app-service;1"]
+ .getService(Ci.nsIMIMEService)
+ .getTypeFromFile(file);
+ }
+ catch (e)
+ {
+ return "application/octet-stream";
+ }
+ },
+
+ /**
+ * Returns the nsILocalFile which corresponds to the path, as determined using
+ * all registered path->directory mappings and any paths which are explicitly
+ * overridden.
+ *
+ * @param path : string
+ * the server path for which a file should be retrieved, e.g. "/foo/bar"
+ * @throws HttpError
+ * when the correct action is the corresponding HTTP error (i.e., because no
+ * mapping was found for a directory in path, the referenced file doesn't
+ * exist, etc.)
+ * @returns nsILocalFile
+ * the file to be sent as the response to a request for the path
+ */
+ _getFileForPath: function(path)
+ {
+ // decode and add underscores as necessary
+ try
+ {
+ path = toInternalPath(path, true);
+ }
+ catch (e)
+ {
+ dumpn("*** toInternalPath threw " + e);
+ throw HTTP_400; // malformed path
+ }
+
+ // next, get the directory which contains this path
+ var pathMap = this._pathDirectoryMap;
+
+ // An example progression of tmp for a path "/foo/bar/baz/" might be:
+ // "foo/bar/baz/", "foo/bar/baz", "foo/bar", "foo", ""
+ var tmp = path.substring(1);
+ while (true)
+ {
+ // do we have a match for current head of the path?
+ var file = pathMap.get(tmp);
+ if (file)
+ {
+ // XXX hack; basically disable showing mapping for /foo/bar/ when the
+ // requested path was /foo/bar, because relative links on the page
+ // will all be incorrect -- we really need the ability to easily
+ // redirect here instead
+ if (tmp == path.substring(1) &&
+ tmp.length != 0 &&
+ tmp.charAt(tmp.length - 1) != "/")
+ file = null;
+ else
+ break;
+ }
+
+ // if we've finished trying all prefixes, exit
+ if (tmp == "")
+ break;
+
+ tmp = tmp.substring(0, tmp.lastIndexOf("/"));
+ }
+
+ // no mapping applies, so 404
+ if (!file)
+ throw HTTP_404;
+
+
+ // last, get the file for the path within the determined directory
+ var parentFolder = file.parent;
+ var dirIsRoot = (parentFolder == null);
+
+ // Strategy here is to append components individually, making sure we
+ // never move above the given directory; this allows paths such as
+ // "<file>/foo/../bar" but prevents paths such as "<file>/../base-sibling";
+ // this component-wise approach also means the code works even on platforms
+ // which don't use "/" as the directory separator, such as Windows
+ var leafPath = path.substring(tmp.length + 1);
+ var comps = leafPath.split("/");
+ for (var i = 0, sz = comps.length; i < sz; i++)
+ {
+ var comp = comps[i];
+
+ if (comp == "..")
+ file = file.parent;
+ else if (comp == "." || comp == "")
+ continue;
+ else
+ file.append(comp);
+
+ if (!dirIsRoot && file.equals(parentFolder))
+ throw HTTP_403;
+ }
+
+ return file;
+ },
+
+ /**
+ * Writes the error page for the given HTTP error code over the given
+ * connection.
+ *
+ * @param errorCode : uint
+ * the HTTP error code to be used
+ * @param connection : Connection
+ * the connection on which the error occurred
+ */
+ handleError: function(errorCode, connection)
+ {
+ var response = new Response(connection);
+
+ dumpn("*** error in request: " + errorCode);
+
+ this._handleError(errorCode, new Request(connection.port), response);
+ },
+
+ /**
+ * Handles a request which generates the given error code, using the
+ * user-defined error handler if one has been set, gracefully falling back to
+ * the x00 status code if the code has no handler, and failing to status code
+ * 500 if all else fails.
+ *
+ * @param errorCode : uint
+ * the HTTP error which is to be returned
+ * @param metadata : Request
+ * metadata for the request, which will often be incomplete since this is an
+ * error
+ * @param response : Response
+ * an uninitialized Response should be initialized when this method
+ * completes with information which represents the desired error code in the
+ * ideal case or a fallback code in abnormal circumstances (i.e., 500 is a
+ * fallback for 505, per HTTP specs)
+ */
+ _handleError: function(errorCode, metadata, response)
+ {
+ if (!metadata)
+ throw Cr.NS_ERROR_NULL_POINTER;
+
+ var errorX00 = errorCode - (errorCode % 100);
+
+ try
+ {
+ if (!(errorCode in HTTP_ERROR_CODES))
+ dumpn("*** WARNING: requested invalid error: " + errorCode);
+
+ // RFC 2616 says that we should try to handle an error by its class if we
+ // can't otherwise handle it -- if that fails, we revert to handling it as
+ // a 500 internal server error, and if that fails we throw and shut down
+ // the server
+
+ // actually handle the error
+ try
+ {
+ if (errorCode in this._overrideErrors)
+ this._overrideErrors[errorCode](metadata, response);
+ else
+ this._defaultErrors[errorCode](metadata, response);
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort(e);
+ return;
+ }
+
+ // don't retry the handler that threw
+ if (errorX00 == errorCode)
+ throw HTTP_500;
+
+ dumpn("*** error in handling for error code " + errorCode + ", " +
+ "falling back to " + errorX00 + "...");
+ response = new Response(response._connection);
+ if (errorX00 in this._overrideErrors)
+ this._overrideErrors[errorX00](metadata, response);
+ else if (errorX00 in this._defaultErrors)
+ this._defaultErrors[errorX00](metadata, response);
+ else
+ throw HTTP_500;
+ }
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort();
+ return;
+ }
+
+ // we've tried everything possible for a meaningful error -- now try 500
+ dumpn("*** error in handling for error code " + errorX00 + ", falling " +
+ "back to 500...");
+
+ try
+ {
+ response = new Response(response._connection);
+ if (500 in this._overrideErrors)
+ this._overrideErrors[500](metadata, response);
+ else
+ this._defaultErrors[500](metadata, response);
+ }
+ catch (e2)
+ {
+ dumpn("*** multiple errors in default error handlers!");
+ dumpn("*** e == " + e + ", e2 == " + e2);
+ response.abort(e2);
+ return;
+ }
+ }
+
+ response.complete();
+ },
+
+ // FIELDS
+
+ /**
+ * This object contains the default handlers for the various HTTP error codes.
+ */
+ _defaultErrors:
+ {
+ 400: function(metadata, response)
+ {
+ // none of the data in metadata is reliable, so hard-code everything here
+ response.setStatusLine("1.1", 400, "Bad Request");
+ response.setHeader("Content-Type", "text/plain;charset=utf-8", false);
+
+ var body = "Bad request\n";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 403: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 403, "Forbidden");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>403 Forbidden</title></head>\
+ <body>\
+ <h1>403 Forbidden</h1>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 404: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 404, "Not Found");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>404 Not Found</title></head>\
+ <body>\
+ <h1>404 Not Found</h1>\
+ <p>\
+ <span style='font-family: monospace;'>" +
+ htmlEscape(metadata.path) +
+ "</span> was not found.\
+ </p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 416: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion,
+ 416,
+ "Requested Range Not Satisfiable");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head>\
+ <title>416 Requested Range Not Satisfiable</title></head>\
+ <body>\
+ <h1>416 Requested Range Not Satisfiable</h1>\
+ <p>The byte range was not valid for the\
+ requested resource.\
+ </p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 500: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion,
+ 500,
+ "Internal Server Error");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>500 Internal Server Error</title></head>\
+ <body>\
+ <h1>500 Internal Server Error</h1>\
+ <p>Something's broken in this server and\
+ needs to be fixed.</p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 501: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 501, "Not Implemented");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>501 Not Implemented</title></head>\
+ <body>\
+ <h1>501 Not Implemented</h1>\
+ <p>This server is not (yet) Apache.</p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 505: function(metadata, response)
+ {
+ response.setStatusLine("1.1", 505, "HTTP Version Not Supported");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>505 HTTP Version Not Supported</title></head>\
+ <body>\
+ <h1>505 HTTP Version Not Supported</h1>\
+ <p>This server only supports HTTP/1.0 and HTTP/1.1\
+ connections.</p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ }
+ },
+
+ /**
+ * Contains handlers for the default set of URIs contained in this server.
+ */
+ _defaultPaths:
+ {
+ "/": function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>httpd.js</title></head>\
+ <body>\
+ <h1>httpd.js</h1>\
+ <p>If you're seeing this page, httpd.js is up and\
+ serving requests! Now set a base path and serve some\
+ files!</p>\
+ </body>\
+ </html>";
+
+ response.bodyOutputStream.write(body, body.length);
+ },
+
+ "/trace": function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+ response.setHeader("Content-Type", "text/plain;charset=utf-8", false);
+
+ var body = "Request-URI: " +
+ metadata.scheme + "://" + metadata.host + ":" + metadata.port +
+ metadata.path + "\n\n";
+ body += "Request (semantically equivalent, slightly reformatted):\n\n";
+ body += metadata.method + " " + metadata.path;
+
+ if (metadata.queryString)
+ body += "?" + metadata.queryString;
+
+ body += " HTTP/" + metadata.httpVersion + "\r\n";
+
+ var headEnum = metadata.headers;
+ while (headEnum.hasMoreElements())
+ {
+ var fieldName = headEnum.getNext()
+ .QueryInterface(Ci.nsISupportsString)
+ .data;
+ body += fieldName + ": " + metadata.getHeader(fieldName) + "\r\n";
+ }
+
+ response.bodyOutputStream.write(body, body.length);
+ }
+ }
+};
+
+
+/**
+ * Maps absolute paths to files on the local file system (as nsILocalFiles).
+ */
+function FileMap()
+{
+ /** Hash which will map paths to nsILocalFiles. */
+ this._map = {};
+}
+FileMap.prototype =
+{
+ // PUBLIC API
+
+ /**
+ * Maps key to a clone of the nsILocalFile value if value is non-null;
+ * otherwise, removes any extant mapping for key.
+ *
+ * @param key : string
+ * string to which a clone of value is mapped
+ * @param value : nsILocalFile
+ * the file to map to key, or null to remove a mapping
+ */
+ put: function(key, value)
+ {
+ if (value)
+ this._map[key] = value.clone();
+ else
+ delete this._map[key];
+ },
+
+ /**
+ * Returns a clone of the nsILocalFile mapped to key, or null if no such
+ * mapping exists.
+ *
+ * @param key : string
+ * key to which the returned file maps
+ * @returns nsILocalFile
+ * a clone of the mapped file, or null if no mapping exists
+ */
+ get: function(key)
+ {
+ var val = this._map[key];
+ return val ? val.clone() : null;
+ }
+};
+
+
+// Response CONSTANTS
+
+// token = *<any CHAR except CTLs or separators>
+// CHAR = <any US-ASCII character (0-127)>
+// CTL = <any US-ASCII control character (0-31) and DEL (127)>
+// separators = "(" | ")" | "<" | ">" | "@"
+// | "," | ";" | ":" | "\" | <">
+// | "/" | "[" | "]" | "?" | "="
+// | "{" | "}" | SP | HT
+const IS_TOKEN_ARRAY =
+ [0, 0, 0, 0, 0, 0, 0, 0, // 0
+ 0, 0, 0, 0, 0, 0, 0, 0, // 8
+ 0, 0, 0, 0, 0, 0, 0, 0, // 16
+ 0, 0, 0, 0, 0, 0, 0, 0, // 24
+
+ 0, 1, 0, 1, 1, 1, 1, 1, // 32
+ 0, 0, 1, 1, 0, 1, 1, 0, // 40
+ 1, 1, 1, 1, 1, 1, 1, 1, // 48
+ 1, 1, 0, 0, 0, 0, 0, 0, // 56
+
+ 0, 1, 1, 1, 1, 1, 1, 1, // 64
+ 1, 1, 1, 1, 1, 1, 1, 1, // 72
+ 1, 1, 1, 1, 1, 1, 1, 1, // 80
+ 1, 1, 1, 0, 0, 0, 1, 1, // 88
+
+ 1, 1, 1, 1, 1, 1, 1, 1, // 96
+ 1, 1, 1, 1, 1, 1, 1, 1, // 104
+ 1, 1, 1, 1, 1, 1, 1, 1, // 112
+ 1, 1, 1, 0, 1, 0, 1]; // 120
+
+
+/**
+ * Determines whether the given character code is a CTL.
+ *
+ * @param code : uint
+ * the character code
+ * @returns boolean
+ * true if code is a CTL, false otherwise
+ */
+function isCTL(code)
+{
+ return (code >= 0 && code <= 31) || (code == 127);
+}
+
+/**
+ * Represents a response to an HTTP request, encapsulating all details of that
+ * response. This includes all headers, the HTTP version, status code and
+ * explanation, and the entity itself.
+ *
+ * @param connection : Connection
+ * the connection over which this response is to be written
+ */
+function Response(connection)
+{
+ /** The connection over which this response will be written. */
+ this._connection = connection;
+
+ /**
+ * The HTTP version of this response; defaults to 1.1 if not set by the
+ * handler.
+ */
+ this._httpVersion = nsHttpVersion.HTTP_1_1;
+
+ /**
+ * The HTTP code of this response; defaults to 200.
+ */
+ this._httpCode = 200;
+
+ /**
+ * The description of the HTTP code in this response; defaults to "OK".
+ */
+ this._httpDescription = "OK";
+
+ /**
+ * An nsIHttpHeaders object in which the headers in this response should be
+ * stored. This property is null after the status line and headers have been
+ * written to the network, and it may be modified up until it is cleared,
+ * except if this._finished is set first (in which case headers are written
+ * asynchronously in response to a finish() call not preceded by
+ * flushHeaders()).
+ */
+ this._headers = new nsHttpHeaders();
+
+ /**
+ * Set to true when this response is ended (completely constructed if possible
+ * and the connection closed); further actions on this will then fail.
+ */
+ this._ended = false;
+
+ /**
+ * A stream used to hold data written to the body of this response.
+ */
+ this._bodyOutputStream = null;
+
+ /**
+ * A stream containing all data that has been written to the body of this
+ * response so far. (Async handlers make the data contained in this
+ * unreliable as a way of determining content length in general, but auxiliary
+ * saved information can sometimes be used to guarantee reliability.)
+ */
+ this._bodyInputStream = null;
+
+ /**
+ * A stream copier which copies data to the network. It is initially null
+ * until replaced with a copier for response headers; when headers have been
+ * fully sent it is replaced with a copier for the response body, remaining
+ * so for the duration of response processing.
+ */
+ this._asyncCopier = null;
+
+ /**
+ * True if this response has been designated as being processed
+ * asynchronously rather than for the duration of a single call to
+ * nsIHttpRequestHandler.handle.
+ */
+ this._processAsync = false;
+
+ /**
+ * True iff finish() has been called on this, signaling that no more changes
+ * to this may be made.
+ */
+ this._finished = false;
+
+ /**
+ * True iff powerSeized() has been called on this, signaling that this
+ * response is to be handled manually by the response handler (which may then
+ * send arbitrary data in response, even non-HTTP responses).
+ */
+ this._powerSeized = false;
+}
+Response.prototype =
+{
+ // PUBLIC CONSTRUCTION API
+
+ //
+ // see nsIHttpResponse.bodyOutputStream
+ //
+ get bodyOutputStream()
+ {
+ if (this._finished)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+
+ if (!this._bodyOutputStream)
+ {
+ var pipe = new Pipe(true, false, Response.SEGMENT_SIZE, PR_UINT32_MAX,
+ null);
+ this._bodyOutputStream = pipe.outputStream;
+ this._bodyInputStream = pipe.inputStream;
+ if (this._processAsync || this._powerSeized)
+ this._startAsyncProcessor();
+ }
+
+ return this._bodyOutputStream;
+ },
+
+ //
+ // see nsIHttpResponse.write
+ //
+ write: function(data)
+ {
+ if (this._finished)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+
+ var dataAsString = String(data);
+ this.bodyOutputStream.write(dataAsString, dataAsString.length);
+ },
+
+ //
+ // see nsIHttpResponse.setStatusLine
+ //
+ setStatusLine: function(httpVersion, code, description)
+ {
+ if (!this._headers || this._finished || this._powerSeized)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ this._ensureAlive();
+
+ if (!(code >= 0 && code < 1000))
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ try
+ {
+ var httpVer;
+ // avoid version construction for the most common cases
+ if (!httpVersion || httpVersion == "1.1")
+ httpVer = nsHttpVersion.HTTP_1_1;
+ else if (httpVersion == "1.0")
+ httpVer = nsHttpVersion.HTTP_1_0;
+ else
+ httpVer = new nsHttpVersion(httpVersion);
+ }
+ catch (e)
+ {
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+
+ // Reason-Phrase = *<TEXT, excluding CR, LF>
+ // TEXT = <any OCTET except CTLs, but including LWS>
+ //
+ // XXX this ends up disallowing octets which aren't Unicode, I think -- not
+ // much to do if description is IDL'd as string
+ if (!description)
+ description = "";
+ for (var i = 0; i < description.length; i++)
+ if (isCTL(description.charCodeAt(i)) && description.charAt(i) != "\t")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ // set the values only after validation to preserve atomicity
+ this._httpDescription = description;
+ this._httpCode = code;
+ this._httpVersion = httpVer;
+ },
+
+ //
+ // see nsIHttpResponse.setHeader
+ //
+ setHeader: function(name, value, merge)
+ {
+ if (!this._headers || this._finished || this._powerSeized)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ this._ensureAlive();
+
+ this._headers.setHeader(name, value, merge);
+ },
+
+ //
+ // see nsIHttpResponse.processAsync
+ //
+ processAsync: function()
+ {
+ if (this._finished)
+ throw Cr.NS_ERROR_UNEXPECTED;
+ if (this._powerSeized)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ if (this._processAsync)
+ return;
+ this._ensureAlive();
+
+ dumpn("*** processing connection " + this._connection.number + " async");
+ this._processAsync = true;
+
+ /*
+ * Either the bodyOutputStream getter or this method is responsible for
+ * starting the asynchronous processor and catching writes of data to the
+ * response body of async responses as they happen, for the purpose of
+ * forwarding those writes to the actual connection's output stream.
+ * If bodyOutputStream is accessed first, calling this method will create
+ * the processor (when it first is clear that body data is to be written
+ * immediately, not buffered). If this method is called first, accessing
+ * bodyOutputStream will create the processor. If only this method is
+ * called, we'll write nothing, neither headers nor the nonexistent body,
+ * until finish() is called. Since that delay is easily avoided by simply
+ * getting bodyOutputStream or calling write(""), we don't worry about it.
+ */
+ if (this._bodyOutputStream && !this._asyncCopier)
+ this._startAsyncProcessor();
+ },
+
+ //
+ // see nsIHttpResponse.seizePower
+ //
+ seizePower: function()
+ {
+ if (this._processAsync)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ if (this._finished)
+ throw Cr.NS_ERROR_UNEXPECTED;
+ if (this._powerSeized)
+ return;
+ this._ensureAlive();
+
+ dumpn("*** forcefully seizing power over connection " +
+ this._connection.number + "...");
+
+ // Purge any already-written data without sending it. We could as easily
+ // swap out the streams entirely, but that makes it possible to acquire and
+ // unknowingly use a stale reference, so we require there only be one of
+ // each stream ever for any response to avoid this complication.
+ if (this._asyncCopier)
+ this._asyncCopier.cancel(Cr.NS_BINDING_ABORTED);
+ this._asyncCopier = null;
+ if (this._bodyOutputStream)
+ {
+ var input = new BinaryInputStream(this._bodyInputStream);
+ var avail;
+ while ((avail = input.available()) > 0)
+ input.readByteArray(avail);
+ }
+
+ this._powerSeized = true;
+ if (this._bodyOutputStream)
+ this._startAsyncProcessor();
+ },
+
+ //
+ // see nsIHttpResponse.finish
+ //
+ finish: function()
+ {
+ if (!this._processAsync && !this._powerSeized)
+ throw Cr.NS_ERROR_UNEXPECTED;
+ if (this._finished)
+ return;
+
+ dumpn("*** finishing connection " + this._connection.number);
+ this._startAsyncProcessor(); // in case bodyOutputStream was never accessed
+ if (this._bodyOutputStream)
+ this._bodyOutputStream.close();
+ this._finished = true;
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpResponse) || iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // POST-CONSTRUCTION API (not exposed externally)
+
+ /**
+ * The HTTP version number of this, as a string (e.g. "1.1").
+ */
+ get httpVersion()
+ {
+ this._ensureAlive();
+ return this._httpVersion.toString();
+ },
+
+ /**
+ * The HTTP status code of this response, as a string of three characters per
+ * RFC 2616.
+ */
+ get httpCode()
+ {
+ this._ensureAlive();
+
+ var codeString = (this._httpCode < 10 ? "0" : "") +
+ (this._httpCode < 100 ? "0" : "") +
+ this._httpCode;
+ return codeString;
+ },
+
+ /**
+ * The description of the HTTP status code of this response, or "" if none is
+ * set.
+ */
+ get httpDescription()
+ {
+ this._ensureAlive();
+
+ return this._httpDescription;
+ },
+
+ /**
+ * The headers in this response, as an nsHttpHeaders object.
+ */
+ get headers()
+ {
+ this._ensureAlive();
+
+ return this._headers;
+ },
+
+ //
+ // see nsHttpHeaders.getHeader
+ //
+ getHeader: function(name)
+ {
+ this._ensureAlive();
+
+ return this._headers.getHeader(name);
+ },
+
+ /**
+ * Determines whether this response may be abandoned in favor of a newly
+ * constructed response. A response may be abandoned only if it is not being
+ * sent asynchronously and if raw control over it has not been taken from the
+ * server.
+ *
+ * @returns boolean
+ * true iff no data has been written to the network
+ */
+ partiallySent: function()
+ {
+ dumpn("*** partiallySent()");
+ return this._processAsync || this._powerSeized;
+ },
+
+ /**
+ * If necessary, kicks off the remaining request processing needed to be done
+ * after a request handler performs its initial work upon this response.
+ */
+ complete: function()
+ {
+ dumpn("*** complete()");
+ if (this._processAsync || this._powerSeized)
+ {
+ NS_ASSERT(this._processAsync ^ this._powerSeized,
+ "can't both send async and relinquish power");
+ return;
+ }
+
+ NS_ASSERT(!this.partiallySent(), "completing a partially-sent response?");
+
+ this._startAsyncProcessor();
+
+ // Now make sure we finish processing this request!
+ if (this._bodyOutputStream)
+ this._bodyOutputStream.close();
+ },
+
+ /**
+ * Abruptly ends processing of this response, usually due to an error in an
+ * incoming request but potentially due to a bad error handler. Since we
+ * cannot handle the error in the usual way (giving an HTTP error page in
+ * response) because data may already have been sent (or because the response
+ * might be expected to have been generated asynchronously or completely from
+ * scratch by the handler), we stop processing this response and abruptly
+ * close the connection.
+ *
+ * @param e : Error
+ * the exception which precipitated this abort, or null if no such exception
+ * was generated
+ */
+ abort: function(e)
+ {
+ dumpn("*** abort(<" + e + ">)");
+
+ // This response will be ended by the processor if one was created.
+ var copier = this._asyncCopier;
+ if (copier)
+ {
+ // We dispatch asynchronously here so that any pending writes of data to
+ // the connection will be deterministically written. This makes it easier
+ // to specify exact behavior, and it makes observable behavior more
+ // predictable for clients. Note that the correctness of this depends on
+ // callbacks in response to _waitToReadData in WriteThroughCopier
+ // happening asynchronously with respect to the actual writing of data to
+ // bodyOutputStream, as they currently do; if they happened synchronously,
+ // an event which ran before this one could write more data to the
+ // response body before we get around to canceling the copier. We have
+ // tests for this in test_seizepower.js, however, and I can't think of a
+ // way to handle both cases without removing bodyOutputStream access and
+ // moving its effective write(data, length) method onto Response, which
+ // would be slower and require more code than this anyway.
+ gThreadManager.currentThread.dispatch({
+ run: function()
+ {
+ dumpn("*** canceling copy asynchronously...");
+ copier.cancel(Cr.NS_ERROR_UNEXPECTED);
+ }
+ }, Ci.nsIThread.DISPATCH_NORMAL);
+ }
+ else
+ {
+ this.end();
+ }
+ },
+
+ /**
+ * Closes this response's network connection, marks the response as finished,
+ * and notifies the server handler that the request is done being processed.
+ */
+ end: function()
+ {
+ NS_ASSERT(!this._ended, "ending this response twice?!?!");
+
+ this._connection.close();
+ if (this._bodyOutputStream)
+ this._bodyOutputStream.close();
+
+ this._finished = true;
+ this._ended = true;
+ },
+
+ // PRIVATE IMPLEMENTATION
+
+ /**
+ * Sends the status line and headers of this response if they haven't been
+ * sent and initiates the process of copying data written to this response's
+ * body to the network.
+ */
+ _startAsyncProcessor: function()
+ {
+ dumpn("*** _startAsyncProcessor()");
+
+ // Handle cases where we're being called a second time. The former case
+ // happens when this is triggered both by complete() and by processAsync(),
+ // while the latter happens when processAsync() in conjunction with sent
+ // data causes abort() to be called.
+ if (this._asyncCopier || this._ended)
+ {
+ dumpn("*** ignoring second call to _startAsyncProcessor");
+ return;
+ }
+
+ // Send headers if they haven't been sent already and should be sent, then
+ // asynchronously continue to send the body.
+ if (this._headers && !this._powerSeized)
+ {
+ this._sendHeaders();
+ return;
+ }
+
+ this._headers = null;
+ this._sendBody();
+ },
+
+ /**
+ * Signals that all modifications to the response status line and headers are
+ * complete and then sends that data over the network to the client. Once
+ * this method completes, a different response to the request that resulted
+ * in this response cannot be sent -- the only possible action in case of
+ * error is to abort the response and close the connection.
+ */
+ _sendHeaders: function()
+ {
+ dumpn("*** _sendHeaders()");
+
+ NS_ASSERT(this._headers);
+ NS_ASSERT(!this._powerSeized);
+
+ // request-line
+ var statusLine = "HTTP/" + this.httpVersion + " " +
+ this.httpCode + " " +
+ this.httpDescription + "\r\n";
+
+ // header post-processing
+
+ var headers = this._headers;
+ headers.setHeader("Connection", "close", false);
+ headers.setHeader("Server", "httpd.js", false);
+ if (!headers.hasHeader("Date"))
+ headers.setHeader("Date", toDateString(Date.now()), false);
+
+ // Any response not being processed asynchronously must have an associated
+ // Content-Length header for reasons of backwards compatibility with the
+ // initial server, which fully buffered every response before sending it.
+ // Beyond that, however, it's good to do this anyway because otherwise it's
+ // impossible to test behaviors that depend on the presence or absence of a
+ // Content-Length header.
+ if (!this._processAsync)
+ {
+ dumpn("*** non-async response, set Content-Length");
+
+ var bodyStream = this._bodyInputStream;
+ var avail = bodyStream ? bodyStream.available() : 0;
+
+ // XXX assumes stream will always report the full amount of data available
+ headers.setHeader("Content-Length", "" + avail, false);
+ }
+
+
+ // construct and send response
+ dumpn("*** header post-processing completed, sending response head...");
+
+ // request-line
+ var preambleData = [statusLine];
+
+ // headers
+ var headEnum = headers.enumerator;
+ while (headEnum.hasMoreElements())
+ {
+ var fieldName = headEnum.getNext()
+ .QueryInterface(Ci.nsISupportsString)
+ .data;
+ var values = headers.getHeaderValues(fieldName);
+ for (var i = 0, sz = values.length; i < sz; i++)
+ preambleData.push(fieldName + ": " + values[i] + "\r\n");
+ }
+
+ // end request-line/headers
+ preambleData.push("\r\n");
+
+ var preamble = preambleData.join("");
+
+ var responseHeadPipe = new Pipe(true, false, 0, PR_UINT32_MAX, null);
+ responseHeadPipe.outputStream.write(preamble, preamble.length);
+
+ var response = this;
+ var copyObserver =
+ {
+ onStartRequest: function(request, cx)
+ {
+ dumpn("*** preamble copying started");
+ },
+
+ onStopRequest: function(request, cx, statusCode)
+ {
+ dumpn("*** preamble copying complete " +
+ "[status=0x" + statusCode.toString(16) + "]");
+
+ if (!Components.isSuccessCode(statusCode))
+ {
+ dumpn("!!! header copying problems: non-success statusCode, " +
+ "ending response");
+
+ response.end();
+ }
+ else
+ {
+ response._sendBody();
+ }
+ },
+
+ QueryInterface: function(aIID)
+ {
+ if (aIID.equals(Ci.nsIRequestObserver) || aIID.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ }
+ };
+
+ var headerCopier = this._asyncCopier =
+ new WriteThroughCopier(responseHeadPipe.inputStream,
+ this._connection.output,
+ copyObserver, null);
+
+ responseHeadPipe.outputStream.close();
+
+ // Forbid setting any more headers or modifying the request line.
+ this._headers = null;
+ },
+
+ /**
+ * Asynchronously writes the body of the response (or the entire response, if
+ * seizePower() has been called) to the network.
+ */
+ _sendBody: function()
+ {
+ dumpn("*** _sendBody");
+
+ NS_ASSERT(!this._headers, "still have headers around but sending body?");
+
+ // If no body data was written, we're done
+ if (!this._bodyInputStream)
+ {
+ dumpn("*** empty body, response finished");
+ this.end();
+ return;
+ }
+
+ var response = this;
+ var copyObserver =
+ {
+ onStartRequest: function(request, context)
+ {
+ dumpn("*** onStartRequest");
+ },
+
+ onStopRequest: function(request, cx, statusCode)
+ {
+ dumpn("*** onStopRequest [status=0x" + statusCode.toString(16) + "]");
+
+ if (statusCode === Cr.NS_BINDING_ABORTED)
+ {
+ dumpn("*** terminating copy observer without ending the response");
+ }
+ else
+ {
+ if (!Components.isSuccessCode(statusCode))
+ dumpn("*** WARNING: non-success statusCode in onStopRequest");
+
+ response.end();
+ }
+ },
+
+ QueryInterface: function(aIID)
+ {
+ if (aIID.equals(Ci.nsIRequestObserver) || aIID.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ }
+ };
+
+ dumpn("*** starting async copier of body data...");
+ this._asyncCopier =
+ new WriteThroughCopier(this._bodyInputStream, this._connection.output,
+ copyObserver, null);
+ },
+
+ /** Ensures that this hasn't been ended. */
+ _ensureAlive: function()
+ {
+ NS_ASSERT(!this._ended, "not handling response lifetime correctly");
+ }
+};
+
+/**
+ * Size of the segments in the buffer used in storing response data and writing
+ * it to the socket.
+ */
+Response.SEGMENT_SIZE = 8192;
+
+/** Serves double duty in WriteThroughCopier implementation. */
+function notImplemented()
+{
+ throw Cr.NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/** Returns true iff the given exception represents stream closure. */
+function streamClosed(e)
+{
+ return e === Cr.NS_BASE_STREAM_CLOSED ||
+ (typeof e === "object" && e.result === Cr.NS_BASE_STREAM_CLOSED);
+}
+
+/** Returns true iff the given exception represents a blocked stream. */
+function wouldBlock(e)
+{
+ return e === Cr.NS_BASE_STREAM_WOULD_BLOCK ||
+ (typeof e === "object" && e.result === Cr.NS_BASE_STREAM_WOULD_BLOCK);
+}
+
+/**
+ * Copies data from source to sink as it becomes available, when that data can
+ * be written to sink without blocking.
+ *
+ * @param source : nsIAsyncInputStream
+ * the stream from which data is to be read
+ * @param sink : nsIAsyncOutputStream
+ * the stream to which data is to be copied
+ * @param observer : nsIRequestObserver
+ * an observer which will be notified when the copy starts and finishes
+ * @param context : nsISupports
+ * context passed to observer when notified of start/stop
+ * @throws NS_ERROR_NULL_POINTER
+ * if source, sink, or observer are null
+ */
+function WriteThroughCopier(source, sink, observer, context)
+{
+ if (!source || !sink || !observer)
+ throw Cr.NS_ERROR_NULL_POINTER;
+
+ /** Stream from which data is being read. */
+ this._source = source;
+
+ /** Stream to which data is being written. */
+ this._sink = sink;
+
+ /** Observer watching this copy. */
+ this._observer = observer;
+
+ /** Context for the observer watching this. */
+ this._context = context;
+
+ /**
+ * True iff this is currently being canceled (cancel has been called, the
+ * callback may not yet have been made).
+ */
+ this._canceled = false;
+
+ /**
+ * False until all data has been read from input and written to output, at
+ * which point this copy is completed and cancel() is asynchronously called.
+ */
+ this._completed = false;
+
+ /** Required by nsIRequest, meaningless. */
+ this.loadFlags = 0;
+ /** Required by nsIRequest, meaningless. */
+ this.loadGroup = null;
+ /** Required by nsIRequest, meaningless. */
+ this.name = "response-body-copy";
+
+ /** Status of this request. */
+ this.status = Cr.NS_OK;
+
+ /** Arrays of byte strings waiting to be written to output. */
+ this._pendingData = [];
+
+ // start copying
+ try
+ {
+ observer.onStartRequest(this, context);
+ this._waitToReadData();
+ this._waitForSinkClosure();
+ }
+ catch (e)
+ {
+ dumpn("!!! error starting copy: " + e +
+ ("lineNumber" in e ? ", line " + e.lineNumber : ""));
+ dumpn(e.stack);
+ this.cancel(Cr.NS_ERROR_UNEXPECTED);
+ }
+}
+WriteThroughCopier.prototype =
+{
+ /* nsISupports implementation */
+
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIInputStreamCallback) ||
+ iid.equals(Ci.nsIOutputStreamCallback) ||
+ iid.equals(Ci.nsIRequest) ||
+ iid.equals(Ci.nsISupports))
+ {
+ return this;
+ }
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // NSIINPUTSTREAMCALLBACK
+
+ /**
+ * Receives a more-data-in-input notification and writes the corresponding
+ * data to the output.
+ *
+ * @param input : nsIAsyncInputStream
+ * the input stream on whose data we have been waiting
+ */
+ onInputStreamReady: function(input)
+ {
+ if (this._source === null)
+ return;
+
+ dumpn("*** onInputStreamReady");
+
+ //
+ // Ordinarily we'll read a non-zero amount of data from input, queue it up
+ // to be written and then wait for further callbacks. The complications in
+ // this method are the cases where we deviate from that behavior when errors
+ // occur or when copying is drawing to a finish.
+ //
+ // The edge cases when reading data are:
+ //
+ // Zero data is read
+ // If zero data was read, we're at the end of available data, so we can
+ // should stop reading and move on to writing out what we have (or, if
+ // we've already done that, onto notifying of completion).
+ // A stream-closed exception is thrown
+ // This is effectively a less kind version of zero data being read; the
+ // only difference is that we notify of completion with that result
+ // rather than with NS_OK.
+ // Some other exception is thrown
+ // This is the least kind result. We don't know what happened, so we
+ // act as though the stream closed except that we notify of completion
+ // with the result NS_ERROR_UNEXPECTED.
+ //
+
+ var bytesWanted = 0, bytesConsumed = -1;
+ try
+ {
+ input = new BinaryInputStream(input);
+
+ bytesWanted = Math.min(input.available(), Response.SEGMENT_SIZE);
+ dumpn("*** input wanted: " + bytesWanted);
+
+ if (bytesWanted > 0)
+ {
+ var data = input.readByteArray(bytesWanted);
+ bytesConsumed = data.length;
+ this._pendingData.push(String.fromCharCode.apply(String, data));
+ }
+
+ dumpn("*** " + bytesConsumed + " bytes read");
+
+ // Handle the zero-data edge case in the same place as all other edge
+ // cases are handled.
+ if (bytesWanted === 0)
+ throw Cr.NS_BASE_STREAM_CLOSED;
+ }
+ catch (e)
+ {
+ if (streamClosed(e))
+ {
+ dumpn("*** input stream closed");
+ e = bytesWanted === 0 ? Cr.NS_OK : Cr.NS_ERROR_UNEXPECTED;
+ }
+ else
+ {
+ dumpn("!!! unexpected error reading from input, canceling: " + e);
+ e = Cr.NS_ERROR_UNEXPECTED;
+ }
+
+ this._doneReadingSource(e);
+ return;
+ }
+
+ var pendingData = this._pendingData;
+
+ NS_ASSERT(bytesConsumed > 0);
+ NS_ASSERT(pendingData.length > 0, "no pending data somehow?");
+ NS_ASSERT(pendingData[pendingData.length - 1].length > 0,
+ "buffered zero bytes of data?");
+
+ NS_ASSERT(this._source !== null);
+
+ // Reading has gone great, and we've gotten data to write now. What if we
+ // don't have a place to write that data, because output went away just
+ // before this read? Drop everything on the floor, including new data, and
+ // cancel at this point.
+ if (this._sink === null)
+ {
+ pendingData.length = 0;
+ this._doneReadingSource(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // Okay, we've read the data, and we know we have a place to write it. We
+ // need to queue up the data to be written, but *only* if none is queued
+ // already -- if data's already queued, the code that actually writes the
+ // data will make sure to wait on unconsumed pending data.
+ try
+ {
+ if (pendingData.length === 1)
+ this._waitToWriteData();
+ }
+ catch (e)
+ {
+ dumpn("!!! error waiting to write data just read, swallowing and " +
+ "writing only what we already have: " + e);
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // Whee! We successfully read some data, and it's successfully queued up to
+ // be written. All that remains now is to wait for more data to read.
+ try
+ {
+ this._waitToReadData();
+ }
+ catch (e)
+ {
+ dumpn("!!! error waiting to read more data: " + e);
+ this._doneReadingSource(Cr.NS_ERROR_UNEXPECTED);
+ }
+ },
+
+
+ // NSIOUTPUTSTREAMCALLBACK
+
+ /**
+ * Callback when data may be written to the output stream without blocking, or
+ * when the output stream has been closed.
+ *
+ * @param output : nsIAsyncOutputStream
+ * the output stream on whose writability we've been waiting, also known as
+ * this._sink
+ */
+ onOutputStreamReady: function(output)
+ {
+ if (this._sink === null)
+ return;
+
+ dumpn("*** onOutputStreamReady");
+
+ var pendingData = this._pendingData;
+ if (pendingData.length === 0)
+ {
+ // There's no pending data to write. The only way this can happen is if
+ // we're waiting on the output stream's closure, so we can respond to a
+ // copying failure as quickly as possible (rather than waiting for data to
+ // be available to read and then fail to be copied). Therefore, we must
+ // be done now -- don't bother to attempt to write anything and wrap
+ // things up.
+ dumpn("!!! output stream closed prematurely, ending copy");
+
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+
+ NS_ASSERT(pendingData[0].length > 0, "queued up an empty quantum?");
+
+ //
+ // Write out the first pending quantum of data. The possible errors here
+ // are:
+ //
+ // The write might fail because we can't write that much data
+ // Okay, we've written what we can now, so re-queue what's left and
+ // finish writing it out later.
+ // The write failed because the stream was closed
+ // Discard pending data that we can no longer write, stop reading, and
+ // signal that copying finished.
+ // Some other error occurred.
+ // Same as if the stream were closed, but notify with the status
+ // NS_ERROR_UNEXPECTED so the observer knows something was wonky.
+ //
+
+ try
+ {
+ var quantum = pendingData[0];
+
+ // XXX |quantum| isn't guaranteed to be ASCII, so we're relying on
+ // undefined behavior! We're only using this because writeByteArray
+ // is unusably broken for asynchronous output streams; see bug 532834
+ // for details.
+ var bytesWritten = output.write(quantum, quantum.length);
+ if (bytesWritten === quantum.length)
+ pendingData.shift();
+ else
+ pendingData[0] = quantum.substring(bytesWritten);
+
+ dumpn("*** wrote " + bytesWritten + " bytes of data");
+ }
+ catch (e)
+ {
+ if (wouldBlock(e))
+ {
+ NS_ASSERT(pendingData.length > 0,
+ "stream-blocking exception with no data to write?");
+ NS_ASSERT(pendingData[0].length > 0,
+ "stream-blocking exception with empty quantum?");
+ this._waitToWriteData();
+ return;
+ }
+
+ if (streamClosed(e))
+ dumpn("!!! output stream prematurely closed, signaling error...");
+ else
+ dumpn("!!! unknown error: " + e + ", quantum=" + quantum);
+
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // The day is ours! Quantum written, now let's see if we have more data
+ // still to write.
+ try
+ {
+ if (pendingData.length > 0)
+ {
+ this._waitToWriteData();
+ return;
+ }
+ }
+ catch (e)
+ {
+ dumpn("!!! unexpected error waiting to write pending data: " + e);
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // Okay, we have no more pending data to write -- but might we get more in
+ // the future?
+ if (this._source !== null)
+ {
+ /*
+ * If we might, then wait for the output stream to be closed. (We wait
+ * only for closure because we have no data to write -- and if we waited
+ * for a specific amount of data, we would get repeatedly notified for no
+ * reason if over time the output stream permitted more and more data to
+ * be written to it without blocking.)
+ */
+ this._waitForSinkClosure();
+ }
+ else
+ {
+ /*
+ * On the other hand, if we can't have more data because the input
+ * stream's gone away, then it's time to notify of copy completion.
+ * Victory!
+ */
+ this._sink = null;
+ this._cancelOrDispatchCancelCallback(Cr.NS_OK);
+ }
+ },
+
+
+ // NSIREQUEST
+
+ /** Returns true if the cancel observer hasn't been notified yet. */
+ isPending: function()
+ {
+ return !this._completed;
+ },
+
+ /** Not implemented, don't use! */
+ suspend: notImplemented,
+ /** Not implemented, don't use! */
+ resume: notImplemented,
+
+ /**
+ * Cancels data reading from input, asynchronously writes out any pending
+ * data, and causes the observer to be notified with the given error code when
+ * all writing has finished.
+ *
+ * @param status : nsresult
+ * the status to pass to the observer when data copying has been canceled
+ */
+ cancel: function(status)
+ {
+ dumpn("*** cancel(" + status.toString(16) + ")");
+
+ if (this._canceled)
+ {
+ dumpn("*** suppressing a late cancel");
+ return;
+ }
+
+ this._canceled = true;
+ this.status = status;
+
+ // We could be in the middle of absolutely anything at this point. Both
+ // input and output might still be around, we might have pending data to
+ // write, and in general we know nothing about the state of the world. We
+ // therefore must assume everything's in progress and take everything to its
+ // final steady state (or so far as it can go before we need to finish
+ // writing out remaining data).
+
+ this._doneReadingSource(status);
+ },
+
+
+ // PRIVATE IMPLEMENTATION
+
+ /**
+ * Stop reading input if we haven't already done so, passing e as the status
+ * when closing the stream, and kick off a copy-completion notice if no more
+ * data remains to be written.
+ *
+ * @param e : nsresult
+ * the status to be used when closing the input stream
+ */
+ _doneReadingSource: function(e)
+ {
+ dumpn("*** _doneReadingSource(0x" + e.toString(16) + ")");
+
+ this._finishSource(e);
+ if (this._pendingData.length === 0)
+ this._sink = null;
+ else
+ NS_ASSERT(this._sink !== null, "null output?");
+
+ // If we've written out all data read up to this point, then it's time to
+ // signal completion.
+ if (this._sink === null)
+ {
+ NS_ASSERT(this._pendingData.length === 0, "pending data still?");
+ this._cancelOrDispatchCancelCallback(e);
+ }
+ },
+
+ /**
+ * Stop writing output if we haven't already done so, discard any data that
+ * remained to be sent, close off input if it wasn't already closed, and kick
+ * off a copy-completion notice.
+ *
+ * @param e : nsresult
+ * the status to be used when closing input if it wasn't already closed
+ */
+ _doneWritingToSink: function(e)
+ {
+ dumpn("*** _doneWritingToSink(0x" + e.toString(16) + ")");
+
+ this._pendingData.length = 0;
+ this._sink = null;
+ this._doneReadingSource(e);
+ },
+
+ /**
+ * Completes processing of this copy: either by canceling the copy if it
+ * hasn't already been canceled using the provided status, or by dispatching
+ * the cancel callback event (with the originally provided status, of course)
+ * if it already has been canceled.
+ *
+ * @param status : nsresult
+ * the status code to use to cancel this, if this hasn't already been
+ * canceled
+ */
+ _cancelOrDispatchCancelCallback: function(status)
+ {
+ dumpn("*** _cancelOrDispatchCancelCallback(" + status + ")");
+
+ NS_ASSERT(this._source === null, "should have finished input");
+ NS_ASSERT(this._sink === null, "should have finished output");
+ NS_ASSERT(this._pendingData.length === 0, "should have no pending data");
+
+ if (!this._canceled)
+ {
+ this.cancel(status);
+ return;
+ }
+
+ var self = this;
+ var event =
+ {
+ run: function()
+ {
+ dumpn("*** onStopRequest async callback");
+
+ self._completed = true;
+ try
+ {
+ self._observer.onStopRequest(self, self._context, self.status);
+ }
+ catch (e)
+ {
+ NS_ASSERT(false,
+ "how are we throwing an exception here? we control " +
+ "all the callers! " + e);
+ }
+ }
+ };
+
+ gThreadManager.currentThread.dispatch(event, Ci.nsIThread.DISPATCH_NORMAL);
+ },
+
+ /**
+ * Kicks off another wait for more data to be available from the input stream.
+ */
+ _waitToReadData: function()
+ {
+ dumpn("*** _waitToReadData");
+ this._source.asyncWait(this, 0, Response.SEGMENT_SIZE,
+ gThreadManager.mainThread);
+ },
+
+ /**
+ * Kicks off another wait until data can be written to the output stream.
+ */
+ _waitToWriteData: function()
+ {
+ dumpn("*** _waitToWriteData");
+
+ var pendingData = this._pendingData;
+ NS_ASSERT(pendingData.length > 0, "no pending data to write?");
+ NS_ASSERT(pendingData[0].length > 0, "buffered an empty write?");
+
+ this._sink.asyncWait(this, 0, pendingData[0].length,
+ gThreadManager.mainThread);
+ },
+
+ /**
+ * Kicks off a wait for the sink to which data is being copied to be closed.
+ * We wait for stream closure when we don't have any data to be copied, rather
+ * than waiting to write a specific amount of data. We can't wait to write
+ * data because the sink might be infinitely writable, and if no data appears
+ * in the source for a long time we might have to spin quite a bit waiting to
+ * write, waiting to write again, &c. Waiting on stream closure instead means
+ * we'll get just one notification if the sink dies. Note that when data
+ * starts arriving from the sink we'll resume waiting for data to be written,
+ * dropping this closure-only callback entirely.
+ */
+ _waitForSinkClosure: function()
+ {
+ dumpn("*** _waitForSinkClosure");
+
+ this._sink.asyncWait(this, Ci.nsIAsyncOutputStream.WAIT_CLOSURE_ONLY, 0,
+ gThreadManager.mainThread);
+ },
+
+ /**
+ * Closes input with the given status, if it hasn't already been closed;
+ * otherwise a no-op.
+ *
+ * @param status : nsresult
+ * status code use to close the source stream if necessary
+ */
+ _finishSource: function(status)
+ {
+ dumpn("*** _finishSource(" + status.toString(16) + ")");
+
+ if (this._source !== null)
+ {
+ this._source.closeWithStatus(status);
+ this._source = null;
+ }
+ }
+};
+
+
+/**
+ * A container for utility functions used with HTTP headers.
+ */
+const headerUtils =
+{
+ /**
+ * Normalizes fieldName (by converting it to lowercase) and ensures it is a
+ * valid header field name (although not necessarily one specified in RFC
+ * 2616).
+ *
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not match the field-name production in RFC 2616
+ * @returns string
+ * fieldName converted to lowercase if it is a valid header, for characters
+ * where case conversion is possible
+ */
+ normalizeFieldName: function(fieldName)
+ {
+ if (fieldName == "")
+ {
+ dumpn("*** Empty fieldName");
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+
+ for (var i = 0, sz = fieldName.length; i < sz; i++)
+ {
+ if (!IS_TOKEN_ARRAY[fieldName.charCodeAt(i)])
+ {
+ dumpn(fieldName + " is not a valid header field name!");
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+ }
+
+ return fieldName.toLowerCase();
+ },
+
+ /**
+ * Ensures that fieldValue is a valid header field value (although not
+ * necessarily as specified in RFC 2616 if the corresponding field name is
+ * part of the HTTP protocol), normalizes the value if it is, and
+ * returns the normalized value.
+ *
+ * @param fieldValue : string
+ * a value to be normalized as an HTTP header field value
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldValue does not match the field-value production in RFC 2616
+ * @returns string
+ * fieldValue as a normalized HTTP header field value
+ */
+ normalizeFieldValue: function(fieldValue)
+ {
+ // field-value = *( field-content | LWS )
+ // field-content = <the OCTETs making up the field-value
+ // and consisting of either *TEXT or combinations
+ // of token, separators, and quoted-string>
+ // TEXT = <any OCTET except CTLs,
+ // but including LWS>
+ // LWS = [CRLF] 1*( SP | HT )
+ //
+ // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
+ // qdtext = <any TEXT except <">>
+ // quoted-pair = "\" CHAR
+ // CHAR = <any US-ASCII character (octets 0 - 127)>
+
+ // Any LWS that occurs between field-content MAY be replaced with a single
+ // SP before interpreting the field value or forwarding the message
+ // downstream (section 4.2); we replace 1*LWS with a single SP
+ var val = fieldValue.replace(/(?:(?:\r\n)?[ \t]+)+/g, " ");
+
+ // remove leading/trailing LWS (which has been converted to SP)
+ val = val.replace(/^ +/, "").replace(/ +$/, "");
+
+ // that should have taken care of all CTLs, so val should contain no CTLs
+ dumpn("*** Normalized value: '" + val + "'");
+ for (var i = 0, len = val.length; i < len; i++)
+ if (isCTL(val.charCodeAt(i)))
+ {
+ dump("*** Char " + i + " has charcode " + val.charCodeAt(i));
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+
+ // XXX disallows quoted-pair where CHAR is a CTL -- will not invalidly
+ // normalize, however, so this can be construed as a tightening of the
+ // spec and not entirely as a bug
+ return val;
+ }
+};
+
+
+
+/**
+ * Converts the given string into a string which is safe for use in an HTML
+ * context.
+ *
+ * @param str : string
+ * the string to make HTML-safe
+ * @returns string
+ * an HTML-safe version of str
+ */
+function htmlEscape(str)
+{
+ // this is naive, but it'll work
+ var s = "";
+ for (var i = 0; i < str.length; i++)
+ s += "&#" + str.charCodeAt(i) + ";";
+ return s;
+}
+
+
+/**
+ * Constructs an object representing an HTTP version (see section 3.1).
+ *
+ * @param versionString
+ * a string of the form "#.#", where # is an non-negative decimal integer with
+ * or without leading zeros
+ * @throws
+ * if versionString does not specify a valid HTTP version number
+ */
+function nsHttpVersion(versionString)
+{
+ var matches = /^(\d+)\.(\d+)$/.exec(versionString);
+ if (!matches)
+ throw "Not a valid HTTP version!";
+
+ /** The major version number of this, as a number. */
+ this.major = parseInt(matches[1], 10);
+
+ /** The minor version number of this, as a number. */
+ this.minor = parseInt(matches[2], 10);
+
+ if (isNaN(this.major) || isNaN(this.minor) ||
+ this.major < 0 || this.minor < 0)
+ throw "Not a valid HTTP version!";
+}
+nsHttpVersion.prototype =
+{
+ /**
+ * Returns the standard string representation of the HTTP version represented
+ * by this (e.g., "1.1").
+ */
+ toString: function ()
+ {
+ return this.major + "." + this.minor;
+ },
+
+ /**
+ * Returns true if this represents the same HTTP version as otherVersion,
+ * false otherwise.
+ *
+ * @param otherVersion : nsHttpVersion
+ * the version to compare against this
+ */
+ equals: function (otherVersion)
+ {
+ return this.major == otherVersion.major &&
+ this.minor == otherVersion.minor;
+ },
+
+ /** True if this >= otherVersion, false otherwise. */
+ atLeast: function(otherVersion)
+ {
+ return this.major > otherVersion.major ||
+ (this.major == otherVersion.major &&
+ this.minor >= otherVersion.minor);
+ }
+};
+
+nsHttpVersion.HTTP_1_0 = new nsHttpVersion("1.0");
+nsHttpVersion.HTTP_1_1 = new nsHttpVersion("1.1");
+
+
+/**
+ * An object which stores HTTP headers for a request or response.
+ *
+ * Note that since headers are case-insensitive, this object converts headers to
+ * lowercase before storing them. This allows the getHeader and hasHeader
+ * methods to work correctly for any case of a header, but it means that the
+ * values returned by .enumerator may not be equal case-sensitively to the
+ * values passed to setHeader when adding headers to this.
+ */
+function nsHttpHeaders()
+{
+ /**
+ * A hash of headers, with header field names as the keys and header field
+ * values as the values. Header field names are case-insensitive, but upon
+ * insertion here they are converted to lowercase. Header field values are
+ * normalized upon insertion to contain no leading or trailing whitespace.
+ *
+ * Note also that per RFC 2616, section 4.2, two headers with the same name in
+ * a message may be treated as one header with the same field name and a field
+ * value consisting of the separate field values joined together with a "," in
+ * their original order. This hash stores multiple headers with the same name
+ * in this manner.
+ */
+ this._headers = {};
+}
+nsHttpHeaders.prototype =
+{
+ /**
+ * Sets the header represented by name and value in this.
+ *
+ * @param name : string
+ * the header name
+ * @param value : string
+ * the header value
+ * @throws NS_ERROR_INVALID_ARG
+ * if name or value is not a valid header component
+ */
+ setHeader: function(fieldName, fieldValue, merge)
+ {
+ var name = headerUtils.normalizeFieldName(fieldName);
+ var value = headerUtils.normalizeFieldValue(fieldValue);
+
+ // The following three headers are stored as arrays because their real-world
+ // syntax prevents joining individual headers into a single header using
+ // ",". See also <http://hg.mozilla.org/mozilla-central/diff/9b2a99adc05e/netwerk/protocol/http/src/nsHttpHeaderArray.cpp#l77>
+ if (merge && name in this._headers)
+ {
+ if (name === "www-authenticate" ||
+ name === "proxy-authenticate" ||
+ name === "set-cookie")
+ {
+ this._headers[name].push(value);
+ }
+ else
+ {
+ this._headers[name][0] += "," + value;
+ NS_ASSERT(this._headers[name].length === 1,
+ "how'd a non-special header have multiple values?")
+ }
+ }
+ else
+ {
+ this._headers[name] = [value];
+ }
+ },
+
+ /**
+ * Returns the value for the header specified by this.
+ *
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not constitute a valid header field name
+ * @throws NS_ERROR_NOT_AVAILABLE
+ * if the given header does not exist in this
+ * @returns string
+ * the field value for the given header, possibly with non-semantic changes
+ * (i.e., leading/trailing whitespace stripped, whitespace runs replaced
+ * with spaces, etc.) at the option of the implementation; multiple
+ * instances of the header will be combined with a comma, except for
+ * the three headers noted in the description of getHeaderValues
+ */
+ getHeader: function(fieldName)
+ {
+ return this.getHeaderValues(fieldName).join("\n");
+ },
+
+ /**
+ * Returns the value for the header specified by fieldName as an array.
+ *
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not constitute a valid header field name
+ * @throws NS_ERROR_NOT_AVAILABLE
+ * if the given header does not exist in this
+ * @returns [string]
+ * an array of all the header values in this for the given
+ * header name. Header values will generally be collapsed
+ * into a single header by joining all header values together
+ * with commas, but certain headers (Proxy-Authenticate,
+ * WWW-Authenticate, and Set-Cookie) violate the HTTP spec
+ * and cannot be collapsed in this manner. For these headers
+ * only, the returned array may contain multiple elements if
+ * that header has been added more than once.
+ */
+ getHeaderValues: function(fieldName)
+ {
+ var name = headerUtils.normalizeFieldName(fieldName);
+
+ if (name in this._headers)
+ return this._headers[name];
+ else
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ },
+
+ /**
+ * Returns true if a header with the given field name exists in this, false
+ * otherwise.
+ *
+ * @param fieldName : string
+ * the field name whose existence is to be determined in this
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not constitute a valid header field name
+ * @returns boolean
+ * true if the header's present, false otherwise
+ */
+ hasHeader: function(fieldName)
+ {
+ var name = headerUtils.normalizeFieldName(fieldName);
+ return (name in this._headers);
+ },
+
+ /**
+ * Returns a new enumerator over the field names of the headers in this, as
+ * nsISupportsStrings. The names returned will be in lowercase, regardless of
+ * how they were input using setHeader (header names are case-insensitive per
+ * RFC 2616).
+ */
+ get enumerator()
+ {
+ var headers = [];
+ for (var i in this._headers)
+ {
+ var supports = new SupportsString();
+ supports.data = i;
+ headers.push(supports);
+ }
+
+ return new nsSimpleEnumerator(headers);
+ }
+};
+
+
+/**
+ * Constructs an nsISimpleEnumerator for the given array of items.
+ *
+ * @param items : Array
+ * the items, which must all implement nsISupports
+ */
+function nsSimpleEnumerator(items)
+{
+ this._items = items;
+ this._nextIndex = 0;
+}
+nsSimpleEnumerator.prototype =
+{
+ hasMoreElements: function()
+ {
+ return this._nextIndex < this._items.length;
+ },
+ getNext: function()
+ {
+ if (!this.hasMoreElements())
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+
+ return this._items[this._nextIndex++];
+ },
+ QueryInterface: function(aIID)
+ {
+ if (Ci.nsISimpleEnumerator.equals(aIID) ||
+ Ci.nsISupports.equals(aIID))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ }
+};
+
+
+/**
+ * A representation of the data in an HTTP request.
+ *
+ * @param port : uint
+ * the port on which the server receiving this request runs
+ */
+function Request(port)
+{
+ /** Method of this request, e.g. GET or POST. */
+ this._method = "";
+
+ /** Path of the requested resource; empty paths are converted to '/'. */
+ this._path = "";
+
+ /** Query string, if any, associated with this request (not including '?'). */
+ this._queryString = "";
+
+ /** Scheme of requested resource, usually http, always lowercase. */
+ this._scheme = "http";
+
+ /** Hostname on which the requested resource resides. */
+ this._host = undefined;
+
+ /** Port number over which the request was received. */
+ this._port = port;
+
+ var bodyPipe = new Pipe(false, false, 0, PR_UINT32_MAX, null);
+
+ /** Stream from which data in this request's body may be read. */
+ this._bodyInputStream = bodyPipe.inputStream;
+
+ /** Stream to which data in this request's body is written. */
+ this._bodyOutputStream = bodyPipe.outputStream;
+
+ /**
+ * The headers in this request.
+ */
+ this._headers = new nsHttpHeaders();
+
+ /**
+ * For the addition of ad-hoc properties and new functionality without having
+ * to change nsIHttpRequest every time; currently lazily created, as its only
+ * use is in directory listings.
+ */
+ this._bag = null;
+}
+Request.prototype =
+{
+ // SERVER METADATA
+
+ //
+ // see nsIHttpRequest.scheme
+ //
+ get scheme()
+ {
+ return this._scheme;
+ },
+
+ //
+ // see nsIHttpRequest.host
+ //
+ get host()
+ {
+ return this._host;
+ },
+
+ //
+ // see nsIHttpRequest.port
+ //
+ get port()
+ {
+ return this._port;
+ },
+
+ // REQUEST LINE
+
+ //
+ // see nsIHttpRequest.method
+ //
+ get method()
+ {
+ return this._method;
+ },
+
+ //
+ // see nsIHttpRequest.httpVersion
+ //
+ get httpVersion()
+ {
+ return this._httpVersion.toString();
+ },
+
+ //
+ // see nsIHttpRequest.path
+ //
+ get path()
+ {
+ return this._path;
+ },
+
+ //
+ // see nsIHttpRequest.queryString
+ //
+ get queryString()
+ {
+ return this._queryString;
+ },
+
+ // HEADERS
+
+ //
+ // see nsIHttpRequest.getHeader
+ //
+ getHeader: function(name)
+ {
+ return this._headers.getHeader(name);
+ },
+
+ //
+ // see nsIHttpRequest.hasHeader
+ //
+ hasHeader: function(name)
+ {
+ return this._headers.hasHeader(name);
+ },
+
+ //
+ // see nsIHttpRequest.headers
+ //
+ get headers()
+ {
+ return this._headers.enumerator;
+ },
+
+ //
+ // see nsIPropertyBag.enumerator
+ //
+ get enumerator()
+ {
+ this._ensurePropertyBag();
+ return this._bag.enumerator;
+ },
+
+ //
+ // see nsIHttpRequest.headers
+ //
+ get bodyInputStream()
+ {
+ return this._bodyInputStream;
+ },
+
+ //
+ // see nsIPropertyBag.getProperty
+ //
+ getProperty: function(name)
+ {
+ this._ensurePropertyBag();
+ return this._bag.getProperty(name);
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpRequest) || iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // PRIVATE IMPLEMENTATION
+
+ /** Ensures a property bag has been created for ad-hoc behaviors. */
+ _ensurePropertyBag: function()
+ {
+ if (!this._bag)
+ this._bag = new WritablePropertyBag();
+ }
+};
+
+
+// XPCOM trappings
+
+this.NSGetFactory = XPCOMUtils.generateNSGetFactory([nsHttpServer]);
+
+/**
+ * Creates a new HTTP server listening for loopback traffic on the given port,
+ * starts it, and runs the server until the server processes a shutdown request,
+ * spinning an event loop so that events posted by the server's socket are
+ * processed.
+ *
+ * This method is primarily intended for use in running this script from within
+ * xpcshell and running a functional HTTP server without having to deal with
+ * non-essential details.
+ *
+ * Note that running multiple servers using variants of this method probably
+ * doesn't work, simply due to how the internal event loop is spun and stopped.
+ *
+ * @note
+ * This method only works with Mozilla 1.9 (i.e., Firefox 3 or trunk code);
+ * you should use this server as a component in Mozilla 1.8.
+ * @param port
+ * the port on which the server will run, or -1 if there exists no preference
+ * for a specific port; note that attempting to use some values for this
+ * parameter (particularly those below 1024) may cause this method to throw or
+ * may result in the server being prematurely shut down
+ * @param basePath
+ * a local directory from which requests will be served (i.e., if this is
+ * "/home/jwalden/" then a request to /index.html will load
+ * /home/jwalden/index.html); if this is omitted, only the default URLs in
+ * this server implementation will be functional
+ */
+function server(port, basePath)
+{
+ if (basePath)
+ {
+ var lp = Cc["@mozilla.org/file/local;1"]
+ .createInstance(Ci.nsILocalFile);
+ lp.initWithPath(basePath);
+ }
+
+ // if you're running this, you probably want to see debugging info
+ DEBUG = true;
+
+ var srv = new nsHttpServer();
+ if (lp)
+ srv.registerDirectory("/", lp);
+ srv.registerContentType("sjs", SJS_TYPE);
+ srv.identity.setPrimary("http", "localhost", port);
+ srv.start(port);
+
+ var thread = gThreadManager.currentThread;
+ while (!srv.isStopped())
+ thread.processNextEvent(true);
+
+ // get rid of any pending requests
+ while (thread.hasPendingEvents())
+ thread.processNextEvent(true);
+
+ DEBUG = false;
+}
diff --git a/test/tests/data/snapshot/img.gif b/test/tests/data/snapshot/img.gif
Binary files differ.
diff --git a/test/tests/data/test.html b/test/tests/data/test.html
@@ -0,0 +1,8 @@
+<html>
+ <head>
+ <meta charset="utf-8"/>
+ </head>
+ <body>
+ <p>This is a test.</p>
+ </body>
+</html>
diff --git a/test/tests/data/test.txt b/test/tests/data/test.txt
@@ -0,0 +1 @@
+This is a test file.
diff --git a/test/tests/itemTest.js b/test/tests/itemTest.js
@@ -537,6 +537,10 @@ describe("Zotero.Item", function () {
file.append(filename);
assert.equal(item.getFilePath(), file.path);
});
+
+ it.skip("should get and set a filename for a base-dir-relative file", function* () {
+
+ })
})
describe("#attachmentPath", function () {
@@ -608,11 +612,13 @@ describe("Zotero.Item", function () {
assert.equal(OS.Path.basename(path), newName)
yield OS.File.exists(path);
+ // File should be flagged for upload
+ // DEBUG: Is this necessary?
assert.equal(
- (yield Zotero.Sync.Storage.getSyncState(item.id)),
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
- assert.isNull(yield Zotero.Sync.Storage.getSyncedHash(item.id));
+ assert.isNull(yield Zotero.Sync.Storage.Local.getSyncedHash(item.id));
})
})
diff --git a/test/tests/storageEngineTest.js b/test/tests/storageEngineTest.js
@@ -0,0 +1,822 @@
+"use strict";
+
+describe("Zotero.Sync.Storage.Engine", function () {
+ Components.utils.import("resource://zotero-unit/httpd.js");
+
+ var win;
+ var apiKey = Zotero.Utilities.randomString(24);
+ var port = 16213;
+ var baseURL = `http://localhost:${port}/`;
+ var server;
+
+ var responses = {};
+
+ var setup = Zotero.Promise.coroutine(function* (options = {}) {
+ server = sinon.fakeServer.create();
+ server.autoRespond = true;
+
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ var caller = new ConcurrentCaller(1);
+ caller.setLogger(msg => Zotero.debug(msg));
+ caller.stopOnError = true;
+
+ Components.utils.import("resource://zotero/config.js");
+ var client = new Zotero.Sync.APIClient({
+ baseURL,
+ apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
+ apiKey,
+ caller,
+ background: options.background || true
+ });
+
+ var engine = new Zotero.Sync.Storage.Engine({
+ apiClient: client,
+ libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
+ stopOnError: true
+ });
+
+ return { engine, client, caller };
+ });
+
+ function setResponse(response) {
+ setHTTPResponse(server, baseURL, response, responses);
+ }
+
+ function parseQueryString(str) {
+ var queryStringParams = str.split('&');
+ var params = {};
+ for (let param of queryStringParams) {
+ let [ key, val ] = param.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ return params;
+ }
+
+ function assertAPIKey(request) {
+ assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
+ }
+
+ //
+ // Tests
+ //
+ before(function* () {
+ })
+ beforeEach(function* () {
+ Zotero.debug("BEFORE HERE");
+ yield resetDB({
+ thisArg: this,
+ skipBundledFiles: true
+ });
+ Zotero.debug("DONE RESET");
+ win = yield loadZoteroPane();
+
+ Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
+
+ this.httpd = new HttpServer();
+ this.httpd.start(port);
+
+ yield Zotero.Users.setCurrentUserID(1);
+ yield Zotero.Users.setCurrentUsername("testuser");
+
+ // Set download-on-sync by default
+ Zotero.Sync.Storage.Local.downloadOnSync(
+ Zotero.Libraries.userLibraryID, true
+ );
+ Zotero.debug("DONE BEFORE");
+ })
+ afterEach(function* () {
+ var defer = new Zotero.Promise.defer();
+ this.httpd.stop(() => defer.resolve());
+ yield defer.promise;
+ win.close();
+ })
+ after(function* () {
+ this.timeout(60000);
+ //yield resetDB();
+ win.close();
+ })
+
+
+ describe("ZFS", function () {
+ describe("Syncing", function () {
+ it("should skip downloads if no last storage sync time", function* () {
+ var { engine, client, caller } = yield setup();
+
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 404
+ });
+ var result = yield engine.start();
+
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.isFalse(Zotero.Libraries.userLibrary.lastStorageSync);
+ })
+
+ it("should skip downloads if unchanged last storage sync time", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ var library = Zotero.Libraries.userLibrary;
+ library.lastStorageSync = newStorageSyncTime;
+ yield library.saveTx();
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ var result = yield engine.start();
+
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.equal(library.lastStorageSync, newStorageSyncTime);
+ })
+
+ it("should ignore a remotely missing file", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 404, null);
+ }
+ }
+ );
+ var result = yield engine.start();
+
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ })
+
+ it("should handle a remotely failing file", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 500, null);
+ }
+ }
+ );
+ // TODO: In stopOnError mode, this the promise is rejected.
+ // This should probably test with stopOnError mode turned off instead.
+ var e = yield getPromiseError(engine.start());
+ assert.equal(e.message, Zotero.Sync.Storage.defaultError);
+ })
+
+ it("should download a missing file", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ // TODO: Test binary data
+ var text = Zotero.Utilities.randomString();
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var mtime = "1441252524905";
+ var md5 = Zotero.Utilities.Internal.md5(text)
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ var s3Path = `pretend-s3/${item.key}`;
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ if (!request.hasHeader('Zotero-API-Key')) {
+ response.setStatusLine(null, 403, "Forbidden");
+ return;
+ }
+ var key = request.getHeader('Zotero-API-Key');
+ if (key != apiKey) {
+ response.setStatusLine(null, 403, "Invalid key");
+ return;
+ }
+ response.setStatusLine(null, 302, "Found");
+ response.setHeader("Zotero-File-Modification-Time", mtime, false);
+ response.setHeader("Zotero-File-MD5", md5, false);
+ response.setHeader("Zotero-File-Compressed", "No", false);
+ response.setHeader("Location", baseURL + s3Path, false);
+ }
+ }
+ );
+ this.httpd.registerPathHandler(
+ "/" + s3Path,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 200, "OK");
+ response.write(text);
+ }
+ }
+ );
+ var result = yield engine.start();
+
+ assert.isTrue(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
+ assert.equal(contents, text);
+ })
+
+ it("should upload new files", function* () {
+ var { engine, client, caller } = yield setup();
+
+ // Single file
+ var file1 = getTestDataDirectory();
+ file1.append('test.png');
+ var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
+ var mtime1 = yield item1.attachmentModificationTime;
+ var hash1 = yield item1.attachmentHash;
+ var path1 = item1.getFilePath();
+ var filename1 = 'test.png';
+ var size1 = (yield OS.File.stat(path1)).size;
+ var contentType1 = 'image/png';
+ var prefix1 = Zotero.Utilities.randomString();
+ var suffix1 = Zotero.Utilities.randomString();
+ var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
+
+ // HTML file with auxiliary image
+ var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
+ var parentItem = yield createDataObject('item');
+ var item2 = yield Zotero.Attachments.importSnapshotFromFile({
+ file: file2,
+ url: 'http://example.com/',
+ parentItemID: parentItem.id,
+ title: 'Test',
+ contentType: 'text/html',
+ charset: 'utf-8'
+ });
+ var mtime2 = yield item2.attachmentModificationTime;
+ var hash2 = yield item2.attachmentHash;
+ var path2 = item2.getFilePath();
+ var filename2 = 'index.html';
+ var size2 = (yield OS.File.stat(path2)).size;
+ var contentType2 = 'text/html';
+ var charset2 = 'utf-8';
+ var prefix2 = Zotero.Utilities.randomString();
+ var suffix2 = Zotero.Utilities.randomString();
+ var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
+
+ var deferreds = [];
+
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 404
+ });
+ // https://github.com/cjohansen/Sinon.JS/issues/607
+ let fixSinonBug = ";charset=utf-8";
+ server.respond(function (req) {
+ // Get upload authorization for single file
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item1.key}/file`
+ && req.requestBody.indexOf('upload=') == -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ assert.equal(params.md5, hash1);
+ assert.equal(params.mtime, mtime1);
+ assert.equal(params.filename, filename1);
+ assert.equal(params.filesize, size1);
+ assert.equal(params.contentType, contentType1);
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json"
+ },
+ JSON.stringify({
+ url: baseURL + "pretend-s3/1",
+ contentType: contentType1,
+ prefix: prefix1,
+ suffix: suffix1,
+ uploadKey: uploadKey1
+ })
+ );
+ }
+ // Get upload authorization for multi-file zip
+ else if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item2.key}/file`
+ && req.requestBody.indexOf('upload=') == -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ // Verify ZIP hash
+ let tmpZipPath = OS.Path.join(
+ Zotero.getTempDirectory().path,
+ item2.key + '.zip'
+ );
+ deferreds.push({
+ promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
+ .then(function (md5) {
+ assert.equal(params.zipMD5, md5);
+ })
+ });
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ Zotero.debug(params);
+ assert.equal(params.md5, hash2);
+ assert.notEqual(params.zipMD5, hash2);
+ assert.equal(params.mtime, mtime2);
+ assert.equal(params.filename, filename2);
+ assert.equal(params.zipFilename, item2.key + ".zip");
+ assert.isTrue(parseInt(params.filesize) == params.filesize);
+ assert.equal(params.contentType, contentType2);
+ assert.equal(params.charset, charset2);
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json"
+ },
+ JSON.stringify({
+ url: baseURL + "pretend-s3/2",
+ contentType: 'application/zip',
+ prefix: prefix2,
+ suffix: suffix2,
+ uploadKey: uploadKey2
+ })
+ );
+ }
+ // Upload single file to S3
+ else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
+ assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
+ assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
+ req.respond(201, {}, "");
+ }
+ // Upload multi-file ZIP to S3
+ else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
+ assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
+
+ // Verify uploaded ZIP file
+ let tmpZipPath = OS.Path.join(
+ Zotero.getTempDirectory().path,
+ Zotero.Utilities.randomString() + '.zip'
+ );
+
+ let deferred = Zotero.Promise.defer();
+ deferreds.push(deferred);
+ var reader = new FileReader();
+ reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
+ try {
+
+ let file = yield OS.File.open(tmpZipPath, {
+ create: true
+ });
+
+ var contents = new Uint8Array(reader.result);
+ contents = contents.slice(prefix2.length, suffix2.length * -1);
+ yield file.write(contents);
+ yield file.close();
+
+ var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
+ .createInstance(Components.interfaces.nsIZipReader);
+ zr.open(Zotero.File.pathToFile(tmpZipPath));
+ zr.test(null);
+ var entries = zr.findEntries('*');
+ var entryNames = [];
+ while (entries.hasMore()) {
+ entryNames.push(entries.getNext());
+ }
+ assert.equal(entryNames.length, 2);
+ assert.sameMembers(entryNames, ['index.html', 'img.gif']);
+ assert.equal(zr.getEntry('index.html').realSize, size2);
+ assert.equal(zr.getEntry('img.gif').realSize, 42);
+
+ deferred.resolve();
+ }
+ catch (e) {
+ deferred.reject(e);
+ }
+ }));
+ reader.readAsArrayBuffer(req.requestBody);
+
+ req.respond(201, {}, "");
+ }
+ // Register single-file upload
+ else if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item1.key}/file`
+ && req.requestBody.indexOf('upload=') != -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ assert.equal(params.upload, uploadKey1);
+
+ req.respond(
+ 204,
+ {
+ "Last-Modified-Version": 10
+ },
+ ""
+ );
+ }
+ // Register multi-file upload
+ else if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item2.key}/file`
+ && req.requestBody.indexOf('upload=') != -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ assert.equal(params.upload, uploadKey2);
+
+ req.respond(
+ 204,
+ {
+ "Last-Modified-Version": 15
+ },
+ ""
+ );
+ }
+ })
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "POST",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+
+ // TODO: One-step uploads
+ /*// https://github.com/cjohansen/Sinon.JS/issues/607
+ let fixSinonBug = ";charset=utf-8";
+ server.respond(function (req) {
+ if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/json" + fixSinonBug
+ );
+
+ let params = JSON.parse(req.requestBody);
+ assert.equal(params.md5, hash);
+ assert.equal(params.mtime, mtime);
+ assert.equal(params.filename, filename);
+ assert.equal(params.size, size);
+ assert.equal(params.contentType, contentType);
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json"
+ },
+ JSON.stringify({
+ url: baseURL + "pretend-s3",
+ headers: {
+ "Content-Type": contentType,
+ "Content-MD5": hash,
+ //"Content-Length": params.size, process but don't return
+ //"x-amz-meta-"
+ },
+ uploadKey
+ })
+ );
+ }
+ else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
+ assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
+ assert.instanceOf(req.requestBody, File);
+ req.respond(201, {}, "");
+ }
+ })*/
+ var result = yield engine.start();
+
+ yield Zotero.Promise.all(deferreds.map(d => d.promise));
+
+ assert.isTrue(result.localChanges);
+ assert.isTrue(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check local objects
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
+ assert.equal(item1.version, 10);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
+ assert.equal(item2.version, 15);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ })
+
+ it("should update local info for file that already exists on the server", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var file = getTestDataDirectory();
+ file.append('test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file: file });
+ item.version = 5;
+ yield item.saveTx();
+ var json = yield item.toJSON();
+ yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
+
+ var mtime = yield item.attachmentModificationTime;
+ var hash = yield item.attachmentHash;
+ var path = item.getFilePath();
+ var filename = 'test.png';
+ var size = (yield OS.File.stat(path)).size;
+ var contentType = 'image/png';
+
+ var newVersion = 10;
+ setResponse({
+ method: "POST",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + (Math.round(new Date().getTime() / 1000) - 50000)
+ });
+ // https://github.com/cjohansen/Sinon.JS/issues/607
+ let fixSinonBug = ";charset=utf-8";
+ server.respond(function (req) {
+ // Get upload authorization for single file
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": newVersion
+ },
+ JSON.stringify({
+ exists: 1,
+ })
+ );
+ }
+ })
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "POST",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+
+ // TODO: One-step uploads
+ var result = yield engine.start();
+
+ assert.isTrue(result.localChanges);
+ assert.isTrue(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check local objects
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
+ assert.equal(item.version, newVersion);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ })
+ })
+
+ describe("#_processUploadFile()", function () {
+ it("should handle 412 with matching version and hash matching local file", function* () {
+ var { engine, client, caller } = yield setup();
+ var zfs = new Zotero.Sync.Storage.ZFS_Module({
+ apiClient: client
+ })
+
+ var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file: filePath });
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ var itemJSON = yield item.toResponseJSON();
+
+ // Set saved hash to a different value, which should be overwritten
+ //
+ // We're also testing cases where a hash isn't set for a file (e.g., if the
+ // storage directory was transferred, the mtime doesn't match, but the file was
+ // never downloaded), but there's no difference in behavior
+ var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
+ });
+
+ server.respond(function (req) {
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1
+ && req.requestHeaders["If-Match"] == dbHash) {
+ req.respond(
+ 412,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": 5
+ },
+ "ETag does not match current version of file"
+ );
+ }
+ })
+ setResponse({
+ method: "GET",
+ url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
+ status: 200,
+ text: JSON.stringify([itemJSON])
+ });
+
+ var result = yield zfs._processUploadFile({
+ name: item.libraryKey
+ });
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncedHash(item.id), itemJSON.data.md5
+ );
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+ assert.isFalse(result.fileSyncRequired);
+ })
+
+ it("should handle 412 with matching version and hash not matching local file", function* () {
+ var { engine, client, caller } = yield setup();
+ var zfs = new Zotero.Sync.Storage.ZFS_Module({
+ apiClient: client
+ })
+
+ var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file: filePath });
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ var fileHash = yield item.attachmentHash;
+ var itemJSON = yield item.toResponseJSON();
+ itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
+
+ server.respond(function (req) {
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1
+ && req.requestHeaders["If-None-Match"] == "*") {
+ req.respond(
+ 412,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": 5
+ },
+ "If-None-Match: * set but file exists"
+ );
+ }
+ })
+ setResponse({
+ method: "GET",
+ url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
+ status: 200,
+ text: JSON.stringify([itemJSON])
+ });
+
+ var result = yield zfs._processUploadFile({
+ name: item.libraryKey
+ });
+ yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncState(item.id),
+ Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+ assert.isTrue(result.fileSyncRequired);
+ })
+
+ it("should handle 412 with greater version", function* () {
+ var { engine, client, caller } = yield setup();
+ var zfs = new Zotero.Sync.Storage.ZFS_Module({
+ apiClient: client
+ })
+
+ var file = getTestDataDirectory();
+ file.append('test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file });
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ server.respond(function (req) {
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1
+ && req.requestHeaders["If-None-Match"] == "*") {
+ req.respond(
+ 412,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": 10
+ },
+ "If-None-Match: * set but file exists"
+ );
+ }
+ })
+
+ var result = yield zfs._processUploadFile({
+ name: item.libraryKey
+ });
+ assert.equal(item.version, 5);
+ assert.equal(item.synced, true);
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isTrue(result.syncRequired);
+ })
+ })
+ })
+})
diff --git a/test/tests/storageLocalTest.js b/test/tests/storageLocalTest.js
@@ -0,0 +1,329 @@
+"use strict";
+
+describe("Zotero.Sync.Storage.Local", function () {
+ var win;
+
+ before(function* () {
+ win = yield loadBrowserWindow();
+ });
+ beforeEach(function* () {
+ yield resetDB({
+ thisArg: this
+ })
+ })
+ after(function () {
+ if (win) {
+ win.close();
+ }
+ });
+
+ describe("#checkForUpdatedFiles()", function () {
+ it("should flag modified file for upload and return it", function* () {
+ // Create attachment
+ let item = yield importFileAttachment('test.txt')
+ var hash = yield item.attachmentHash;
+ // Set file mtime to the past (without milliseconds, which aren't used on OS X)
+ var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
+ yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
+
+ // Mark as synced, so it will be checked
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Update mtime and contents
+ var path = yield item.getFilePathAsync();
+ yield OS.File.setDates(path);
+ yield Zotero.File.putContentsAsync(path, Zotero.Utilities.randomString());
+
+ // File should be returned
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
+
+ yield item.eraseTx();
+
+ assert.equal(changed, true);
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
+ );
+ })
+
+ it("should skip a file if mod time hasn't changed", function* () {
+ // Create attachment
+ let item = yield importFileAttachment('test.txt')
+ var hash = yield item.attachmentHash;
+ var mtime = yield item.attachmentModificationTime;
+
+ // Mark as synced, so it will be checked
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
+ var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
+
+ yield item.eraseTx();
+
+ assert.isFalse(changed);
+ assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ })
+
+ it("should skip a file if mod time has changed but contents haven't", function* () {
+ // Create attachment
+ let item = yield importFileAttachment('test.txt')
+ var hash = yield item.attachmentHash;
+ // Set file mtime to the past (without milliseconds, which aren't used on OS X)
+ var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
+ yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
+
+ // Mark as synced, so it will be checked
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Update mtime, but not contents
+ var path = yield item.getFilePathAsync();
+ yield OS.File.setDates(path);
+
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
+ var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
+ var syncedModTime = yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id);
+ var newModTime = yield item.attachmentModificationTime;
+
+ yield item.eraseTx();
+
+ assert.isFalse(changed);
+ assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ assert.equal(syncedModTime, newModTime);
+ })
+ })
+
+ describe("#processDownload()", function () {
+ var file1Name = 'index.html';
+ var file1Contents = '<html><body>Test</body></html>';
+ var file2Name = 'test.txt';
+ var file2Contents = 'Test';
+
+ var createZIP = Zotero.Promise.coroutine(function* (zipFile) {
+ var tmpDir = Zotero.getTempDirectory().path;
+ var zipDir = OS.Path.join(tmpDir, Zotero.Utilities.randomString());
+ yield OS.File.makeDir(zipDir);
+
+ yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file1Name), file1Contents);
+ yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file2Name), file2Contents);
+
+ yield Zotero.File.zipDirectory(zipDir, zipFile);
+ yield OS.File.removeDir(zipDir);
+ });
+
+ it("should download and extract a ZIP file into the attachment directory", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var parentItem = yield createDataObject('item');
+ var key = Zotero.DataObjectUtilities.generateKey();
+
+ var tmpDir = Zotero.getTempDirectory().path;
+ var zipFile = OS.Path.join(tmpDir, key + '.tmp');
+ yield createZIP(zipFile);
+
+ var md5 = Zotero.Utilities.Internal.md5(Zotero.File.pathToFile(zipFile));
+ var mtime = 1445667239000;
+
+ var json = {
+ key,
+ version: 10,
+ itemType: 'attachment',
+ linkMode: 'imported_url',
+ url: 'https://example.com',
+ filename: file1Name,
+ contentType: 'text/html',
+ charset: 'utf-8',
+ md5,
+ mtime
+ };
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ 'item', libraryID, [json]
+ );
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+ var item = yield Zotero.Items.getByLibraryAndKeyAsync(libraryID, key);
+ yield Zotero.Sync.Storage.Local.processDownload({
+ item,
+ md5,
+ mtime,
+ compressed: true
+ });
+ yield OS.File.remove(zipFile);
+
+ yield assert.eventually.equal(
+ item.attachmentHash, Zotero.Utilities.Internal.md5(file1Contents)
+ );
+ yield assert.eventually.equal(item.attachmentModificationTime, mtime);
+ })
+ })
+
+ describe("#_deleteExistingAttachmentFiles()", function () {
+ it("should delete all files", function* () {
+ var item = yield importFileAttachment('test.html');
+ var path = OS.Path.dirname(item.getFilePath());
+ var files = ['a', 'b', 'c', 'd'];
+ for (let file of files) {
+ yield Zotero.File.putContentsAsync(OS.Path.join(path, file), file);
+ }
+ yield Zotero.Sync.Storage.Local._deleteExistingAttachmentFiles(item);
+ for (let file of files) {
+ assert.isFalse(
+ (yield OS.File.exists(OS.Path.join(path, file))),
+ `File '${file}' doesn't exist`
+ );
+ }
+ })
+ })
+
+ describe("#getConflicts()", function () {
+ it("should return an array of objects for attachments in conflict", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+
+ var item1 = yield importFileAttachment('test.png');
+ item1.version = 10;
+ yield item1.saveTx();
+ var item2 = yield importFileAttachment('test.txt');
+ var item3 = yield importFileAttachment('test.html');
+ item3.version = 11;
+ yield item3.saveTx();
+
+ var json1 = yield item1.toJSON();
+ var json3 = yield item3.toJSON();
+ // Change remote mtimes
+ // Round to nearest second because OS X doesn't support ms resolution
+ var now = Math.round(new Date().getTime() / 1000) * 1000;
+ json1.mtime = now - 10000;
+ json3.mtime = now - 20000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
+
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+
+ var conflicts = yield Zotero.Sync.Storage.Local.getConflicts(libraryID);
+ assert.lengthOf(conflicts, 2);
+
+ var item1Conflict = conflicts.find(x => x.left.key == item1.key);
+ assert.equal(
+ item1Conflict.left.dateModified,
+ Zotero.Date.dateToISO(new Date(yield item1.attachmentModificationTime))
+ );
+ assert.equal(
+ item1Conflict.right.dateModified,
+ Zotero.Date.dateToISO(new Date(json1.mtime))
+ );
+
+ var item3Conflict = conflicts.find(x => x.left.key == item3.key);
+ assert.equal(
+ item3Conflict.left.dateModified,
+ Zotero.Date.dateToISO(new Date(yield item3.attachmentModificationTime))
+ );
+ assert.equal(
+ item3Conflict.right.dateModified,
+ Zotero.Date.dateToISO(new Date(json3.mtime))
+ );
+ })
+ })
+
+ describe("#resolveConflicts()", function () {
+ it("should show the conflict resolution window on attachment conflicts", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+
+ var item1 = yield importFileAttachment('test.png');
+ item1.version = 10;
+ yield item1.saveTx();
+ var item2 = yield importFileAttachment('test.txt');
+ var item3 = yield importFileAttachment('test.html');
+ item3.version = 11;
+ yield item3.saveTx();
+
+ var json1 = yield item1.toJSON();
+ var json3 = yield item3.toJSON();
+ // Change remote mtimes
+ json1.mtime = new Date().getTime() + 10000;
+ json3.mtime = new Date().getTime() - 10000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
+
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+
+ var promise = waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
+ var doc = dialog.document;
+ var wizard = doc.documentElement;
+ var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
+
+ // 1 (remote)
+ // Later remote version should be selected
+ assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
+
+ // Check checkbox text
+ assert.equal(
+ doc.getElementById('resolve-all').label,
+ Zotero.getString('sync.conflict.resolveAllRemote')
+ );
+
+ // Select local object
+ mergeGroup.leftpane.click();
+ assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
+
+ wizard.getButton('next').click();
+
+ // 2 (local)
+ // Later local version should be selected
+ assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
+ // Select remote object
+ mergeGroup.rightpane.click();
+ assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
+
+ if (Zotero.isMac) {
+ assert.isTrue(wizard.getButton('next').hidden);
+ assert.isFalse(wizard.getButton('finish').hidden);
+ }
+ else {
+ // TODO
+ }
+ wizard.getButton('finish').click();
+ })
+ yield Zotero.Sync.Storage.Local.resolveConflicts(libraryID);
+ yield promise;
+
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncState(item1.id),
+ Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
+ );
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncState(item3.id),
+ Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
+ );
+ })
+ })
+
+
+})
diff --git a/test/tests/storageRequestTest.js b/test/tests/storageRequestTest.js
@@ -0,0 +1,22 @@
+"use strict";
+
+describe("Zotero.Sync.Storage.Request", function () {
+ describe("#run()", function () {
+ it("should run a request and wait for it to complete", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var count = 0;
+ var request = new Zotero.Sync.Storage.Request({
+ type: 'download',
+ libraryID,
+ name: "1/AAAAAAAA",
+ onStart: Zotero.Promise.coroutine(function* () {
+ yield Zotero.Promise.delay(25);
+ count++;
+ return new Zotero.Sync.Storage.Result;
+ })
+ });
+ var results = yield request.start();
+ assert.equal(count, 1);
+ })
+ })
+})
diff --git a/test/tests/syncEngineTest.js b/test/tests/syncEngineTest.js
@@ -19,28 +19,20 @@ describe("Zotero.Sync.Data.Engine", function () {
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
- caller.onError = function (e) {
- Zotero.logError(e);
- if (options.onError) {
- options.onError(e);
- }
- if (e.fatal) {
- caller.stop();
- throw e;
- }
- };
+ Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
- baseURL: baseURL,
+ baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
- apiKey: apiKey,
- concurrentCaller: caller,
+ apiKey,
+ caller,
background: options.background || true
});
var engine = new Zotero.Sync.Data.Engine({
apiClient: client,
- libraryID: options.libraryID || Zotero.Libraries.userLibraryID
+ libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
+ stopOnError: true
});
return { engine, client, caller };
diff --git a/test/tests/syncLocalTest.js b/test/tests/syncLocalTest.js
@@ -4,7 +4,7 @@ describe("Zotero.Sync.Data.Local", function() {
describe("#processSyncCacheForObjectType()", function () {
var types = Zotero.DataObjectUtilities.getTypes();
- it("should update local version number if remote version is identical", function* () {
+ it("should update local version number and mark as synced if remote version is identical", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
for (let type of types) {
@@ -24,11 +24,167 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
- assert.equal(
- objectsClass.getByLibraryAndKey(libraryID, obj.key).version, 10
- );
+ let localObj = objectsClass.getByLibraryAndKey(libraryID, obj.key);
+ assert.equal(localObj.version, 10);
+ assert.isTrue(localObj.synced);
}
})
+
+ it("should keep local item changes while applying non-conflicting remote changes", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+
+ var type = 'item';
+ let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
+ let obj = yield createDataObject(type, { version: 5 });
+ let data = yield obj.toJSON();
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ type, libraryID, [data]
+ );
+
+ // Change local title
+ yield modifyDataObject(obj)
+ var changedTitle = obj.getField('title');
+
+ // Save remote version to cache without title but with changed place
+ data.key = obj.key;
+ data.version = 10;
+ var changedPlace = data.place = 'New York';
+ let json = {
+ key: obj.key,
+ version: 10,
+ data: data
+ };
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ type, libraryID, [json]
+ );
+
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, type, { stopOnError: true }
+ );
+ assert.equal(obj.version, 10);
+ assert.equal(obj.getField('title'), changedTitle);
+ assert.equal(obj.getField('place'), changedPlace);
+ })
+
+ it("should mark new attachment items for download", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
+
+ var key = Zotero.DataObjectUtilities.generateKey();
+ var version = 10;
+ var json = {
+ key,
+ version,
+ data: {
+ key,
+ version,
+ itemType: 'attachment',
+ linkMode: 'imported_file',
+ md5: '57f8a4fda823187b91e1191487b87fe6',
+ mtime: 1442261130615
+ }
+ };
+
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ 'item', Zotero.Libraries.userLibraryID, [json]
+ );
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+ var id = Zotero.Items.getIDFromLibraryAndKey(libraryID, key);
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ })
+
+ it("should mark updated attachment items for download", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
+
+ var item = yield importFileAttachment('test.png');
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ // Set file as synced
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, (yield item.attachmentModificationTime)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncedHash(
+ item.id, (yield item.attachmentHash)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Simulate download of version with updated attachment
+ var json = yield item.toResponseJSON();
+ json.version = 10;
+ json.data.version = 10;
+ json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
+ json.data.mtime = new Date().getTime() + 10000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ 'item', Zotero.Libraries.userLibraryID, [json]
+ );
+
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ })
+
+ it("should ignore attachment metadata when resolving metadata conflict", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
+
+ var item = yield importFileAttachment('test.png');
+ item.version = 5;
+ yield item.saveTx();
+ var json = yield item.toResponseJSON();
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
+
+ // Set file as synced
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, (yield item.attachmentModificationTime)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncedHash(
+ item.id, (yield item.attachmentHash)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Modify title locally, leaving item unsynced
+ var newTitle = Zotero.Utilities.randomString();
+ item.setField('title', newTitle);
+ yield item.saveTx();
+
+ // Simulate download of version with original title but updated attachment
+ json.version = 10;
+ json.data.version = 10;
+ json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
+ json.data.mtime = new Date().getTime() + 10000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
+
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+
+ assert.equal(item.getField('title'), newTitle);
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ })
})
describe("Conflict Resolution", function () {
@@ -232,7 +388,10 @@ describe("Zotero.Sync.Data.Local", function() {
jsonData.title = Zotero.Utilities.randomString();
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
+ var windowOpened = false;
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
+ windowOpened = true;
+
var doc = dialog.document;
var wizard = doc.documentElement;
var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
@@ -240,12 +399,14 @@ describe("Zotero.Sync.Data.Local", function() {
// Remote version should be selected by default
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
assert.ok(mergeGroup.leftpane.pane.onclick);
+ // Select local deleted version
mergeGroup.leftpane.pane.click();
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
+ assert.isTrue(windowOpened);
obj = objectsClass.getByLibraryAndKey(libraryID, key);
assert.isFalse(obj);
@@ -825,15 +986,28 @@ describe("Zotero.Sync.Data.Local", function() {
assert.sameDeepMembers(
result.conflicts,
[
- {
- field: "place",
- op: "delete"
- },
- {
- field: "date",
- op: "add",
- value: "2015-05-15"
- }
+ [
+ {
+ field: "place",
+ op: "add",
+ value: "Place"
+ },
+ {
+ field: "place",
+ op: "delete"
+ }
+ ],
+ [
+ {
+ field: "date",
+ op: "delete"
+ },
+ {
+ field: "date",
+ op: "add",
+ value: "2015-05-15"
+ }
+ ]
]
);
})
@@ -1296,4 +1470,68 @@ describe("Zotero.Sync.Data.Local", function() {
})
})
})
+
+
+ describe("#reconcileChangesWithoutCache()", function () {
+ it("should return conflict for conflicting fields", function () {
+ var json1 = {
+ key: "AAAAAAAA",
+ version: 1234,
+ title: "Title 1",
+ pages: 10,
+ dateModified: "2015-05-14 14:12:34"
+ };
+ var json2 = {
+ key: "AAAAAAAA",
+ version: 1235,
+ title: "Title 2",
+ place: "New York",
+ dateModified: "2015-05-14 13:45:12"
+ };
+ var ignoreFields = ['dateAdded', 'dateModified'];
+ var result = Zotero.Sync.Data.Local._reconcileChangesWithoutCache(
+ 'item', json1, json2, ignoreFields
+ );
+ assert.lengthOf(result.changes, 0);
+ assert.sameDeepMembers(
+ result.conflicts,
+ [
+ [
+ {
+ field: "title",
+ op: "add",
+ value: "Title 1"
+ },
+ {
+ field: "title",
+ op: "add",
+ value: "Title 2"
+ }
+ ],
+ [
+ {
+ field: "pages",
+ op: "add",
+ value: 10
+ },
+ {
+ field: "pages",
+ op: "delete"
+ }
+ ],
+ [
+ {
+ field: "place",
+ op: "delete"
+ },
+ {
+ field: "place",
+ op: "add",
+ value: "New York"
+ }
+ ]
+ ]
+ );
+ })
+ })
})
diff --git a/test/tests/syncRunnerTest.js b/test/tests/syncRunnerTest.js
@@ -5,7 +5,7 @@ describe("Zotero.Sync.Runner", function () {
var apiKey = Zotero.Utilities.randomString(24);
var baseURL = "http://local.zotero/";
- var userLibraryID, publicationsLibraryID, runner, caller, server, client, stub, spy;
+ var userLibraryID, publicationsLibraryID, runner, caller, server, stub, spy;
var responses = {
keyInfo: {
@@ -129,15 +129,7 @@ describe("Zotero.Sync.Runner", function () {
}
};
- var client = new Zotero.Sync.APIClient({
- baseURL: baseURL,
- apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
- apiKey: apiKey,
- concurrentCaller: caller,
- background: options.background || true
- });
-
- return { runner, caller, client };
+ return { runner, caller };
})
function setResponse(response) {
@@ -160,7 +152,7 @@ describe("Zotero.Sync.Runner", function () {
server = sinon.fakeServer.create();
server.autoRespond = true;
- ({ runner, caller, client } = yield setup());
+ ({ runner, caller } = yield setup());
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
@@ -180,7 +172,7 @@ describe("Zotero.Sync.Runner", function () {
it("should check key access", function* () {
spy = sinon.spy(runner, "checkUser");
setResponse('keyInfo.fullAccess');
- var json = yield runner.checkAccess(client);
+ var json = yield runner.checkAccess(runner.getAPIClient());
sinon.assert.calledWith(spy, 1, "Username");
var compare = {};
Object.assign(compare, responses.keyInfo.fullAccess.json);
@@ -216,7 +208,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
assert.sameMembers(
@@ -240,19 +232,25 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json, [userLibraryID]
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json, [userLibraryID]
);
assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [userLibraryID]);
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json, [userLibraryID, publicationsLibraryID]
+ runner.getAPIClient(),
+ false,
+ responses.keyInfo.fullAccess.json,
+ [userLibraryID, publicationsLibraryID]
);
assert.lengthOf(libraries, 2);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID]);
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json, [group1.libraryID]
+ runner.getAPIClient(),
+ false,
+ responses.keyInfo.fullAccess.json,
+ [group1.libraryID]
);
assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [group1.libraryID]);
@@ -277,7 +275,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
assert.sameMembers(
@@ -318,7 +316,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
- client,
+ runner.getAPIClient(),
false,
responses.keyInfo.fullAccess.json,
[group1.libraryID, group2.libraryID]
@@ -339,7 +337,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
var groupData1 = responses.groups.ownerGroup;
@@ -370,7 +368,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group1.name);
});
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group2.libraryID]);
@@ -388,7 +386,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name);
}, "extra1");
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group.libraryID]);
@@ -405,7 +403,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name);
}, "cancel");
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 0);
assert.isTrue(Zotero.Groups.exists(groupData.json.id));
@@ -656,6 +654,11 @@ describe("Zotero.Sync.Runner", function () {
Zotero.Libraries.getVersion(Zotero.Groups.getLibraryIDFromGroupID(2694172)),
20
);
+
+ // Last sync time should be within the last second
+ var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
+ assert.isAbove(lastSyncTime, new Date().getTime() - 1000);
+ assert.isBelow(lastSyncTime, new Date().getTime());
})
})
})
diff --git a/test/tests/zoteroPaneTest.js b/test/tests/zoteroPaneTest.js
@@ -1,3 +1,5 @@
+"use strict";
+
describe("ZoteroPane", function() {
var win, doc, zp;
@@ -90,4 +92,96 @@ describe("ZoteroPane", function() {
);
})
})
+
+ describe("#viewAttachment", function () {
+ Components.utils.import("resource://zotero-unit/httpd.js");
+ var apiKey = Zotero.Utilities.randomString(24);
+ var port = 16213;
+ var baseURL = `http://localhost:${port}/`;
+ var server;
+ var responses = {};
+
+ var setup = Zotero.Promise.coroutine(function* (options = {}) {
+ server = sinon.fakeServer.create();
+ server.autoRespond = true;
+ });
+
+ function setResponse(response) {
+ setHTTPResponse(server, baseURL, response, responses);
+ }
+
+ before(function () {
+ Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
+
+ Zotero.Sync.Runner.apiKey = apiKey;
+ Zotero.Sync.Runner.baseURL = baseURL;
+ })
+ beforeEach(function* () {
+ this.httpd = new HttpServer();
+ this.httpd.start(port);
+
+ yield Zotero.Users.setCurrentUserID(1);
+ yield Zotero.Users.setCurrentUsername("testuser");
+ })
+ afterEach(function* () {
+ var defer = new Zotero.Promise.defer();
+ this.httpd.stop(() => defer.resolve());
+ yield defer.promise;
+ })
+
+ it("should download an attachment on-demand", function* () {
+ yield setup();
+ Zotero.Sync.Storage.Local.downloadAsNeeded(Zotero.Libraries.userLibraryID, true);
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ // TODO: Test binary data
+ var text = Zotero.Utilities.randomString();
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var mtime = "1441252524000";
+ var md5 = Zotero.Utilities.Internal.md5(text)
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ var s3Path = `pretend-s3/${item.key}`;
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 302, "Found");
+ response.setHeader("Zotero-File-Modification-Time", mtime, false);
+ response.setHeader("Zotero-File-MD5", md5, false);
+ response.setHeader("Zotero-File-Compressed", "No", false);
+ response.setHeader("Location", baseURL + s3Path, false);
+ }
+ }
+ );
+ this.httpd.registerPathHandler(
+ "/" + s3Path,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 200, "OK");
+ response.write(text);
+ }
+ }
+ );
+
+ yield zp.viewAttachment(item.id);
+
+ assert.equal((yield item.attachmentHash), md5);
+ assert.equal((yield item.attachmentModificationTime), mtime);
+ var path = yield item.getFilePathAsync();
+ assert.equal((yield Zotero.File.getContentsAsync(path)), text);
+ })
+ })
})
