zfs.js (29910B)
1 /* 2 ***** BEGIN LICENSE BLOCK ***** 3 4 Copyright © 2009 Center for History and New Media 5 George Mason University, Fairfax, Virginia, USA 6 http://zotero.org 7 8 This file is part of Zotero. 9 10 Zotero is free software: you can redistribute it and/or modify 11 it under the terms of the GNU Affero General Public License as published by 12 the Free Software Foundation, either version 3 of the License, or 13 (at your option) any later version. 14 15 Zotero is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU Affero General Public License for more details. 19 20 You should have received a copy of the GNU Affero General Public License 21 along with Zotero. If not, see <http://www.gnu.org/licenses/>. 22 23 ***** END LICENSE BLOCK ***** 24 */ 25 26 if (!Zotero.Sync.Storage.Mode) { 27 Zotero.Sync.Storage.Mode = {}; 28 } 29 30 Zotero.Sync.Storage.Mode.ZFS = function (options) { 31 this.options = options; 32 this.apiClient = options.apiClient; 33 34 this._s3Backoff = 1; 35 this._s3ConsecutiveFailures = 0; 36 this._maxS3Backoff = 60; 37 this._maxS3ConsecutiveFailures = options.maxS3ConsecutiveFailures !== undefined 38 ? options.maxS3ConsecutiveFailures : 5; 39 }; 40 Zotero.Sync.Storage.Mode.ZFS.prototype = { 41 mode: "zfs", 42 name: "ZFS", 43 verified: true, 44 45 46 /** 47 * Begin download process for individual file 48 * 49 * @param {Zotero.Sync.Storage.Request} request 50 * @return {Promise<Zotero.Sync.Storage.Result>} 51 */ 52 downloadFile: Zotero.Promise.coroutine(function* (request) { 53 var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); 54 if (!item) { 55 throw new Error("Item '" + request.name + "' not found"); 56 } 57 58 var path = item.getFilePath(); 59 if (!path) { 60 Zotero.debug(`Cannot download file for attachment ${item.libraryKey} with no path`); 61 return new Zotero.Sync.Storage.Result; 62 } 63 64 var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp'); 65 66 // saveURI() below appears not to create empty files for Content-Length: 0, 67 // so we create one here just in case, which also lets us check file access 68 try { 69 let file = yield OS.File.open(destPath, { 70 truncate: true 71 }); 72 file.close(); 73 } 74 catch (e) { 75 Zotero.File.checkFileAccessError(e, destPath, 'create'); 76 } 77 78 var deferred = Zotero.Promise.defer(); 79 var requestData = {item}; 80 81 var listener = new Zotero.Sync.Storage.StreamListener( 82 { 83 onStart: function (req) { 84 if (request.isFinished()) { 85 Zotero.debug("Download request " + request.name 86 + " stopped before download started -- closing channel"); 87 req.cancel(Components.results.NS_BINDING_ABORTED); 88 deferred.resolve(new Zotero.Sync.Storage.Result); 89 } 90 }, 91 onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) { 92 // These will be used in processDownload() if the download succeeds 93 oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel); 94 95 Zotero.debug("CHANNEL HERE FOR " + item.libraryKey + " WITH " + oldChannel.status); 96 Zotero.debug(oldChannel.URI.spec); 97 Zotero.debug(newChannel.URI.spec); 98 99 var header; 100 try { 101 header = "Zotero-File-Modification-Time"; 102 requestData.mtime = parseInt(oldChannel.getResponseHeader(header)); 103 header = "Zotero-File-MD5"; 104 requestData.md5 = oldChannel.getResponseHeader(header); 105 header = "Zotero-File-Compressed"; 106 requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes'; 107 } 108 catch (e) { 109 deferred.reject(new Error(`${header} header not set in file request for ${item.libraryKey}`)); 110 return false; 111 } 112 113 if (!(yield OS.File.exists(path))) { 114 return true; 115 } 116 117 var updateHash = false; 118 var fileModTime = yield item.attachmentModificationTime; 119 if (requestData.mtime == fileModTime) { 120 Zotero.debug("File mod time matches remote file -- skipping download of " 121 + item.libraryKey); 122 } 123 // If not compressed, check hash, in case only timestamp changed 124 else if (!requestData.compressed && (yield item.attachmentHash) == requestData.md5) { 125 Zotero.debug("File hash matches remote file -- skipping download of " 126 + item.libraryKey); 127 updateHash = true; 128 } 129 else { 130 return true; 131 } 132 133 // Update local metadata and stop request, skipping file download 134 yield OS.File.setDates(path, null, new Date(requestData.mtime)); 135 item.attachmentSyncedModificationTime = requestData.mtime; 136 if (updateHash) { 137 item.attachmentSyncedHash = requestData.md5; 138 } 139 item.attachmentSyncState = "in_sync"; 140 yield item.saveTx({ skipAll: true }); 141 142 deferred.resolve(new Zotero.Sync.Storage.Result({ 143 localChanges: true 144 })); 145 146 return false; 147 }), 148 onProgress: function (req, progress, progressMax) { 149 request.onProgress(progress, progressMax); 150 }, 151 onStop: function (req, status, res) { 152 request.setChannel(false); 153 154 if (status != 200) { 155 if (status == 404) { 156 Zotero.debug("Remote file not found for item " + item.libraryKey); 157 deferred.resolve(new Zotero.Sync.Storage.Result); 158 return; 159 } 160 161 // If S3 connection is interrupted, delay and retry, or bail if too many 162 // consecutive failures 163 if (status == 0 || status == 500 || status == 503) { 164 if (++this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) { 165 let libraryKey = item.libraryKey; 166 let msg = "S3 returned 0 for " + libraryKey + " -- retrying download" 167 Components.utils.reportError(msg); 168 Zotero.debug(msg, 1); 169 if (this._s3Backoff < this._maxS3Backoff) { 170 this._s3Backoff *= 2; 171 } 172 Zotero.debug("Delaying " + libraryKey + " download for " 173 + this._s3Backoff + " seconds", 2); 174 Zotero.Promise.delay(this._s3Backoff * 1000) 175 .then(function () { 176 deferred.resolve(this.downloadFile(request)); 177 }.bind(this)); 178 return; 179 } 180 181 Zotero.debug(this._s3ConsecutiveFailures 182 + " consecutive S3 failures -- aborting", 1); 183 this._s3ConsecutiveFailures = 0; 184 } 185 186 var msg = "Unexpected status code " + status + " for GET " + uri; 187 Zotero.debug(msg, 1); 188 Components.utils.reportError(msg); 189 // Output saved content, in case an error was captured 190 try { 191 let sample = Zotero.File.getContents(destPath, null, 4096); 192 if (sample) { 193 Zotero.debug(sample, 1); 194 } 195 } 196 catch (e) { 197 Zotero.debug(e, 1); 198 } 199 deferred.reject(new Error(Zotero.Sync.Storage.defaultError)); 200 return; 201 } 202 203 // Don't try to process if the request has been cancelled 204 if (request.isFinished()) { 205 Zotero.debug("Download request " + request.name 206 + " is no longer running after file download", 2); 207 deferred.resolve(new Zotero.Sync.Storage.Result); 208 return; 209 } 210 211 Zotero.debug("Finished download of " + destPath); 212 213 try { 214 deferred.resolve( 215 Zotero.Sync.Storage.Local.processDownload(requestData) 216 ); 217 } 218 catch (e) { 219 deferred.reject(e); 220 } 221 }.bind(this), 222 onCancel: function (req, status) { 223 Zotero.debug("Request cancelled"); 224 if (deferred.promise.isPending()) { 225 deferred.resolve(new Zotero.Sync.Storage.Result); 226 } 227 } 228 } 229 ); 230 231 var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`); 232 var uri = this.apiClient.buildRequestURI(params); 233 var headers = this.apiClient.getHeaders(); 234 235 Zotero.debug('Saving ' + uri); 236 const nsIWBP = Components.interfaces.nsIWebBrowserPersist; 237 var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"] 238 .createInstance(nsIWBP); 239 wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE; 240 wbp.progressListener = listener; 241 Zotero.Utilities.Internal.saveURI(wbp, uri, destPath, headers); 242 243 return deferred.promise; 244 }), 245 246 247 uploadFile: Zotero.Promise.coroutine(function* (request) { 248 var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); 249 if (yield Zotero.Attachments.hasMultipleFiles(item)) { 250 let created = yield Zotero.Sync.Storage.Utilities.createUploadFile(request); 251 if (!created) { 252 return new Zotero.Sync.Storage.Result; 253 } 254 } 255 return this._processUploadFile(request); 256 }), 257 258 259 /** 260 * Remove all synced files from the server 261 */ 262 purgeDeletedStorageFiles: Zotero.Promise.coroutine(function* (libraryID) { 263 if (libraryID != Zotero.Libraries.userLibraryID) return; 264 265 var sql = "SELECT value FROM settings WHERE setting=? AND key=?"; 266 var values = yield Zotero.DB.columnQueryAsync(sql, ['storage', 'zfsPurge']); 267 if (!values.length) { 268 return false; 269 } 270 271 Zotero.debug("Unlinking synced files on ZFS"); 272 273 var params = this._getRequestParams(libraryID, "removestoragefiles"); 274 var uri = this.apiClient.buildRequestURI(params); 275 276 yield Zotero.HTTP.request("POST", uri, ""); 277 278 var sql = "DELETE FROM settings WHERE setting=? AND key=?"; 279 yield Zotero.DB.queryAsync(sql, ['storage', 'zfsPurge']); 280 }), 281 282 283 // 284 // Private methods 285 // 286 _getRequestParams: function (libraryID, target) { 287 var library = Zotero.Libraries.get(libraryID); 288 return { 289 libraryType: library.libraryType, 290 libraryTypeID: library.libraryTypeID, 291 target 292 }; 293 }, 294 295 296 /** 297 * Get authorization from API for uploading file 298 * 299 * @param {Zotero.Item} item 300 * @return {Object|String} - Object with upload params or 'exists' 301 */ 302 _getFileUploadParameters: Zotero.Promise.coroutine(function* (item) { 303 var funcName = "Zotero.Sync.Storage.ZFS._getFileUploadParameters()"; 304 305 var path = item.getFilePath(); 306 var filename = OS.Path.basename(path); 307 var zip = yield Zotero.Attachments.hasMultipleFiles(item); 308 if (zip) { 309 var uploadPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip'); 310 } 311 else { 312 var uploadPath = path; 313 } 314 315 var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`); 316 var uri = this.apiClient.buildRequestURI(params); 317 318 // TODO: One-step uploads 319 /*var headers = { 320 "Content-Type": "application/json" 321 }; 322 var storedHash = yield Zotero.Sync.Storage.Local.getSyncedHash(item.id); 323 //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id); 324 if (storedHash) { 325 headers["If-Match"] = storedHash; 326 } 327 else { 328 headers["If-None-Match"] = "*"; 329 } 330 var mtime = yield item.attachmentModificationTime; 331 var hash = Zotero.Utilities.Internal.md5(file); 332 var json = { 333 md5: hash, 334 mtime, 335 filename, 336 size: file.fileSize 337 }; 338 if (zip) { 339 json.zip = true; 340 } 341 342 try { 343 var req = yield this.apiClient.makeRequest( 344 "POST", uri, { body: JSON.stringify(json), headers, debug: true } 345 ); 346 }*/ 347 348 var headers = { 349 "Content-Type": "application/x-www-form-urlencoded" 350 }; 351 var storedHash = item.attachmentSyncedHash; 352 //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id); 353 if (storedHash) { 354 headers["If-Match"] = storedHash; 355 } 356 else { 357 headers["If-None-Match"] = "*"; 358 } 359 360 // Build POST body 361 var params = { 362 mtime: yield item.attachmentModificationTime, 363 md5: yield item.attachmentHash, 364 filename, 365 filesize: (yield OS.File.stat(uploadPath)).size 366 }; 367 if (zip) { 368 params.zipMD5 = yield Zotero.Utilities.Internal.md5Async(uploadPath); 369 params.zipFilename = OS.Path.basename(uploadPath); 370 } 371 var body = []; 372 for (let i in params) { 373 body.push(i + "=" + encodeURIComponent(params[i])); 374 } 375 body = body.join('&'); 376 377 var req; 378 while (true) { 379 try { 380 req = yield this.apiClient.makeRequest( 381 "POST", 382 uri, 383 { 384 body, 385 headers, 386 // This should include all errors in _handleUploadAuthorizationFailure() 387 successCodes: [200, 201, 204, 403, 404, 412, 413], 388 debug: true 389 } 390 ); 391 } 392 catch (e) { 393 if (e instanceof Zotero.HTTP.UnexpectedStatusException) { 394 let msg = "Unexpected status code " + e.status + " in " + funcName 395 + " (" + item.libraryKey + ")"; 396 Zotero.logError(msg); 397 Zotero.debug(e.xmlhttp.getAllResponseHeaders()); 398 throw new Error(Zotero.Sync.Storage.defaultError); 399 } 400 throw e; 401 } 402 403 let result = yield this._handleUploadAuthorizationFailure(req, item); 404 if (result instanceof Zotero.Sync.Storage.Result) { 405 return result; 406 } 407 // If remote attachment exists but has no hash (which can happen for an old (pre-4.0?) 408 // attachment with just an mtime, or after a storage purge), send again with If-None-Match 409 else if (result == "ERROR_412_WITHOUT_VERSION") { 410 if (headers["If-None-Match"]) { 411 throw new Error("412 returned for request with If-None-Match"); 412 } 413 delete headers["If-Match"]; 414 headers["If-None-Match"] = "*"; 415 storedHash = null; 416 Zotero.debug("Retrying with If-None-Match"); 417 } 418 else { 419 break; 420 } 421 } 422 423 try { 424 var json = JSON.parse(req.responseText); 425 } 426 catch (e) { 427 Zotero.logError(e); 428 Zotero.debug(req.responseText, 1); 429 } 430 if (!json) { 431 throw new Error("Invalid response retrieving file upload parameters"); 432 } 433 434 if (!json.uploadKey && !json.exists) { 435 throw new Error("Invalid response retrieving file upload parameters"); 436 } 437 438 if (json.exists) { 439 let version = req.getResponseHeader('Last-Modified-Version'); 440 if (!version) { 441 throw new Error("Last-Modified-Version not provided"); 442 } 443 json.version = version; 444 } 445 446 // TEMP 447 // 448 // Passed through to _updateItemFileInfo() 449 json.mtime = params.mtime; 450 json.md5 = params.md5; 451 if (storedHash) { 452 json.storedHash = storedHash; 453 } 454 455 return json; 456 }), 457 458 459 /** 460 * Handle known errors from upload authorization request 461 * 462 * These must be included in successCodes in _getFileUploadParameters() 463 */ 464 _handleUploadAuthorizationFailure: Zotero.Promise.coroutine(function* (req, item) { 465 // 466 // These must be included in successCodes above. 467 // TODO: 429? 468 if (req.status == 403) { 469 let groupID = Zotero.Groups.getGroupIDFromLibraryID(item.libraryID); 470 let e = new Zotero.Error( 471 "File editing denied for group", 472 "ZFS_FILE_EDITING_DENIED", 473 { 474 groupID: groupID 475 } 476 ); 477 throw e; 478 } 479 // This shouldn't happen, but if it does, mark item for upload and restart sync 480 else if (req.status == 404) { 481 Zotero.logError(`Item ${item.libraryID}/${item.key} not found in upload authorization ` 482 + 'request -- marking for upload'); 483 yield Zotero.Sync.Data.Local.markObjectAsUnsynced(item); 484 return new Zotero.Sync.Storage.Result({ 485 syncRequired: true 486 }); 487 } 488 else if (req.status == 412) { 489 let version = req.getResponseHeader('Last-Modified-Version'); 490 if (!version) { 491 return "ERROR_412_WITHOUT_VERSION"; 492 } 493 if (version > item.version) { 494 return new Zotero.Sync.Storage.Result({ 495 syncRequired: true 496 }); 497 } 498 499 // Get updated item metadata 500 let library = Zotero.Libraries.get(item.libraryID); 501 let json = yield this.apiClient.downloadObjects( 502 library.libraryType, 503 library.libraryTypeID, 504 'item', 505 [item.key] 506 )[0]; 507 if (!Array.isArray(json)) { 508 Zotero.logError(json); 509 throw new Error(Zotero.Sync.Storage.defaultError); 510 } 511 if (json.length > 1) { 512 throw new Error("More than one result for item lookup"); 513 } 514 515 yield Zotero.Sync.Data.Local.saveCacheObjects('item', item.libraryID, json); 516 json = json[0]; 517 518 if (json.data.version > item.version) { 519 return new Zotero.Sync.Storage.Result({ 520 syncRequired: true 521 }); 522 } 523 524 let fileHash = yield item.attachmentHash; 525 let fileModTime = yield item.attachmentModificationTime; 526 527 Zotero.debug("MD5"); 528 Zotero.debug(json.data.md5); 529 Zotero.debug(fileHash); 530 531 if (json.data.md5 == fileHash) { 532 item.attachmentSyncedModificationTime = fileModTime; 533 item.attachmentSyncedHash = fileHash; 534 item.attachmentSyncState = "in_sync"; 535 yield item.saveTx({ skipAll: true }); 536 537 return new Zotero.Sync.Storage.Result; 538 } 539 540 item.attachmentSyncState = "in_conflict"; 541 yield item.saveTx({ skipAll: true }); 542 543 return new Zotero.Sync.Storage.Result({ 544 fileSyncRequired: true 545 }); 546 } 547 else if (req.status == 413) { 548 let retry = req.getResponseHeader('Retry-After'); 549 if (retry) { 550 let minutes = Math.round(retry / 60); 551 throw new Zotero.Error( 552 Zotero.getString('sync.storage.error.zfs.tooManyQueuedUploads', minutes), 553 "ZFS_UPLOAD_QUEUE_LIMIT" 554 ); 555 } 556 557 let text, buttonText = null, buttonCallback; 558 let libraryType = item.library.libraryType; 559 560 // Group file 561 if (libraryType == 'group') { 562 var group = Zotero.Groups.getByLibraryID(item.libraryID); 563 text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n" 564 + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2'); 565 } 566 // Personal file 567 else { 568 text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n" 569 + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2'); 570 buttonText = Zotero.getString('sync.storage.openAccountSettings'); 571 buttonCallback = function () { 572 var url = "https://www.zotero.org/settings/storage"; 573 574 var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] 575 .getService(Components.interfaces.nsIWindowMediator); 576 var win = wm.getMostRecentWindow("navigator:browser"); 577 win.ZoteroPane.loadURI(url, { metaKey: true, ctrlKey: true, shiftKey: true }); 578 } 579 } 580 581 var filename = item.attachmentFilename; 582 var fileSize = (yield OS.File.stat(item.getFilePath())).size; 583 584 text += "\n\n" + filename + " (" + Math.round(fileSize / 1024) + "KB)"; 585 586 let e = new Zotero.Error( 587 text, 588 "ZFS_OVER_QUOTA", 589 { 590 dialogButtonText: buttonText, 591 dialogButtonCallback: buttonCallback 592 } 593 ); 594 e.errorType = 'warning'; 595 Zotero.debug(e, 2); 596 Components.utils.reportError(e); 597 throw e; 598 } 599 }), 600 601 602 /** 603 * Given parameters from authorization, upload file to S3 604 */ 605 _uploadFile: Zotero.Promise.coroutine(function* (request, item, params) { 606 if (request.isFinished()) { 607 Zotero.debug("Upload request " + request.name + " is no longer running after getting " 608 + "upload parameters"); 609 return new Zotero.Sync.Storage.Result; 610 } 611 612 var file = yield this._getUploadFile(item); 613 614 Components.utils.importGlobalProperties(["File"]); 615 file = File.createFromFileName ? File.createFromFileName(file.path) : new File(file); 616 // File.createFromFileName() returns a Promise in Fx54+ 617 if (file.then) { 618 file = yield file; 619 } 620 621 var blob = new Blob([params.prefix, file, params.suffix]); 622 623 try { 624 var req = yield Zotero.HTTP.request( 625 "POST", 626 params.url, 627 { 628 headers: { 629 "Content-Type": params.contentType 630 }, 631 body: blob, 632 requestObserver: function (req) { 633 request.setChannel(req.channel); 634 req.upload.addEventListener("progress", function (event) { 635 if (event.lengthComputable) { 636 request.onProgress(event.loaded, event.total); 637 } 638 }); 639 }, 640 debug: true, 641 successCodes: [201] 642 } 643 ); 644 } 645 catch (e) { 646 // Certificate error 647 if (e instanceof Zotero.Error) { 648 throw e; 649 } 650 651 // For timeouts and failures from S3, which happen intermittently, 652 // wait a little and try again 653 let timeoutMessage = "Your socket connection to the server was not read from or " 654 + "written to within the timeout period."; 655 if (e.status == 0 656 || (e.status == 400 && e.xmlhttp.responseText.indexOf(timeoutMessage) != -1)) { 657 if (this._s3ConsecutiveFailures >= this._maxS3ConsecutiveFailures) { 658 Zotero.debug(this._s3ConsecutiveFailures 659 + " consecutive S3 failures -- aborting", 1); 660 this._s3ConsecutiveFailures = 0; 661 } 662 else { 663 let msg = "S3 returned " + e.status + " (" + item.libraryKey + ") " 664 + "-- retrying upload" 665 Zotero.logError(msg); 666 Zotero.debug(e.xmlhttp.responseText, 1); 667 if (this._s3Backoff < this._maxS3Backoff) { 668 this._s3Backoff *= 2; 669 } 670 this._s3ConsecutiveFailures++; 671 Zotero.debug("Delaying " + item.libraryKey + " upload for " 672 + this._s3Backoff + " seconds", 2); 673 yield Zotero.Promise.delay(this._s3Backoff * 1000); 674 return this._uploadFile(request, item, params); 675 } 676 } 677 else if (e.status == 500) { 678 // TODO: localize 679 throw new Error("File upload failed. Please try again."); 680 } 681 else { 682 Zotero.logError(`Unexpected file upload status ${e.status} (${item.libraryKey})`); 683 Zotero.debug(e, 1); 684 Components.utils.reportError(e.xmlhttp.responseText); 685 throw new Error(Zotero.Sync.Storage.defaultError); 686 } 687 688 // TODO: Detect cancel? 689 //onUploadCancel(httpRequest, status, data) 690 //deferred.resolve(false); 691 } 692 693 request.setChannel(false); 694 return this._onUploadComplete(req, request, item, params); 695 }), 696 697 698 /** 699 * Post-upload file registration with API 700 */ 701 _onUploadComplete: Zotero.Promise.coroutine(function* (req, request, item, params) { 702 var uploadKey = params.uploadKey; 703 704 Zotero.debug("Upload of attachment " + item.key + " finished with status code " + req.status); 705 Zotero.debug(req.responseText); 706 707 // Decrease backoff delay on successful upload 708 if (this._s3Backoff > 1) { 709 this._s3Backoff /= 2; 710 } 711 // And reset consecutive failures 712 this._s3ConsecutiveFailures = 0; 713 714 var requestParams = this._getRequestParams(item.libraryID, `items/${item.key}/file`); 715 var uri = this.apiClient.buildRequestURI(requestParams); 716 var headers = { 717 "Content-Type": "application/x-www-form-urlencoded" 718 }; 719 if (params.storedHash) { 720 headers["If-Match"] = params.storedHash; 721 } 722 else { 723 headers["If-None-Match"] = "*"; 724 } 725 var body = "upload=" + uploadKey; 726 727 // Register upload on server 728 try { 729 req = yield this.apiClient.makeRequest( 730 "POST", 731 uri, 732 { 733 body, 734 headers, 735 successCodes: [204], 736 requestObserver: function (xmlhttp) { 737 request.setChannel(xmlhttp.channel); 738 }, 739 debug: true 740 } 741 ); 742 } 743 catch (e) { 744 let msg = `Unexpected file registration status ${e.status} (${item.libraryKey})`; 745 Zotero.logError(msg); 746 Zotero.logError(e.xmlhttp.responseText); 747 Zotero.debug(e.xmlhttp.getAllResponseHeaders()); 748 throw new Error(Zotero.Sync.Storage.defaultError); 749 } 750 751 var version = req.getResponseHeader('Last-Modified-Version'); 752 if (!version) { 753 throw new Error("Last-Modified-Version not provided"); 754 } 755 params.version = version; 756 757 yield this._updateItemFileInfo(item, params); 758 759 return new Zotero.Sync.Storage.Result({ 760 localChanges: true, 761 remoteChanges: true 762 }); 763 }), 764 765 766 /** 767 * Update the local attachment item with the mtime and hash of the uploaded file and the 768 * library version returned by the upload request, and save a modified version of the item 769 * to the sync cache 770 */ 771 _updateItemFileInfo: Zotero.Promise.coroutine(function* (item, params) { 772 // Mark as in-sync 773 yield Zotero.DB.executeTransaction(function* () { 774 // Store file mod time and hash 775 item.attachmentSyncedModificationTime = params.mtime; 776 item.attachmentSyncedHash = params.md5; 777 item.attachmentSyncState = "in_sync"; 778 yield item.save({ skipAll: true }); 779 780 // Update sync cache with new file metadata and version from server 781 var json = yield Zotero.Sync.Data.Local.getCacheObject( 782 'item', item.libraryID, item.key, item.version 783 ); 784 if (json) { 785 json.version = params.version; 786 json.data.version = params.version; 787 json.data.mtime = params.mtime; 788 json.data.md5 = params.md5; 789 yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json); 790 } 791 // Update item with new version from server 792 yield Zotero.Items.updateVersion([item.id], params.version); 793 794 // TODO: Can filename, contentType, and charset change the attachment item? 795 }); 796 797 try { 798 if (yield Zotero.Attachments.hasMultipleFiles(item)) { 799 var file = Zotero.getTempDirectory(); 800 file.append(item.key + '.zip'); 801 yield OS.File.remove(file.path); 802 } 803 } 804 catch (e) { 805 Components.utils.reportError(e); 806 } 807 }), 808 809 810 _onUploadCancel: Zotero.Promise.coroutine(function* (httpRequest, status, data) { 811 var request = data.request; 812 var item = data.item; 813 814 Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status); 815 816 try { 817 if (yield Zotero.Attachments.hasMultipleFiles(item)) { 818 var file = Zotero.getTempDirectory(); 819 file.append(item.key + '.zip'); 820 file.remove(false); 821 } 822 } 823 catch (e) { 824 Components.utils.reportError(e); 825 } 826 }), 827 828 829 _getUploadFile: Zotero.Promise.coroutine(function* (item) { 830 if (yield Zotero.Attachments.hasMultipleFiles(item)) { 831 var file = Zotero.getTempDirectory(); 832 var filename = item.key + '.zip'; 833 file.append(filename); 834 } 835 else { 836 var file = item.getFile(); 837 } 838 return file; 839 }), 840 841 842 /** 843 * Get attachment item metadata on storage server 844 * 845 * @param {Zotero.Item} item 846 * @param {Zotero.Sync.Storage.Request} request 847 * @return {Promise<Object>|false} - Promise for object with 'hash', 'filename', 'mtime', 848 * 'compressed', or false if item not found 849 */ 850 _getStorageFileInfo: Zotero.Promise.coroutine(function* (item, request) { 851 var funcName = "Zotero.Sync.Storage.ZFS._getStorageFileInfo()"; 852 853 var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`); 854 var uri = this.apiClient.buildRequestURI(params); 855 856 try { 857 let req = yield this.apiClient.makeRequest( 858 "GET", 859 uri, 860 { 861 successCodes: [200, 404], 862 requestObserver: function (xmlhttp) { 863 request.setChannel(xmlhttp.channel); 864 } 865 } 866 ); 867 if (req.status == 404) { 868 return new Zotero.Sync.Storage.Result; 869 } 870 871 let info = {}; 872 info.hash = req.getResponseHeader('ETag'); 873 if (!info.hash) { 874 let msg = `Hash not found in info response in ${funcName} (${item.libraryKey})`; 875 Zotero.debug(msg, 1); 876 Zotero.debug(req.status); 877 Zotero.debug(req.responseText); 878 Components.utils.reportError(msg); 879 try { 880 Zotero.debug(req.getAllResponseHeaders()); 881 } 882 catch (e) { 883 Zotero.debug("Response headers unavailable"); 884 } 885 let e = Zotero.getString('sync.storage.error.zfs.restart', Zotero.appName); 886 throw new Error(e); 887 } 888 info.filename = req.getResponseHeader('X-Zotero-Filename'); 889 let mtime = req.getResponseHeader('X-Zotero-Modification-Time'); 890 info.mtime = parseInt(mtime); 891 info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes'; 892 Zotero.debug(info); 893 894 return info; 895 } 896 catch (e) { 897 if (e instanceof Zotero.HTTP.UnexpectedStatusException) { 898 if (e.xmlhttp.status == 0) { 899 var msg = "Request cancelled getting storage file info"; 900 } 901 else { 902 var msg = "Unexpected status code " + e.xmlhttp.status 903 + " getting storage file info for item " + item.libraryKey; 904 } 905 Zotero.debug(msg, 1); 906 Zotero.debug(e.xmlhttp.responseText); 907 Components.utils.reportError(msg); 908 throw new Error(Zotero.Sync.Storage.defaultError); 909 } 910 911 throw e; 912 } 913 }), 914 915 916 /** 917 * Upload the file to the server 918 * 919 * @param {Zotero.Sync.Storage.Request} request 920 * @return {Promise} 921 */ 922 _processUploadFile: Zotero.Promise.coroutine(function* (request) { 923 /* 924 updateSizeMultiplier( 925 (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100 926 ); 927 */ 928 929 var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); 930 931 932 /*var info = yield this._getStorageFileInfo(item, request); 933 934 if (request.isFinished()) { 935 Zotero.debug("Upload request '" + request.name 936 + "' is no longer running after getting file info"); 937 return false; 938 } 939 940 // Check for conflict 941 if (item.attachmentSyncState 942 != Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD) { 943 if (info) { 944 // Local file time 945 var fmtime = yield item.attachmentModificationTime; 946 // Remote mod time 947 var mtime = info.mtime; 948 949 var useLocal = false; 950 var same = !(yield Zotero.Sync.Storage.checkFileModTime(item, fmtime, mtime)); 951 952 // Ignore maxed-out 32-bit ints, from brief problem after switch to 32-bit servers 953 if (!same && mtime == 2147483647) { 954 Zotero.debug("Remote mod time is invalid -- uploading local file version"); 955 useLocal = true; 956 } 957 958 if (same) { 959 yield Zotero.DB.executeTransaction(function* () { 960 yield Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime); 961 yield Zotero.Sync.Storage.setSyncState( 962 item.id, Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC 963 ); 964 }); 965 return { 966 localChanges: true, 967 remoteChanges: false 968 }; 969 } 970 971 let smtime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id); 972 if (!useLocal && smtime != mtime) { 973 Zotero.debug("Conflict -- last synced file mod time " 974 + "does not match time on storage server" 975 + " (" + smtime + " != " + mtime + ")"); 976 return { 977 localChanges: false, 978 remoteChanges: false, 979 conflict: { 980 local: { modTime: fmtime }, 981 remote: { modTime: mtime } 982 } 983 }; 984 } 985 } 986 else { 987 Zotero.debug("Remote file not found for item " + item.libraryKey); 988 } 989 }*/ 990 991 var result = yield this._getFileUploadParameters(item); 992 if (result.exists) { 993 yield this._updateItemFileInfo(item, result); 994 return new Zotero.Sync.Storage.Result({ 995 localChanges: true, 996 remoteChanges: true 997 }); 998 } 999 else if (result instanceof Zotero.Sync.Storage.Result) { 1000 return result; 1001 } 1002 return this._uploadFile(request, item, result); 1003 }) 1004 }