zfsTest.js (31214B)
1 "use strict"; 2 3 describe("Zotero.Sync.Storage.Mode.ZFS", function () { 4 // 5 // Setup 6 // 7 Components.utils.import("resource://zotero-unit/httpd.js"); 8 9 var apiKey = Zotero.Utilities.randomString(24); 10 var port = 16213; 11 var baseURL = `http://localhost:${port}/`; 12 13 var win, server, requestCount; 14 var responses = {}; 15 16 function setResponse(response) { 17 setHTTPResponse(server, baseURL, response, responses); 18 } 19 20 function resetRequestCount() { 21 requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length; 22 } 23 24 function assertRequestCount(count) { 25 assert.equal( 26 server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount, 27 count 28 ); 29 } 30 31 function parseQueryString(str) { 32 var queryStringParams = str.split('&'); 33 var params = {}; 34 for (let param of queryStringParams) { 35 let [ key, val ] = param.split('='); 36 params[key] = decodeURIComponent(val); 37 } 38 return params; 39 } 40 41 function assertAPIKey(request) { 42 assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey); 43 } 44 45 // 46 // Tests 47 // 48 beforeEach(function* () { 49 yield resetDB({ 50 thisArg: this, 51 skipBundledFiles: true 52 }); 53 win = yield loadZoteroPane(); 54 55 Zotero.HTTP.mock = sinon.FakeXMLHttpRequest; 56 server = sinon.fakeServer.create(); 57 server.autoRespond = true; 58 59 this.httpd = new HttpServer(); 60 this.httpd.start(port); 61 62 yield Zotero.Users.setCurrentUserID(1); 63 yield Zotero.Users.setCurrentUsername("testuser"); 64 65 Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'zfs'); 66 67 // Set download-on-sync by default 68 Zotero.Sync.Storage.Local.downloadOnSync( 69 Zotero.Libraries.userLibraryID, true 70 ); 71 72 resetRequestCount(); 73 }) 74 75 var setup = Zotero.Promise.coroutine(function* (options = {}) { 76 Components.utils.import("resource://zotero/concurrentCaller.js"); 77 var caller = new ConcurrentCaller(1); 78 caller.setLogger(msg => Zotero.debug(msg)); 79 caller.stopOnError = true; 80 81 Components.utils.import("resource://zotero/config.js"); 82 var client = new Zotero.Sync.APIClient({ 83 baseURL, 84 apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION, 85 apiKey, 86 caller, 87 background: options.background || true 88 }); 89 90 var engine = new Zotero.Sync.Storage.Engine({ 91 libraryID: options.libraryID || Zotero.Libraries.userLibraryID, 92 controller: new Zotero.Sync.Storage.Mode.ZFS({ 93 apiClient: client, 94 maxS3ConsecutiveFailures: 2 95 }), 96 stopOnError: true 97 }); 98 99 return { engine, client, caller }; 100 }) 101 102 afterEach(function* () { 103 var defer = new Zotero.Promise.defer(); 104 this.httpd.stop(() => defer.resolve()); 105 yield defer.promise; 106 win.close(); 107 }) 108 109 after(function* () { 110 this.timeout(60000); 111 //yield resetDB(); 112 win.close(); 113 }) 114 115 116 describe("Syncing", function () { 117 it("should skip downloads if not marked as needed", function* () { 118 var { engine, client, caller } = yield setup(); 119 120 var library = Zotero.Libraries.userLibrary; 121 library.libraryVersion = 5; 122 yield library.saveTx(); 123 124 var result = yield engine.start(); 125 126 assertRequestCount(0); 127 128 assert.isFalse(result.localChanges); 129 assert.isFalse(result.remoteChanges); 130 assert.isFalse(result.syncRequired); 131 132 assert.equal(library.storageVersion, library.libraryVersion); 133 }) 134 135 it("should ignore download for a remotely missing file", function* () { 136 var { engine, client, caller } = yield setup(); 137 138 var library = Zotero.Libraries.userLibrary; 139 library.libraryVersion = 5; 140 yield library.saveTx(); 141 library.storageDownloadNeeded = true; 142 143 var item = new Zotero.Item("attachment"); 144 item.attachmentLinkMode = 'imported_file'; 145 item.attachmentPath = 'storage:test.txt'; 146 item.attachmentSyncState = "to_download"; 147 yield item.saveTx(); 148 149 this.httpd.registerPathHandler( 150 `/users/1/items/${item.key}/file`, 151 { 152 handle: function (request, response) { 153 response.setStatusLine(null, 404, null); 154 } 155 } 156 ); 157 var result = yield engine.start(); 158 159 assert.isFalse(result.localChanges); 160 assert.isFalse(result.remoteChanges); 161 assert.isFalse(result.syncRequired); 162 163 assert.isFalse(library.storageDownloadNeeded); 164 assert.equal(library.storageVersion, library.libraryVersion); 165 }) 166 167 it("shouldn't update storageVersion if stopped", function* () { 168 var { engine, client, caller } = yield setup(); 169 170 var library = Zotero.Libraries.userLibrary; 171 library.libraryVersion = 5; 172 yield library.saveTx(); 173 library.storageDownloadNeeded = true; 174 175 var items = []; 176 for (let i = 0; i < 5; i++) { 177 let item = new Zotero.Item("attachment"); 178 item.attachmentLinkMode = 'imported_file'; 179 item.attachmentPath = 'storage:test.txt'; 180 item.attachmentSyncState = "to_download"; 181 yield item.saveTx(); 182 items.push(item); 183 } 184 185 var call = 0; 186 var stub = sinon.stub(engine.controller, 'downloadFile').callsFake(function () { 187 call++; 188 if (call == 1) { 189 engine.stop(); 190 } 191 return new Zotero.Sync.Storage.Result; 192 }); 193 194 var result = yield engine.start(); 195 196 stub.restore(); 197 198 assert.equal(library.storageVersion, 0); 199 }); 200 201 it("should handle a remotely failing file", function* () { 202 var { engine, client, caller } = yield setup(); 203 204 var library = Zotero.Libraries.userLibrary; 205 library.libraryVersion = 5; 206 yield library.saveTx(); 207 library.storageDownloadNeeded = true; 208 209 var item = new Zotero.Item("attachment"); 210 item.attachmentLinkMode = 'imported_file'; 211 item.attachmentPath = 'storage:test.txt'; 212 item.attachmentSyncState = "to_download"; 213 yield item.saveTx(); 214 215 this.httpd.registerPathHandler( 216 `/users/1/items/${item.key}/file`, 217 { 218 handle: function (request, response) { 219 response.setStatusLine(null, 500, null); 220 } 221 } 222 ); 223 // TODO: In stopOnError mode, this the promise is rejected. 224 // This should probably test with stopOnError mode turned off instead. 225 var e = yield getPromiseError(engine.start()); 226 assert.equal(e.message, Zotero.Sync.Storage.defaultError); 227 228 assert.isTrue(library.storageDownloadNeeded); 229 assert.equal(library.storageVersion, 0); 230 }) 231 232 it("should download a missing file", function* () { 233 var { engine, client, caller } = yield setup(); 234 235 var library = Zotero.Libraries.userLibrary; 236 library.libraryVersion = 5; 237 yield library.saveTx(); 238 library.storageDownloadNeeded = true; 239 240 var item = new Zotero.Item("attachment"); 241 item.attachmentLinkMode = 'imported_file'; 242 item.attachmentPath = 'storage:test.txt'; 243 // TODO: Test binary data 244 var text = Zotero.Utilities.randomString(); 245 item.attachmentSyncState = "to_download"; 246 yield item.saveTx(); 247 248 var mtime = "1441252524905"; 249 var md5 = Zotero.Utilities.Internal.md5(text) 250 251 var s3Path = `pretend-s3/${item.key}`; 252 this.httpd.registerPathHandler( 253 `/users/1/items/${item.key}/file`, 254 { 255 handle: function (request, response) { 256 if (!request.hasHeader('Zotero-API-Key')) { 257 response.setStatusLine(null, 403, "Forbidden"); 258 return; 259 } 260 var key = request.getHeader('Zotero-API-Key'); 261 if (key != apiKey) { 262 response.setStatusLine(null, 403, "Invalid key"); 263 return; 264 } 265 response.setStatusLine(null, 302, "Found"); 266 response.setHeader("Zotero-File-Modification-Time", mtime, false); 267 response.setHeader("Zotero-File-MD5", md5, false); 268 response.setHeader("Zotero-File-Compressed", "No", false); 269 response.setHeader("Location", baseURL + s3Path, false); 270 } 271 } 272 ); 273 this.httpd.registerPathHandler( 274 "/" + s3Path, 275 { 276 handle: function (request, response) { 277 response.setStatusLine(null, 200, "OK"); 278 response.write(text); 279 } 280 } 281 ); 282 var result = yield engine.start(); 283 284 assert.isTrue(result.localChanges); 285 assert.isFalse(result.remoteChanges); 286 assert.isFalse(result.syncRequired); 287 288 var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync()); 289 assert.equal(contents, text); 290 291 assert.isFalse(library.storageDownloadNeeded); 292 assert.equal(library.storageVersion, library.libraryVersion); 293 }) 294 295 it("should upload new files", function* () { 296 var { engine, client, caller } = yield setup(); 297 298 // Single file 299 var file1 = getTestDataDirectory(); 300 file1.append('test.png'); 301 var item1 = yield Zotero.Attachments.importFromFile({ file: file1 }); 302 var mtime1 = yield item1.attachmentModificationTime; 303 var hash1 = yield item1.attachmentHash; 304 var path1 = item1.getFilePath(); 305 var filename1 = 'test.png'; 306 var size1 = (yield OS.File.stat(path1)).size; 307 var contentType1 = 'image/png'; 308 var prefix1 = Zotero.Utilities.randomString(); 309 var suffix1 = Zotero.Utilities.randomString(); 310 var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789'); 311 312 let file1Blob = File.createFromFileName ? File.createFromFileName(file1.path) : new File(file1); 313 if (file1Blob.then) { 314 file1Blob = yield file1Blob; 315 } 316 317 // HTML file with auxiliary image 318 var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html'); 319 var parentItem = yield createDataObject('item'); 320 var item2 = yield Zotero.Attachments.importSnapshotFromFile({ 321 file: file2, 322 url: 'http://example.com/', 323 parentItemID: parentItem.id, 324 title: 'Test', 325 contentType: 'text/html', 326 charset: 'utf-8' 327 }); 328 var mtime2 = yield item2.attachmentModificationTime; 329 var hash2 = yield item2.attachmentHash; 330 var path2 = item2.getFilePath(); 331 var filename2 = 'index.html'; 332 var size2 = (yield OS.File.stat(path2)).size; 333 var contentType2 = 'text/html'; 334 var charset2 = 'utf-8'; 335 var prefix2 = Zotero.Utilities.randomString(); 336 var suffix2 = Zotero.Utilities.randomString(); 337 var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789'); 338 339 var deferreds = []; 340 341 // https://github.com/cjohansen/Sinon.JS/issues/607 342 let fixSinonBug = ";charset=utf-8"; 343 server.respond(function (req) { 344 // Get upload authorization for single file 345 if (req.method == "POST" 346 && req.url == `${baseURL}users/1/items/${item1.key}/file` 347 && req.requestBody.indexOf('upload=') == -1) { 348 assertAPIKey(req); 349 assert.equal(req.requestHeaders["If-None-Match"], "*"); 350 assert.equal( 351 req.requestHeaders["Content-Type"], 352 "application/x-www-form-urlencoded" + fixSinonBug 353 ); 354 355 let parts = req.requestBody.split('&'); 356 let params = {}; 357 for (let part of parts) { 358 let [key, val] = part.split('='); 359 params[key] = decodeURIComponent(val); 360 } 361 assert.equal(params.md5, hash1); 362 assert.equal(params.mtime, mtime1); 363 assert.equal(params.filename, filename1); 364 assert.equal(params.filesize, size1); 365 366 req.respond( 367 200, 368 { 369 "Content-Type": "application/json" 370 }, 371 JSON.stringify({ 372 url: baseURL + "pretend-s3/1", 373 contentType: contentType1, 374 prefix: prefix1, 375 suffix: suffix1, 376 uploadKey: uploadKey1 377 }) 378 ); 379 } 380 // Get upload authorization for multi-file zip 381 else if (req.method == "POST" 382 && req.url == `${baseURL}users/1/items/${item2.key}/file` 383 && req.requestBody.indexOf('upload=') == -1) { 384 assertAPIKey(req); 385 assert.equal(req.requestHeaders["If-None-Match"], "*"); 386 assert.equal( 387 req.requestHeaders["Content-Type"], 388 "application/x-www-form-urlencoded" + fixSinonBug 389 ); 390 391 // Verify ZIP hash 392 let tmpZipPath = OS.Path.join( 393 Zotero.getTempDirectory().path, 394 item2.key + '.zip' 395 ); 396 deferreds.push({ 397 promise: Zotero.Utilities.Internal.md5Async(tmpZipPath) 398 .then(function (md5) { 399 assert.equal(params.zipMD5, md5); 400 }) 401 }); 402 403 let parts = req.requestBody.split('&'); 404 let params = {}; 405 for (let part of parts) { 406 let [key, val] = part.split('='); 407 params[key] = decodeURIComponent(val); 408 } 409 Zotero.debug(params); 410 assert.equal(params.md5, hash2); 411 assert.notEqual(params.zipMD5, hash2); 412 assert.equal(params.mtime, mtime2); 413 assert.equal(params.filename, filename2); 414 assert.equal(params.zipFilename, item2.key + ".zip"); 415 assert.isTrue(parseInt(params.filesize) == params.filesize); 416 417 req.respond( 418 200, 419 { 420 "Content-Type": "application/json" 421 }, 422 JSON.stringify({ 423 url: baseURL + "pretend-s3/2", 424 contentType: 'application/zip', 425 prefix: prefix2, 426 suffix: suffix2, 427 uploadKey: uploadKey2 428 }) 429 ); 430 } 431 // Upload single file to S3 432 else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") { 433 assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug); 434 assert.equal( 435 req.requestBody.size, 436 (new Blob( 437 [ 438 prefix1, 439 file1Blob, 440 suffix1 441 ] 442 ).size) 443 ); 444 req.respond(201, {}, ""); 445 } 446 // Upload multi-file ZIP to S3 447 else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") { 448 assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug); 449 450 // Verify uploaded ZIP file 451 let tmpZipPath = OS.Path.join( 452 Zotero.getTempDirectory().path, 453 Zotero.Utilities.randomString() + '.zip' 454 ); 455 456 let deferred = Zotero.Promise.defer(); 457 deferreds.push(deferred); 458 var reader = new FileReader(); 459 reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () { 460 try { 461 462 let file = yield OS.File.open(tmpZipPath, { 463 create: true 464 }); 465 466 var contents = new Uint8Array(reader.result); 467 contents = contents.slice(prefix2.length, suffix2.length * -1); 468 yield file.write(contents); 469 yield file.close(); 470 471 var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"] 472 .createInstance(Components.interfaces.nsIZipReader); 473 zr.open(Zotero.File.pathToFile(tmpZipPath)); 474 zr.test(null); 475 var entries = zr.findEntries('*'); 476 var entryNames = []; 477 while (entries.hasMore()) { 478 entryNames.push(entries.getNext()); 479 } 480 assert.equal(entryNames.length, 2); 481 assert.sameMembers(entryNames, ['index.html', 'img.gif']); 482 assert.equal(zr.getEntry('index.html').realSize, size2); 483 assert.equal(zr.getEntry('img.gif').realSize, 42); 484 485 deferred.resolve(); 486 } 487 catch (e) { 488 deferred.reject(e); 489 } 490 })); 491 reader.readAsArrayBuffer(req.requestBody); 492 493 req.respond(201, {}, ""); 494 } 495 // Register single-file upload 496 else if (req.method == "POST" 497 && req.url == `${baseURL}users/1/items/${item1.key}/file` 498 && req.requestBody.indexOf('upload=') != -1) { 499 assertAPIKey(req); 500 assert.equal(req.requestHeaders["If-None-Match"], "*"); 501 assert.equal( 502 req.requestHeaders["Content-Type"], 503 "application/x-www-form-urlencoded" + fixSinonBug 504 ); 505 506 let parts = req.requestBody.split('&'); 507 let params = {}; 508 for (let part of parts) { 509 let [key, val] = part.split('='); 510 params[key] = decodeURIComponent(val); 511 } 512 assert.equal(params.upload, uploadKey1); 513 514 req.respond( 515 204, 516 { 517 "Last-Modified-Version": 10 518 }, 519 "" 520 ); 521 } 522 // Register multi-file upload 523 else if (req.method == "POST" 524 && req.url == `${baseURL}users/1/items/${item2.key}/file` 525 && req.requestBody.indexOf('upload=') != -1) { 526 assertAPIKey(req); 527 assert.equal(req.requestHeaders["If-None-Match"], "*"); 528 assert.equal( 529 req.requestHeaders["Content-Type"], 530 "application/x-www-form-urlencoded" + fixSinonBug 531 ); 532 533 let parts = req.requestBody.split('&'); 534 let params = {}; 535 for (let part of parts) { 536 let [key, val] = part.split('='); 537 params[key] = decodeURIComponent(val); 538 } 539 assert.equal(params.upload, uploadKey2); 540 541 req.respond( 542 204, 543 { 544 "Last-Modified-Version": 15 545 }, 546 "" 547 ); 548 } 549 }) 550 551 // TODO: One-step uploads 552 /*// https://github.com/cjohansen/Sinon.JS/issues/607 553 let fixSinonBug = ";charset=utf-8"; 554 server.respond(function (req) { 555 if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) { 556 assert.equal(req.requestHeaders["If-None-Match"], "*"); 557 assert.equal( 558 req.requestHeaders["Content-Type"], 559 "application/json" + fixSinonBug 560 ); 561 562 let params = JSON.parse(req.requestBody); 563 assert.equal(params.md5, hash); 564 assert.equal(params.mtime, mtime); 565 assert.equal(params.filename, filename); 566 assert.equal(params.size, size); 567 assert.equal(params.contentType, contentType); 568 569 req.respond( 570 200, 571 { 572 "Content-Type": "application/json" 573 }, 574 JSON.stringify({ 575 url: baseURL + "pretend-s3", 576 headers: { 577 "Content-Type": contentType, 578 "Content-MD5": hash, 579 //"Content-Length": params.size, process but don't return 580 //"x-amz-meta-" 581 }, 582 uploadKey 583 }) 584 ); 585 } 586 else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") { 587 assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug); 588 assert.instanceOf(req.requestBody, File); 589 req.respond(201, {}, ""); 590 } 591 })*/ 592 var result = yield engine.start(); 593 594 yield Zotero.Promise.all(deferreds.map(d => d.promise)); 595 596 assert.isTrue(result.localChanges); 597 assert.isTrue(result.remoteChanges); 598 assert.isFalse(result.syncRequired); 599 600 // Check local objects 601 assert.equal(item1.attachmentSyncedModificationTime, mtime1); 602 assert.equal(item1.attachmentSyncedHash, hash1); 603 assert.equal(item1.version, 10); 604 assert.equal(item2.attachmentSyncedModificationTime, mtime2); 605 assert.equal(item2.attachmentSyncedHash, hash2); 606 assert.equal(item2.version, 15); 607 }) 608 609 it("should update local info for remotely updated file that matches local file", function* () { 610 var { engine, client, caller } = yield setup(); 611 612 var library = Zotero.Libraries.userLibrary; 613 library.libraryVersion = 5; 614 yield library.saveTx(); 615 library.storageDownloadNeeded = true; 616 617 var file = getTestDataDirectory(); 618 file.append('test.txt'); 619 var item = yield Zotero.Attachments.importFromFile({ file }); 620 item.version = 5; 621 item.attachmentSyncState = "to_download"; 622 yield item.saveTx(); 623 var path = yield item.getFilePathAsync(); 624 yield OS.File.setDates(path, null, new Date() - 100000); 625 626 var json = item.toJSON(); 627 yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json); 628 629 var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) + ""; 630 var md5 = Zotero.Utilities.Internal.md5(file) 631 632 var s3Path = `pretend-s3/${item.key}`; 633 this.httpd.registerPathHandler( 634 `/users/1/items/${item.key}/file`, 635 { 636 handle: function (request, response) { 637 if (!request.hasHeader('Zotero-API-Key')) { 638 response.setStatusLine(null, 403, "Forbidden"); 639 return; 640 } 641 var key = request.getHeader('Zotero-API-Key'); 642 if (key != apiKey) { 643 response.setStatusLine(null, 403, "Invalid key"); 644 return; 645 } 646 response.setStatusLine(null, 302, "Found"); 647 response.setHeader("Zotero-File-Modification-Time", mtime, false); 648 response.setHeader("Zotero-File-MD5", md5, false); 649 response.setHeader("Zotero-File-Compressed", "No", false); 650 response.setHeader("Location", baseURL + s3Path, false); 651 } 652 } 653 ); 654 var result = yield engine.start(); 655 656 assert.equal(item.attachmentSyncedModificationTime, mtime); 657 yield assert.eventually.equal(item.attachmentModificationTime, mtime); 658 assert.isTrue(result.localChanges); 659 assert.isFalse(result.remoteChanges); 660 assert.isFalse(result.syncRequired); 661 }) 662 663 it("should update local info for file that already exists on the server", function* () { 664 var { engine, client, caller } = yield setup(); 665 666 var file = getTestDataDirectory(); 667 file.append('test.png'); 668 var item = yield Zotero.Attachments.importFromFile({ file: file }); 669 item.version = 5; 670 yield item.saveTx(); 671 var json = item.toJSON(); 672 yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json); 673 674 var mtime = yield item.attachmentModificationTime; 675 var hash = yield item.attachmentHash; 676 var path = item.getFilePath(); 677 var filename = 'test.png'; 678 var size = (yield OS.File.stat(path)).size; 679 var contentType = 'image/png'; 680 681 var newVersion = 10; 682 // https://github.com/cjohansen/Sinon.JS/issues/607 683 let fixSinonBug = ";charset=utf-8"; 684 server.respond(function (req) { 685 // Get upload authorization for single file 686 if (req.method == "POST" 687 && req.url == `${baseURL}users/1/items/${item.key}/file` 688 && req.requestBody.indexOf('upload=') == -1) { 689 assertAPIKey(req); 690 assert.equal(req.requestHeaders["If-None-Match"], "*"); 691 assert.equal( 692 req.requestHeaders["Content-Type"], 693 "application/x-www-form-urlencoded" + fixSinonBug 694 ); 695 696 req.respond( 697 200, 698 { 699 "Content-Type": "application/json", 700 "Last-Modified-Version": newVersion 701 }, 702 JSON.stringify({ 703 exists: 1, 704 }) 705 ); 706 } 707 }) 708 709 // TODO: One-step uploads 710 var result = yield engine.start(); 711 712 assert.isTrue(result.localChanges); 713 assert.isTrue(result.remoteChanges); 714 assert.isFalse(result.syncRequired); 715 716 // Check local objects 717 assert.equal(item.attachmentSyncedModificationTime, mtime); 718 assert.equal(item.attachmentSyncedHash, hash); 719 assert.equal(item.version, newVersion); 720 }) 721 722 723 it("should retry with If-None-Match on 412 with missing remote hash", function* () { 724 var { engine, client, caller } = yield setup(); 725 var zfs = new Zotero.Sync.Storage.Mode.ZFS({ 726 apiClient: client 727 }) 728 729 var file = getTestDataDirectory(); 730 file.append('test.png'); 731 var item = yield Zotero.Attachments.importFromFile({ file }); 732 item.version = 5; 733 item.synced = true; 734 item.attachmentSyncedModificationTime = Date.now(); 735 item.attachmentSyncedHash = 'bd4c33e03798a7e8bc0b46f8bda74fac' 736 yield item.saveTx(); 737 738 var contentType = 'image/png'; 739 var prefix = Zotero.Utilities.randomString(); 740 var suffix = Zotero.Utilities.randomString(); 741 var uploadKey = Zotero.Utilities.randomString(32, 'abcdef0123456789'); 742 743 var called = 0; 744 // https://github.com/cjohansen/Sinon.JS/issues/607 745 let fixSinonBug = ";charset=utf-8"; 746 server.respond(function (req) { 747 // Try with If-Match 748 if (req.method == "POST" 749 && req.url == `${baseURL}users/1/items/${item.key}/file` 750 && !req.requestBody.includes('upload=') 751 && req.requestHeaders["If-Match"] == item.attachmentSyncedHash) { 752 called++; 753 req.respond( 754 412, 755 { 756 "Content-Type": "application/json" 757 }, 758 "If-Match set but file does not exist" 759 ); 760 } 761 // Retry with If-None-Match 762 else if (req.method == "POST" 763 && req.url == `${baseURL}users/1/items/${item.key}/file` 764 && !req.requestBody.includes('upload=') 765 && req.requestHeaders["If-None-Match"] == "*") { 766 assert.equal(called++, 1); 767 req.respond( 768 200, 769 { 770 "Content-Type": "application/json" 771 }, 772 JSON.stringify({ 773 url: baseURL + "pretend-s3/1", 774 contentType: contentType, 775 prefix: prefix, 776 suffix: suffix, 777 uploadKey: uploadKey 778 }) 779 ); 780 } 781 // Upload file to S3 782 else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") { 783 assert.equal(called++, 2); 784 req.respond(201, {}, ""); 785 } 786 // Use If-None-Match when registering upload 787 else if (req.method == "POST" 788 && req.url == `${baseURL}users/1/items/${item.key}/file` 789 && req.requestBody.includes('upload=')) { 790 assert.equal(called++, 3); 791 assert.equal(req.requestHeaders["If-None-Match"], "*"); 792 req.respond( 793 204, 794 { 795 "Last-Modified-Version": 10 796 }, 797 "" 798 ); 799 } 800 }); 801 802 var result = yield engine.start(); 803 assert.equal(called, 4); 804 }); 805 }) 806 807 808 describe("#_processUploadFile()", function () { 809 it("should handle 404 from upload authorization request", function* () { 810 var { engine, client, caller } = yield setup(); 811 var zfs = new Zotero.Sync.Storage.Mode.ZFS({ 812 apiClient: client 813 }) 814 815 var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png'); 816 var item = yield Zotero.Attachments.importFromFile({ file: filePath }); 817 item.version = 5; 818 item.synced = true; 819 yield item.saveTx(); 820 821 var itemJSON = item.toResponseJSON(); 822 itemJSON.data.mtime = yield item.attachmentModificationTime; 823 itemJSON.data.md5 = yield item.attachmentHash; 824 825 server.respond(function (req) { 826 if (req.method == "POST" 827 && req.url == `${baseURL}users/1/items/${item.key}/file` 828 && !req.requestBody.includes('upload=')) { 829 req.respond( 830 404, 831 { 832 "Last-Modified-Version": 5 833 }, 834 "Not Found" 835 ); 836 } 837 }) 838 839 var result = yield zfs._processUploadFile({ 840 name: item.libraryKey 841 }); 842 assert.isTrue(result.syncRequired); 843 }); 844 845 it("should handle 412 with matching version and hash matching local file", function* () { 846 var { engine, client, caller } = yield setup(); 847 var zfs = new Zotero.Sync.Storage.Mode.ZFS({ 848 apiClient: client 849 }) 850 851 var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png'); 852 var item = yield Zotero.Attachments.importFromFile({ file: filePath }); 853 item.version = 5; 854 item.synced = true; 855 yield item.saveTx(); 856 857 var itemJSON = item.toResponseJSON(); 858 itemJSON.data.mtime = yield item.attachmentModificationTime; 859 itemJSON.data.md5 = yield item.attachmentHash; 860 861 // Set saved hash to a different value, which should be overwritten 862 // 863 // We're also testing cases where a hash isn't set for a file (e.g., if the 864 // storage directory was transferred, the mtime doesn't match, but the file was 865 // never downloaded), but there's no difference in behavior 866 var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; 867 item.attachmentSyncedHash = dbHash; 868 yield item.saveTx({ skipAll: true }); 869 870 server.respond(function (req) { 871 if (req.method == "POST" 872 && req.url == `${baseURL}users/1/items/${item.key}/file` 873 && req.requestBody.indexOf('upload=') == -1 874 && req.requestHeaders["If-Match"] == dbHash) { 875 req.respond( 876 412, 877 { 878 "Content-Type": "application/json", 879 "Last-Modified-Version": 5 880 }, 881 "ETag does not match current version of file" 882 ); 883 } 884 }) 885 setResponse({ 886 method: "GET", 887 url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`, 888 status: 200, 889 text: JSON.stringify([itemJSON]) 890 }); 891 892 var result = yield zfs._processUploadFile({ 893 name: item.libraryKey 894 }); 895 assert.equal(item.attachmentSyncedHash, (yield item.attachmentHash)); 896 assert.isFalse(result.localChanges); 897 assert.isFalse(result.remoteChanges); 898 assert.isFalse(result.syncRequired); 899 assert.isFalse(result.fileSyncRequired); 900 }) 901 902 it("should handle 412 with matching version and hash not matching local file", function* () { 903 var { engine, client, caller } = yield setup(); 904 var zfs = new Zotero.Sync.Storage.Mode.ZFS({ 905 apiClient: client 906 }) 907 908 var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png'); 909 var item = yield Zotero.Attachments.importFromFile({ file: filePath }); 910 item.version = 5; 911 item.synced = true; 912 yield item.saveTx(); 913 914 var fileHash = yield item.attachmentHash; 915 var itemJSON = item.toResponseJSON(); 916 itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa' 917 918 server.respond(function (req) { 919 if (req.method == "POST" 920 && req.url == `${baseURL}users/1/items/${item.key}/file` 921 && req.requestBody.indexOf('upload=') == -1 922 && req.requestHeaders["If-None-Match"] == "*") { 923 req.respond( 924 412, 925 { 926 "Content-Type": "application/json", 927 "Last-Modified-Version": 5 928 }, 929 "If-None-Match: * set but file exists" 930 ); 931 } 932 }) 933 setResponse({ 934 method: "GET", 935 url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`, 936 status: 200, 937 text: JSON.stringify([itemJSON]) 938 }); 939 940 var result = yield zfs._processUploadFile({ 941 name: item.libraryKey 942 }); 943 assert.isNull(item.attachmentSyncedHash); 944 assert.equal(item.attachmentSyncState, Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT); 945 assert.isFalse(result.localChanges); 946 assert.isFalse(result.remoteChanges); 947 assert.isFalse(result.syncRequired); 948 assert.isTrue(result.fileSyncRequired); 949 }) 950 951 it("should handle 412 with greater version", function* () { 952 var { engine, client, caller } = yield setup(); 953 var zfs = new Zotero.Sync.Storage.Mode.ZFS({ 954 apiClient: client 955 }) 956 957 var file = getTestDataDirectory(); 958 file.append('test.png'); 959 var item = yield Zotero.Attachments.importFromFile({ file }); 960 item.version = 5; 961 item.synced = true; 962 yield item.saveTx(); 963 964 server.respond(function (req) { 965 if (req.method == "POST" 966 && req.url == `${baseURL}users/1/items/${item.key}/file` 967 && req.requestBody.indexOf('upload=') == -1 968 && req.requestHeaders["If-None-Match"] == "*") { 969 req.respond( 970 412, 971 { 972 "Content-Type": "application/json", 973 "Last-Modified-Version": 10 974 }, 975 "If-None-Match: * set but file exists" 976 ); 977 } 978 }) 979 980 var result = yield zfs._processUploadFile({ 981 name: item.libraryKey 982 }); 983 assert.equal(item.version, 5); 984 assert.equal(item.synced, true); 985 assert.isFalse(result.localChanges); 986 assert.isFalse(result.remoteChanges); 987 assert.isTrue(result.syncRequired); 988 }); 989 990 991 it("should handle 413 on quota limit", function* () { 992 var { engine, client, caller } = yield setup(); 993 var zfs = new Zotero.Sync.Storage.Mode.ZFS({ 994 apiClient: client 995 }) 996 997 var file = getTestDataDirectory(); 998 file.append('test.png'); 999 var item = yield Zotero.Attachments.importFromFile({ file }); 1000 item.version = 5; 1001 item.synced = true; 1002 yield item.saveTx(); 1003 1004 server.respond(function (req) { 1005 if (req.method == "POST" 1006 && req.url == `${baseURL}users/1/items/${item.key}/file` 1007 && req.requestBody.indexOf('upload=') == -1 1008 && req.requestHeaders["If-None-Match"] == "*") { 1009 req.respond( 1010 413, 1011 { 1012 "Content-Type": "application/json", 1013 "Last-Modified-Version": 10 1014 }, 1015 "File would exceed quota (299.7 + 0.5 > 300)" 1016 ); 1017 } 1018 }) 1019 1020 var e = yield getPromiseError(zfs._processUploadFile({ 1021 name: item.libraryKey 1022 })); 1023 assert.ok(e); 1024 assert.equal(e.errorType, 'warning'); 1025 assert.include(e.message, 'test.png'); 1026 assert.equal(e.dialogButtonText, Zotero.getString('sync.storage.openAccountSettings')); 1027 }) 1028 }) 1029 })