syncFullTextEngineTest.js (13008B)
1 "use strict"; 2 3 describe("Zotero.Sync.Data.FullTextEngine", function () { 4 Components.utils.import("resource://zotero/config.js"); 5 6 var apiKey = Zotero.Utilities.randomString(24); 7 var baseURL = "http://local.zotero/"; 8 var engine, server, client, caller, stub, spy; 9 10 var responses = {}; 11 12 var setup = Zotero.Promise.coroutine(function* (options = {}) { 13 server = sinon.fakeServer.create(); 14 server.autoRespond = true; 15 16 Components.utils.import("resource://zotero/concurrentCaller.js"); 17 var caller = new ConcurrentCaller(1); 18 caller.setLogger(msg => Zotero.debug(msg)); 19 caller.stopOnError = true; 20 21 var client = new Zotero.Sync.APIClient({ 22 baseURL, 23 apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION, 24 apiKey, 25 caller, 26 background: options.background || true 27 }); 28 29 var engine = new Zotero.Sync.Data.FullTextEngine({ 30 apiClient: client, 31 libraryID: options.libraryID || Zotero.Libraries.userLibraryID, 32 stopOnError: true 33 }); 34 35 return { engine, client, caller }; 36 }); 37 38 function setResponse(response) { 39 setHTTPResponse(server, baseURL, response, responses); 40 } 41 42 function generateContent() { 43 return new Array(10).fill("").map(x => Zotero.Utilities.randomString()).join(" "); 44 } 45 46 // 47 // Tests 48 // 49 beforeEach(function* () { 50 yield resetDB({ 51 thisArg: this, 52 skipBundledFiles: true 53 }); 54 55 Zotero.HTTP.mock = sinon.FakeXMLHttpRequest; 56 57 yield Zotero.Users.setCurrentUserID(1); 58 yield Zotero.Users.setCurrentUsername("testuser"); 59 }) 60 61 describe("Full-Text Syncing", function () { 62 it("should skip full-text download if main library version is the same", function* () { 63 ({ engine, client, caller } = yield setup()); 64 var library = Zotero.Libraries.userLibrary; 65 library.libraryVersion = 10; 66 yield library.saveTx(); 67 yield Zotero.Fulltext.setLibraryVersion(library.id, 10); 68 yield engine.start(); 69 }); 70 71 it("should download full-text into a new library and subsequent updates", function* () { 72 ({ engine, client, caller } = yield setup()); 73 74 var item = yield createDataObject('item'); 75 var attachment = new Zotero.Item('attachment'); 76 attachment.parentItemID = item.id; 77 attachment.attachmentLinkMode = 'imported_file'; 78 attachment.attachmentContentType = 'application/pdf'; 79 attachment.attachmentFilename = 'test.pdf'; 80 yield attachment.saveTx(); 81 82 var content = generateContent() 83 var spy = sinon.spy(Zotero.Fulltext, "registerContentProcessor") 84 85 var itemFullTextVersion = 10; 86 var libraryVersion = 15; 87 88 // Set main library version to new version 89 var library = Zotero.Libraries.userLibrary; 90 library.libraryVersion = libraryVersion; 91 yield library.saveTx(); 92 93 setResponse({ 94 method: "GET", 95 url: "users/1/fulltext?format=versions", 96 status: 200, 97 headers: { 98 "Last-Modified-Version": libraryVersion 99 }, 100 json: { 101 [attachment.key]: itemFullTextVersion 102 } 103 }); 104 setResponse({ 105 method: "GET", 106 url: `users/1/items/${attachment.key}/fulltext`, 107 status: 200, 108 headers: { 109 "Last-Modified-Version": itemFullTextVersion 110 }, 111 json: { 112 content, 113 indexedPages: 1, 114 totalPages: 1 115 } 116 }); 117 yield engine.start(); 118 119 var dir = Zotero.Attachments.getStorageDirectory(attachment).path; 120 var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed'); 121 assert.isTrue(yield OS.File.exists(unprocessed)); 122 var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed)); 123 assert.propertyVal(data, 'text', content); 124 assert.propertyVal(data, 'indexedPages', 1); 125 assert.propertyVal(data, 'totalPages', 1); 126 assert.propertyVal(data, 'version', itemFullTextVersion); 127 yield assert.eventually.equal( 128 Zotero.FullText.getLibraryVersion(item.libraryID), 129 libraryVersion 130 ); 131 132 sinon.assert.calledOnce(spy); 133 spy.restore(); 134 135 // 136 // Get new content 137 // 138 ({ engine, client, caller } = yield setup()); 139 140 item = yield createDataObject('item'); 141 attachment = new Zotero.Item('attachment'); 142 attachment.parentItemID = item.id; 143 attachment.attachmentLinkMode = 'imported_file'; 144 attachment.attachmentContentType = 'application/pdf'; 145 attachment.attachmentFilename = 'test.pdf'; 146 yield attachment.saveTx(); 147 148 content = generateContent() 149 spy = sinon.spy(Zotero.Fulltext, "registerContentProcessor") 150 151 itemFullTextVersion = 17; 152 var lastLibraryVersion = libraryVersion; 153 libraryVersion = 20; 154 155 // Set main library version to new version 156 library.libraryVersion = libraryVersion; 157 yield library.saveTx(); 158 159 setResponse({ 160 method: "GET", 161 url: "users/1/fulltext?format=versions&since=" + lastLibraryVersion, 162 status: 200, 163 headers: { 164 "Last-Modified-Version": libraryVersion 165 }, 166 json: { 167 [attachment.key]: itemFullTextVersion 168 } 169 }); 170 setResponse({ 171 method: "GET", 172 url: `users/1/items/${attachment.key}/fulltext`, 173 status: 200, 174 headers: { 175 "Last-Modified-Version": itemFullTextVersion 176 }, 177 json: { 178 content, 179 indexedPages: 1, 180 totalPages: 1 181 } 182 }); 183 yield engine.start(); 184 185 var dir = Zotero.Attachments.getStorageDirectory(attachment).path; 186 var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed'); 187 assert.isTrue(yield OS.File.exists(unprocessed)); 188 var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed)); 189 assert.propertyVal(data, 'text', content); 190 assert.propertyVal(data, 'indexedPages', 1); 191 assert.propertyVal(data, 'totalPages', 1); 192 assert.propertyVal(data, 'version', itemFullTextVersion); 193 yield assert.eventually.equal( 194 Zotero.FullText.getLibraryVersion(item.libraryID), 195 libraryVersion 196 ); 197 198 sinon.assert.calledOnce(spy); 199 spy.restore(); 200 }) 201 202 it("should handle remotely missing full-text content", function* () { 203 ({ engine, client, caller } = yield setup()); 204 205 var item = yield createDataObject('item'); 206 var attachment = new Zotero.Item('attachment'); 207 attachment.parentItemID = item.id; 208 attachment.attachmentLinkMode = 'imported_file'; 209 attachment.attachmentContentType = 'application/pdf'; 210 attachment.attachmentFilename = 'test.pdf'; 211 yield attachment.saveTx(); 212 213 var itemFullTextVersion = 10; 214 var libraryVersion = 15; 215 setResponse({ 216 method: "GET", 217 url: "users/1/fulltext?format=versions", 218 status: 200, 219 headers: { 220 "Last-Modified-Version": libraryVersion 221 }, 222 json: { 223 [attachment.key]: itemFullTextVersion 224 } 225 }); 226 setResponse({ 227 method: "GET", 228 url: `users/1/items/${attachment.key}/fulltext`, 229 status: 404, 230 headers: { 231 "Last-Modified-Version": itemFullTextVersion 232 }, 233 text: "" 234 }); 235 yield engine.start(); 236 }) 237 238 it("should upload new full-text content and subsequent updates", function* () { 239 // https://github.com/cjohansen/Sinon.JS/issues/607 240 var fixSinonBug = ";charset=utf-8"; 241 242 var library = Zotero.Libraries.userLibrary; 243 var libraryID = library.id; 244 library.libraryVersion = 5; 245 yield library.saveTx(); 246 247 ({ engine, client, caller } = yield setup()); 248 249 var item = yield createDataObject('item'); 250 251 var attachment1 = new Zotero.Item('attachment'); 252 attachment1.parentItemID = item.id; 253 attachment1.attachmentLinkMode = 'imported_file'; 254 attachment1.attachmentContentType = 'text/html'; 255 attachment1.attachmentFilename = 'test.html'; 256 attachment1.attachmentCharset = 'utf-8'; 257 attachment1.synced = true; 258 yield attachment1.saveTx(); 259 yield Zotero.Attachments.createDirectoryForItem(attachment1); 260 var path = attachment1.getFilePath(); 261 var content1 = "A" + generateContent() 262 yield Zotero.File.putContentsAsync(path, content1); 263 264 var attachment2 = new Zotero.Item('attachment'); 265 attachment2.parentItemID = item.id; 266 attachment2.attachmentLinkMode = 'imported_file'; 267 attachment2.attachmentContentType = 'text/html'; 268 attachment2.attachmentFilename = 'test.html'; 269 attachment2.attachmentCharset = 'utf-8'; 270 attachment2.synced = true; 271 yield attachment2.saveTx(); 272 yield Zotero.Attachments.createDirectoryForItem(attachment2); 273 path = attachment2.getFilePath(); 274 var content2 = "B" + generateContent() 275 yield Zotero.File.putContentsAsync(path, content2); 276 277 yield Zotero.Fulltext.indexItems([attachment1.id, attachment2.id]); 278 279 var libraryVersion = 15; 280 281 var count = 1; 282 setResponse({ 283 method: "GET", 284 url: "users/1/fulltext?format=versions", 285 status: 200, 286 headers: { 287 "Last-Modified-Version": libraryVersion 288 }, 289 json: {} 290 }); 291 server.respond(function (req) { 292 if (req.method == "POST") { 293 if (req.url == `${baseURL}users/1/fulltext`) { 294 assert.propertyVal( 295 req.requestHeaders, 296 'Content-Type', 297 'application/json' + fixSinonBug 298 ); 299 300 let json = JSON.parse(req.requestBody); 301 assert.lengthOf(json, 2); 302 303 json.sort((a, b) => a.content < b.content ? -1 : 1); 304 assert.propertyVal(json[0], 'key', attachment1.key); 305 assert.propertyVal(json[0], 'content', content1); 306 assert.propertyVal(json[0], 'indexedChars', content1.length); 307 assert.propertyVal(json[0], 'totalChars', content1.length); 308 assert.propertyVal(json[0], 'indexedPages', 0); 309 assert.propertyVal(json[0], 'totalPages', 0); 310 assert.propertyVal(json[1], 'key', attachment2.key); 311 assert.propertyVal(json[1], 'content', content2); 312 assert.propertyVal(json[1], 'indexedChars', content2.length); 313 assert.propertyVal(json[1], 'totalChars', content2.length); 314 assert.propertyVal(json[1], 'indexedPages', 0); 315 assert.propertyVal(json[1], 'totalPages', 0); 316 317 req.respond( 318 200, 319 { 320 "Content-Type": "application/json", 321 "Last-Modified-Version": ++libraryVersion 322 }, 323 JSON.stringify({ 324 "successful": { 325 "0": { 326 key: attachment1.key 327 }, 328 "1": { 329 key: attachment2.key 330 } 331 }, 332 "unchanged": {}, 333 "failed": {} 334 }) 335 ); 336 count--; 337 } 338 } 339 }) 340 341 yield engine.start(); 342 assert.equal(count, 0); 343 yield assert.eventually.equal(Zotero.FullText.getItemVersion(attachment1.id), libraryVersion); 344 yield assert.eventually.equal(Zotero.FullText.getItemVersion(attachment2.id), libraryVersion); 345 yield assert.eventually.equal(Zotero.Fulltext.getLibraryVersion(libraryID), libraryVersion); 346 assert.equal(Zotero.Libraries.userLibrary.libraryVersion, libraryVersion); 347 348 // 349 // Upload new content 350 // 351 ({ engine, client, caller } = yield setup()); 352 library.libraryVersion = libraryVersion; 353 yield library.saveTx(); 354 355 var attachment3 = new Zotero.Item('attachment'); 356 attachment3.parentItemID = item.id; 357 attachment3.attachmentLinkMode = 'imported_file'; 358 attachment3.attachmentContentType = 'text/html'; 359 attachment3.attachmentFilename = 'test.html'; 360 attachment3.attachmentCharset = 'utf-8'; 361 attachment3.synced = true; 362 yield attachment3.saveTx(); 363 yield Zotero.Attachments.createDirectoryForItem(attachment3); 364 365 path = attachment3.getFilePath(); 366 var content3 = generateContent() 367 yield Zotero.File.putContentsAsync(path, content3); 368 yield Zotero.Fulltext.indexItems([attachment3.id]); 369 370 count = 1; 371 setResponse({ 372 method: "GET", 373 url: "users/1/fulltext?format=versions&since=" + libraryVersion, 374 status: 200, 375 headers: { 376 "Last-Modified-Version": libraryVersion 377 }, 378 json: {} 379 }); 380 server.respond(function (req) { 381 if (req.method == "POST") { 382 if (req.url == `${baseURL}users/1/fulltext`) { 383 assert.propertyVal(req.requestHeaders, 'Zotero-API-Key', apiKey); 384 assert.propertyVal( 385 req.requestHeaders, 386 'Content-Type', 387 'application/json' + fixSinonBug 388 ); 389 390 let json = JSON.parse(req.requestBody); 391 assert.lengthOf(json, 1); 392 json = json[0]; 393 assert.propertyVal(json, 'key', attachment3.key); 394 assert.propertyVal(json, 'content', content3); 395 assert.propertyVal(json, 'indexedChars', content3.length); 396 assert.propertyVal(json, 'totalChars', content3.length); 397 assert.propertyVal(json, 'indexedPages', 0); 398 assert.propertyVal(json, 'totalPages', 0); 399 400 req.respond( 401 200, 402 { 403 "Content-Type": "application/json", 404 "Last-Modified-Version": ++libraryVersion 405 }, 406 JSON.stringify({ 407 "successful": { 408 "0": { 409 key: attachment3.key 410 } 411 }, 412 "unchanged": {}, 413 "failed": {} 414 }) 415 ); 416 count--; 417 } 418 } 419 }) 420 421 yield engine.start(); 422 assert.equal(count, 0); 423 yield assert.eventually.equal(Zotero.FullText.getItemVersion(attachment3.id), libraryVersion); 424 yield assert.eventually.equal(Zotero.Fulltext.getLibraryVersion(libraryID), libraryVersion); 425 assert.equal(Zotero.Libraries.userLibrary.libraryVersion, libraryVersion); 426 }) 427 }); 428 })