storageEngine.js (10537B)
1 /* 2 ***** BEGIN LICENSE BLOCK ***** 3 4 Copyright © 2015 Center for History and New Media 5 George Mason University, Fairfax, Virginia, USA 6 http://zotero.org 7 8 This file is part of Zotero. 9 10 Zotero is free software: you can redistribute it and/or modify 11 it under the terms of the GNU Affero General Public License as published by 12 the Free Software Foundation, either version 3 of the License, or 13 (at your option) any later version. 14 15 Zotero is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU Affero General Public License for more details. 19 20 You should have received a copy of the GNU Affero General Public License 21 along with Zotero. If not, see <http://www.gnu.org/licenses/>. 22 23 ***** END LICENSE BLOCK ***** 24 */ 25 26 27 if (!Zotero.Sync.Storage) { 28 Zotero.Sync.Storage = {}; 29 } 30 31 /** 32 * An Engine manages file sync processes for a given library 33 * 34 * @param {Object} options 35 * @param {Integer} options.libraryID 36 * @param {Object} options.controller - Storage controller instance (ZFS_Controller/WebDAV_Controller) 37 * @param {Function} [onProgress] - Function to run when a request finishes: f(progress, progressMax) 38 * @param {Function} [onError] - Function to run on error 39 * @param {Boolean} [stopOnError] 40 */ 41 Zotero.Sync.Storage.Engine = function (options) { 42 if (options.libraryID == undefined) { 43 throw new Error("options.libraryID not set"); 44 } 45 if (options.controller == undefined) { 46 throw new Error("options.controller not set"); 47 } 48 49 this.background = options.background; 50 this.firstInSession = options.firstInSession; 51 this.lastFullFileCheck = options.lastFullFileCheck; 52 this.libraryID = options.libraryID; 53 this.library = Zotero.Libraries.get(options.libraryID); 54 this.controller = options.controller; 55 56 this.numRequests = 0; 57 this.requestsRemaining = 0; 58 59 this.local = Zotero.Sync.Storage.Local; 60 this.utils = Zotero.Sync.Storage.Utilities; 61 62 this.setStatus = options.setStatus || function () {}; 63 this.onError = options.onError || function (e) {}; 64 this.onProgress = options.onProgress || function (progress, progressMax) {}; 65 this.stopOnError = options.stopOnError || false; 66 67 this.queues = []; 68 ['download', 'upload'].forEach(function (type) { 69 this.queues[type] = new ConcurrentCaller({ 70 id: `${this.libraryID}/${type}`, 71 numConcurrent: Zotero.Prefs.get( 72 'sync.storage.max' + Zotero.Utilities.capitalize(type) + 's' 73 ), 74 onError: this.onError, 75 stopOnError: this.stopOnError, 76 logger: Zotero.debug 77 }); 78 }.bind(this)) 79 80 this.maxCheckAge = 10800; // maximum age in seconds for upload modification check (3 hours) 81 } 82 83 Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function* () { 84 var libraryID = this.libraryID; 85 if (!Zotero.Sync.Storage.Local.getEnabledForLibrary(libraryID)) { 86 Zotero.debug("File sync is not enabled for " + this.library.name); 87 return false; 88 } 89 90 Zotero.debug("Starting file sync for " + this.library.name); 91 92 if (!this.controller.verified) { 93 Zotero.debug(`${this.controller.name} file sync is not active -- verifying`); 94 95 try { 96 yield this.controller.checkServer(); 97 } 98 catch (e) { 99 let wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] 100 .getService(Components.interfaces.nsIWindowMediator); 101 let lastWin = wm.getMostRecentWindow("navigator:browser"); 102 103 let success = yield this.controller.handleVerificationError(e, lastWin, true); 104 if (!success) { 105 Zotero.debug(this.controller.name + " verification failed", 2); 106 107 throw new Zotero.Error( 108 Zotero.getString('sync.storage.error.verificationFailed', this.controller.name), 109 0, 110 { 111 dialogButtonText: Zotero.getString('sync.openSyncPreferences'), 112 dialogButtonCallback: function () { 113 let wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] 114 .getService(Components.interfaces.nsIWindowMediator); 115 let lastWin = wm.getMostRecentWindow("navigator:browser"); 116 lastWin.ZoteroPane.openPreferences('zotero-prefpane-sync'); 117 } 118 } 119 ); 120 } 121 } 122 } 123 124 if (this.controller.cacheCredentials) { 125 yield this.controller.cacheCredentials(); 126 } 127 128 var lastSyncTime = null; 129 var downloadAll = this.local.downloadOnSync(libraryID); 130 // 131 // TODO: If files are persistently missing, don't try to download them each time 132 133 var filesEditable = Zotero.Libraries.get(libraryID).filesEditable; 134 this.requestsRemaining = 0; 135 136 // Check for updated files to upload 137 if (!filesEditable) { 138 Zotero.debug("No file editing access -- skipping file modification check for " 139 + this.library.name); 140 } 141 // If this is a background sync, it's not the first sync of the session, the library has had 142 // at least one full check this session, and it's been less than maxCheckAge since the last 143 // full check of this library, check only files that were previously modified or opened 144 // recently 145 else if (this.background 146 && !this.firstInSession 147 && this.local.lastFullFileCheck[libraryID] 148 && (this.local.lastFullFileCheck[libraryID] 149 + (this.maxCheckAge * 1000)) > new Date().getTime()) { 150 let itemIDs = this.local.getFilesToCheck(libraryID, this.maxCheckAge); 151 yield this.local.checkForUpdatedFiles(libraryID, itemIDs); 152 } 153 // Otherwise check all files in library 154 else { 155 this.local.lastFullFileCheck[libraryID] = new Date().getTime(); 156 yield this.local.checkForUpdatedFiles(libraryID); 157 } 158 159 yield this.local.resolveConflicts(libraryID); 160 161 var downloadForced = yield this.local.checkForForcedDownloads(libraryID); 162 163 // If we don't have any forced downloads, we can skip downloads if no storage metadata has 164 // changed (meaning nothing else has uploaded files since the last successful file sync) 165 if (downloadAll && !downloadForced) { 166 if (this.library.storageVersion == this.library.libraryVersion) { 167 Zotero.debug("No remote storage changes for " + this.library.name 168 + " -- skipping file downloads"); 169 downloadAll = false; 170 } 171 } 172 173 // Get files to download 174 if (downloadAll || downloadForced) { 175 let itemIDs = yield this.local.getFilesToDownload(libraryID, !downloadAll); 176 if (itemIDs.length) { 177 Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to " 178 + "download for " + this.library.name); 179 for (let itemID of itemIDs) { 180 let item = yield Zotero.Items.getAsync(itemID); 181 yield this.queueItem(item); 182 } 183 } 184 else { 185 Zotero.debug("No files to download for " + this.library.name); 186 } 187 } 188 189 // Get files to upload 190 if (filesEditable) { 191 let itemIDs = yield this.local.getFilesToUpload(libraryID); 192 if (itemIDs.length) { 193 Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to " 194 + "upload for " + this.library.name); 195 for (let itemID of itemIDs) { 196 let item = yield Zotero.Items.getAsync(itemID, { noCache: true }); 197 yield this.queueItem(item); 198 } 199 } 200 else { 201 Zotero.debug("No files to upload for " + this.library.name); 202 } 203 } 204 else { 205 Zotero.debug("No file editing access -- skipping file uploads for " + this.library.name); 206 } 207 208 var promises = { 209 download: this.queues.download.runAll(), 210 upload: this.queues.upload.runAll() 211 } 212 213 // Process the results 214 var downloadSuccessful = false; 215 var changes = new Zotero.Sync.Storage.Result; 216 for (let type of ['download', 'upload']) { 217 let results = yield promises[type]; 218 let succeeded = 0; 219 let failed = 0; 220 221 for (let p of results) { 222 if (p.isFulfilled()) { 223 succeeded++; 224 } 225 else if (!p.isPending()) { 226 if (this.stopOnError) { 227 let e = p.reason(); 228 Zotero.debug(`File ${type} sync failed for ${this.library.name}`); 229 throw e; 230 } 231 failed++; 232 } 233 } 234 235 Zotero.debug(`File ${type} sync finished for ${this.library.name} ` 236 + `(${succeeded} succeeded, ${failed} failed)`); 237 238 changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value())); 239 240 if (type == 'download' 241 // Not stopped 242 && this.requestsRemaining == 0 243 // No errors 244 && results.every(p => !p.isRejected())) { 245 downloadSuccessful = true; 246 } 247 } 248 249 if (downloadSuccessful) { 250 this.library.storageDownloadNeeded = false; 251 this.library.storageVersion = this.library.libraryVersion; 252 yield this.library.saveTx(); 253 } 254 255 // For ZFS, this purges all files on server based on flag set when switching from ZFS 256 // to WebDAV in prefs. For WebDAV, this purges locally deleted files on server. 257 try { 258 yield this.controller.purgeDeletedStorageFiles(libraryID); 259 } 260 catch (e) { 261 Zotero.logError(e); 262 } 263 264 // If WebDAV sync, purge orphaned files 265 if (downloadSuccessful && this.controller.mode == 'webdav') { 266 try { 267 yield this.controller.purgeOrphanedStorageFiles(libraryID); 268 } 269 catch (e) { 270 Zotero.logError(e); 271 } 272 } 273 274 if (!changes.localChanges) { 275 Zotero.debug("No local changes made during file sync"); 276 } 277 278 Zotero.debug("Done with file sync for " + this.library.name); 279 280 return changes; 281 }) 282 283 284 Zotero.Sync.Storage.Engine.prototype.stop = function () { 285 Zotero.debug("Stopping file sync for " + this.library.name); 286 for (let type in this.queues) { 287 this.queues[type].stop(); 288 } 289 } 290 291 Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) { 292 switch (item.attachmentSyncState) { 293 case Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD: 294 case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_DOWNLOAD: 295 var type = 'download'; 296 var fn = 'downloadFile'; 297 break; 298 299 case Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD: 300 case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD: 301 var type = 'upload'; 302 var fn = 'uploadFile'; 303 break; 304 305 case false: 306 Zotero.debug("Sync state for item " + item.id + " not found", 2); 307 return; 308 309 default: 310 throw new Error("Invalid sync state " + item.attachmentSyncState); 311 } 312 313 if (type == 'upload') { 314 if (!(yield item.fileExists())) { 315 Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping"); 316 return; 317 } 318 } 319 this.queues[type].add(() => { 320 var request = new Zotero.Sync.Storage.Request({ 321 type, 322 libraryID: this.libraryID, 323 name: item.libraryKey, 324 onStart: request => this.controller[fn](request), 325 onStop: () => { 326 this.requestsRemaining--; 327 this.onProgress(this.numRequests - this.requestsRemaining, this.numRequests); 328 } 329 }); 330 return request.start(); 331 }); 332 this.numRequests++; 333 this.requestsRemaining++; 334 })