feed.js (17683B)
1 /* 2 ***** BEGIN LICENSE BLOCK ***** 3 4 Copyright © 2015 Center for History and New Media 5 George Mason University, Fairfax, Virginia, USA 6 http://zotero.org 7 8 This file is part of Zotero. 9 10 Zotero is free software: you can redistribute it and/or modify 11 it under the terms of the GNU Affero General Public License as published by 12 the Free Software Foundation, either version 3 of the License, or 13 (at your option) any later version. 14 15 Zotero is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU Affero General Public License for more details. 19 20 You should have received a copy of the GNU Affero General Public License 21 along with Zotero. If not, see <http://www.gnu.org/licenses/>. 22 23 ***** END LICENSE BLOCK ***** 24 */ 25 26 /** 27 * Zotero.Feed, extends Zotero.Library 28 * 29 * Custom parameters: 30 * - name - name of the feed displayed in the collection tree 31 * - url 32 * - cleanupReadAfter - number of days after which read items should be removed 33 * - cleanupUnreadAfter - number of days after which unread items should be removed 34 * - refreshInterval - in terms of hours 35 * 36 * @param params 37 * @returns Zotero.Feed 38 * @constructor 39 */ 40 Zotero.Feed = function(params = {}) { 41 params.libraryType = 'feed'; 42 Zotero.Feed._super.call(this, params); 43 44 this._feedCleanupReadAfter = null; 45 this._feedCleanupUnreadAfter = null; 46 this._feedRefreshInterval = null; 47 this._feedUnreadCount = null; 48 this._updating = false; 49 this._previousURL = null; 50 51 // Feeds are not editable by the user. Remove the setter 52 this.editable = false; 53 Zotero.defineProperty(this, 'editable', { 54 get: function() { return this._get('_libraryEditable'); } 55 }); 56 57 // Feeds are not filesEditable by the user. Remove the setter 58 this.filesEditable = false; 59 Zotero.defineProperty(this, 'filesEditable', { 60 get: function() { return this._get('_libraryFilesEditable'); } 61 }); 62 63 Zotero.Utilities.assignProps(this, params, 64 ['name', 'url', 'refreshInterval', 'cleanupReadAfter', 'cleanupUnreadAfter']); 65 66 // Return a proxy so that we can disable the object once it's deleted 67 return new Proxy(this, { 68 get: function(obj, prop) { 69 if (obj._disabled && !(prop == 'libraryID' || prop == 'id' || prop == 'treeViewID')) { 70 throw new Error("Feed " + obj.libraryID + " has been disabled"); 71 } 72 return obj[prop]; 73 } 74 }); 75 } 76 77 Zotero.Feed._colToProp = function(c) { 78 return "_feed" + Zotero.Utilities.capitalize(c); 79 } 80 81 Zotero.extendClass(Zotero.Library, Zotero.Feed); 82 83 Zotero.defineProperty(Zotero.Feed, '_unreadCountSQL', { 84 value: "(SELECT COUNT(*) FROM items I JOIN feedItems FI USING (itemID)" 85 + " WHERE I.libraryID=F.libraryID AND FI.readTime IS NULL) AS _feedUnreadCount" 86 }); 87 88 Zotero.defineProperty(Zotero.Feed, '_dbColumns', { 89 value: Object.freeze(['name', 'url', 'lastUpdate', 'lastCheck', 90 'lastCheckError', 'cleanupUnreadAfter', 'cleanupReadAfter', 'refreshInterval']) 91 }); 92 93 Zotero.defineProperty(Zotero.Feed, '_primaryDataSQLParts'); 94 95 Zotero.defineProperty(Zotero.Feed, '_rowSQLSelect', { 96 value: Zotero.Library._rowSQLSelect + ", " 97 + Zotero.Feed._dbColumns.map(c => "F." + c + " AS " + Zotero.Feed._colToProp(c)).join(", ") 98 + ", " + Zotero.Feed._unreadCountSQL 99 }); 100 101 Zotero.defineProperty(Zotero.Feed, '_rowSQL', { 102 value: "SELECT " + Zotero.Feed._rowSQLSelect 103 + " FROM feeds F JOIN libraries L USING (libraryID)" 104 }); 105 106 Zotero.defineProperty(Zotero.Feed.prototype, '_objectType', { 107 value: 'feed' 108 }); 109 110 Zotero.defineProperty(Zotero.Feed.prototype, 'isFeed', { 111 value: true 112 }); 113 114 Zotero.defineProperty(Zotero.Feed.prototype, 'allowsLinkedFiles', { 115 value: false 116 }); 117 118 Zotero.defineProperty(Zotero.Feed.prototype, 'libraryTypes', { 119 value: Object.freeze(Zotero.Feed._super.prototype.libraryTypes.concat(['feed'])) 120 }); 121 122 Zotero.defineProperty(Zotero.Feed.prototype, 'libraryTypeID', { 123 get: () => undefined 124 }); 125 126 Zotero.defineProperty(Zotero.Feed.prototype, 'unreadCount', { 127 get: function() { return this._feedUnreadCount; } 128 }); 129 Zotero.defineProperty(Zotero.Feed.prototype, 'updating', { 130 get: function() { return !!this._updating; } 131 }); 132 133 (function() { 134 // Create accessors 135 let accessors = ['name', 'url', 'refreshInterval', 'cleanupUnreadAfter', 'cleanupReadAfter']; 136 for (let i=0; i<accessors.length; i++) { 137 let name = accessors[i]; 138 let prop = Zotero.Feed._colToProp(name); 139 Zotero.defineProperty(Zotero.Feed.prototype, name, { 140 get: function() { return this._get(prop); }, 141 set: function(v) { return this._set(prop, v); } 142 }) 143 } 144 let getters = ['lastCheck', 'lastUpdate', 'lastCheckError']; 145 for (let i=0; i<getters.length; i++) { 146 let name = getters[i]; 147 let prop = Zotero.Feed._colToProp(name); 148 Zotero.defineProperty(Zotero.Feed.prototype, name, { 149 get: function() { return this._get(prop); } 150 }) 151 } 152 })() 153 154 Zotero.Feed.prototype._isValidFeedProp = function(prop) { 155 let preffix = '_feed'; 156 if (prop.indexOf(preffix) != 0 || prop.length == preffix.length) { 157 return false; 158 } 159 160 let col = prop.substr(preffix.length); 161 col = col.charAt(0).toLowerCase() + col.substr(1); 162 163 return Zotero.Feed._dbColumns.indexOf(col) != -1; 164 } 165 166 Zotero.Feed.prototype._isValidProp = function(prop) { 167 return this._isValidFeedProp(prop) 168 || Zotero.Feed._super.prototype._isValidProp.call(this, prop); 169 } 170 171 Zotero.Feed.prototype._set = function (prop, val) { 172 switch (prop) { 173 case '_feedName': 174 if (!val || typeof val != 'string') { 175 throw new Error(prop + " must be a non-empty string"); 176 } 177 break; 178 case '_feedUrl': 179 let uri, 180 invalidUrlError = "Invalid feed URL " + val; 181 try { 182 uri = Components.classes["@mozilla.org/network/io-service;1"] 183 .getService(Components.interfaces.nsIIOService) 184 .newURI(val, null, null); 185 val = uri.spec; 186 } catch(e) { 187 throw new Error(invalidUrlError); 188 } 189 190 if (uri.scheme !== 'http' && uri.scheme !== 'https') { 191 throw new Error(invalidUrlError); 192 } 193 this._previousURL = this.url; 194 break; 195 case '_feedRefreshInterval': 196 case '_feedCleanupReadAfter': 197 case '_feedCleanupUnreadAfter': 198 if (val === null) break; 199 200 let newVal = Number.parseInt(val, 10); 201 if (newVal != val || !newVal || newVal <= 0) { 202 throw new Error(`${prop} must be null or a positive integer, but is ${val}`); 203 } 204 break; 205 case '_feedLastCheckError': 206 if (!val) { 207 val = null; 208 break; 209 } 210 211 if (typeof val !== 'string') { 212 throw new Error(`${prop} must be null or a string, but is ${val}`); 213 } 214 break; 215 } 216 217 return Zotero.Feed._super.prototype._set.call(this, prop, val); 218 } 219 220 Zotero.Feed.prototype._loadDataFromRow = function(row) { 221 Zotero.Feed._super.prototype._loadDataFromRow.call(this, row); 222 223 this._feedName = row._feedName; 224 this._feedUrl = row._feedUrl; 225 this._feedLastCheckError = row._feedLastCheckError || null; 226 this._feedLastCheck = row._feedLastCheck || null; 227 this._feedLastUpdate = row._feedLastUpdate || null; 228 this._feedCleanupReadAfter = parseInt(row._feedCleanupReadAfter) || null; 229 this._feedCleanupUnreadAfter = parseInt(row._feedCleanupUnreadAfter) || null; 230 this._feedRefreshInterval = parseInt(row._feedRefreshInterval) || null; 231 this._feedUnreadCount = parseInt(row._feedUnreadCount); 232 } 233 234 Zotero.Feed.prototype._reloadFromDB = Zotero.Promise.coroutine(function* () { 235 let sql = Zotero.Feed._rowSQL + " WHERE F.libraryID=?"; 236 let row = yield Zotero.DB.rowQueryAsync(sql, [this.libraryID]); 237 this._loadDataFromRow(row); 238 }); 239 240 Zotero.defineProperty(Zotero.Feed.prototype, '_childObjectTypes', { 241 value: Object.freeze(['feedItem', 'item']) 242 }); 243 244 Zotero.Feed.prototype._initSave = Zotero.Promise.coroutine(function* (env) { 245 let proceed = yield Zotero.Feed._super.prototype._initSave.call(this, env); 246 if (!proceed) return false; 247 248 if (!this._feedName) throw new Error("Feed name not set"); 249 if (!this._feedUrl) throw new Error("Feed URL not set"); 250 if (!this.refreshInterval) this.refreshInterval = Zotero.Prefs.get('feeds.defaultTTL') * 60; 251 if (!this.cleanupReadAfter) this.cleanupReadAfter = Zotero.Prefs.get('feeds.defaultCleanupReadAfter'); 252 if (!this.cleanupUnreadAfter) this.cleanupUnreadAfter = Zotero.Prefs.get('feeds.defaultCleanupUnreadAfter'); 253 254 if (env.isNew) { 255 // Make sure URL is unique 256 if (Zotero.Feeds.existsByURL(this._feedUrl)) { 257 throw new Error('Feed for URL already exists: ' + this._feedUrl); 258 } 259 } 260 261 return true; 262 }); 263 264 Zotero.Feed.prototype._saveData = Zotero.Promise.coroutine(function* (env) { 265 yield Zotero.Feed._super.prototype._saveData.apply(this, arguments); 266 267 Zotero.debug("Saving feed data for library " + this.id); 268 269 let changedCols = [], params = []; 270 for (let i=0; i<Zotero.Feed._dbColumns.length; i++) { 271 let col = Zotero.Feed._dbColumns[i]; 272 let prop = Zotero.Feed._colToProp(col); 273 274 if (!this._changed[prop]) continue; 275 276 changedCols.push(col); 277 params.push(this[prop]); 278 } 279 280 if (env.isNew) { 281 changedCols.push('libraryID'); 282 params.push(this.libraryID); 283 284 let sql = "INSERT INTO feeds (" + changedCols.join(', ') + ") " 285 + "VALUES (" + Array(params.length).fill('?').join(', ') + ")"; 286 yield Zotero.DB.queryAsync(sql, params); 287 288 Zotero.Notifier.queue( 289 'add', 'feed', this.libraryID, env.notifierData, env.options.notifierQueue 290 ); 291 } 292 else if (changedCols.length) { 293 let sql = "UPDATE feeds SET " + changedCols.map(v => v + '=?').join(', ') 294 + " WHERE libraryID=?"; 295 params.push(this.libraryID); 296 yield Zotero.DB.queryAsync(sql, params); 297 298 if (!env.options.skipNotifier) { 299 Zotero.Notifier.queue( 300 'modify', 'feed', this.libraryID, env.notifierData, env.options.notifierQueue 301 ); 302 } 303 } 304 else { 305 Zotero.debug("Feed data did not change for feed " + this.libraryID, 5); 306 } 307 }); 308 309 Zotero.Feed.prototype._finalizeSave = Zotero.Promise.coroutine(function* (env) { 310 let syncedDataChanged = 311 ['_feedName', '_feedCleanupReadAfter', '_feedCleanupUnreadAfter', '_feedRefreshInterval'].some((val) => this._changed[val]); 312 313 yield Zotero.Feed._super.prototype._finalizeSave.apply(this, arguments); 314 315 if (!env.isNew && this._previousURL) { 316 // Re-register library if URL changed 317 Zotero.Feeds.unregister(this.libraryID); 318 319 let syncedFeeds = Zotero.SyncedSettings.get(Zotero.Libraries.userLibraryID, 'feeds') || {}; 320 delete syncedFeeds[this._previousURL]; 321 yield Zotero.SyncedSettings.set(Zotero.Libraries.userLibraryID, 'feeds', syncedFeeds); 322 } 323 if (syncedDataChanged || env.isNew || this._previousURL) { 324 yield this.storeSyncedSettings(); 325 if (env.isNew || this._previousURL) { 326 Zotero.Feeds.register(this); 327 } 328 } 329 this._previousURL = null; 330 331 }); 332 333 Zotero.Feed.prototype._finalizeErase = Zotero.Promise.coroutine(function* (env) { 334 let notifierData = {}; 335 notifierData[this.libraryID] = { 336 libraryID: this.libraryID 337 }; 338 Zotero.Notifier.queue('delete', 'feed', this.id, notifierData, env.options.notifierQueue); 339 Zotero.Feeds.unregister(this.libraryID); 340 341 let syncedFeeds = Zotero.SyncedSettings.get(Zotero.Libraries.userLibraryID, 'feeds') || {}; 342 delete syncedFeeds[this.url]; 343 if (Object.keys(syncedFeeds).length == 0) { 344 yield Zotero.SyncedSettings.clear(Zotero.Libraries.userLibraryID, 'feeds'); 345 } else { 346 yield Zotero.SyncedSettings.set(Zotero.Libraries.userLibraryID, 'feeds', syncedFeeds); 347 } 348 349 return Zotero.Feed._super.prototype._finalizeErase.apply(this, arguments); 350 }); 351 352 Zotero.Feed.prototype.erase = Zotero.Promise.coroutine(function* (options = {}) { 353 let childItemIDs = yield Zotero.FeedItems.getAll(this.id, false, false, true); 354 yield Zotero.FeedItems.erase(childItemIDs); 355 356 yield Zotero.Feed._super.prototype.erase.call(this, options); 357 }); 358 359 Zotero.Feed.prototype.storeSyncedSettings = Zotero.Promise.coroutine(function* () { 360 let syncedFeeds = Zotero.SyncedSettings.get(Zotero.Libraries.userLibraryID, 'feeds') || {}; 361 syncedFeeds[this.url] = [this.name, this.cleanupReadAfter, this.cleanupUnreadAfter, this.refreshInterval]; 362 return Zotero.SyncedSettings.set(Zotero.Libraries.userLibraryID, 'feeds', syncedFeeds); 363 }); 364 365 Zotero.Feed.prototype.getExpiredFeedItemIDs = Zotero.Promise.coroutine(function* () { 366 let sql = "SELECT itemID AS id FROM feedItems " 367 + "LEFT JOIN items I USING (itemID) " 368 + "WHERE I.libraryID=? " 369 + "AND (" 370 + "(readTime IS NOT NULL AND julianday('now', 'utc') - (julianday(readTime, 'utc') + ?) > 0) " 371 + "OR (readTime IS NULL AND julianday('now', 'utc') - (julianday(dateModified, 'utc') + ?) > 0)" 372 + ")"; 373 return Zotero.DB.columnQueryAsync(sql, [this.id, {int: this.cleanupReadAfter}, {int: this.cleanupUnreadAfter}]); 374 }); 375 376 /** 377 * Clearing conditions for an item: 378 * - Has been read at least feed.cleanupReadAfter earlier OR is unread and older than feed.cleanupUnreadAfter 379 * - AND Does not exist in the RSS feed anymore 380 * 381 * If we clear items once they've been read, we may potentially end up 382 * with empty feeds for those that do not update very frequently. 383 */ 384 Zotero.Feed.prototype.clearExpiredItems = Zotero.Promise.coroutine(function* (itemsInFeedIDs) { 385 itemsInFeedIDs = itemsInFeedIDs || new Set(); 386 try { 387 // Clear expired items 388 let expiredItems = yield this.getExpiredFeedItemIDs(); 389 let toClear = expiredItems; 390 if (itemsInFeedIDs.size) { 391 toClear = []; 392 for (let id of expiredItems) { 393 if (!itemsInFeedIDs.has(id)) { 394 toClear.push(id); 395 } 396 } 397 } 398 Zotero.debug("Clearing up read feed items..."); 399 if (toClear.length) { 400 Zotero.debug(toClear.join(', ')); 401 yield Zotero.FeedItems.erase(toClear); 402 } else { 403 Zotero.debug("No expired feed items"); 404 } 405 } catch(e) { 406 Zotero.debug("Error clearing expired feed items"); 407 Zotero.debug(e); 408 } 409 }); 410 411 Zotero.Feed.prototype._updateFeed = Zotero.Promise.coroutine(function* () { 412 var toSave = [], attachmentsToAdd = [], feedItemIDs = new Set(); 413 if (this._updating) { 414 return this._updating; 415 } 416 let deferred = Zotero.Promise.defer(); 417 this._updating = deferred.promise; 418 yield Zotero.Notifier.trigger('statusChanged', 'feed', this.id); 419 this._set('_feedLastCheckError', null); 420 421 try { 422 let fr = new Zotero.FeedReader(this.url); 423 yield fr.process(); 424 let itemIterator = new fr.ItemIterator(); 425 let item, processedGUIDs = new Set(); 426 while (item = yield itemIterator.next().value) { 427 if (processedGUIDs.has(item.guid)) { 428 Zotero.debug("Feed item " + item.guid + " already processed from feed"); 429 continue; 430 } 431 processedGUIDs.add(item.guid); 432 433 Zotero.debug("Feed item retrieved:", 5); 434 Zotero.debug(item, 5); 435 436 let feedItem = yield Zotero.FeedItems.getAsyncByGUID(item.guid); 437 if (feedItem) { 438 feedItemIDs.add(feedItem.id); 439 } 440 if (!feedItem) { 441 Zotero.debug("Creating new feed item " + item.guid); 442 feedItem = new Zotero.FeedItem(); 443 feedItem.guid = item.guid; 444 feedItem.libraryID = this.id; 445 } else if (!feedItem.isTranslated) { 446 // TODO: maybe handle enclosed items on update better 447 item.enclosedItems = []; 448 449 // TODO figure out a better GUID collision resolution system 450 // that works with sync. 451 if (feedItem.libraryID != this.libraryID) { 452 let otherFeed = Zotero.Feeds.get(feedItem.libraryID); 453 Zotero.debug("Feed item " + feedItem.url + " from " + this.url + 454 " exists in a different feed " + otherFeed.url + ". Skipping"); 455 continue; 456 } 457 458 Zotero.debug("Feed item " + item.guid + " already in library"); 459 Zotero.debug("Updating metadata"); 460 } else { 461 // Not new and has been translated 462 Zotero.debug("Feed item " + item.guid + " is not new and has already been translated. Skipping"); 463 continue; 464 } 465 466 for (let enclosedItem of item.enclosedItems) { 467 enclosedItem.parentItem = feedItem; 468 attachmentsToAdd.push(enclosedItem); 469 } 470 471 // Delete invalid data 472 delete item.guid; 473 delete item.enclosedItems; 474 feedItem.fromJSON(item); 475 476 if (!feedItem.hasChanged()) { 477 Zotero.debug("Feed item " + feedItem.guid + " has not changed"); 478 continue 479 } 480 feedItem.isRead = false; 481 toSave.push(feedItem); 482 } 483 } 484 catch (e) { 485 if (e.message) { 486 Zotero.logError("Error processing feed from " + this.url + ":\n\n" + e); 487 } 488 this._set('_feedLastCheckError', e.message || 'Error processing feed'); 489 } 490 if (toSave.length) { 491 yield Zotero.DB.executeTransaction(function* () { 492 // Save in reverse order 493 for (let i=toSave.length-1; i>=0; i--) { 494 yield toSave[i].save(); 495 } 496 497 }); 498 this._set('_feedLastUpdate', Zotero.Date.dateToSQL(new Date(), true)); 499 } 500 for (let attachment of attachmentsToAdd) { 501 if (attachment.url.indexOf('pdf') != -1 || attachment.contentType.indexOf('pdf') != -1) { 502 attachment.parentItemID = attachment.parentItem.id; 503 attachment.title = Zotero.getString('fileTypes.pdf'); 504 yield Zotero.Attachments.linkFromURL(attachment); 505 } 506 } 507 yield this.clearExpiredItems(feedItemIDs); 508 this._set('_feedLastCheck', Zotero.Date.dateToSQL(new Date(), true)); 509 yield this.saveTx(); 510 yield this.updateUnreadCount(); 511 deferred.resolve(); 512 this._updating = false; 513 yield Zotero.Notifier.trigger('statusChanged', 'feed', this.id); 514 }); 515 516 Zotero.Feed.prototype.updateFeed = Zotero.Promise.coroutine(function* () { 517 try { 518 let result = yield this._updateFeed(); 519 return result; 520 } finally { 521 Zotero.Feeds.scheduleNextFeedCheck(); 522 } 523 }); 524 525 Zotero.Feed.prototype.updateUnreadCount = Zotero.Promise.coroutine(function* () { 526 let sql = "SELECT " + Zotero.Feed._unreadCountSQL 527 + " FROM feeds F JOIN libraries L USING (libraryID)" 528 + " WHERE L.libraryID=?"; 529 let newCount = yield Zotero.DB.valueQueryAsync(sql, [this.id]); 530 531 if (newCount != this._feedUnreadCount) { 532 this._feedUnreadCount = newCount; 533 yield Zotero.Notifier.trigger('unreadCountUpdated', 'feed', this.id); 534 } 535 });