commit bddb583e43b78e4ba865ab7959d3a0b9eef76769
parent 903d37c434c5da8cb807456e616d6a7f79182fd7
Author: Simon Kornblith <simon@simonster.com>
Date: Mon, 28 Jun 2010 09:07:44 +0000
- cache translators in DB to reduce startup times on Windows
- fix some error handling in translate.js
Diffstat:
2 files changed, 135 insertions(+), 67 deletions(-)
diff --git a/chrome/content/zotero/xpcom/translate.js b/chrome/content/zotero/xpcom/translate.js
@@ -54,13 +54,38 @@ Zotero.Translators = new function() {
_cache = {"import":[], "export":[], "web":[], "search":[]};
_translators = {};
+ var dbCacheResults = Zotero.DB.query("SELECT leafName, translatorJSON, "+
+ "code, lastModifiedTime FROM translatorCache");
+ var dbCache = {};
+ for each(var cacheEntry in dbCacheResults) {
+ dbCache[cacheEntry.leafName] = cacheEntry;
+ }
+
var i = 0;
+ var filesInCache = {};
var contents = Zotero.getTranslatorsDirectory().directoryEntries;
while(contents.hasMoreElements()) {
var file = contents.getNext().QueryInterface(Components.interfaces.nsIFile);
- if(!file.leafName || file.leafName[0] == ".") continue;
+ var leafName = file.leafName;
+ if(!leafName || leafName[0] == ".") continue;
+ var lastModifiedTime = file.lastModifiedTime;
+
+ var dbCacheEntry = false;
+ if(dbCache[leafName]) {
+ filesInCache[leafName] = true;
+ if(dbCache[leafName].lastModifiedTime == lastModifiedTime) {
+ dbCacheEntry = dbCache[file.leafName];
+ }
+ }
- var translator = new Zotero.Translator(file);
+ if(dbCacheEntry) {
+ // get JSON from cache if possible
+ var translator = new Zotero.Translator(file, dbCacheEntry.translatorJSON, dbCacheEntry.code);
+ filesInCache[leafName] = true;
+ } else {
+ // otherwise, load from file
+ var translator = new Zotero.Translator(file);
+ }
if(translator.translatorID) {
if(_translators[translator.translatorID]) {
@@ -76,12 +101,25 @@ Zotero.Translators = new function() {
_cache[type].push(translator);
}
}
+
+ if(!dbCacheEntry) {
+ // Add cache misses to DB
+ Zotero.Translators.cacheInDB(leafName, translator.metadataString, translator.cacheCode ? translator.code : null, lastModifiedTime);
+ delete translator.metadataString;
+ }
}
}
i++;
}
+ // Remove translators from DB as necessary
+ for(var leafName in dbCache) {
+ if(!filesInCache[leafName]) {
+ Zotero.DB.query("DELETE FROM translatorCache WHERE leafName = ?", [leafName]);
+ }
+ }
+
// Sort by priority
var collation = Zotero.getLocaleCollation();
var cmp = function (a, b) {
@@ -93,7 +131,7 @@ Zotero.Translators = new function() {
}
return collation.compareString(1, a.label, b.label);
}
- for (var type in _cache) {
+ for(var type in _cache) {
_cache[type].sort(cmp);
}
@@ -116,7 +154,6 @@ Zotero.Translators = new function() {
return _cache[type].slice(0);
}
-
/**
* @param {String} label
* @return {String}
@@ -214,6 +251,11 @@ Zotero.Translators = new function() {
return destFile;
}
+
+ this.cacheInDB = function(fileName, metadataJSON, code, lastModifiedTime) {
+ Zotero.DB.query("REPLACE INTO translatorCache VALUES (?, ?, ?, ?)",
+ [fileName, metadataJSON, code, lastModifiedTime]);
+ }
}
/**
@@ -232,47 +274,54 @@ Zotero.Translators = new function() {
* @property {String} lastUpdated SQL-style date and time of translator's last update
* @property {String} code The executable JavaScript for the translator
*/
-Zotero.Translator = function(file) {
+Zotero.Translator = function(file, json, code) {
+ const codeGetterFunction = function() { return Zotero.File.getContents(this.file); }
// Maximum length for the info JSON in a translator
const MAX_INFO_LENGTH = 4096;
const infoRe = /{(?:(?:"(?:[^"\r\n]*(?:\\")?)*")*[^}"]*)*}/;
this.file = file;
- var fStream = Components.classes["@mozilla.org/network/file-input-stream;1"].
- createInstance(Components.interfaces.nsIFileInputStream);
- var cStream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
- createInstance(Components.interfaces.nsIConverterInputStream);
- fStream.init(file, -1, -1, 0);
- cStream.init(fStream, "UTF-8", 8192,
- Components.interfaces.nsIConverterInputStream.DEFAULT_REPLACEMENT_CHARACTER);
-
- var str = {};
- cStream.readString(MAX_INFO_LENGTH, str);
-
- // We assume lastUpdated is at the end to avoid running the regexp on more than necessary
- var lastUpdatedIndex = str.value.indexOf('"lastUpdated"');
- if (lastUpdatedIndex == -1) {
- this.logError("Invalid or missing translator metadata JSON object");
- fStream.close();
- return;
- }
-
- // Add 50 characters to clear lastUpdated timestamp and final "}"
- var header = str.value.substr(0, lastUpdatedIndex + 50);
- var m = infoRe.exec(header);
- if (!m) {
- this.logError("Invalid or missing translator metadata JSON object");
- fStream.close();
- return;
- }
-
- try {
- var info = Zotero.JSON.unserialize(m[0]);
- } catch(e) {
- this.logError("Invalid or missing translator metadata JSON object");
- fStream.close();
- return;
+ if(json) {
+ var info = Zotero.JSON.unserialize(json);
+ } else {
+ var fStream = Components.classes["@mozilla.org/network/file-input-stream;1"].
+ createInstance(Components.interfaces.nsIFileInputStream);
+ var cStream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
+ createInstance(Components.interfaces.nsIConverterInputStream);
+ fStream.init(file, -1, -1, 0);
+ cStream.init(fStream, "UTF-8", 8192,
+ Components.interfaces.nsIConverterInputStream.DEFAULT_REPLACEMENT_CHARACTER);
+
+ var str = {};
+ cStream.readString(MAX_INFO_LENGTH, str);
+
+ // We assume lastUpdated is at the end to avoid running the regexp on more than necessary
+ var lastUpdatedIndex = str.value.indexOf('"lastUpdated"');
+ if (lastUpdatedIndex == -1) {
+ this.logError("Invalid or missing translator metadata JSON object");
+ fStream.close();
+ return;
+ }
+
+ // Add 50 characters to clear lastUpdated timestamp and final "}"
+ var header = str.value.substr(0, lastUpdatedIndex + 50);
+ var m = infoRe.exec(header);
+ if (!m) {
+ this.logError("Invalid or missing translator metadata JSON object");
+ fStream.close();
+ return;
+ }
+
+ this.metadataString = m[0];
+
+ try {
+ var info = Zotero.JSON.unserialize(this.metadataString);
+ } catch(e) {
+ this.logError("Invalid or missing translator metadata JSON object");
+ fStream.close();
+ return;
+ }
}
var haveMetadata = true;
@@ -291,39 +340,47 @@ Zotero.Translator = function(file) {
return;
}
+ this.detectXPath = info["detectXPath"] ? info["detectXPath"] : null;
+
+ /**
+ * g = Gecko (Firefox)
+ * c = Google Chrome (WebKit & V8)
+ * s = Safari (WebKit & Nitro/Squirrelfish Extreme)
+ * i = Internet Explorer
+ * a = All
+ */
+ this.browserSupport = info["browserSupport"] ? info["browserSupport"] : "g";
+
if(this.translatorType & TRANSLATOR_TYPES["import"]) {
// compile import regexp to match only file extension
this.importRegexp = this.target ? new RegExp("\\."+this.target+"$", "i") : null;
}
+
+ this.cacheCode = false;
if(this.translatorType & TRANSLATOR_TYPES["web"]) {
// compile web regexp
this.webRegexp = this.target ? new RegExp(this.target, "i") : null;
if(!this.target) {
- // for translators used on every page, cache code in memory
- var strs = [str.value];
- var amountRead;
- while(amountRead = cStream.readString(8192, str)) strs.push(str.value);
- this._code = strs.join("");
-
+ this.cacheCode = true;
+
+ if(json) {
+ // if have JSON, also have code
+ this.code = code;
+ } else {
+ // for translators used on every page, cache code in memory
+ var strs = [str.value];
+ var amountRead;
+ while(amountRead = cStream.readString(8192, str)) strs.push(str.value);
+ this.code = strs.join("");
+ }
}
}
- fStream.close();
+ if(!this.cacheCode) this.__defineGetter__("code", codeGetterFunction);
+ if(!json) cStream.close();
}
-
-Zotero.Translator.prototype.__defineGetter__("code",
-/**
- * Getter for "code" property
- * @return {String} Code of translator
- * @inner
- */
-function() {
- if(this._code) return this._code;
- return Zotero.File.getContents(this.file);
-});
-
/**
* Log a translator-related error
* @param {String} message The error message
@@ -692,11 +749,14 @@ Zotero.Translate.prototype.getTranslators = function() {
// see which translators can translate
this._translatorSearch = new Zotero.Translate.TranslatorSearch(this, translators);
- // erroring should call complete
- this.error = function(value, error) { this._translatorSearch.complete(value, error) };
-
- // return translators if asynchronous
- if(!this._translatorSearch.asyncMode) return this._translatorSearch.foundTranslators;
+ if(this._translatorSearch.asyncMode) {
+ // erroring should call complete
+ var me = this;
+ this.error = function(value, error) { me._translatorSearch.complete(value, error); };
+ } else {
+ // return translators if synchronous
+ return this._translatorSearch.foundTranslators;
+ }
}
/*
@@ -774,7 +834,8 @@ Zotero.Translate.prototype.translate = function(libraryID, saveAttachments) {
}
// erroring should end
- this.error = this._translationComplete;
+ var me = this;
+ this.error = function(value, error) { me._translationComplete(value, error); }
if(!this._loadTranslator()) {
return;
@@ -901,7 +962,7 @@ Zotero.Translate.prototype._generateSandbox = function() {
var translation = new Zotero.Translate(type);
translation._parentTranslator = me;
- if(type == "export" && (this.type == "web" || this.type == "search")) {
+ if(type == "export" && (me.type == "web" || me.type == "search")) {
throw("for security reasons, web and search translators may not call export translators");
}
@@ -2743,6 +2804,7 @@ Zotero.Translate.TranslatorSearch.prototype.complete = function(returnValue, err
// reset done function
this.translate._sandbox.Zotero.done = undefined;
this.translate.waitForCompletion = false;
+ this.asyncMode = false;
if(returnValue) {
this.processReturnValue(this.currentTranslator, returnValue);
@@ -2753,7 +2815,6 @@ Zotero.Translate.TranslatorSearch.prototype.complete = function(returnValue, err
}
this.currentTranslator = undefined;
- this.asyncMode = false;
// resume execution
this.execute();
diff --git a/system.sql b/system.sql
@@ -1,4 +1,4 @@
--- 29
+-- 30
-- Copyright (c) 2009 Center for History and New Media
-- George Mason University, Fairfax, Virginia, USA
@@ -198,6 +198,13 @@ CREATE TABLE transactionLog (
FOREIGN KEY (transactionID) REFERENCES transactions(transactionID)
);
+DROP TABLE IF EXISTS translatorCache;
+CREATE TABLE translatorCache (
+ leafName TEXT PRIMARY KEY,
+ translatorJSON TEXT,
+ code TEXT,
+ lastModifiedTime INT
+);
-- unused
INSERT INTO "fieldFormats" VALUES(1, '.*', 0);
