commit 4c9b5935e8d0b731242ae058d194f00932c73b28
parent 55c331e68b1c44c7824694568a66017edb0b6284
Author: Simon Kornblith <simon@simonster.com>
Date: Sat, 16 Jul 2011 20:47:17 +0000
- Improvements to server.js for translation-server
- Optimizations. The biggest of these is to simplify our mechanism of wrapping functions for Fx 4+, which gives us roughly a 3x speed boost in RIS import. However, zotero-node is still ~20% faster than translation-server, and RDF import/export may still be too slow for very large numbers of references. A large part of the RDF overhead seems to come from the number of function calls we make, which numbers in the hundreds of thousands for a 2.5 MB file.
Diffstat:
4 files changed, 144 insertions(+), 69 deletions(-)
diff --git a/chrome/content/zotero/xpcom/server.js b/chrome/content/zotero/xpcom/server.js
@@ -39,7 +39,7 @@ Zotero.Server = new function() {
/**
* initializes a very rudimentary web server
*/
- this.init = function() {
+ this.init = function(port, bindAllAddr, maxConcurrentConnections) {
if (Zotero.HTTP.browserIsOffline()) {
Zotero.debug('Browser is offline -- not initializing HTTP server');
_registerOnlineObserver();
@@ -51,10 +51,10 @@ Zotero.Server = new function() {
.createInstance(Components.interfaces.nsIServerSocket);
try {
// bind to a random port on loopback only
- serv.init(Zotero.Prefs.get('httpServer.port'), true, -1);
+ serv.init(port ? port : Zotero.Prefs.get('httpServer.port'), !bindAllAddr, -1);
serv.asyncListen(Zotero.Server.SocketListener);
- Zotero.debug("HTTP server listening on 127.0.0.1:"+serv.port);
+ Zotero.debug("HTTP server listening on "+(bindAllAddr ? "*": " 127.0.0.1")+":"+serv.port);
} catch(e) {
Zotero.debug("Not initializing HTTP server");
}
@@ -82,6 +82,20 @@ Zotero.Server = new function() {
return response;
}
+ /**
+ * Parses a query string into a key => value object
+ * @param {String} queryString Query string
+ */
+ this.decodeQueryString = function(queryString) {
+ var splitData = queryString.split("&");
+ var decodedData = {};
+ for each(var variable in splitData) {
+ var splitIndex = variable.indexOf("=");
+ decodedData[decodeURIComponent(variable.substr(0, splitIndex))] = decodeURIComponent(variable.substr(splitIndex+1));
+ }
+ return decodedData;
+ }
+
function _registerOnlineObserver() {
if (_onlineObserverRegistered) {
return;
@@ -218,7 +232,7 @@ Zotero.Server.DataListener.prototype.onDataAvailable = function(request, context
Zotero.Server.DataListener.prototype._headerFinished = function() {
this.headerFinished = true;
- Zotero.debug(this.header);
+ Zotero.debug(this.header, 5);
const methodRe = /^([A-Z]+) ([^ \r\n?]+)(\?[^ \r\n]+)?/;
const contentTypeRe = /[\r\n]Content-Type: +([^ \r\n]+)/i;
@@ -233,33 +247,35 @@ Zotero.Server.DataListener.prototype._headerFinished = function() {
}
if(!method) {
- this._requestFinished(Zotero.Server.generateResponse(400));
+ this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Invalid method specified\n"));
return;
}
if(!Zotero.Server.Endpoints[method[2]]) {
- this._requestFinished(Zotero.Server.generateResponse(404));
+ this._requestFinished(Zotero.Server.generateResponse(404, "text/plain", "No endpoint found\n"));
return;
}
+ this.pathname = method[2];
this.endpoint = Zotero.Server.Endpoints[method[2]];
+ this.query = method[3];
if(method[1] == "HEAD" || method[1] == "OPTIONS") {
this._requestFinished(Zotero.Server.generateResponse(200));
} else if(method[1] == "GET") {
- this._requestFinished(this._processEndpoint("GET", method[3]));
+ this._processEndpoint("GET", null);
} else if(method[1] == "POST") {
const contentLengthRe = /[\r\n]Content-Length: +([0-9]+)/i;
// parse content length
var m = contentLengthRe.exec(this.header);
if(!m) {
- this._requestFinished(Zotero.Server.generateResponse(400));
+ this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Content-length not provided\n"));
return;
}
this.bodyLength = parseInt(m[1]);
this._bodyData();
} else {
- this._requestFinished(Zotero.Server.generateResponse(501));
+ this._requestFinished(Zotero.Server.generateResponse(501, "text/plain", "Method not implemented\n"));
return;
}
}
@@ -297,29 +313,30 @@ Zotero.Server.DataListener.prototype._processEndpoint = function(method, postDat
var endpoint = new this.endpoint;
// check that endpoint supports method
- if(endpoint.supportedMethods.indexOf(method) === -1) {
- this._requestFinished(Zotero.Server.generateResponse(400));
+ if(endpoint.supportedMethods && endpoint.supportedMethods.indexOf(method) === -1) {
+ this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Endpoint does not support method\n"));
return;
}
var decodedData = null;
if(postData && this.contentType) {
// check that endpoint supports contentType
- if(endpoint.supportedDataTypes.indexOf(this.contentType) === -1) {
- this._requestFinished(Zotero.Server.generateResponse(400));
+ var supportedDataTypes = endpoint.supportedDataTypes;
+ if(supportedDataTypes && supportedDataTypes.indexOf(this.contentType) === -1) {
+ this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Endpoint does not support content-type\n"));
return;
}
// decode JSON or urlencoded post data, and pass through anything else
- if(this.contentType === "application/json") {
- decodedData = JSON.parse(postData);
- } else if(this.contentType === "application/x-www-urlencoded") {
- var splitData = postData.split("&");
- decodedData = {};
- for each(var variable in splitData) {
- var splitIndex = variable.indexOf("=");
- data[decodeURIComponent(variable.substr(0, splitIndex))] = decodeURIComponent(variable.substr(splitIndex+1));
+ if(supportedDataTypes && this.contentType === "application/json") {
+ try {
+ decodedData = JSON.parse(postData);
+ } catch(e) {
+ this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Invalid JSON provided\n"));
+ return;
}
+ } else if(supportedDataTypes && this.contentType === "application/x-www-urlencoded") {
+ decodedData = Zotero.Server.decodeQueryString(postData);
} else {
decodedData = postData;
}
@@ -332,10 +349,19 @@ Zotero.Server.DataListener.prototype._processEndpoint = function(method, postDat
}
// pass to endpoint
- endpoint.init(decodedData, sendResponseCallback);
+ if((endpoint.init.length ? endpoint.init.length : endpoint.init.arity) === 3) {
+ var url = {
+ "pathname":this.pathname,
+ "query":this.query ? Zotero.Server.decodeQueryString(this.query.substr(1)) : {}
+ };
+
+ endpoint.init(url, decodedData, sendResponseCallback);
+ } else {
+ endpoint.init(decodedData, sendResponseCallback);
+ }
} catch(e) {
Zotero.debug(e);
- this._requestFinished(Zotero.Server.generateResponse(500));
+ this._requestFinished(Zotero.Server.generateResponse(500), "text/plain", "An error occurred\n");
throw e;
}
}
@@ -356,7 +382,7 @@ Zotero.Server.DataListener.prototype._requestFinished = function(response) {
intlStream.init(this.oStream, "UTF-8", 1024, "?".charCodeAt(0));
// write response
- Zotero.debug(response);
+ Zotero.debug(response, 5);
intlStream.writeString(response);
} finally {
intlStream.close();
diff --git a/chrome/content/zotero/xpcom/translation/translate.js b/chrome/content/zotero/xpcom/translation/translate.js
@@ -81,7 +81,7 @@ Zotero.Translate.Sandbox = {
* @param {SandboxItem} An item created using the Zotero.Item class from the sandbox
*/
"_itemDone":function(translate, item) {
- Zotero.debug("Translate: Saving item");
+ //Zotero.debug("Translate: Saving item");
// warn if itemDone called after translation completed
if(translate._complete) {
@@ -1859,7 +1859,7 @@ Zotero.Translate.IO = {
}
if(nodes.getElementsByTagName("parsererror").length) {
- throw new Error("DOMParser error: loading data into data store failed");
+ throw "DOMParser error: loading data into data store failed";
}
return nodes;
@@ -1901,8 +1901,14 @@ Zotero.Translate.IO.String.prototype = {
this.RDF = new Zotero.Translate.IO._RDFSandbox(this._dataStore);
if(this._string.length) {
+ try {
+ var xml = Zotero.Translate.IO.parseDOMXML(this._string);
+ } catch(e) {
+ this._xmlInvalid = true;
+ throw e;
+ }
var parser = new Zotero.RDF.AJAW.RDFParser(this._dataStore);
- parser.parse(Zotero.Translate.IO.parseDOMXML(this._string), this._uri);
+ parser.parse(xml, this._uri);
callback(true);
}
},
@@ -1931,19 +1937,23 @@ Zotero.Translate.IO.String.prototype = {
var oldPointer = this._stringPointer;
var lfIndex = this._string.indexOf("\n", this._stringPointer);
- if(lfIndex != -1) {
+ if(lfIndex !== -1) {
// in case we have a CRLF
this._stringPointer = lfIndex+1;
- if(this._string.length > lfIndex && this._string[lfIndex-1] == "\r") {
+ if(this._string.length > lfIndex && this._string[lfIndex-1] === "\r") {
lfIndex--;
}
return this._string.substr(oldPointer, lfIndex-oldPointer);
}
- var crIndex = this._string.indexOf("\r", this._stringPointer);
- if(crIndex != -1) {
- this._stringPointer = crIndex+1;
- return this._string.substr(oldPointer, crIndex-oldPointer-1);
+ if(!this._noCR) {
+ var crIndex = this._string.indexOf("\r", this._stringPointer);
+ if(crIndex === -1) {
+ this._noCR = true;
+ } else {
+ this._stringPointer = crIndex+1;
+ return this._string.substr(oldPointer, crIndex-oldPointer-1);
+ }
}
this._stringPointer = this._string.length;
@@ -1957,7 +1967,12 @@ Zotero.Translate.IO.String.prototype = {
"_getXML":function() {
if(this._mode == "xml/dom") {
- return Zotero.Translate.IO.parseDOMXML(this._string);
+ try {
+ return Zotero.Translate.IO.parseDOMXML(this._string);
+ } catch(e) {
+ this._xmlInvalid = true;
+ throw e;
+ }
} else {
return this._string.replace(/<\?xml[^>]+\?>/, "");
}
@@ -1965,9 +1980,13 @@ Zotero.Translate.IO.String.prototype = {
"init":function(newMode, callback) {
this._stringPointer = 0;
+ this._noCR = undefined;
this._mode = newMode;
- if(Zotero.Translate.IO.rdfDataModes.indexOf(this._mode) !== -1) {
+ if(newMode && (Zotero.Translate.IO.rdfDataModes.indexOf(newMode) !== -1
+ || newMode.substr(0, 3) === "xml") && this._xmlInvalid) {
+ throw "XML known invalid";
+ } else if(Zotero.Translate.IO.rdfDataModes.indexOf(this._mode) !== -1) {
this._initRDF(callback);
} else {
callback(true);
diff --git a/chrome/content/zotero/xpcom/translation/translate_firefox.js b/chrome/content/zotero/xpcom/translation/translate_firefox.js
@@ -148,22 +148,44 @@ Zotero.Translate.SandboxManager.prototype = {
if(newExposedProps) newExposedProps[localKey] = "r";
// magical XPCSafeJSObjectWrappers for sandbox
- if(typeof object[localKey] === "function" || typeof object[localKey] === "object") {
- if(attachTo == this.sandbox) Zotero.debug(localKey);
- attachTo[localKey] = function() {
- var args = (passAsFirstArgument ? [passAsFirstArgument] : []);
- for(var i=0; i<arguments.length; i++) {
- args.push((typeof arguments[i] === "object" && arguments[i] !== null)
- || typeof arguments[i] === "function"
- ? new XPCSafeJSObjectWrapper(arguments[i]) : arguments[i]);
+ var type = typeof object[localKey];
+ var isFunction = type === "function";
+ var isObject = typeof object[localKey] === "object";
+ if(isFunction || isObject) {
+ if(isFunction) {
+ if(Zotero.isFx4) {
+ if(passAsFirstArgument) {
+ attachTo[localKey] = object[localKey].bind(object, passAsFirstArgument);
+ } else {
+ attachTo[localKey] = object[localKey].bind(object);
+ }
+ } else {
+ attachTo[localKey] = function() {
+ if(passAsFirstArgument) {
+ var args = new Array(arguments.length+1);
+ args[0] = passAsFirstArgument;
+ var offset = 1;
+ } else {
+ var args = new Array(arguments.length);
+ var offset = 0;
+ }
+
+ for(var i=0, nArgs=arguments.length; i<nArgs; i++) {
+ args[i+offset] = (((typeof arguments[i] === "object" && arguments[i] !== null)
+ || typeof arguments[i] === "function")
+ ? new XPCSafeJSObjectWrapper(arguments[i]) : arguments[i]);
+ }
+
+ return object[localKey].apply(object, args);
+ };
}
-
- return object[localKey].apply(object, args);
- };
+ } else {
+ attachTo[localKey] = {};
+ }
// attach members
if(!(object instanceof Components.interfaces.nsISupports)) {
- this.importObject(object[localKey], passAsFirstArgument ? passAsFirstArgument : null, attachTo[localKey]);
+ this.importObject(object[localKey], passAsFirstArgument, attachTo[localKey]);
}
} else {
attachTo[localKey] = object[localKey];
diff --git a/chrome/content/zotero/xpcom/utilities.js b/chrome/content/zotero/xpcom/utilities.js
@@ -218,12 +218,17 @@ Zotero.Utilities = {
* @type String
*/
"unescapeHTML":function(/**String*/ str) {
+ // If no tags, no need to unescape
+ if(str.indexOf("<") === -1 && str.indexOf("&") === -1) return str;
+
if(Zotero.isFx) {
- var nsISUHTML = Components.classes["@mozilla.org/feed-unescapehtml;1"]
- .getService(Components.interfaces.nsIScriptableUnescapeHTML);
- return nsISUHTML.unescape(str);
+ if(!Zotero.Utilities._nsISUHTML) {
+ Zotero.Utilities._nsISUHTML = Components.classes["@mozilla.org/feed-unescapehtml;1"]
+ .getService(Components.interfaces.nsIScriptableUnescapeHTML);
+ }
+ return Zotero.Utilities._nsISUHTML.unescape(str);
} else if(Zotero.isNode) {
- var doc = require('jsdom').jsdom(str, null, {
+ /*var doc = require('jsdom').jsdom(str, null, {
"features":{
"FetchExternalResources":false,
"ProcessExternalResources":false,
@@ -232,7 +237,8 @@ Zotero.Utilities = {
}
});
if(!doc.documentElement) return str;
- return doc.documentElement.textContent;
+ return doc.documentElement.textContent;*/
+ return Zotero.Utilities.cleanTags(str);
} else {
var node = document.createElement("div");
node.innerHTML = str;
@@ -847,7 +853,6 @@ Zotero.Utilities = {
* Converts an item from toArray() format to content=json format used by the server
*/
"itemToServerJSON":function(item) {
- const IGNORE_FIELDS = ["seeAlso", "attachments", "complete"];
var newItem = {};
var typeID = Zotero.ItemTypes.getID(item.itemType);
@@ -857,9 +862,10 @@ Zotero.Utilities = {
typeID = Zotero.ItemTypes.getID(item.itemType);
}
- var fieldID;
+ var fieldID, itemFieldID;
for(var field in item) {
- if(IGNORE_FIELDS.indexOf(field) !== -1) continue;
+ if(field === "complete" || field === "itemID" || field === "attachments"
+ || field === "seeAlso") continue;
var val = item[field];
@@ -867,8 +873,9 @@ Zotero.Utilities = {
newItem[field] = val;
} else if(field === "creators") {
// normalize creators
- var newCreators = newItem.creators = [];
- for(var j in val) {
+ var n = val.length;
+ var newCreators = newItem.creators = new Array(n);
+ for(var j=0; j<n; j++) {
var creator = val[j];
// Single-field mode
@@ -886,7 +893,6 @@ Zotero.Utilities = {
}
// ensure creatorType is present and valid
- newCreator.creatorType = "author";
if(creator.creatorType) {
if(Zotero.CreatorTypes.getID(creator.creatorType)) {
newCreator.creatorType = creator.creatorType;
@@ -894,13 +900,15 @@ Zotero.Utilities = {
Zotero.debug("Translate: Invalid creator type "+creator.creatorType+"; falling back to author");
}
}
+ if(!newCreator.creatorType) newCreator.creatorType = "author";
- newCreators.push(newCreator);
+ newCreators[j] = newCreator;
}
} else if(field === "tags") {
// normalize tags
- var newTags = newItem.tags = [];
- for(var j in val) {
+ var n = val.length;
+ var newTags = newItem.tags = new Array(n);
+ for(var j=0; j<n; j++) {
var tag = val[j];
if(typeof tag === "object") {
if(tag.tag) {
@@ -912,12 +920,13 @@ Zotero.Utilities = {
continue;
}
}
- newTags.push({"tag":tag.toString(), "type":1})
+ newTags[j] = {"tag":tag.toString(), "type":1};
}
} else if(field === "notes") {
// normalize notes
- var newNotes = newItem.notes = [];
- for(var j in val) {
+ var n = val.length;
+ var newNotes = newItem.notes = new Array(n);
+ for(var j=0; j<n; j++) {
var note = val[j];
if(typeof note === "object") {
if(!note.note) {
@@ -926,9 +935,9 @@ Zotero.Utilities = {
}
note = note.note;
}
- newNotes.push({"itemType":"note", "note":note.toString()});
+ newNotes[j] = {"itemType":"note", "note":note.toString()};
}
- } else if(fieldID = Zotero.ItemFields.getID(field)) {
+ } else if((fieldID = Zotero.ItemFields.getID(field))) {
// if content is not a string, either stringify it or delete it
if(typeof val !== "string") {
if(val || val === 0) {
@@ -939,8 +948,7 @@ Zotero.Utilities = {
}
// map from base field if possible
- var itemFieldID = Zotero.ItemFields.getFieldIDFromTypeAndBase(typeID, fieldID);
- if(itemFieldID) {
+ if((itemFieldID = Zotero.ItemFields.getFieldIDFromTypeAndBase(typeID, fieldID))) {
newItem[Zotero.ItemFields.getName(itemFieldID)] = val;
continue; // already know this is valid
}
@@ -951,7 +959,7 @@ Zotero.Utilities = {
} else {
Zotero.debug("Translate: Discarded field "+field+": field not valid for type "+item.itemType, 3);
}
- } else if(field !== "complete") {
+ } else {
Zotero.debug("Translate: Discarded unknown field "+field, 3);
}
}
