commit 747c11c917fa96ecf5c30d63e8276600bdc4cce5 parent c2ebcc9dbc52d123331f3eb6935345b8335b82b2 Author: Adomas Ven <adomas.ven@gmail.com> Date: Mon, 12 Dec 2016 14:29:59 +0200 Improves proxy support (#1129) Improves proxy support - Automatically detect and dehyphenise https proxies which use EZProxy HttpsHyphens - Web translators now pass around Zotero.Proxy instances which can proxify/deproxify urls passed to `translate.setLocation()` before calling `translate.getTranslators()`/ translate.detect()`. The proxy passing is done within connector background/injected processes and between standalone and connectors. - Proxy protocol unified with connectors. Connectors can now pass proxies to `/connector/save_items`. The proxies will be used to resolve true item and attachment urls when saving. Closes zotero/zotero#578, zotero/zotero#721 Relevant zotero/zotero#34, zotero/zotero#556 Diffstat:
16 files changed, 392 insertions(+), 124 deletions(-)
diff --git a/chrome/content/zotero/xpcom/connector/connector.js b/chrome/content/zotero/xpcom/connector/connector.js @@ -157,14 +157,11 @@ Zotero.Connector = new function() { options = {method: options}; } var method = options.method; - var sendRequest = (data === null || data === undefined) - ? Zotero.HTTP.doGet.bind(Zotero.HTTP) - : Zotero.HTTP.doPost.bind(Zotero.HTTP); var headers = Object.assign({ "Content-Type":"application/json", "X-Zotero-Version":Zotero.version, "X-Zotero-Connector-API-Version":CONNECTOR_API_VERSION - }, options.headers); + }, options.headers || {}); var queryString = options.queryString ? ("?" + options.queryString) : ""; var newCallback = function(req) { @@ -224,7 +221,11 @@ Zotero.Connector = new function() { if (headers["Content-Type"] == 'application/json') { data = JSON.stringify(data); } - sendRequest(uri, data, newCallback, headers); + if (data == null || data == undefined) { + Zotero.HTTP.doGet(uri, newCallback, headers); + } else { + Zotero.HTTP.doPost(uri, data, newCallback, headers); + } } }, diff --git a/chrome/content/zotero/xpcom/connector/translate_item.js b/chrome/content/zotero/xpcom/connector/translate_item.js @@ -26,9 +26,6 @@ /** * Save translator items. * - * In the connector these options are actually irrelevent. We're just passing the items to standalone or - * saving to server. - * * @constructor * @param {Object} options * <li>libraryID - ID of library in which items should be saved</li> @@ -36,11 +33,14 @@ * <li>attachmentMode - One of Zotero.Translate.ItemSaver.ATTACHMENT_* specifying how attachments should be saved</li> * <li>forceTagType - Force tags to specified tag type</li> * <li>cookieSandbox - Cookie sandbox for attachment requests</li> + * <li>proxy - A proxy to deproxify item URLs</li> * <li>baseURI - URI to which attachment paths should be relative</li> * */ Zotero.Translate.ItemSaver = function(options) { this.newItems = []; + this._proxy = options.proxy; + this._baseURI = options.baseURI; // Add listener for callbacks, but only for Safari or the bookmarklet. In Chrome, we // (have to) save attachments from the inject page. @@ -80,7 +80,12 @@ Zotero.Translate.ItemSaver.prototype = { saveItems: function (items, attachmentCallback) { var deferred = Zotero.Promise.defer(); // first try to save items via connector - var payload = {"items":items}; + var payload = { items, uri: this._baseURI }; + if (Zotero.isSafari) { + // This is the best in terms of cookies we can do in Safari + payload.cookie = document.cookie; + } + payload.proxy = this._proxy && this._proxy.toJSON(); Zotero.Connector.setCookiesThenSaveItems(payload, function(data, status) { if(data !== false) { Zotero.debug("Translate: Save via Standalone succeeded"); @@ -179,6 +184,10 @@ Zotero.Translate.ItemSaver.prototype = { for(var i=0, n=items.length; i<n; i++) { var item = items[i]; + // deproxify url + if (this._proxy && item.url) { + item.url = this._proxy.toProper(item.url); + } itemIndices[i] = newItems.length; newItems = newItems.concat(Zotero.Utilities.itemToServerJSON(item)); if(typedArraysSupported) { @@ -239,7 +248,6 @@ Zotero.Translate.ItemSaver.prototype = { * on failure or attachmentCallback(attachment, progressPercent) periodically during saving. */ "_saveAttachmentsToServer":function(itemKey, baseName, attachments, prefs, attachmentCallback) { - Zotero.debug("saveattachmentstoserver"); var me = this, uploadAttachments = [], retrieveHeadersForAttachments = attachments.length; @@ -255,6 +263,10 @@ Zotero.Translate.ItemSaver.prototype = { var attachmentPayload = []; for(var i=0; i<uploadAttachments.length; i++) { var attachment = uploadAttachments[i]; + // deproxify url + if (this._proxy && attachment.url) { + attachment.url = this._proxy.toProper(attachment.url); + } attachmentPayload.push({ "itemType":"attachment", "parentItem":itemKey, diff --git a/chrome/content/zotero/xpcom/connector/translator.js b/chrome/content/zotero/xpcom/connector/translator.js @@ -146,10 +146,10 @@ Zotero.Translators = new function() { if(!_initialized) Zotero.Translators.init(); var allTranslators = _cache["web"]; var potentialTranslators = []; - var converterFunctions = []; + var proxies = []; - var rootSearchURIs = this.getSearchURIs(rootURI); - var frameSearchURIs = isFrame ? this.getSearchURIs(URI) : rootSearchURIs; + var rootSearchURIs = Zotero.Proxies.getPotentialProxies(rootURI); + var frameSearchURIs = isFrame ? Zotero.Proxies.getPotentialProxies(URI) : rootSearchURIs; Zotero.debug("Translators: Looking for translators for "+Object.keys(frameSearchURIs).join(', ')); @@ -174,14 +174,14 @@ Zotero.Translators = new function() { if (frameURIMatches) { potentialTranslators.push(translator); - converterFunctions.push(frameSearchURIs[frameSearchURI]); + proxies.push(frameSearchURIs[frameSearchURI]); // prevent adding the translator multiple times break rootURIsLoop; } } } else if(!isFrame && (isGeneric || rootURIMatches)) { potentialTranslators.push(translator); - converterFunctions.push(rootSearchURIs[rootSearchURI]); + proxies.push(rootSearchURIs[rootSearchURI]); break; } } @@ -189,51 +189,10 @@ Zotero.Translators = new function() { var codeGetter = new Zotero.Translators.CodeGetter(potentialTranslators); return codeGetter.getAll().then(function () { - return [potentialTranslators, converterFunctions]; + return [potentialTranslators, proxies]; }); }); - - /** - * Get the array of searchURIs and related proxy converter functions - * - * @param {String} URI to get searchURIs and converterFunctions for - */ - this.getSearchURIs = function(URI) { - var searchURIs = {}; - searchURIs[URI] = null; - - // if there is a subdomain that is also a TLD, also test against URI with the domain - // dropped after the TLD - // (i.e., www.nature.com.mutex.gmu.edu => www.nature.com) - var m = /^(https?:\/\/)([^\/]+)/i.exec(URI); - if (m) { - // First, drop the 0- if it exists (this is an III invention) - var host = m[2]; - if(host.substr(0, 2) === "0-") host = host.substr(2); - var hostnames = host.split("."); - for (var i=1; i<hostnames.length-2; i++) { - if (TLDS[hostnames[i].toLowerCase()]) { - var properHost = hostnames.slice(0, i+1).join("."); - var proxyHost = hostnames.slice(i+1).join("."); - var searchURI = m[1]+properHost+URI.substr(m[0].length); - if(Zotero.isBrowserExt || Zotero.isSafari) { - // in Chrome/Safari, the converterFunction needs to be passed as JSON, so - // just push an array with the proper and proxyHosts - searchURIs[searchURI] = [properHost, proxyHost]; - } else { - // in Firefox, add a converterFunction - searchURIs[searchURI] = new function() { - var re = new RegExp('^https?://(?:[^/]+\\.)?'+Zotero.Utilities.quotemeta(properHost)+'(?=/)', "gi"); - var _proxyHost = proxyHost.replace(/\$/g, "$$$$"); - return function(uri) { return uri.replace(re, "$&."+_proxyHost) }; - }; - } - } - } - } - return searchURIs; - }; - + /** * Converts translators to JSON-serializable objects */ diff --git a/chrome/content/zotero/xpcom/db.js b/chrome/content/zotero/xpcom/db.js @@ -674,6 +674,13 @@ Zotero.DBConnection.prototype.queryAsync = Zotero.Promise.coroutine(function* (s Zotero.debug(msg, 1); throw new Error(msg); } + }, + has: function(target, name) { + try { + return !!target.getResultByName(name); + } catch (e) { + return false; + } } }; for (let i=0, len=rows.length; i<len; i++) { diff --git a/chrome/content/zotero/xpcom/proxy.js b/chrome/content/zotero/xpcom/proxy.js @@ -75,8 +75,8 @@ Zotero.Proxies = new function() { * @return {Promise<Zotero.Proxy>} */ this.newProxyFromRow = Zotero.Promise.coroutine(function* (row) { - var proxy = new Zotero.Proxy; - yield proxy._loadFromRow(row); + var proxy = new Zotero.Proxy(row); + yield proxy.loadHosts(); return proxy; }); @@ -368,6 +368,65 @@ Zotero.Proxies = new function() { } /** + * Check the url for potential proxies and deproxify, providing a scheme to build + * a proxy object. + * + * @param URL + * @returns {Object} Unproxied url to proxy object + */ + this.getPotentialProxies = function(URL) { + var urlToProxy = {}; + // If it's a known proxied URL just return it + if (Zotero.Proxies.transparent) { + for (var proxy of Zotero.Proxies.proxies) { + if (proxy.regexp) { + var m = proxy.regexp.exec(URL); + if (m) { + let proper = proxy.toProper(m); + urlToProxy[proper] = proxy.toJSON(); + return urlToProxy; + } + } + } + } + urlToProxy[URL] = null; + + // if there is a subdomain that is also a TLD, also test against URI with the domain + // dropped after the TLD + // (i.e., www.nature.com.mutex.gmu.edu => www.nature.com) + var m = /^(https?:\/\/)([^\/]+)/i.exec(URL); + if (m) { + // First, drop the 0- if it exists (this is an III invention) + var host = m[2]; + if (host.substr(0, 2) === "0-") host = host.substr(2); + var hostnameParts = [host.split(".")]; + if (m[1] == 'https://' && host.replace(/-/g, '.') != host) { + // try replacing hyphens with dots for https protocol + // to account for EZProxy HttpsHypens mode + hostnameParts.push(host.replace(/-/g, '.').split('.')); + } + + for (let i=0; i < hostnameParts.length; i++) { + let parts = hostnameParts[i]; + // If hostnameParts has two entries, then the second one is with replaced hyphens + let dotsToHyphens = i == 1; + // skip the lowest level subdomain, domain and TLD + for (let j=1; j<parts.length-2; j++) { + // if a part matches a TLD, everything up to it is probably the true URL + if (TLDS[parts[j].toLowerCase()]) { + var properHost = parts.slice(0, j+1).join("."); + // protocol + properHost + /path + var properURL = m[1]+properHost+URL.substr(m[0].length); + var proxyHost = parts.slice(j+1).join('.'); + urlToProxy[properURL] = {scheme: m[1] + '%h.' + proxyHost + '/%p', dotsToHyphens}; + } + } + } + } + return urlToProxy; + }; + + /** * Determines whether a host is blacklisted, i.e., whether we should refuse to save transparent * proxy entries for this host. This is necessary because EZProxy offers to proxy all Google and * Wikipedia subdomains, but in practice, this would get really annoying. @@ -521,9 +580,35 @@ Zotero.Proxies = new function() { * @constructor * @class Represents an individual proxy server */ -Zotero.Proxy = function () { +Zotero.Proxy = function (row) { this.hosts = []; - this.multiHost = false; + this._loadFromRow(row); +} + +/** + * Loads a proxy object from a DB row + * @private + */ +Zotero.Proxy.prototype._loadFromRow = function (row) { + this.proxyID = row.proxyID; + this.multiHost = row.scheme && row.scheme.indexOf('%h') != -1 || !!row.multiHost; + this.autoAssociate = !!row.autoAssociate; + this.scheme = row.scheme; + // Database query results will throw as this option is only present when the proxy comes along with the translator + if ('dotsToHyphens' in row) { + this.dotsToHyphens = !!row.dotsToHyphens; + } + + if (this.scheme) { + this.compileRegexp(); + } +}; + +Zotero.Proxy.prototype.toJSON = function() { + if (!this.scheme) { + throw Error('Cannot convert proxy to JSON - no scheme'); + } + return {id: this.id, scheme: this.scheme, dotsToHyphens: this.dotsToHyphens}; } /** @@ -556,7 +641,7 @@ const Zotero_Proxy_schemeParameterRegexps = { Zotero.Proxy.prototype.compileRegexp = function() { // take host only if flagged as multiHost var parametersToCheck = Zotero_Proxy_schemeParameters; - if(this.multiHost) parametersToCheck["%h"] = "([a-zA-Z0-9]+\\.[a-zA-Z0-9\.]+)"; + if(this.multiHost) parametersToCheck["%h"] = "([a-zA-Z0-9]+[.\\-][a-zA-Z0-9.\\-]+)"; var indices = this.indices = {}; this.parameters = []; @@ -686,7 +771,8 @@ Zotero.Proxy.prototype.save = Zotero.Promise.coroutine(function* (transparent) { Zotero.Proxy.prototype.revert = Zotero.Promise.coroutine(function* () { if (!this.proxyID) throw new Error("Cannot revert an unsaved proxy"); var row = yield Zotero.DB.rowQueryAsync("SELECT * FROM proxies WHERE proxyID = ?", [this.proxyID]); - yield this._loadFromRow(row); + this._loadFromRow(row); + yield this.loadHosts(); }); /** @@ -706,14 +792,29 @@ Zotero.Proxy.prototype.erase = Zotero.Promise.coroutine(function* () { /** * Converts a proxied URL to an unproxied URL using this proxy * - * @param m {Array} The match from running this proxy's regexp against a URL spec - * @type String + * @param m {String|Array} The URL or the match from running this proxy's regexp against a URL spec + * @return {String} The unproxified URL if was proxified or the unchanged URL */ Zotero.Proxy.prototype.toProper = function(m) { + if (!Array.isArray(m)) { + let match = this.regexp.exec(m); + if (!match) { + return m + } else { + m = match; + } + } + let scheme = this.scheme.indexOf('https') == -1 ? 'http://' : 'https://'; if(this.multiHost) { - var properURL = "http://"+m[this.parameters.indexOf("%h")+1]+"/"; + var properURL = scheme+m[this.parameters.indexOf("%h")+1]+"/"; } else { - var properURL = "http://"+this.hosts[0]+"/"; + var properURL = scheme+this.hosts[0]+"/"; + } + + // Replace `-` with `.` in https to support EZProxy HttpsHyphens. + // Potentially troublesome with domains that contain dashes + if (this.dotsToHyphens) { + properURL = properURL.replace(/-/g, '.'); } if(this.indices["%p"]) { @@ -731,17 +832,23 @@ Zotero.Proxy.prototype.toProper = function(m) { /** * Converts an unproxied URL to a proxied URL using this proxy * - * @param {nsIURI} uri The nsIURI corresponding to the unproxied URL - * @type String + * @param {String|nsIURI} uri The URL as a string or the nsIURI corresponding to the unproxied URL + * @return {String} The proxified URL if was unproxified or the unchanged url */ Zotero.Proxy.prototype.toProxy = function(uri) { + if (typeof uri == "string") { + uri = Services.io.newURI(uri, null, null); + } + if (this.regexp.exec(uri.spec)) { + return uri.spec; + } var proxyURL = this.scheme; for(var i=this.parameters.length-1; i>=0; i--) { var param = this.parameters[i]; var value = ""; if(param == "%h") { - value = uri.hostPort; + value = this.dotsToHyphens ? uri.hostPort.replace(/-/g, '.') : uri.hostPort; } else if(param == "%p") { value = uri.path.substr(1); } else if(param == "%d") { @@ -756,19 +863,13 @@ Zotero.Proxy.prototype.toProxy = function(uri) { return proxyURL; } -/** - * Loads a proxy object from a DB row - * @private - */ -Zotero.Proxy.prototype._loadFromRow = Zotero.Promise.coroutine(function* (row) { - this.proxyID = row.proxyID; - this.multiHost = !!row.multiHost; - this.autoAssociate = !!row.autoAssociate; - this.scheme = row.scheme; +Zotero.Proxy.prototype.loadHosts = Zotero.Promise.coroutine(function* () { + if (!this.proxyID) { + throw Error("Cannot load hosts without a proxyID") + } this.hosts = yield Zotero.DB.columnQueryAsync( - "SELECT hostname FROM proxyHosts WHERE proxyID = ? ORDER BY hostname", row.proxyID + "SELECT hostname FROM proxyHosts WHERE proxyID = ? ORDER BY hostname", this.proxyID ); - this.compileRegexp(); }); /** diff --git a/chrome/content/zotero/xpcom/server_connector.js b/chrome/content/zotero/xpcom/server_connector.js @@ -177,17 +177,18 @@ Zotero.Server.Connector.Detect.prototype = { }, /** - * Callback to be executed when list of translators becomes available. Sends response with - * item types, translator IDs, labels, and icons for available translators. + * Callback to be executed when list of translators becomes available. Sends standard + * translator passing properties with proxies where available for translators. * @param {Zotero.Translate} translate * @param {Zotero.Translator[]} translators */ - _translatorsAvailable: function(obj, translators) { - var jsons = []; - for (let translator of translators) { - jsons.push(translator.serialize(TRANSLATOR_PASSING_PROPERTIES)); - } - this.sendResponse(200, "application/json", JSON.stringify(jsons)); + _translatorsAvailable: function(translate, translators) { + translators = translators.map(function(translator) { + translator = translator.serialize(TRANSLATOR_PASSING_PROPERTIES.concat('proxy')); + translator.proxy = translator.proxy ? translator.proxy.toJSON() : null; + return translator; + }); + this.sendResponse(200, "application/json", JSON.stringify(translators)); Zotero.Browser.deleteHiddenBrowser(this._browser); } @@ -371,13 +372,15 @@ Zotero.Server.Connector.SaveItem.prototype = { Zotero.Server.Connector.AttachmentProgressManager.add(data.items[i].attachments); } + let proxy = data.proxy && new Zotero.Proxy(data.proxy); // save items var itemSaver = new Zotero.Translate.ItemSaver({ libraryID, collections: collection ? [collection.id] : undefined, attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD, forceTagType: 1, - cookieSandbox + cookieSandbox, + proxy }); try { let items = yield itemSaver.saveItems( diff --git a/chrome/content/zotero/xpcom/translation/translate.js b/chrome/content/zotero/xpcom/translation/translate.js @@ -1120,18 +1120,21 @@ Zotero.Translate.Base.prototype = { // if detection returns immediately, return found translators return potentialTranslators.then(function(result) { var allPotentialTranslators = result[0]; - var properToProxyFunctions = result[1]; + var proxies = result[1]; // this gets passed out by Zotero.Translators.getWebTranslatorsForLocation() because it is // specific for each translator, but we want to avoid making a copy of a translator whenever // possible. - this._properToProxyFunctions = properToProxyFunctions ? properToProxyFunctions : null; + this._proxies = proxies ? [] : null; this._waitingForRPC = false; for(var i=0, n=allPotentialTranslators.length; i<n; i++) { var translator = allPotentialTranslators[i]; if(translator.runMode === Zotero.Translator.RUN_MODE_IN_BROWSER) { this._potentialTranslators.push(translator); + if (proxies) { + this._proxies.push(proxies[i]); + } } else if (this instanceof Zotero.Translate.Web && Zotero.Connector) { this._waitingForRPC = true; } @@ -1166,6 +1169,7 @@ Zotero.Translate.Base.prototype = { for(var i=0, n=rpcTranslators.length; i<n; i++) { rpcTranslators[i] = new Zotero.Translator(rpcTranslators[i]); rpcTranslators[i].runMode = Zotero.Translator.RUN_MODE_ZOTERO_STANDALONE; + rpcTranslators[i].proxy = rpcTranslators[i].proxy ? new Zotero.Proxy(rpcTranslators[i].proxy) : null; } this._foundTranslators = this._foundTranslators.concat(rpcTranslators); } @@ -1378,8 +1382,8 @@ Zotero.Translate.Base.prototype = { // convert proxy to proper if applicable if(!dontUseProxy && this.translator && this.translator[0] - && this.translator[0].properToProxy) { - var proxiedURL = this.translator[0].properToProxy(resolved); + && this._proxy) { + var proxiedURL = this._proxy.toProxy(resolved); if (proxiedURL != resolved) { Zotero.debug("Translate: proxified to " + proxiedURL); } @@ -1440,13 +1444,13 @@ Zotero.Translate.Base.prototype = { if(this._currentState === "detect") { if(this._potentialTranslators.length) { var lastTranslator = this._potentialTranslators.shift(); - var lastProperToProxyFunction = this._properToProxyFunctions ? this._properToProxyFunctions.shift() : null; + var lastProxy = this._proxies ? this._proxies.shift() : null; - if(returnValue) { - var dupeTranslator = {"properToProxy":lastProperToProxyFunction}; + if (returnValue) { + var dupeTranslator = {proxy: lastProxy ? new Zotero.Proxy(lastProxy) : null}; - for(var i in lastTranslator) dupeTranslator[i] = lastTranslator[i]; - if(Zotero.isBookmarklet && returnValue === "server") { + for (var i in lastTranslator) dupeTranslator[i] = lastTranslator[i]; + if (Zotero.isBookmarklet && returnValue === "server") { // In the bookmarklet, the return value from detectWeb can be "server" to // indicate the translator should be run on the Zotero server dupeTranslator.runMode = Zotero.Translator.RUN_MODE_ZOTERO_SERVER; @@ -1689,6 +1693,13 @@ Zotero.Translate.Base.prototype = { } this._currentTranslator = translator; + + // Pass on the proxy of the parent translate + if (this._parentTranslator) { + this._proxy = this._parentTranslator._proxy; + } else { + this._proxy = translator.proxy; + } this._runningAsyncProcesses = 0; this._returnValue = undefined; this._aborted = false; @@ -1950,12 +1961,13 @@ Zotero.Translate.Web.prototype._getParameters = function() { */ Zotero.Translate.Web.prototype._prepareTranslation = Zotero.Promise.method(function () { this._itemSaver = new Zotero.Translate.ItemSaver({ - "libraryID":this._libraryID, - "collections": this._collections, - "attachmentMode":Zotero.Translate.ItemSaver[(this._saveAttachments ? "ATTACHMENT_MODE_DOWNLOAD" : "ATTACHMENT_MODE_IGNORE")], - "forceTagType":1, - "cookieSandbox":this._cookieSandbox, - "baseURI":this.location + libraryID: this._libraryID, + collections: this._collections, + attachmentMode: Zotero.Translate.ItemSaver[(this._saveAttachments ? "ATTACHMENT_MODE_DOWNLOAD" : "ATTACHMENT_MODE_IGNORE")], + forceTagType: 1, + cookieSandbox: this._cookieSandbox, + proxy: this._proxy, + baseURI: this.location }); this.newItems = []; }); @@ -1987,11 +1999,12 @@ Zotero.Translate.Web.prototype._translateTranslatorLoaded = function() { (runMode === Zotero.Translator.RUN_MODE_ZOTERO_SERVER && Zotero.Connector.isOnline)) { var me = this; Zotero.Connector.callMethod("savePage", { - "uri":this.location.toString(), - "translatorID":(typeof this.translator[0] === "object" + uri: this.location.toString(), + translatorID: (typeof this.translator[0] === "object" ? this.translator[0].translatorID : this.translator[0]), - "cookie":this.document.cookie, - "html":this.document.documentElement.innerHTML + cookie: this.document.cookie, + proxy: this._proxy ? this._proxy.toJSON() : null, + html: this.document.documentElement.innerHTML }, function(obj) { me._translateRPCComplete(obj) }); } else if(runMode === Zotero.Translator.RUN_MODE_ZOTERO_SERVER) { var me = this; diff --git a/chrome/content/zotero/xpcom/translation/translate_item.js b/chrome/content/zotero/xpcom/translation/translate_item.js @@ -33,6 +33,7 @@ * <li>attachmentMode - One of Zotero.Translate.ItemSaver.ATTACHMENT_* specifying how attachments should be saved</li> * <li>forceTagType - Force tags to specified tag type</li> * <li>cookieSandbox - Cookie sandbox for attachment requests</li> + * <li>proxy - A proxy to deproxify item URLs</li> * <li>baseURI - URI to which attachment paths should be relative</li> */ Zotero.Translate.ItemSaver = function(options) { @@ -53,6 +54,7 @@ Zotero.Translate.ItemSaver = function(options) { Zotero.Translate.ItemSaver.ATTACHMENT_MODE_IGNORE; this._forceTagType = options.forceTagType; this._cookieSandbox = options.cookieSandbox; + this._proxy = options.proxy; // the URI to which other URIs are assumed to be relative if(typeof baseURI === "object" && baseURI instanceof Components.interfaces.nsIURI) { @@ -109,6 +111,13 @@ Zotero.Translate.ItemSaver.prototype = { }; newItem.fromJSON(this._deleteIrrelevantFields(item)); + // deproxify url + if (this._proxy && item.url) { + let url = this._proxy.toProper(item.url); + Zotero.debug(`Deproxifying item url ${item.url} with scheme ${this._proxy.scheme} to ${url}`, 5); + newItem.setField('url', url); + } + if (this._collections) { newItem.setCollections(this._collections); } @@ -253,6 +262,12 @@ Zotero.Translate.ItemSaver.prototype = { } if (!newAttachment) return false; // attachmentCallback should not have been called in this case + + // deproxify url + let url = newAttachment.getField('url'); + if (this._proxy && url) { + newAttachment.setField('url', this._proxy.toProper(url)); + } // save fields if (attachment.accessDate) newAttachment.setField("accessDate", attachment.accessDate); diff --git a/chrome/content/zotero/xpcom/translation/translator.js b/chrome/content/zotero/xpcom/translation/translator.js @@ -23,6 +23,9 @@ ***** END LICENSE BLOCK ***** */ +// Enumeration of types of translators +var TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8}; + // Properties required for every translator var TRANSLATOR_REQUIRED_PROPERTIES = ["translatorID", "translatorType", "label", "creator", "target", "priority", "lastUpdated"]; diff --git a/chrome/content/zotero/xpcom/translation/translators.js b/chrome/content/zotero/xpcom/translation/translators.js @@ -25,9 +25,6 @@ "use strict"; -// Enumeration of types of translators -var TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8}; - /** * Singleton to handle loading and caching of translators * @namespace @@ -297,10 +294,10 @@ Zotero.Translators = new function() { return this.getAllForType(type).then(function(allTranslators) { var potentialTranslators = []; - var converterFunctions = []; + var proxies = []; - var rootSearchURIs = this.getSearchURIs(rootURI); - var frameSearchURIs = isFrame ? this.getSearchURIs(URI) : rootSearchURIs; + var rootSearchURIs = Zotero.Proxies.getPotentialProxies(rootURI); + var frameSearchURIs = isFrame ? Zotero.Proxies.getPotentialProxies(URI) : rootSearchURIs; Zotero.debug("Translators: Looking for translators for "+Object.keys(frameSearchURIs).join(', ')); @@ -316,7 +313,7 @@ Zotero.Translators = new function() { if (frameURIMatches) { potentialTranslators.push(translator); - converterFunctions.push(frameSearchURIs[frameSearchURI]); + proxies.push(frameSearchURIs[frameSearchURI]); // prevent adding the translator multiple times break rootURIsLoop; } @@ -324,13 +321,13 @@ Zotero.Translators = new function() { } else if(!isFrame && (isGeneric || rootURIMatches)) { potentialTranslators.push(translator); - converterFunctions.push(rootSearchURIs[rootSearchURI]); + proxies.push(rootSearchURIs[rootSearchURI]); break; } } } - return [potentialTranslators, converterFunctions]; + return [potentialTranslators, proxies]; }.bind(this)); }, diff --git a/chrome/content/zotero/xpcom/utilities_translate.js b/chrome/content/zotero/xpcom/utilities_translate.js @@ -253,8 +253,8 @@ Zotero.Utilities.Translate.prototype.processDocuments = function(urls, processor } for(var i=0; i<urls.length; i++) { - if(this._translate.document && this._translate.document.location - && this._translate.document.location.toString() === urls[i]) { + if(translate.document && translate.document.location + && translate.document.location.toString() === urls[i]) { // Document is attempting to reload itself Zotero.debug("Translate: Attempted to load the current document using processDocuments; using loaded document instead"); // This fixes document permissions issues in translation-server when translators call @@ -374,6 +374,18 @@ Zotero.Utilities.Translate.prototype.doPost = function(url, body, onDone, header }, headers, responseCharset, translate.cookieSandbox ? translate.cookieSandbox : undefined); } +Zotero.Utilities.Translate.prototype.urlToProxy = function(url) { + var proxy = this._translate._proxy; + if (proxy) return proxy.toProxy(url); + return url; +}; + +Zotero.Utilities.Translate.prototype.urlToProper = function(url) { + var proxy = this._translate._proxy; + if (proxy) return proxy.toProper(url); + return url; +}; + Zotero.Utilities.Translate.prototype.__exposedProps__ = {"HTTP":"r"}; for(var j in Zotero.Utilities.Translate.prototype) { if(typeof Zotero.Utilities.Translate.prototype[j] === "function" && j[0] !== "_" && j != "Translate") { diff --git a/components/zotero-service.js b/components/zotero-service.js @@ -44,6 +44,7 @@ const xpcomFilesAll = [ 'ipc', 'profile', 'progressWindow', + 'proxy', 'translation/translate', 'translation/translate_firefox', 'translation/translator', @@ -98,7 +99,6 @@ const xpcomFilesLocal = [ 'locateManager', 'mime', 'notifier', - 'proxy', 'quickCopy', 'report', 'router', diff --git a/test/content/support.js b/test/content/support.js @@ -788,7 +788,7 @@ function buildDummyTranslator(translatorType, code, info={}) { "lastUpdated":"0000-00-00 00:00:00", }, info); let translator = new Zotero.Translator(info); - translator.code = code; + translator.code = JSON.stringify(info) + "\n" + code; return translator; } diff --git a/test/tests/proxyTest.js b/test/tests/proxyTest.js @@ -0,0 +1,31 @@ +"use strict"; + +describe("Zotero.Proxies", function(){ + describe("#getPotentialProxies", function() { + it("should return the provided url mapped to null when url is not proxied", function() { + let url = "http://www.example.com"; + let proxies = Zotero.Proxies.getPotentialProxies(url); + let expectedProxies = {}; + expectedProxies[url] = null; + assert.deepEqual(proxies, expectedProxies); + }); + + it("should return the provided url and deproxied url", function() { + let url = "https://www.example.com.proxy.example.com"; + let proxies = Zotero.Proxies.getPotentialProxies(url); + let expectedProxies = {}; + expectedProxies[url] = null; + expectedProxies["https://www.example.com"] = {scheme: "https://%h.proxy.example.com/%p", dotsToHyphens: false}; + assert.deepEqual(proxies, expectedProxies); + }); + + it("should return the provided url and deproxied url with replaced hyphens", function() { + let url = "https://www-example-com.proxy.example.com"; + let proxies = Zotero.Proxies.getPotentialProxies(url); + let expectedProxies = {}; + expectedProxies[url] = null; + expectedProxies["https://www.example.com"] = {scheme: "https://%h.proxy.example.com/%p", dotsToHyphens: true}; + assert.deepEqual(proxies, expectedProxies); + }); + }); +}); diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js @@ -57,13 +57,41 @@ describe("Connector Server", function () { ); assert.isTrue(Zotero.Translators.get.calledWith('dummy-translator')); - assert.equal(response.response, code); + let translatorCode = yield translator.getCode(); + assert.equal(response.response, translatorCode); Zotero.Translators.get.restore(); }) }); + describe("/connector/detect", function() { + it("should return relevant translators with proxies", function* () { + var code = 'function detectWeb() {return "newspaperArticle";}\nfunction doWeb() {}'; + var translator = buildDummyTranslator("web", code, {target: "https://www.example.com/.*"}); + sinon.stub(Zotero.Translators, 'getAllForType').resolves([translator]); + + var response = yield Zotero.HTTP.request( + 'POST', + connectorServerPath + "/connector/detect", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + uri: "https://www-example-com.proxy.example.com/article", + html: "<head><title>Owl</title></head><body><p>🦉</p></body>" + }) + } + ); + + assert.equal(JSON.parse(response.response)[0].proxy.scheme, 'https://%h.proxy.example.com/%p'); + + Zotero.Translators.getAllForType.restore(); + }); + }); + + describe("/connector/saveItems", function () { // TODO: Test cookies it("should save a translated item to the current selected collection", function* () { @@ -185,6 +213,49 @@ describe("Connector Server", function () { win.ZoteroPane.collectionsView.getSelectedLibraryID(), Zotero.Libraries.userLibraryID ); }); + + it("should use the provided proxy to deproxify item url", function* () { + yield selectLibrary(win, Zotero.Libraries.userLibraryID); + yield waitForItemsLoad(win); + + var body = { + items: [ + { + itemType: "newspaperArticle", + title: "Title", + creators: [ + { + firstName: "First", + lastName: "Last", + creatorType: "author" + } + ], + attachments: [], + url: "https://www-example-com.proxy.example.com/path" + } + ], + uri: "https://www-example-com.proxy.example.com/path", + proxy: {scheme: 'https://%h.proxy.example.com/%p', dotsToHyphens: true} + }; + + var promise = waitForItemEvent('add'); + var req = yield Zotero.HTTP.request( + 'POST', + connectorServerPath + "/connector/saveItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify(body) + } + ); + + // Check item + var ids = yield promise; + assert.lengthOf(ids, 1); + var item = Zotero.Items.get(ids[0]); + assert.equal(item.getField('url'), 'https://www.example.com/path'); + }); }); describe("/connector/saveSnapshot", function () { diff --git a/test/tests/translateTest.js b/test/tests/translateTest.js @@ -680,6 +680,49 @@ describe("Zotero.Translate", function() { assert.isNumber(translation.newItems[0].id); assert.ok(collection.hasItem(translation.newItems[0].id)); }); + + }); + describe('#saveItems', function() { + it("should deproxify item and attachment urls when proxy provided", function* (){ + var itemID; + var item = loadSampleData('journalArticle'); + item = item.journalArticle; + item.url = 'https://www-example-com.proxy.example.com/'; + item.attachments = [{ + url: 'https://www-example-com.proxy.example.com/pdf.pdf', + mimeType: 'application/pdf', + title: 'Example PDF'}]; + var itemSaver = new Zotero.Translate.ItemSaver({ + libraryID: Zotero.Libraries.userLibraryID, + attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_FILE, + proxy: new Zotero.Proxy({scheme: 'https://%h.proxy.example.com/%p', dotsToHyphens: true}) + }); + var itemDeferred = Zotero.Promise.defer(); + var attachmentDeferred = Zotero.Promise.defer(); + itemSaver.saveItems([item], Zotero.Promise.coroutine(function* (attachment, progressPercentage) { + // ItemSaver returns immediately without waiting for attachments, so we use the callback + // to test attachments + if (progressPercentage != 100) return; + try { + yield itemDeferred.promise; + let item = Zotero.Items.get(itemID); + attachment = Zotero.Items.get(item.getAttachments()[0]); + assert.equal(attachment.getField('url'), 'https://www.example.com/pdf.pdf'); + attachmentDeferred.resolve(); + } catch (e) { + attachmentDeferred.reject(e); + } + })).then(function(items) { + try { + assert.equal(items[0].getField('url'), 'https://www.example.com/'); + itemID = items[0].id; + itemDeferred.resolve(); + } catch (e) { + itemDeferred.reject(e); + } + }); + yield Zotero.Promise.all([itemDeferred.promise, attachmentDeferred.promise]); + }); }); }); });