www

commit f008843fc5b15a0a21fcdadcf573de8699a3436c
parent 499ab285397e55446005d72a5844f35fbc743289
Author: Dan Stillman <dstillman@zotero.org>
Date:   Thu, 24 Apr 2014 17:51:47 -0400

Automatically retry intermittent Amazon S3 upload timeouts

"Your socket connection to the server was not read from or written to
within the timeout period."

I can't reproduce these reliably, so this is fairly untested, but it
seems to work. It backs off exponentially when the error occurs, and
halves the delay on successful requests.

Eventually all 50x file sync errors should also be retried
automatically, but that can't really happen in 4.0.

Diffstat:
M
chrome/content/zotero/xpcom/storage/zfs.js
 | 
32
+++++++++++++++++++++++++++++---

1 file changed, 29 insertions(+), 3 deletions(-)
diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js
@@ -31,6 +31,8 @@ Zotero.Sync.Storage.ZFS = (function () {
 		"Zotero-API-Version" : ZOTERO_CONFIG.API_VERSION
 	};
 	var _cachedCredentials = false;
+	var _s3Backoff = 1;
+	var _maxS3Backoff = 60;
 	
 	/**
 	 * Get file metadata on storage server
@@ -463,9 +465,29 @@ Zotero.Sync.Storage.ZFS = (function () {
 				onStop: function (httpRequest, status, response, data) {
 					data.request.setChannel(false);
 					
-					deferred.resolve(
-						onUploadComplete(httpRequest, status, response, data)
-					);
+					// For timeouts from S3, which seem to happen intermittently,
+					// wait a little and try again
+					let timeoutMessage = "Your socket connection to the server was not read from or "
+						+ "written to within the timeout period.";
+					if (status == 400 && response.indexOf(timeoutMessage) != -1) {
+						let libraryKey = Zotero.Items.getLibraryKeyHash(item);
+						let msg = "S3 returned 400 in Zotero.Sync.Storage.ZFS.onUploadComplete()"
+							+ " (" + libraryKey + ") -- retrying"
+						Components.utils.reportError(msg);
+						Zotero.debug(msg, 1);
+						Zotero.debug(response, 1);
+						if (_s3Backoff < _maxS3Backoff) {
+							_s3Backoff *= 2;
+						}
+						Zotero.debug("Delaying " + libraryKey + " upload for " + _s3Backoff + " seconds");
+						Q.delay(_s3Backoff * 1000)
+						.then(function () {
+							deferred.resolve(postFile(request, item, url, uploadKey, params));
+						});
+						return;
+					}
+					
+					deferred.resolve(onUploadComplete(httpRequest, status, response, data));
 				},
 				onCancel: function (httpRequest, status, data) {
 					onUploadCancel(httpRequest, status, data)
@@ -504,6 +526,10 @@ Zotero.Sync.Storage.ZFS = (function () {
 			
 			switch (status) {
 				case 201:
+					// Decrease backoff delay on successful upload
+					if (_s3Backoff > 1) {
+						_s3Backoff /= 2;
+					}
 					break;
 				
 				case 500: