commit f5b5617885bba1c4a76e812853f3f6a0fe92c434
parent 4464e8ed9ed026813aa8048ee28818589b4a0ef8
Author: Dan Stillman <dstillman@zotero.org>
Date: Thu, 18 Sep 2014 16:23:49 -0400
Improve long-filename handling during syncing
This will hopefully fix some remaining issues with long filenames during
syncing, particularly on Linux with encrypted filenames (which have a
filename length of 143).
(This may have reintroduced some edge case bugs, so it needs some
testing.)
Diffstat:
3 files changed, 185 insertions(+), 167 deletions(-)
diff --git a/chrome/content/zotero/xpcom/file.js b/chrome/content/zotero/xpcom/file.js
@@ -311,6 +311,111 @@ Zotero.File = new function(){
}
+ this.createShortened = function (file, type, mode, maxBytes) {
+ if (!maxBytes) {
+ maxBytes = 255;
+ }
+
+ // Limit should be 255, but leave room for unique numbering if necessary
+ var padding = 3;
+
+ while (true) {
+ var newLength = maxBytes - padding;
+
+ try {
+ file.create(type, mode);
+ }
+ catch (e) {
+ let pathError = false;
+
+ let pathByteLength = Zotero.Utilities.Internal.byteLength(file.path);
+ let fileNameByteLength = Zotero.Utilities.Internal.byteLength(file.leafName);
+
+ // Windows API only allows paths of 260 characters
+ if (e.name == "NS_ERROR_FILE_NOT_FOUND" && pathByteLength > 260) {
+ Zotero.debug("Path is " + file.path);
+ pathError = true;
+ }
+ // ext3/ext4/HFS+ have a filename length limit of ~254 bytes
+ else if ((e.name == "NS_ERROR_FAILURE" || e.name == "NS_ERROR_FILE_NAME_TOO_LONG")
+ && (fileNameByteLength >= 254 || (Zotero.isLinux && fileNameByteLength > 143))) {
+ Zotero.debug("Filename is '" + file.leafName + "'");
+ }
+ else {
+ Zotero.debug("Path is " + file.path);
+ throw e;
+ }
+
+ // Preserve extension
+ var matches = file.leafName.match(/\.[a-z0-9]{0,20}$/);
+ var ext = matches ? matches[0] : "";
+
+ if (pathError) {
+ let pathLength = pathByteLength - fileNameByteLength;
+ newLength -= pathLength;
+
+ if (newLength < 5) {
+ throw new Error("Path is too long");
+ }
+ }
+
+ // Shorten the filename
+ //
+ // Shortened file could already exist if there was another file with a
+ // similar name that was also longer than the limit, so we do this in a
+ // loop, adding numbers if necessary
+ var uniqueFile = file.clone();
+ var step = 0;
+ while (step < 100) {
+ let newBaseName = uniqueFile.leafName.substr(0, newLength - ext.length);
+ if (step == 0) {
+ var newName = newBaseName + ext;
+ }
+ else {
+ var newName = newBaseName + "-" + step + ext;
+ }
+
+ // Check actual byte length, and shorten more if necessary
+ if (Zotero.Utilities.Internal.byteLength(newName) > maxBytes) {
+ step = 0;
+ newLength--;
+ continue;
+ }
+
+ uniqueFile.leafName = newName;
+ if (!uniqueFile.exists()) {
+ break;
+ }
+
+ step++;
+ }
+
+ var msg = "Shortening filename to '" + newName + "'";
+ Zotero.debug(msg, 2);
+ Zotero.log(msg, 'warning');
+
+ try {
+ uniqueFile.create(Components.interfaces.nsIFile.type, mode);
+ }
+ catch (e) {
+ // On Linux, try 143, which is the max filename length with eCryptfs
+ if (e.name == "NS_ERROR_FILE_NAME_TOO_LONG" && Zotero.isLinux && uniqueFile.leafName.length > 143) {
+ Zotero.debug("Trying shorter filename in case of filesystem encryption", 2);
+ maxBytes = 143;
+ continue;
+ }
+ else {
+ throw e;
+ }
+ }
+
+ file.leafName = uniqueFile.leafName;
+ }
+ break;
+ }
+ }
+
+
this.copyToUnique = function (file, newFile) {
newFile.createUnique(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
var newName = newFile.leafName;
@@ -546,10 +651,13 @@ Zotero.File = new function(){
var opWord = Zotero.getString('file.accessError.updated');
}
+ Zotero.debug(file.path);
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+
if (e.name == 'NS_ERROR_FILE_ACCESS_DENIED' || e.name == 'NS_ERROR_FILE_IS_LOCKED'
// These show up on some Windows systems
|| e.name == 'NS_ERROR_FAILURE' || e.name == 'NS_ERROR_FILE_NOT_FOUND') {
- Zotero.debug(e);
str = str + " " + Zotero.getString('file.accessError.cannotBe') + " " + opWord + ".";
var checkFileWindows = Zotero.getString('file.accessError.message.windows');
var checkFileOther = Zotero.getString('file.accessError.message.other');
diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js
@@ -1232,7 +1232,7 @@ Zotero.Sync.Storage = new function () {
// If library isn't editable but filename was changed, update
// database without updating the item's mod time, which would result
// in a library access error
- if (!Zotero.Items.editCheck(item)) {
+ if (!Zotero.Items.isEditable(item)) {
Zotero.debug("File renamed without library access -- "
+ "updating itemAttachments path", 3);
item.relinkAttachmentFile(newFile, true);
@@ -1501,80 +1501,50 @@ Zotero.Sync.Storage = new function () {
Zotero.debug("Moving download file " + tempFile.leafName + " into attachment directory as '" + fileName + "'");
try {
- tempFile.moveTo(parentDir, fileName);
+ var destFile = parentDir.clone();
+ destFile.append(fileName);
+ Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
}
catch (e) {
- var destFile = file.clone();
-
- var windowsLength = false;
- var nameLength = false;
-
- // Windows API only allows paths of 260 characters
- if (e.name == "NS_ERROR_FILE_NOT_FOUND" && destFile.path.length > 255) {
- windowsLength = true;
- }
- // ext3/ext4/HFS+ have a filename length limit of ~254 bytes
- //
- // These filenames will almost always be ASCII ad files,
- // but allow an extra 10 bytes anyway
- else if (e.name == "NS_ERROR_FAILURE" && destFile.leafName.length >= 244) {
- nameLength = true;
- }
- // Filesystem encryption (or, more specifically, filename encryption)
- // can result in a lower limit -- not much we can do about this,
- // but log a warning and skip the file
- else if (e.name == "NS_ERROR_FAILURE" && Zotero.isLinux && destFile.leafName.length > 130) {
- Zotero.debug(e);
- var msg = Zotero.getString('sync.storage.error.encryptedFilenames', destFile.leafName);
- Components.utils.reportError(msg);
- return;
- }
+ Zotero.File.checkFileAccessError(e, destFile, 'create');
+ }
+
+ if (destFile.leafName != fileName) {
+ Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
- if (windowsLength || nameLength) {
- // Preserve extension
- var matches = destFile.leafName.match(/\.[a-z0-9]{0,8}$/);
- var ext = matches ? matches[0] : "";
-
- if (windowsLength) {
- var pathLength = destFile.path.length - destFile.leafName.length;
- var newLength = 255 - pathLength;
- // Require 40 available characters in path -- this is arbitrary,
- // but otherwise filenames are going to end up being cut off
- if (newLength < 40) {
- var msg = "Due to a Windows path length limitation, your Zotero data directory "
- + "is too deep in the filesystem for syncing to work reliably. "
- + "Please relocate your Zotero data to a higher directory.";
- throw (msg);
- }
- }
- else {
- var newLength = 254;
+ // Abort if Windows path limitation would cause filenames to be overly truncated
+ if (Zotero.isWin && destFile.leafName.length < 40) {
+ try {
+ destFile.remove(false);
}
-
- // Shorten file if it's too long -- we don't relink it, but this should
- // be pretty rare and probably only occurs on extraneous files with
- // gibberish for filenames
- var fileName = destFile.leafName.substr(0, newLength - (ext.length + 1)) + ext;
- var msg = "Shortening filename to '" + fileName + "'";
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg);
-
- tempFile.moveTo(parentDir, fileName);
- renamed = true;
+ catch (e) {}
+ var msg = "Due to a Windows path length limitation, your Zotero data directory "
+ + "is too deep in the filesystem for syncing to work reliably. "
+ + "Please relocate your Zotero data to a higher directory.";
+ Zotero.debug(msg, 1);
+ throw new Error(msg);
}
- else {
- Components.utils.reportError(e);
- var msg = Zotero.getString('sync.storage.error.fileNotCreated', parentDir.leafName + '/' + fileName);
- throw(msg);
+
+ renamed = true;
+ }
+
+ try {
+ tempFile.moveTo(parentDir, destFile.leafName);
+ }
+ catch (e) {
+ try {
+ destFile.remove(false);
}
+ catch (e) {}
+
+ Zotero.File.checkFileAccessError(e, destFile, 'create');
}
var returnFile = null;
// processDownload() needs to know that we're renaming the file
if (renamed) {
- var returnFile = file.clone();
+ var returnFile = destFile.clone();
}
-
return returnFile;
}
@@ -1704,124 +1674,47 @@ Zotero.Sync.Storage = new function () {
}
try {
- destFile.create(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
+ Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
}
catch (e) {
Zotero.debug(e, 1);
+ Components.utils.reportError(e);
- var windowsLength = false;
- var nameLength = false;
+ zipReader.close();
- // Windows API only allows paths of 260 characters
- if (e.name == "NS_ERROR_FILE_NOT_FOUND" && destFile.path.length > 255) {
- Zotero.debug("Path is " + destFile.path);
- windowsLength = true;
- }
- // ext3/ext4/HFS+ have a filename length limit of ~254 bytes
- //
- // These filenames will almost always be ASCII ad files,
- // but allow an extra 10 bytes anyway
- else if (e.name == "NS_ERROR_FAILURE" && destFile.leafName.length >= 244) {
- Zotero.debug("Filename is " + destFile.leafName);
- nameLength = true;
- }
- // Filesystem encryption (or, more specifically, filename encryption)
- // can result in a lower limit -- not much we can do about this,
- // but log a warning and skip the file
- else if (e.name == "NS_ERROR_FAILURE" && Zotero.isLinux && destFile.leafName.length > 130) {
- var msg = Zotero.getString('sync.storage.error.encryptedFilenames', destFile.leafName);
- Components.utils.reportError(msg);
- continue;
- }
- else {
- Zotero.debug("Path is " + destFile.path);
- }
+ Zotero.File.checkFileAccessError(e, destFile, 'create');
+ }
+
+ if (destFile.leafName != fileName) {
+ Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
- if (windowsLength || nameLength) {
- // Preserve extension
- var matches = destFile.leafName.match(/\.[a-z0-9]{0,8}$/);
- var ext = matches ? matches[0] : "";
-
- if (windowsLength) {
- var pathLength = destFile.path.length - destFile.leafName.length;
- // Limit should be 255, but a shorter limit seems to be
- // enforced for nsIZipReader.extract() below on
- // non-English systems
- var newLength = 240 - pathLength;
- // Require 40 available characters in path -- this is arbitrary,
- // but otherwise filenames are going to end up being cut off
- if (newLength < 40) {
- zipReader.close();
- var msg = "Due to a Windows path length limitation, your Zotero data directory "
- + "is too deep in the filesystem for syncing to work reliably. "
- + "Please relocate your Zotero data to a higher directory.";
- throw (msg);
- }
- }
- else {
- var newLength = 240;
- }
-
- // Shorten file if it's too long -- we don't relink it, but this should
- // be pretty rare and probably only occurs on extraneous files with
- // gibberish for filenames
- //
- // Shortened file could already exist if there was another file with a
- // similar name that was also longer than the limit, so we do this in a
- // loop, adding numbers if necessary
- var step = 0;
- do {
- if (step == 0) {
- var newName = destFile.leafName.substr(0, newLength - ext.length) + ext;
- }
- else {
- var newName = destFile.leafName.substr(0, newLength - ext.length) + "-" + step + ext;
- }
- destFile.leafName = newName;
- step++;
- }
- while (destFile.exists());
-
- var msg = "Shortening filename to '" + newName + "'";
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg);
-
+ // Abort if Windows path limitation would cause filenames to be overly truncated
+ if (Zotero.isWin && destFile.leafName.length < 40) {
try {
- destFile.create(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
- }
- catch (e) {
- // See above
- if (e.name == "NS_ERROR_FAILURE" && Zotero.isLinux && destFile.leafName.length > 130) {
- Zotero.debug(e);
- var msg = Zotero.getString('sync.storage.error.encryptedFilenames', destFile.leafName);
- Components.utils.reportError(msg);
- continue;
- }
-
- zipReader.close();
-
- Components.utils.reportError(e);
- var msg = Zotero.getString('sync.storage.error.fileNotCreated', parentDir.leafName + '/' + fileName);
- throw(msg);
- }
-
- if (primaryFile) {
- renamed = true;
+ destFile.remove(false);
}
- }
- else {
+ catch (e) {}
zipReader.close();
-
- Components.utils.reportError(e);
- var msg = Zotero.getString('sync.storage.error.fileNotCreated', parentDir.leafName + '/' + fileName);
- throw(msg);
+ var msg = "Due to a Windows path length limitation, your Zotero data directory "
+ + "is too deep in the filesystem for syncing to work reliably. "
+ + "Please relocate your Zotero data to a higher directory.";
+ Zotero.debug(msg, 1);
+ throw new Error(msg);
+ }
+
+ if (primaryFile) {
+ renamed = true;
}
}
+
try {
zipReader.extract(entryName, destFile);
}
catch (e) {
- Zotero.debug(destFile.path);
+ try {
+ destFile.remove(false);
+ }
+ catch (e) {}
// For advertising junk files, ignore a bug on Windows where
// destFile.create() works but zipReader.extract() doesn't
diff --git a/chrome/content/zotero/xpcom/utilities_internal.js b/chrome/content/zotero/xpcom/utilities_internal.js
@@ -195,6 +195,23 @@ Zotero.Utilities.Internal = {
/**
+ * Return the byte length of a UTF-8 string
+ *
+ * http://stackoverflow.com/a/23329386
+ */
+ byteLength: function (str) {
+ var s = str.length;
+ for (var i=str.length-1; i>=0; i--) {
+ var code = str.charCodeAt(i);
+ if (code > 0x7f && code <= 0x7ff) s++;
+ else if (code > 0x7ff && code <= 0xffff) s+=2;
+ if (code >= 0xDC00 && code <= 0xDFFF) i--; //trail surrogate
+ }
+ return s;
+ },
+
+
+ /**
* Display a prompt from an error with custom buttons and a callback
*/
"errorPrompt":function(title, e) {
