www

commit 8931b3050dc21363b6762065c32276e47f82a49d
parent dd0529b64e27bbe34f0da352ac0a98a05b7ab74d
Author: Dan Stillman <dstillman@zotero.org>
Date:   Mon,  1 Feb 2010 08:28:52 +0000

- Fix error indexing documents containing words with invalid characters
- Increase number of words inserted per statement up to max allowed


Diffstat:
M
chrome/content/zotero/xpcom/fulltext.js
 | 
8
+++++++-

1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/chrome/content/zotero/xpcom/fulltext.js b/chrome/content/zotero/xpcom/fulltext.js
@@ -199,7 +199,7 @@ Zotero.Fulltext = new function(){
 		
 		var existing = [];
 		var done = 0;
-		var maxWords = 500;
+		var maxWords = 999; // compiled limit
 		var numWords = words.length;
 		
 		Zotero.DB.beginTransaction();
@@ -234,11 +234,17 @@ Zotero.Fulltext = new function(){
 		Zotero.DB.query("REPLACE INTO fulltextItems (itemID, version) VALUES (?,?)",
 			[itemID, FULLTEXT_VERSION]);
 		
+		
 		// Handle bound parameters manually for optimal speed
 		var statement1 = Zotero.DB.getStatement("INSERT INTO fulltextWords (word) VALUES (?)");
 		var statement2 = Zotero.DB.getStatement("INSERT OR IGNORE INTO fulltextItemWords VALUES (?,?)");
 		
 		for each(var word in origWords) {
+			// Skip words containing invalid characters
+			if (word.match(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\ud800-\udfff\ufffe\uffff]/)) {
+				Zotero.debug("Skipping word '" + word + "' due to invalid characters");
+				continue;
+			}
 			if (existing['_' + word]){
 				var wordID = existing['_' + word];
 			}