Merge branch 'main' into better_spellcheck

nw/core/index.py

@@ -271,16 +271,16 @@ class NWIndex():
         theRoot = self.theProject.projTree.getRootItem(tHandle)
 
         if theItem is None:
-            logger.error("Not indexing unknown item %s" % tHandle)
+            logger.info("Not indexing unknown item %s" % tHandle)
             return False
         if theItem.itemType != nwItemType.FILE:
-            logger.error("Not indexing non-file item %s" % tHandle)
+            logger.info("Not indexing non-file item %s" % tHandle)
             return False
         if theItem.itemLayout == nwItemLayout.NO_LAYOUT:
-            logger.error("Not indexing no-layout item %s" % tHandle)
+            logger.info("Not indexing no-layout item %s" % tHandle)
             return False
         if theItem.parHandle is None:
-            logger.error("Not indexing orphaned item %s" % tHandle)
+            logger.info("Not indexing orphaned item %s" % tHandle)
             return False
 
         # Run word counter for the whole text
@@ -289,10 +289,10 @@ class NWIndex():
 
         # If the file is archived or trashed, we don't index the file itself
         if self.theProject.projTree.isTrashRoot(theItem.parHandle):
-            logger.error("Not indexing trash item %s" % tHandle)
+            logger.info("Not indexing trash item %s" % tHandle)
             return False
         if theRoot.itemClass == nwItemClass.ARCHIVE:
-            logger.error("Not indexing archived item %s" % tHandle)
+            logger.info("Not indexing archived item %s" % tHandle)
             return False
 
         itemClass  = theItem.itemClass

nw/core/tokenizer.py

@@ -33,7 +33,7 @@ from PyQt5.QtCore import QRegularExpression
 
 from nw.core.document import NWDoc
 from nw.core.tools import numberToWord, numberToRoman
-from nw.constants import nwItemLayout, nwItemType, nwRegEx
+from nw.constants import nwConst, nwItemLayout, nwItemType, nwRegEx
 
 logger = logging.getLogger(__name__)
 
@@ -120,6 +120,9 @@ class Tokenizer():
         self.isNote  = False
         self.isNovel = False
 
+        # Error Handling
+        self.errData = []
+
         return
 
     ##
@@ -212,6 +215,14 @@ class Tokenizer():
             theDocument  = NWDoc(self.theProject, self.theParent)
             self.theText = theDocument.openDocument(theHandle)
 
+        docSize = len(self.theText)
+        if docSize > nwConst.maxDocSize:
+            errVal = "Document '%s' is too big (%.2f MB). Skipping." % (
+                self.theItem.itemName, docSize/1.0e6
+            )
+            self.theText = "# ERROR\n\n%s\n\n" % errVal
+            self.errData.append(errVal)
+
         self.isNone  = self.theItem.itemLayout == nwItemLayout.NO_LAYOUT
         self.isTitle = self.theItem.itemLayout == nwItemLayout.TITLE
         self.isBook  = self.theItem.itemLayout == nwItemLayout.BOOK
@@ -230,6 +241,11 @@ class Tokenizer():
         """
         return self.theResult
 
+    def getResultSize(self):
+        """Return the size of the result from the conversion.
+        """
+        return len(self.theResult)
+
     def getFilteredMarkdown(self):
         """Return the novelWriter markdown after the filters have been applied.
         """

nw/core/tools.py

@@ -29,6 +29,8 @@
 
 import logging
 
+from nw.constants import nwUnicode
+
 logger = logging.getLogger(__name__)
 
 # =============================================================================================== #
@@ -44,6 +46,15 @@ def countWords(theText):
     paraCount = 0
     prevEmpty = True
 
+    # We need to treat dashes as word separators for counting words.
+    # The check+replace apprach is much faster that direct replace for
+    # large texts, and a bit slower for small texts, but in the latter
+    # case it doesn't matter.
+    if nwUnicode.U_ENDASH in theText:
+        theText = theText.replace(nwUnicode.U_ENDASH, " ")
+    if nwUnicode.U_EMDASH in theText:
+        theText = theText.replace(nwUnicode.U_EMDASH, " ")
+
     for aLine in theText.splitlines():
 
         countPara = True
@@ -72,8 +83,7 @@ def countWords(theText):
             charCount -= 2
             countPara = False
 
-        theBuff = aLine.replace("–", " ").replace("—", " ")
-        wordCount += len(theBuff.split())
+        wordCount += len(aLine.split())
         charCount += theLen
         if countPara and prevEmpty:
             paraCount += 1