From b3b3c581f0984de89888615e18188e0ce49108c7 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 13:18:08 +0200
Subject: [PATCH] Speedup of word counter for very large documents

---
 nw/core/tools.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/nw/core/tools.py b/nw/core/tools.py
index ce30de96..59c141d9 100644
--- a/nw/core/tools.py
+++ b/nw/core/tools.py
@@ -29,6 +29,8 @@
 
 import logging
 
+from nw.constants import nwUnicode
+
 logger = logging.getLogger(__name__)
 
 # =============================================================================================== #
@@ -44,6 +46,15 @@ def countWords(theText):
     paraCount = 0
     prevEmpty = True
 
+    # We need to treat dashes as word separators for counting words.
+    # The check+replace apprach is much faster that direct replace for
+    # large texts, and a bit slower for small texts, but in the latter
+    # case it doesn't matter.
+    if nwUnicode.U_ENDASH in theText:
+        theText = theText.replace(nwUnicode.U_ENDASH, " ")
+    if nwUnicode.U_EMDASH in theText:
+        theText = theText.replace(nwUnicode.U_EMDASH, " ")
+
     for aLine in theText.splitlines():
 
         countPara = True
@@ -72,8 +83,7 @@ def countWords(theText):
             charCount -= 2
             countPara = False
 
-        theBuff = aLine.replace("–", " ").replace("—", " ")
-        wordCount += len(theBuff.split())
+        wordCount += len(aLine.split())
         charCount += theLen
         if countPara and prevEmpty:
             paraCount += 1