Move main word counter to new module

.gitignore

@@ -50,3 +50,6 @@ ToC.txt
 # Coverage
 /.coverage
 /coverage.*
+
+# Other
+/test.py

novelwriter/constants.py

@@ -23,8 +23,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
 from __future__ import annotations
 
-import re
-
 from PyQt5.QtCore import QCoreApplication, QT_TRANSLATE_NOOP
 
 from novelwriter.enum import nwBuildFmt, nwItemClass, nwItemLayout, nwOutline
@@ -70,9 +68,6 @@ class nwRegEx:
     FMT_SC = r"(?i)(?<!\\)(\[[\/\!]?(?:i|b|s|u|m|sup|sub)\])"
     FMT_SV = r"(?<!\\)(\[(?i)(?:fn|footnote):)(.+?)(?<!\\)(\])"
 
-    # Pre-Compiled RegEx
-    RX_SC = re.compile(FMT_SC)
-
 # END Class nwRegEx
 
 

novelwriter/core/index.py

@@ -3,7 +3,6 @@ novelWriter – Project Index
 ===========================
 
 File History:
-Created: 2019-04-22 [0.0.1]  countWords
 Created: 2019-05-27 [0.1.4]  NWIndex
 Created: 2022-05-28 [2.0rc1] IndexItem
 Created: 2022-05-28 [2.0rc1] IndexHeading
@@ -40,7 +39,8 @@ from novelwriter import SHARED
 from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
 from novelwriter.error import logException
 from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode
-from novelwriter.constants import nwFiles, nwKeyWords, nwRegEx, nwUnicode, nwHeaders
+from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
+from novelwriter.text.counting import standardCounter
 
 if TYPE_CHECKING:  # pragma: no cover
     from novelwriter.core.item import NWItem
@@ -266,7 +266,7 @@ class NWIndex:
         self._itemIndex.add(tHandle, tItem)
 
         # Run word counter for the whole text
-        cC, wC, pC = countWords(text)
+        cC, wC, pC = standardCounter(text)
         tItem.setCharCount(cC)
         tItem.setWordCount(wC)
         tItem.setParaCount(pC)
@@ -400,7 +400,7 @@ class NWIndex:
 
     def _indexWordCounts(self, tHandle: str, text: str, sTitle: str) -> None:
         """Count text stats and save the counts to the index."""
-        cC, wC, pC = countWords(text)
+        cC, wC, pC = standardCounter(text)
         self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC)
         return
 
@@ -1315,86 +1315,3 @@ def processComment(text: str) -> tuple[nwComment, str, int]:
     if content and (clean := classifier.strip().lower()) in CLASSIFIERS:
         return CLASSIFIERS[clean], content.strip(), text.find(":") + 1
     return nwComment.PLAIN, check, 0
-
-
-def countWords(text: str) -> tuple[int, int, int]:
-    """Count words in a piece of text, skipping special syntax and
-    comments.
-    """
-    charCount = 0
-    wordCount = 0
-    paraCount = 0
-    prevEmpty = True
-
-    if not isinstance(text, str):
-        return charCount, wordCount, paraCount
-
-    # We need to treat dashes as word separators for counting words.
-    # The check+replace approach is much faster than direct replace for
-    # large texts, and a bit slower for small texts, but in the latter
-    # case it doesn't really matter.
-    if nwUnicode.U_ENDASH in text:
-        text = text.replace(nwUnicode.U_ENDASH, " ")
-    if nwUnicode.U_EMDASH in text:
-        text = text.replace(nwUnicode.U_EMDASH, " ")
-
-    # Strip shortcodes
-    if "[" in text:
-        text = nwRegEx.RX_SC.sub("", text)
-
-    for line in text.splitlines():
-
-        countPara = True
-
-        if not line:
-            prevEmpty = True
-            continue
-
-        if line[0] == "@" or line[0] == "%":
-            continue
-
-        if line[0] == "[":
-            check = line.lower()
-            if check.startswith(("[newpage]", "[new page]", "[vspace]")):
-                continue
-            elif check.startswith("[vspace:") and line.endswith("]"):
-                continue
-
-        elif line[0] == "#":
-            if line[:5] == "#### ":
-                line = line[5:]
-                countPara = False
-            elif line[:4] == "### ":
-                line = line[4:]
-                countPara = False
-            elif line[:3] == "## ":
-                line = line[3:]
-                countPara = False
-            elif line[:2] == "# ":
-                line = line[2:]
-                countPara = False
-            elif line[:3] == "#! ":
-                line = line[3:]
-                countPara = False
-            elif line[:4] == "##! ":
-                line = line[4:]
-                countPara = False
-
-        elif line[0] == ">" or line[-1] == "<":
-            if line[:2] == ">>":
-                line = line[2:].lstrip(" ")
-            elif line[:1] == ">":
-                line = line[1:].lstrip(" ")
-            if line[-2:] == "<<":
-                line = line[:-2].rstrip(" ")
-            elif line[-1:] == "<":
-                line = line[:-1].rstrip(" ")
-
-        wordCount += len(line.split())
-        charCount += len(line)
-        if countPara and prevEmpty:
-            paraCount += 1
-
-        prevEmpty = not countPara
-
-    return charCount, wordCount, paraCount

novelwriter/gui/doceditor.py

@@ -56,9 +56,9 @@ from novelwriter import CONFIG, SHARED
 from novelwriter.enum import nwDocAction, nwDocInsert, nwDocMode, nwItemClass, nwTrinary
 from novelwriter.common import minmax, transferCase
 from novelwriter.constants import nwKeyWords, nwLabels, nwShortcode, nwUnicode, trConst
-from novelwriter.core.index import countWords
 from novelwriter.tools.lipsum import GuiLipsum
 from novelwriter.core.document import NWDocument
+from novelwriter.text.counting import standardCounter
 from novelwriter.gui.dochighlight import GuiDocHighlighter
 from novelwriter.gui.editordocument import GuiTextDocument
 from novelwriter.extensions.eventfilters import WheelEventFilter
@@ -462,7 +462,7 @@ class GuiDocEditor(QPlainTextEdit):
             return False
 
         docText = self.getText()
-        cC, wC, pC = countWords(docText)
+        cC, wC, pC = standardCounter(docText)
         self._updateDocCounts(cC, wC, pC)
 
         self.saveCursorPosition()
@@ -2226,7 +2226,7 @@ class BackgroundWordCounter(QRunnable):
         else:
             text = self._docEditor.getText()
 
-        cC, wC, pC = countWords(text)
+        cC, wC, pC = standardCounter(text)
         self.signals.countsReady.emit(cC, wC, pC)
         self._isRunning = False
 

novelwriter/text/__init__.py

@@ -0,0 +1,3 @@
+"""
+novelWriter – Text Analysis Tools
+"""

novelwriter/text/counting.py

@@ -0,0 +1,118 @@
+"""
+novelWriter – Text Counting Functions
+=====================================
+
+File History:
+Created:   2019-04-22 [0.0.1] standardCounter
+Rewritten: 2024-02-27 [2.4b1] preProcessText, standardCounter
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+from __future__ import annotations
+
+import re
+
+from novelwriter.constants import nwRegEx, nwUnicode
+
+RX_SC = re.compile(nwRegEx.FMT_SC)
+RX_LO = re.compile(r"(?i)(?<!\\)(\[(?:vspace|newpage|new page)(:\d+)?)(?<!\\)(\])")
+RX_IN = re.compile(r"^>{1,2}\s*|\s*<{1,2}$")
+
+
+def preProcessText(text: str, keepHeaders: bool = True) -> list[str]:
+    """Strip formatting codes from the text and split into lines."""
+    if not isinstance(text, str):
+        return []
+
+    # We need to treat dashes as word separators for counting words.
+    # The check+replace approach is much faster than direct replace for
+    # large texts, and a bit slower for small texts, but in the latter
+    # case it doesn't really matter.
+    if nwUnicode.U_ENDASH in text:
+        text = text.replace(nwUnicode.U_ENDASH, " ")
+    if nwUnicode.U_EMDASH in text:
+        text = text.replace(nwUnicode.U_EMDASH, " ")
+
+    ignore = "%@" if keepHeaders else "%@#"
+
+    result = []
+    for line in text.splitlines():
+        line = line.rstrip()
+        if line:
+            if line[0] in ignore:
+                continue
+            if line[0] == ">":
+                line = line.lstrip(">").lstrip(" ")
+            if line[-1] == "<":
+                line = line.rstrip("<").rstrip(" ")
+            if "[" in line:
+                # Strip shortcodes and special formatting
+                # RegEx is slow, so we do this only when necessary
+                line = RX_SC.sub("", line)
+                line = RX_LO.sub("", line)
+
+        result.append(line)
+
+    return result
+
+
+def standardCounter(text: str) -> tuple[int, int, int]:
+    """A counter that counts paragraphs, words and characters.
+    This is the standard counter that includes headers in the word and
+    character counts.
+    """
+    charCount = 0
+    wordCount = 0
+    paraCount = 0
+    prevEmpty = True
+
+    for line in preProcessText(text):
+
+        countPara = True
+
+        if not line:
+            prevEmpty = True
+            continue
+
+        if line[0] == "#":
+            if line[:5] == "#### ":
+                line = line[5:]
+                countPara = False
+            elif line[:4] == "### ":
+                line = line[4:]
+                countPara = False
+            elif line[:3] == "## ":
+                line = line[3:]
+                countPara = False
+            elif line[:2] == "# ":
+                line = line[2:]
+                countPara = False
+            elif line[:3] == "#! ":
+                line = line[3:]
+                countPara = False
+            elif line[:4] == "##! ":
+                line = line[4:]
+                countPara = False
+
+        wordCount += len(line.split())
+        charCount += len(line)
+        if countPara and prevEmpty:
+            paraCount += 1
+
+        prevEmpty = not countPara
+
+    return charCount, wordCount, paraCount

tests/test_core/test_core_index.py

@@ -31,7 +31,7 @@ from tools import C, buildTestProject, cmpFiles, writeFile
 
 from novelwriter.enum import nwComment, nwItemClass, nwItemLayout
 from novelwriter.constants import nwFiles
-from novelwriter.core.index import IndexItem, NWIndex, countWords, TagsIndex, processComment
+from novelwriter.core.index import IndexItem, NWIndex, TagsIndex, processComment
 from novelwriter.core.project import NWProject
 
 
@@ -1319,105 +1319,3 @@ def testCoreIndex_processComment():
     assert processComment("% \t  SHORT  : Hi:You") == (nwComment.SHORT, "Hi:You", 13)
 
 # END Test testCoreIndex_processComment
-
-
-@pytest.mark.core
-def testCoreIndex_countWords():
-    """Test the word counter and the exclusion filers."""
-    # Non-Text
-    assert countWords(None) == (0, 0, 0)  # type: ignore
-    assert countWords(1234) == (0, 0, 0)  # type: ignore
-
-    # General Text
-    cC, wC, pC = countWords((
-        "# Heading One\n"
-        "## Heading Two\n"
-        "### Heading Three\n"
-        "#### Heading Four\n\n"
-        "@tag: value\n\n"
-        "% A comment that should not be counted.\n\n"
-        "The first paragraph.\n\n"
-        "The second paragraph.\n\n\n"
-        "The third paragraph.\n\n"
-        "Dashes\u2013and even longer\u2014dashes."
-    ))
-    assert cC == 138
-    assert wC == 22
-    assert pC == 4
-
-    # Text Alignment
-    cC, wC, pC = countWords((
-        "# Title\n\n"
-        "Left aligned<<\n\n"
-        "Left aligned <<\n\n"
-        "Right indent<\n\n"
-        "Right indent <\n\n"
-    ))
-    assert cC == 53
-    assert wC == 9
-    assert pC == 4
-
-    cC, wC, pC = countWords((
-        "# Title\n\n"
-        ">>Right aligned\n\n"
-        ">> Right aligned\n\n"
-        ">Left indent\n\n"
-        "> Left indent\n\n"
-    ))
-    assert cC == 53
-    assert wC == 9
-    assert pC == 4
-
-    cC, wC, pC = countWords((
-        "# Title\n\n"
-        ">>Centre aligned<<\n\n"
-        ">> Centre aligned <<\n\n"
-        ">Double indent<\n\n"
-        "> Double indent <\n\n"
-    ))
-    assert cC == 59
-    assert wC == 9
-    assert pC == 4
-
-    # Formatting Codes, Upper Case (Old Implementation)
-    cC, wC, pC = countWords((
-        "Some text\n\n"
-        "[NEWPAGE]\n\n"
-        "more text\n\n"
-        "[NEW PAGE]]\n\n"
-        "even more text\n\n"
-        "[VSPACE]\n\n"
-        "and some final text\n\n"
-        "[VSPACE:4]\n\n"
-        "THE END\n\n"
-    ))
-    assert cC == 58
-    assert wC == 13
-    assert pC == 5
-
-    # Formatting Codes, Lower Case (Current Implementation)
-    cC, wC, pC = countWords((
-        "Some text\n\n"
-        "[newpage]\n\n"
-        "more text\n\n"
-        "[new page]]\n\n"
-        "even more text\n\n"
-        "[vspace]\n\n"
-        "and some final text\n\n"
-        "[vspace:4]\n\n"
-        "THE END\n\n"
-    ))
-    assert cC == 58
-    assert wC == 13
-    assert pC == 5
-
-    # Check ShortCodes
-    cC, wC, pC = countWords((
-        "Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n"
-        "Text with [b][i] nested [/i] emphasis [/b] in it.\n\n"
-    ))
-    assert cC == 78
-    assert wC == 14
-    assert pC == 2
-
-# END Test testCoreIndex_countWords

tests/test_gui/test_gui_doceditor.py

@@ -32,8 +32,8 @@ from PyQt5.QtWidgets import QAction, QMenu, qApp
 from novelwriter import CONFIG, SHARED
 from novelwriter.enum import nwDocAction, nwDocInsert, nwItemLayout, nwTrinary, nwWidget
 from novelwriter.constants import nwKeyWords, nwUnicode
-from novelwriter.core.index import countWords
 from novelwriter.gui.doceditor import GuiDocEditor, GuiDocToolBar
+from novelwriter.text.counting import standardCounter
 from novelwriter.dialogs.editlabel import GuiEditLabel
 
 KEY_DELAY = 1
@@ -1673,7 +1673,7 @@ def testGuiEditor_WordCounters(qtbot, monkeypatch, nwGUI, projPath, ipsumText, m
     assert nwGUI.openDocument(C.hSceneDoc) is True
 
     text = "\n\n".join(ipsumText)
-    cC, wC, pC = countWords(text)
+    cC, wC, pC = standardCounter(text)
     nwGUI.docEditor.replaceText(text)
 
     # Check that a busy counter is blocked

tests/test_text/test_core_counting.py

@@ -0,0 +1,127 @@
+"""
+novelWriter – Counter Module Tester
+===================================
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+from __future__ import annotations
+
+import pytest
+
+from novelwriter.text.counting import standardCounter
+
+
+@pytest.mark.core
+def testTextCounting_standardCounter():
+    """Test the word counter and the exclusion filers."""
+    # Non-Text
+    assert standardCounter(None) == (0, 0, 0)  # type: ignore
+    assert standardCounter(1234) == (0, 0, 0)  # type: ignore
+
+    # General Text
+    cC, wC, pC = standardCounter((
+        "# Heading One\n"
+        "## Heading Two\n"
+        "### Heading Three\n"
+        "#### Heading Four\n\n"
+        "@tag: value\n\n"
+        "% A comment that should not be counted.\n\n"
+        "The first paragraph.\n\n"
+        "The second paragraph.\n\n\n"
+        "The third paragraph.\n\n"
+        "Dashes\u2013and even longer\u2014dashes."
+    ))
+    assert cC == 138
+    assert wC == 22
+    assert pC == 4
+
+    # Text Alignment
+    cC, wC, pC = standardCounter((
+        "# Title\n\n"
+        "Left aligned<<\n\n"
+        "Left aligned <<\n\n"
+        "Right indent<\n\n"
+        "Right indent <\n\n"
+    ))
+    assert cC == 53
+    assert wC == 9
+    assert pC == 4
+
+    cC, wC, pC = standardCounter((
+        "# Title\n\n"
+        ">>Right aligned\n\n"
+        ">> Right aligned\n\n"
+        ">Left indent\n\n"
+        "> Left indent\n\n"
+    ))
+    assert cC == 53
+    assert wC == 9
+    assert pC == 4
+
+    cC, wC, pC = standardCounter((
+        "# Title\n\n"
+        ">>Centre aligned<<\n\n"
+        ">> Centre aligned <<\n\n"
+        ">Double indent<\n\n"
+        "> Double indent <\n\n"
+    ))
+    assert cC == 59
+    assert wC == 9
+    assert pC == 4
+
+    # Formatting Codes, Upper Case (Old Implementation)
+    cC, wC, pC = standardCounter((
+        "Some text\n\n"
+        "[NEWPAGE]\n\n"
+        "more text\n\n"
+        "[NEW PAGE]\n\n"
+        "even more text\n\n"
+        "[VSPACE]\n\n"
+        "and some final text\n\n"
+        "[VSPACE:4]\n\n"
+        "THE END\n\n"
+    ))
+    assert cC == 58
+    assert wC == 13
+    assert pC == 5
+
+    # Formatting Codes, Lower Case (Current Implementation)
+    cC, wC, pC = standardCounter((
+        "Some text\n\n"
+        "[newpage]\n\n"
+        "more text\n\n"
+        "[new page]\n\n"
+        "even more text\n\n"
+        "[vspace]\n\n"
+        "and some final text\n\n"
+        "[vspace:4]\n\n"
+        "THE END\n\n"
+    ))
+    assert cC == 58
+    assert wC == 13
+    assert pC == 5
+
+    # Check ShortCodes
+    cC, wC, pC = standardCounter((
+        "Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n"
+        "Text with [b][i] nested [/i] emphasis [/b] in it.\n\n"
+    ))
+    assert cC == 78
+    assert wC == 14
+    assert pC == 2
+
+# END Test testTextCounting_standardCounter