From 3c4673724e253088c76b00a8aae6d302a7e4a5d6 Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Tue, 27 Feb 2024 17:02:29 +0100
Subject: [PATCH] Move main word counter to new module
---
.gitignore | 3 +
novelwriter/constants.py | 5 -
novelwriter/core/index.py | 91 +-----------------
novelwriter/gui/doceditor.py | 6 +-
novelwriter/text/__init__.py | 3 +
novelwriter/text/counting.py | 118 ++++++++++++++++++++++++
tests/test_core/test_core_index.py | 104 +--------------------
tests/test_gui/test_gui_doceditor.py | 4 +-
tests/test_text/test_core_counting.py | 127 ++++++++++++++++++++++++++
9 files changed, 261 insertions(+), 200 deletions(-)
create mode 100644 novelwriter/text/__init__.py
create mode 100644 novelwriter/text/counting.py
create mode 100644 tests/test_text/test_core_counting.py
diff --git a/.gitignore b/.gitignore
index a7e67336..a69e4f00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,3 +50,6 @@ ToC.txt
# Coverage
/.coverage
/coverage.*
+
+# Other
+/test.py
diff --git a/novelwriter/constants.py b/novelwriter/constants.py
index 0ba85a57..130378bd 100644
--- a/novelwriter/constants.py
+++ b/novelwriter/constants.py
@@ -23,8 +23,6 @@ along with this program. If not, see .
"""
from __future__ import annotations
-import re
-
from PyQt5.QtCore import QCoreApplication, QT_TRANSLATE_NOOP
from novelwriter.enum import nwBuildFmt, nwItemClass, nwItemLayout, nwOutline
@@ -70,9 +68,6 @@ class nwRegEx:
FMT_SC = r"(?i)(? None:
"""Count text stats and save the counts to the index."""
- cC, wC, pC = countWords(text)
+ cC, wC, pC = standardCounter(text)
self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC)
return
@@ -1315,86 +1315,3 @@ def processComment(text: str) -> tuple[nwComment, str, int]:
if content and (clean := classifier.strip().lower()) in CLASSIFIERS:
return CLASSIFIERS[clean], content.strip(), text.find(":") + 1
return nwComment.PLAIN, check, 0
-
-
-def countWords(text: str) -> tuple[int, int, int]:
- """Count words in a piece of text, skipping special syntax and
- comments.
- """
- charCount = 0
- wordCount = 0
- paraCount = 0
- prevEmpty = True
-
- if not isinstance(text, str):
- return charCount, wordCount, paraCount
-
- # We need to treat dashes as word separators for counting words.
- # The check+replace approach is much faster than direct replace for
- # large texts, and a bit slower for small texts, but in the latter
- # case it doesn't really matter.
- if nwUnicode.U_ENDASH in text:
- text = text.replace(nwUnicode.U_ENDASH, " ")
- if nwUnicode.U_EMDASH in text:
- text = text.replace(nwUnicode.U_EMDASH, " ")
-
- # Strip shortcodes
- if "[" in text:
- text = nwRegEx.RX_SC.sub("", text)
-
- for line in text.splitlines():
-
- countPara = True
-
- if not line:
- prevEmpty = True
- continue
-
- if line[0] == "@" or line[0] == "%":
- continue
-
- if line[0] == "[":
- check = line.lower()
- if check.startswith(("[newpage]", "[new page]", "[vspace]")):
- continue
- elif check.startswith("[vspace:") and line.endswith("]"):
- continue
-
- elif line[0] == "#":
- if line[:5] == "#### ":
- line = line[5:]
- countPara = False
- elif line[:4] == "### ":
- line = line[4:]
- countPara = False
- elif line[:3] == "## ":
- line = line[3:]
- countPara = False
- elif line[:2] == "# ":
- line = line[2:]
- countPara = False
- elif line[:3] == "#! ":
- line = line[3:]
- countPara = False
- elif line[:4] == "##! ":
- line = line[4:]
- countPara = False
-
- elif line[0] == ">" or line[-1] == "<":
- if line[:2] == ">>":
- line = line[2:].lstrip(" ")
- elif line[:1] == ">":
- line = line[1:].lstrip(" ")
- if line[-2:] == "<<":
- line = line[:-2].rstrip(" ")
- elif line[-1:] == "<":
- line = line[:-1].rstrip(" ")
-
- wordCount += len(line.split())
- charCount += len(line)
- if countPara and prevEmpty:
- paraCount += 1
-
- prevEmpty = not countPara
-
- return charCount, wordCount, paraCount
diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py
index 33f6672b..e715ca8f 100644
--- a/novelwriter/gui/doceditor.py
+++ b/novelwriter/gui/doceditor.py
@@ -56,9 +56,9 @@ from novelwriter import CONFIG, SHARED
from novelwriter.enum import nwDocAction, nwDocInsert, nwDocMode, nwItemClass, nwTrinary
from novelwriter.common import minmax, transferCase
from novelwriter.constants import nwKeyWords, nwLabels, nwShortcode, nwUnicode, trConst
-from novelwriter.core.index import countWords
from novelwriter.tools.lipsum import GuiLipsum
from novelwriter.core.document import NWDocument
+from novelwriter.text.counting import standardCounter
from novelwriter.gui.dochighlight import GuiDocHighlighter
from novelwriter.gui.editordocument import GuiTextDocument
from novelwriter.extensions.eventfilters import WheelEventFilter
@@ -462,7 +462,7 @@ class GuiDocEditor(QPlainTextEdit):
return False
docText = self.getText()
- cC, wC, pC = countWords(docText)
+ cC, wC, pC = standardCounter(docText)
self._updateDocCounts(cC, wC, pC)
self.saveCursorPosition()
@@ -2226,7 +2226,7 @@ class BackgroundWordCounter(QRunnable):
else:
text = self._docEditor.getText()
- cC, wC, pC = countWords(text)
+ cC, wC, pC = standardCounter(text)
self.signals.countsReady.emit(cC, wC, pC)
self._isRunning = False
diff --git a/novelwriter/text/__init__.py b/novelwriter/text/__init__.py
new file mode 100644
index 00000000..69bc7fa4
--- /dev/null
+++ b/novelwriter/text/__init__.py
@@ -0,0 +1,3 @@
+"""
+novelWriter – Text Analysis Tools
+"""
diff --git a/novelwriter/text/counting.py b/novelwriter/text/counting.py
new file mode 100644
index 00000000..e92f4dc8
--- /dev/null
+++ b/novelwriter/text/counting.py
@@ -0,0 +1,118 @@
+"""
+novelWriter – Text Counting Functions
+=====================================
+
+File History:
+Created: 2019-04-22 [0.0.1] standardCounter
+Rewritten: 2024-02-27 [2.4b1] preProcessText, standardCounter
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+"""
+from __future__ import annotations
+
+import re
+
+from novelwriter.constants import nwRegEx, nwUnicode
+
+RX_SC = re.compile(nwRegEx.FMT_SC)
+RX_LO = re.compile(r"(?i)(?{1,2}\s*|\s*<{1,2}$")
+
+
+def preProcessText(text: str, keepHeaders: bool = True) -> list[str]:
+ """Strip formatting codes from the text and split into lines."""
+ if not isinstance(text, str):
+ return []
+
+ # We need to treat dashes as word separators for counting words.
+ # The check+replace approach is much faster than direct replace for
+ # large texts, and a bit slower for small texts, but in the latter
+ # case it doesn't really matter.
+ if nwUnicode.U_ENDASH in text:
+ text = text.replace(nwUnicode.U_ENDASH, " ")
+ if nwUnicode.U_EMDASH in text:
+ text = text.replace(nwUnicode.U_EMDASH, " ")
+
+ ignore = "%@" if keepHeaders else "%@#"
+
+ result = []
+ for line in text.splitlines():
+ line = line.rstrip()
+ if line:
+ if line[0] in ignore:
+ continue
+ if line[0] == ">":
+ line = line.lstrip(">").lstrip(" ")
+ if line[-1] == "<":
+ line = line.rstrip("<").rstrip(" ")
+ if "[" in line:
+ # Strip shortcodes and special formatting
+ # RegEx is slow, so we do this only when necessary
+ line = RX_SC.sub("", line)
+ line = RX_LO.sub("", line)
+
+ result.append(line)
+
+ return result
+
+
+def standardCounter(text: str) -> tuple[int, int, int]:
+ """A counter that counts paragraphs, words and characters.
+ This is the standard counter that includes headers in the word and
+ character counts.
+ """
+ charCount = 0
+ wordCount = 0
+ paraCount = 0
+ prevEmpty = True
+
+ for line in preProcessText(text):
+
+ countPara = True
+
+ if not line:
+ prevEmpty = True
+ continue
+
+ if line[0] == "#":
+ if line[:5] == "#### ":
+ line = line[5:]
+ countPara = False
+ elif line[:4] == "### ":
+ line = line[4:]
+ countPara = False
+ elif line[:3] == "## ":
+ line = line[3:]
+ countPara = False
+ elif line[:2] == "# ":
+ line = line[2:]
+ countPara = False
+ elif line[:3] == "#! ":
+ line = line[3:]
+ countPara = False
+ elif line[:4] == "##! ":
+ line = line[4:]
+ countPara = False
+
+ wordCount += len(line.split())
+ charCount += len(line)
+ if countPara and prevEmpty:
+ paraCount += 1
+
+ prevEmpty = not countPara
+
+ return charCount, wordCount, paraCount
diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py
index f225d0f2..b8daa6c6 100644
--- a/tests/test_core/test_core_index.py
+++ b/tests/test_core/test_core_index.py
@@ -31,7 +31,7 @@ from tools import C, buildTestProject, cmpFiles, writeFile
from novelwriter.enum import nwComment, nwItemClass, nwItemLayout
from novelwriter.constants import nwFiles
-from novelwriter.core.index import IndexItem, NWIndex, countWords, TagsIndex, processComment
+from novelwriter.core.index import IndexItem, NWIndex, TagsIndex, processComment
from novelwriter.core.project import NWProject
@@ -1319,105 +1319,3 @@ def testCoreIndex_processComment():
assert processComment("% \t SHORT : Hi:You") == (nwComment.SHORT, "Hi:You", 13)
# END Test testCoreIndex_processComment
-
-
-@pytest.mark.core
-def testCoreIndex_countWords():
- """Test the word counter and the exclusion filers."""
- # Non-Text
- assert countWords(None) == (0, 0, 0) # type: ignore
- assert countWords(1234) == (0, 0, 0) # type: ignore
-
- # General Text
- cC, wC, pC = countWords((
- "# Heading One\n"
- "## Heading Two\n"
- "### Heading Three\n"
- "#### Heading Four\n\n"
- "@tag: value\n\n"
- "% A comment that should not be counted.\n\n"
- "The first paragraph.\n\n"
- "The second paragraph.\n\n\n"
- "The third paragraph.\n\n"
- "Dashes\u2013and even longer\u2014dashes."
- ))
- assert cC == 138
- assert wC == 22
- assert pC == 4
-
- # Text Alignment
- cC, wC, pC = countWords((
- "# Title\n\n"
- "Left aligned<<\n\n"
- "Left aligned <<\n\n"
- "Right indent<\n\n"
- "Right indent <\n\n"
- ))
- assert cC == 53
- assert wC == 9
- assert pC == 4
-
- cC, wC, pC = countWords((
- "# Title\n\n"
- ">>Right aligned\n\n"
- ">> Right aligned\n\n"
- ">Left indent\n\n"
- "> Left indent\n\n"
- ))
- assert cC == 53
- assert wC == 9
- assert pC == 4
-
- cC, wC, pC = countWords((
- "# Title\n\n"
- ">>Centre aligned<<\n\n"
- ">> Centre aligned <<\n\n"
- ">Double indent<\n\n"
- "> Double indent <\n\n"
- ))
- assert cC == 59
- assert wC == 9
- assert pC == 4
-
- # Formatting Codes, Upper Case (Old Implementation)
- cC, wC, pC = countWords((
- "Some text\n\n"
- "[NEWPAGE]\n\n"
- "more text\n\n"
- "[NEW PAGE]]\n\n"
- "even more text\n\n"
- "[VSPACE]\n\n"
- "and some final text\n\n"
- "[VSPACE:4]\n\n"
- "THE END\n\n"
- ))
- assert cC == 58
- assert wC == 13
- assert pC == 5
-
- # Formatting Codes, Lower Case (Current Implementation)
- cC, wC, pC = countWords((
- "Some text\n\n"
- "[newpage]\n\n"
- "more text\n\n"
- "[new page]]\n\n"
- "even more text\n\n"
- "[vspace]\n\n"
- "and some final text\n\n"
- "[vspace:4]\n\n"
- "THE END\n\n"
- ))
- assert cC == 58
- assert wC == 13
- assert pC == 5
-
- # Check ShortCodes
- cC, wC, pC = countWords((
- "Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n"
- "Text with [b][i] nested [/i] emphasis [/b] in it.\n\n"
- ))
- assert cC == 78
- assert wC == 14
- assert pC == 2
-
-# END Test testCoreIndex_countWords
diff --git a/tests/test_gui/test_gui_doceditor.py b/tests/test_gui/test_gui_doceditor.py
index 8ce95710..419e4bf9 100644
--- a/tests/test_gui/test_gui_doceditor.py
+++ b/tests/test_gui/test_gui_doceditor.py
@@ -32,8 +32,8 @@ from PyQt5.QtWidgets import QAction, QMenu, qApp
from novelwriter import CONFIG, SHARED
from novelwriter.enum import nwDocAction, nwDocInsert, nwItemLayout, nwTrinary, nwWidget
from novelwriter.constants import nwKeyWords, nwUnicode
-from novelwriter.core.index import countWords
from novelwriter.gui.doceditor import GuiDocEditor, GuiDocToolBar
+from novelwriter.text.counting import standardCounter
from novelwriter.dialogs.editlabel import GuiEditLabel
KEY_DELAY = 1
@@ -1673,7 +1673,7 @@ def testGuiEditor_WordCounters(qtbot, monkeypatch, nwGUI, projPath, ipsumText, m
assert nwGUI.openDocument(C.hSceneDoc) is True
text = "\n\n".join(ipsumText)
- cC, wC, pC = countWords(text)
+ cC, wC, pC = standardCounter(text)
nwGUI.docEditor.replaceText(text)
# Check that a busy counter is blocked
diff --git a/tests/test_text/test_core_counting.py b/tests/test_text/test_core_counting.py
new file mode 100644
index 00000000..3c4caaa3
--- /dev/null
+++ b/tests/test_text/test_core_counting.py
@@ -0,0 +1,127 @@
+"""
+novelWriter – Counter Module Tester
+===================================
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+"""
+from __future__ import annotations
+
+import pytest
+
+from novelwriter.text.counting import standardCounter
+
+
+@pytest.mark.core
+def testTextCounting_standardCounter():
+ """Test the word counter and the exclusion filers."""
+ # Non-Text
+ assert standardCounter(None) == (0, 0, 0) # type: ignore
+ assert standardCounter(1234) == (0, 0, 0) # type: ignore
+
+ # General Text
+ cC, wC, pC = standardCounter((
+ "# Heading One\n"
+ "## Heading Two\n"
+ "### Heading Three\n"
+ "#### Heading Four\n\n"
+ "@tag: value\n\n"
+ "% A comment that should not be counted.\n\n"
+ "The first paragraph.\n\n"
+ "The second paragraph.\n\n\n"
+ "The third paragraph.\n\n"
+ "Dashes\u2013and even longer\u2014dashes."
+ ))
+ assert cC == 138
+ assert wC == 22
+ assert pC == 4
+
+ # Text Alignment
+ cC, wC, pC = standardCounter((
+ "# Title\n\n"
+ "Left aligned<<\n\n"
+ "Left aligned <<\n\n"
+ "Right indent<\n\n"
+ "Right indent <\n\n"
+ ))
+ assert cC == 53
+ assert wC == 9
+ assert pC == 4
+
+ cC, wC, pC = standardCounter((
+ "# Title\n\n"
+ ">>Right aligned\n\n"
+ ">> Right aligned\n\n"
+ ">Left indent\n\n"
+ "> Left indent\n\n"
+ ))
+ assert cC == 53
+ assert wC == 9
+ assert pC == 4
+
+ cC, wC, pC = standardCounter((
+ "# Title\n\n"
+ ">>Centre aligned<<\n\n"
+ ">> Centre aligned <<\n\n"
+ ">Double indent<\n\n"
+ "> Double indent <\n\n"
+ ))
+ assert cC == 59
+ assert wC == 9
+ assert pC == 4
+
+ # Formatting Codes, Upper Case (Old Implementation)
+ cC, wC, pC = standardCounter((
+ "Some text\n\n"
+ "[NEWPAGE]\n\n"
+ "more text\n\n"
+ "[NEW PAGE]\n\n"
+ "even more text\n\n"
+ "[VSPACE]\n\n"
+ "and some final text\n\n"
+ "[VSPACE:4]\n\n"
+ "THE END\n\n"
+ ))
+ assert cC == 58
+ assert wC == 13
+ assert pC == 5
+
+ # Formatting Codes, Lower Case (Current Implementation)
+ cC, wC, pC = standardCounter((
+ "Some text\n\n"
+ "[newpage]\n\n"
+ "more text\n\n"
+ "[new page]\n\n"
+ "even more text\n\n"
+ "[vspace]\n\n"
+ "and some final text\n\n"
+ "[vspace:4]\n\n"
+ "THE END\n\n"
+ ))
+ assert cC == 58
+ assert wC == 13
+ assert pC == 5
+
+ # Check ShortCodes
+ cC, wC, pC = standardCounter((
+ "Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n"
+ "Text with [b][i] nested [/i] emphasis [/b] in it.\n\n"
+ ))
+ assert cC == 78
+ assert wC == 14
+ assert pC == 2
+
+# END Test testTextCounting_standardCounter