Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move main word counter to new module

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-02-27 17:02:29 +01:00
parent a87a94ffeb
commit 3c4673724e
9 changed files with 261 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -50,3 +50,6 @@ ToC.txt
# Coverage
/.coverage
/coverage.*
# Other
/test.py
novelwriter/constants.py
-5
View File
@@ -23,8 +23,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import re
from PyQt5.QtCore import QCoreApplication, QT_TRANSLATE_NOOP
from novelwriter.enum import nwBuildFmt, nwItemClass, nwItemLayout, nwOutline
@@ -70,9 +68,6 @@ class nwRegEx:
FMT_SC = r"(?i)(?<!\\)(\[[\/\!]?(?:i|b|s|u|m|sup|sub)\])"
FMT_SV = r"(?<!\\)(\[(?i)(?:fn|footnote):)(.+?)(?<!\\)(\])"
# Pre-Compiled RegEx
RX_SC = re.compile(FMT_SC)
# END Class nwRegEx
novelwriter/core/index.py
+4 -87
View File
@@ -3,7 +3,6 @@ novelWriter – Project Index
===========================
File History:
Created: 2019-04-22 [0.0.1] countWords
Created: 2019-05-27 [0.1.4] NWIndex
Created: 2022-05-28 [2.0rc1] IndexItem
Created: 2022-05-28 [2.0rc1] IndexHeading
@@ -40,7 +39,8 @@ from novelwriter import SHARED
from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
from novelwriter.error import logException
from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode
from novelwriter.constants import nwFiles, nwKeyWords, nwRegEx, nwUnicode, nwHeaders
from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
from novelwriter.text.counting import standardCounter
if TYPE_CHECKING: # pragma: no cover
from novelwriter.core.item import NWItem
@@ -266,7 +266,7 @@ class NWIndex:
self._itemIndex.add(tHandle, tItem)
# Run word counter for the whole text
cC, wC, pC = countWords(text)
cC, wC, pC = standardCounter(text)
tItem.setCharCount(cC)
tItem.setWordCount(wC)
tItem.setParaCount(pC)
@@ -400,7 +400,7 @@ class NWIndex:
def _indexWordCounts(self, tHandle: str, text: str, sTitle: str) -> None:
"""Count text stats and save the counts to the index."""
cC, wC, pC = countWords(text)
cC, wC, pC = standardCounter(text)
self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC)
return
@@ -1315,86 +1315,3 @@ def processComment(text: str) -> tuple[nwComment, str, int]:
if content and (clean := classifier.strip().lower()) in CLASSIFIERS:
return CLASSIFIERS[clean], content.strip(), text.find(":") + 1
return nwComment.PLAIN, check, 0
def countWords(text: str) -> tuple[int, int, int]:
"""Count words in a piece of text, skipping special syntax and
comments.
"""
charCount = 0
wordCount = 0
paraCount = 0
prevEmpty = True
if not isinstance(text, str):
return charCount, wordCount, paraCount
# We need to treat dashes as word separators for counting words.
# The check+replace approach is much faster than direct replace for
# large texts, and a bit slower for small texts, but in the latter
# case it doesn't really matter.
if nwUnicode.U_ENDASH in text:
text = text.replace(nwUnicode.U_ENDASH, " ")
if nwUnicode.U_EMDASH in text:
text = text.replace(nwUnicode.U_EMDASH, " ")
# Strip shortcodes
if "[" in text:
text = nwRegEx.RX_SC.sub("", text)
for line in text.splitlines():
countPara = True
if not line:
prevEmpty = True
continue
if line[0] == "@" or line[0] == "%":
continue
if line[0] == "[":
check = line.lower()
if check.startswith(("[newpage]", "[new page]", "[vspace]")):
continue
elif check.startswith("[vspace:") and line.endswith("]"):
continue
elif line[0] == "#":
if line[:5] == "#### ":
line = line[5:]
countPara = False
elif line[:4] == "### ":
line = line[4:]
countPara = False
elif line[:3] == "## ":
line = line[3:]
countPara = False
elif line[:2] == "# ":
line = line[2:]
countPara = False
elif line[:3] == "#! ":
line = line[3:]
countPara = False
elif line[:4] == "##! ":
line = line[4:]
countPara = False
elif line[0] == ">" or line[-1] == "<":
if line[:2] == ">>":
line = line[2:].lstrip(" ")
elif line[:1] == ">":
line = line[1:].lstrip(" ")
if line[-2:] == "<<":
line = line[:-2].rstrip(" ")
elif line[-1:] == "<":
line = line[:-1].rstrip(" ")
wordCount += len(line.split())
charCount += len(line)
if countPara and prevEmpty:
paraCount += 1
prevEmpty = not countPara
return charCount, wordCount, paraCount
novelwriter/gui/doceditor.py
+3 -3
View File
@@ -56,9 +56,9 @@ from novelwriter import CONFIG, SHARED
from novelwriter.enum import nwDocAction, nwDocInsert, nwDocMode, nwItemClass, nwTrinary
from novelwriter.common import minmax, transferCase
from novelwriter.constants import nwKeyWords, nwLabels, nwShortcode, nwUnicode, trConst
from novelwriter.core.index import countWords
from novelwriter.tools.lipsum import GuiLipsum
from novelwriter.core.document import NWDocument
from novelwriter.text.counting import standardCounter
from novelwriter.gui.dochighlight import GuiDocHighlighter
from novelwriter.gui.editordocument import GuiTextDocument
from novelwriter.extensions.eventfilters import WheelEventFilter
@@ -462,7 +462,7 @@ class GuiDocEditor(QPlainTextEdit):
return False
docText = self.getText()
cC, wC, pC = countWords(docText)
cC, wC, pC = standardCounter(docText)
self._updateDocCounts(cC, wC, pC)
self.saveCursorPosition()
@@ -2226,7 +2226,7 @@ class BackgroundWordCounter(QRunnable):
else:
text = self._docEditor.getText()
cC, wC, pC = countWords(text)
cC, wC, pC = standardCounter(text)
self.signals.countsReady.emit(cC, wC, pC)
self._isRunning = False
novelwriter/text/__init__.py
+3
View File
@@ -0,0 +1,3 @@
"""
novelWriter – Text Analysis Tools
"""
novelwriter/text/counting.py
+118
View File
@@ -0,0 +1,118 @@
"""
novelWriter – Text Counting Functions
=====================================
File History:
Created: 2019-04-22 [0.0.1] standardCounter
Rewritten: 2024-02-27 [2.4b1] preProcessText, standardCounter
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import re
from novelwriter.constants import nwRegEx, nwUnicode
RX_SC = re.compile(nwRegEx.FMT_SC)
RX_LO = re.compile(r"(?i)(?<!\\)(\[(?:vspace|newpage|new page)(:\d+)?)(?<!\\)(\])")
RX_IN = re.compile(r"^>{1,2}\s*|\s*<{1,2}$")
def preProcessText(text: str, keepHeaders: bool = True) -> list[str]:
"""Strip formatting codes from the text and split into lines."""
if not isinstance(text, str):
return []
# We need to treat dashes as word separators for counting words.
# The check+replace approach is much faster than direct replace for
# large texts, and a bit slower for small texts, but in the latter
# case it doesn't really matter.
if nwUnicode.U_ENDASH in text:
text = text.replace(nwUnicode.U_ENDASH, " ")
if nwUnicode.U_EMDASH in text:
text = text.replace(nwUnicode.U_EMDASH, " ")
ignore = "%@" if keepHeaders else "%@#"
result = []
for line in text.splitlines():
line = line.rstrip()
if line:
if line[0] in ignore:
continue
if line[0] == ">":
line = line.lstrip(">").lstrip(" ")
if line[-1] == "<":
line = line.rstrip("<").rstrip(" ")
if "[" in line:
# Strip shortcodes and special formatting
# RegEx is slow, so we do this only when necessary
line = RX_SC.sub("", line)
line = RX_LO.sub("", line)
result.append(line)
return result
def standardCounter(text: str) -> tuple[int, int, int]:
"""A counter that counts paragraphs, words and characters.
This is the standard counter that includes headers in the word and
character counts.
"""
charCount = 0
wordCount = 0
paraCount = 0
prevEmpty = True
for line in preProcessText(text):
countPara = True
if not line:
prevEmpty = True
continue
if line[0] == "#":
if line[:5] == "#### ":
line = line[5:]
countPara = False
elif line[:4] == "### ":
line = line[4:]
countPara = False
elif line[:3] == "## ":
line = line[3:]
countPara = False
elif line[:2] == "# ":
line = line[2:]
countPara = False
elif line[:3] == "#! ":
line = line[3:]
countPara = False
elif line[:4] == "##! ":
line = line[4:]
countPara = False
wordCount += len(line.split())
charCount += len(line)
if countPara and prevEmpty:
paraCount += 1
prevEmpty = not countPara
return charCount, wordCount, paraCount
tests/test_core/test_core_index.py
+1 -103
View File
@@ -31,7 +31,7 @@ from tools import C, buildTestProject, cmpFiles, writeFile
from novelwriter.enum import nwComment, nwItemClass, nwItemLayout
from novelwriter.constants import nwFiles
from novelwriter.core.index import IndexItem, NWIndex, countWords, TagsIndex, processComment
from novelwriter.core.index import IndexItem, NWIndex, TagsIndex, processComment
from novelwriter.core.project import NWProject
@@ -1319,105 +1319,3 @@ def testCoreIndex_processComment():
assert processComment("% \t SHORT : Hi:You") == (nwComment.SHORT, "Hi:You", 13)
# END Test testCoreIndex_processComment
@pytest.mark.core
def testCoreIndex_countWords():
"""Test the word counter and the exclusion filers."""
# Non-Text
assert countWords(None) == (0, 0, 0) # type: ignore
assert countWords(1234) == (0, 0, 0) # type: ignore
# General Text
cC, wC, pC = countWords((
"# Heading One\n"
"## Heading Two\n"
"### Heading Three\n"
"#### Heading Four\n\n"
"@tag: value\n\n"
"% A comment that should not be counted.\n\n"
"The first paragraph.\n\n"
"The second paragraph.\n\n\n"
"The third paragraph.\n\n"
"Dashes\u2013and even longer\u2014dashes."
))
assert cC == 138
assert wC == 22
assert pC == 4
# Text Alignment
cC, wC, pC = countWords((
"# Title\n\n"
"Left aligned<<\n\n"
"Left aligned <<\n\n"
"Right indent<\n\n"
"Right indent <\n\n"
))
assert cC == 53
assert wC == 9
assert pC == 4
cC, wC, pC = countWords((
"# Title\n\n"
">>Right aligned\n\n"
">> Right aligned\n\n"
">Left indent\n\n"
"> Left indent\n\n"
))
assert cC == 53
assert wC == 9
assert pC == 4
cC, wC, pC = countWords((
"# Title\n\n"
">>Centre aligned<<\n\n"
">> Centre aligned <<\n\n"
">Double indent<\n\n"
"> Double indent <\n\n"
))
assert cC == 59
assert wC == 9
assert pC == 4
# Formatting Codes, Upper Case (Old Implementation)
cC, wC, pC = countWords((
"Some text\n\n"
"[NEWPAGE]\n\n"
"more text\n\n"
"[NEW PAGE]]\n\n"
"even more text\n\n"
"[VSPACE]\n\n"
"and some final text\n\n"
"[VSPACE:4]\n\n"
"THE END\n\n"
))
assert cC == 58
assert wC == 13
assert pC == 5
# Formatting Codes, Lower Case (Current Implementation)
cC, wC, pC = countWords((
"Some text\n\n"
"[newpage]\n\n"
"more text\n\n"
"[new page]]\n\n"
"even more text\n\n"
"[vspace]\n\n"
"and some final text\n\n"
"[vspace:4]\n\n"
"THE END\n\n"
))
assert cC == 58
assert wC == 13
assert pC == 5
# Check ShortCodes
cC, wC, pC = countWords((
"Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n"
"Text with [b][i] nested [/i] emphasis [/b] in it.\n\n"
))
assert cC == 78
assert wC == 14
assert pC == 2
# END Test testCoreIndex_countWords
tests/test_gui/test_gui_doceditor.py
+2 -2
View File
@@ -32,8 +32,8 @@ from PyQt5.QtWidgets import QAction, QMenu, qApp
from novelwriter import CONFIG, SHARED
from novelwriter.enum import nwDocAction, nwDocInsert, nwItemLayout, nwTrinary, nwWidget
from novelwriter.constants import nwKeyWords, nwUnicode
from novelwriter.core.index import countWords
from novelwriter.gui.doceditor import GuiDocEditor, GuiDocToolBar
from novelwriter.text.counting import standardCounter
from novelwriter.dialogs.editlabel import GuiEditLabel
KEY_DELAY = 1
@@ -1673,7 +1673,7 @@ def testGuiEditor_WordCounters(qtbot, monkeypatch, nwGUI, projPath, ipsumText, m
assert nwGUI.openDocument(C.hSceneDoc) is True
text = "\n\n".join(ipsumText)
cC, wC, pC = countWords(text)
cC, wC, pC = standardCounter(text)
nwGUI.docEditor.replaceText(text)
# Check that a busy counter is blocked
tests/test_text/test_core_counting.py
+127
View File
@@ -0,0 +1,127 @@
"""
novelWriter – Counter Module Tester
===================================
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import pytest
from novelwriter.text.counting import standardCounter
@pytest.mark.core
def testTextCounting_standardCounter():
"""Test the word counter and the exclusion filers."""
# Non-Text
assert standardCounter(None) == (0, 0, 0) # type: ignore
assert standardCounter(1234) == (0, 0, 0) # type: ignore
# General Text
cC, wC, pC = standardCounter((
"# Heading One\n"
"## Heading Two\n"
"### Heading Three\n"
"#### Heading Four\n\n"
"@tag: value\n\n"
"% A comment that should not be counted.\n\n"
"The first paragraph.\n\n"
"The second paragraph.\n\n\n"
"The third paragraph.\n\n"
"Dashes\u2013and even longer\u2014dashes."
))
assert cC == 138
assert wC == 22
assert pC == 4
# Text Alignment
cC, wC, pC = standardCounter((
"# Title\n\n"
"Left aligned<<\n\n"
"Left aligned <<\n\n"
"Right indent<\n\n"
"Right indent <\n\n"
))
assert cC == 53
assert wC == 9
assert pC == 4
cC, wC, pC = standardCounter((
"# Title\n\n"
">>Right aligned\n\n"
">> Right aligned\n\n"
">Left indent\n\n"
"> Left indent\n\n"
))
assert cC == 53
assert wC == 9
assert pC == 4
cC, wC, pC = standardCounter((
"# Title\n\n"
">>Centre aligned<<\n\n"
">> Centre aligned <<\n\n"
">Double indent<\n\n"
"> Double indent <\n\n"
))
assert cC == 59
assert wC == 9
assert pC == 4
# Formatting Codes, Upper Case (Old Implementation)
cC, wC, pC = standardCounter((
"Some text\n\n"
"[NEWPAGE]\n\n"
"more text\n\n"
"[NEW PAGE]\n\n"
"even more text\n\n"
"[VSPACE]\n\n"
"and some final text\n\n"
"[VSPACE:4]\n\n"
"THE END\n\n"
))
assert cC == 58
assert wC == 13
assert pC == 5
# Formatting Codes, Lower Case (Current Implementation)
cC, wC, pC = standardCounter((
"Some text\n\n"
"[newpage]\n\n"
"more text\n\n"
"[new page]\n\n"
"even more text\n\n"
"[vspace]\n\n"
"and some final text\n\n"
"[vspace:4]\n\n"
"THE END\n\n"
))
assert cC == 58
assert wC == 13
assert pC == 5
# Check ShortCodes
cC, wC, pC = standardCounter((
"Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n"
"Text with [b][i] nested [/i] emphasis [/b] in it.\n\n"
))
assert cC == 78
assert wC == 14
assert pC == 2
# END Test testTextCounting_standardCounter