Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move main word counter to new module

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-02-27 17:02:29 +01:00
parent a87a94ffeb
commit 3c4673724e
9 changed files with 261 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/constants.py
-5
View File
@@ -23,8 +23,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import re
from PyQt5.QtCore import QCoreApplication, QT_TRANSLATE_NOOP
from novelwriter.enum import nwBuildFmt, nwItemClass, nwItemLayout, nwOutline
@@ -70,9 +68,6 @@ class nwRegEx:
FMT_SC = r"(?i)(?<!\\)(\[[\/\!]?(?:i|b|s|u|m|sup|sub)\])"
FMT_SV = r"(?<!\\)(\[(?i)(?:fn|footnote):)(.+?)(?<!\\)(\])"
# Pre-Compiled RegEx
RX_SC = re.compile(FMT_SC)
# END Class nwRegEx
novelwriter/core/index.py
+4 -87
View File
@@ -3,7 +3,6 @@ novelWriter – Project Index
===========================
File History:
Created: 2019-04-22 [0.0.1] countWords
Created: 2019-05-27 [0.1.4] NWIndex
Created: 2022-05-28 [2.0rc1] IndexItem
Created: 2022-05-28 [2.0rc1] IndexHeading
@@ -40,7 +39,8 @@ from novelwriter import SHARED
from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
from novelwriter.error import logException
from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode
from novelwriter.constants import nwFiles, nwKeyWords, nwRegEx, nwUnicode, nwHeaders
from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
from novelwriter.text.counting import standardCounter
if TYPE_CHECKING: # pragma: no cover
from novelwriter.core.item import NWItem
@@ -266,7 +266,7 @@ class NWIndex:
self._itemIndex.add(tHandle, tItem)
# Run word counter for the whole text
cC, wC, pC = countWords(text)
cC, wC, pC = standardCounter(text)
tItem.setCharCount(cC)
tItem.setWordCount(wC)
tItem.setParaCount(pC)
@@ -400,7 +400,7 @@ class NWIndex:
def _indexWordCounts(self, tHandle: str, text: str, sTitle: str) -> None:
"""Count text stats and save the counts to the index."""
cC, wC, pC = countWords(text)
cC, wC, pC = standardCounter(text)
self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC)
return
@@ -1315,86 +1315,3 @@ def processComment(text: str) -> tuple[nwComment, str, int]:
if content and (clean := classifier.strip().lower()) in CLASSIFIERS:
return CLASSIFIERS[clean], content.strip(), text.find(":") + 1
return nwComment.PLAIN, check, 0
def countWords(text: str) -> tuple[int, int, int]:
"""Count words in a piece of text, skipping special syntax and
comments.
"""
charCount = 0
wordCount = 0
paraCount = 0
prevEmpty = True
if not isinstance(text, str):
return charCount, wordCount, paraCount
# We need to treat dashes as word separators for counting words.
# The check+replace approach is much faster than direct replace for
# large texts, and a bit slower for small texts, but in the latter
# case it doesn't really matter.
if nwUnicode.U_ENDASH in text:
text = text.replace(nwUnicode.U_ENDASH, " ")
if nwUnicode.U_EMDASH in text:
text = text.replace(nwUnicode.U_EMDASH, " ")
# Strip shortcodes
if "[" in text:
text = nwRegEx.RX_SC.sub("", text)
for line in text.splitlines():
countPara = True
if not line:
prevEmpty = True
continue
if line[0] == "@" or line[0] == "%":
continue
if line[0] == "[":
check = line.lower()
if check.startswith(("[newpage]", "[new page]", "[vspace]")):
continue
elif check.startswith("[vspace:") and line.endswith("]"):
continue
elif line[0] == "#":
if line[:5] == "#### ":
line = line[5:]
countPara = False
elif line[:4] == "### ":
line = line[4:]
countPara = False
elif line[:3] == "## ":
line = line[3:]
countPara = False
elif line[:2] == "# ":
line = line[2:]
countPara = False
elif line[:3] == "#! ":
line = line[3:]
countPara = False
elif line[:4] == "##! ":
line = line[4:]
countPara = False
elif line[0] == ">" or line[-1] == "<":
if line[:2] == ">>":
line = line[2:].lstrip(" ")
elif line[:1] == ">":
line = line[1:].lstrip(" ")
if line[-2:] == "<<":
line = line[:-2].rstrip(" ")
elif line[-1:] == "<":
line = line[:-1].rstrip(" ")
wordCount += len(line.split())
charCount += len(line)
if countPara and prevEmpty:
paraCount += 1
prevEmpty = not countPara
return charCount, wordCount, paraCount
novelwriter/gui/doceditor.py
+3 -3
View File
@@ -56,9 +56,9 @@ from novelwriter import CONFIG, SHARED
from novelwriter.enum import nwDocAction, nwDocInsert, nwDocMode, nwItemClass, nwTrinary
from novelwriter.common import minmax, transferCase
from novelwriter.constants import nwKeyWords, nwLabels, nwShortcode, nwUnicode, trConst
from novelwriter.core.index import countWords
from novelwriter.tools.lipsum import GuiLipsum
from novelwriter.core.document import NWDocument
from novelwriter.text.counting import standardCounter
from novelwriter.gui.dochighlight import GuiDocHighlighter
from novelwriter.gui.editordocument import GuiTextDocument
from novelwriter.extensions.eventfilters import WheelEventFilter
@@ -462,7 +462,7 @@ class GuiDocEditor(QPlainTextEdit):
return False
docText = self.getText()
cC, wC, pC = countWords(docText)
cC, wC, pC = standardCounter(docText)
self._updateDocCounts(cC, wC, pC)
self.saveCursorPosition()
@@ -2226,7 +2226,7 @@ class BackgroundWordCounter(QRunnable):
else:
text = self._docEditor.getText()
cC, wC, pC = countWords(text)
cC, wC, pC = standardCounter(text)
self.signals.countsReady.emit(cC, wC, pC)
self._isRunning = False
novelwriter/text/__init__.py
+3
View File
@@ -0,0 +1,3 @@
"""
novelWriter – Text Analysis Tools
"""
novelwriter/text/counting.py
+118
View File
@@ -0,0 +1,118 @@
"""
novelWriter – Text Counting Functions
=====================================
File History:
Created: 2019-04-22 [0.0.1] standardCounter
Rewritten: 2024-02-27 [2.4b1] preProcessText, standardCounter
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import re
from novelwriter.constants import nwRegEx, nwUnicode
RX_SC = re.compile(nwRegEx.FMT_SC)
RX_LO = re.compile(r"(?i)(?<!\\)(\[(?:vspace|newpage|new page)(:\d+)?)(?<!\\)(\])")
RX_IN = re.compile(r"^>{1,2}\s*|\s*<{1,2}$")
def preProcessText(text: str, keepHeaders: bool = True) -> list[str]:
"""Strip formatting codes from the text and split into lines."""
if not isinstance(text, str):
return []
# We need to treat dashes as word separators for counting words.
# The check+replace approach is much faster than direct replace for
# large texts, and a bit slower for small texts, but in the latter
# case it doesn't really matter.
if nwUnicode.U_ENDASH in text:
text = text.replace(nwUnicode.U_ENDASH, " ")
if nwUnicode.U_EMDASH in text:
text = text.replace(nwUnicode.U_EMDASH, " ")
ignore = "%@" if keepHeaders else "%@#"
result = []
for line in text.splitlines():
line = line.rstrip()
if line:
if line[0] in ignore:
continue
if line[0] == ">":
line = line.lstrip(">").lstrip(" ")
if line[-1] == "<":
line = line.rstrip("<").rstrip(" ")
if "[" in line:
# Strip shortcodes and special formatting
# RegEx is slow, so we do this only when necessary
line = RX_SC.sub("", line)
line = RX_LO.sub("", line)
result.append(line)
return result
def standardCounter(text: str) -> tuple[int, int, int]:
"""A counter that counts paragraphs, words and characters.
This is the standard counter that includes headers in the word and
character counts.
"""
charCount = 0
wordCount = 0
paraCount = 0
prevEmpty = True
for line in preProcessText(text):
countPara = True
if not line:
prevEmpty = True
continue
if line[0] == "#":
if line[:5] == "#### ":
line = line[5:]
countPara = False
elif line[:4] == "### ":
line = line[4:]
countPara = False
elif line[:3] == "## ":
line = line[3:]
countPara = False
elif line[:2] == "# ":
line = line[2:]
countPara = False
elif line[:3] == "#! ":
line = line[3:]
countPara = False
elif line[:4] == "##! ":
line = line[4:]
countPara = False
wordCount += len(line.split())
charCount += len(line)
if countPara and prevEmpty:
paraCount += 1
prevEmpty = not countPara
return charCount, wordCount, paraCount