From cca309d7d7f3dd2417e56b87ec389e2d78431e92 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Thu, 29 Feb 2024 22:40:14 +0100 Subject: [PATCH] Add all words char count to tokenizer --- novelwriter/core/tokenizer.py | 29 ++++++++++++++++++++------ tests/test_core/test_core_tokenizer.py | 18 ++++++++-------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 76baad82..1bd3d794 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -772,6 +772,8 @@ class Tokenizer(ABC): allChars = self._counts.get("allChars", 0) textChars = self._counts.get("textChars", 0) titleChars = self._counts.get("titleChars", 0) + + allWordChars = self._counts.get("allWordChars", 0) textWordChars = self._counts.get("textWordChars", 0) titleWordChars = self._counts.get("titleWordChars", 0) @@ -783,6 +785,7 @@ class Tokenizer(ABC): tWords = tText.split() nWords = len(tWords) nChars = len(tText) + nWChars = len("".join(tWords)) if tType == self.T_EMPTY: if len(para) > 0: @@ -790,13 +793,15 @@ class Tokenizer(ABC): tPWords = tTemp.split() nPWords = len(tPWords) nPChars = len(tTemp) + nPWChars = len("".join(tPWords)) paragraphCount += 1 allWords += nPWords textWords += nPWords allChars += nPChars textChars += nPChars - textWordChars += len("".join(tPWords)) + allWordChars += nPWChars + textWordChars += nPWChars para = [] elif tType in self.L_HEADINGS: @@ -804,38 +809,48 @@ class Tokenizer(ABC): allWords += nWords titleWords += nWords allChars += nChars + allWordChars += nWChars titleChars += nChars - titleWordChars += len("".join(tWords)) + titleWordChars += nWChars elif tType == self.T_SEP: allWords += nWords allChars += nChars + allWordChars += nWChars elif tType == self.T_TEXT: para.append(tText.rstrip()) elif tType == self.T_SYNOPSIS and self._doSynopsis: text = "{0}: {1}".format(self._localLookup("Synopsis"), tText) - allWords += len(text.split()) + words = text.split() + allWords += len(words) allChars += len(text) + allWordChars += len("".join(words)) elif tType == self.T_SHORT and self._doSynopsis: text = "{0}: {1}".format(self._localLookup("Short Description"), tText) - allWords += len(text.split()) + words = text.split() + allWords += len(words) allChars += len(text) + allWordChars += len("".join(words)) elif tType == self.T_COMMENT and self._doComments: text = "{0}: {1}".format(self._localLookup("Comment"), tText) - allWords += len(text.split()) + words = text.split() + allWords += len(words) allChars += len(text) + allWordChars += len("".join(words)) elif tType == self.T_KEYWORD and self._doKeywords: valid, bits, _ = self._project.index.scanThis("@"+tText) if valid and bits: key = self._localLookup(nwLabels.KEY_NAME[bits[0]]) text = "{0}: {1}".format(key, ", ".join(bits[1:])) - allWords += len(text.split()) + words = text.split() + allWords += len(words) allChars += len(text) + allWordChars += len("".join(words)) self._counts["titleCount"] = titleCount self._counts["paragraphCount"] = paragraphCount @@ -847,6 +862,8 @@ class Tokenizer(ABC): self._counts["allChars"] = allChars self._counts["textChars"] = textChars self._counts["titleChars"] = titleChars + + self._counts["allWordChars"] = allWordChars self._counts["textWordChars"] = textWordChars self._counts["titleWordChars"] = titleWordChars diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 2e4c1bda..f722c89f 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -1278,7 +1278,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 1, "paragraphCount": 0, "allWords": 3, "textWords": 0, "titleWords": 3, "allChars": 15, "textChars": 0, "titleChars": 15, - "textWordChars": 0, "titleWordChars": 13 + "allWordChars": 13, "textWordChars": 0, "titleWordChars": 13 } # Header w/Format @@ -1294,7 +1294,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 1, "paragraphCount": 0, "allWords": 5, "textWords": 0, "titleWords": 5, "allChars": 20, "textChars": 0, "titleChars": 20, - "textWordChars": 0, "titleWordChars": 16 + "allWordChars": 16, "textWordChars": 0, "titleWordChars": 16 } # Two Paragraphs @@ -1308,7 +1308,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 0, "paragraphCount": 2, "allWords": 9, "textWords": 9, "titleWords": 0, "allChars": 47, "textChars": 47, "titleChars": 0, - "textWordChars": 40, "titleWordChars": 0 + "allWordChars": 40, "textWordChars": 40, "titleWordChars": 0 } # Two Scenes w/Separator @@ -1326,7 +1326,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 1, "paragraphCount": 2, "allWords": 6, "textWords": 2, "titleWords": 1, "allChars": 20, "textChars": 8, "titleChars": 7, - "textWordChars": 8, "titleWordChars": 7 + "allWordChars": 18, "textWordChars": 8, "titleWordChars": 7 } # Scene w/Synopsis @@ -1346,7 +1346,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 1, "paragraphCount": 1, "allWords": 4, "textWords": 1, "titleWords": 1, "allChars": 26, "textChars": 4, "titleChars": 7, - "textWordChars": 4, "titleWordChars": 7 + "allWordChars": 25, "textWordChars": 4, "titleWordChars": 7 } # Scene w/Short @@ -1366,7 +1366,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 1, "paragraphCount": 1, "allWords": 5, "textWords": 1, "titleWords": 1, "allChars": 35, "textChars": 4, "titleChars": 7, - "textWordChars": 4, "titleWordChars": 7 + "allWordChars": 33, "textWordChars": 4, "titleWordChars": 7 } # Scene w/Comment @@ -1386,7 +1386,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 1, "paragraphCount": 1, "allWords": 4, "textWords": 1, "titleWords": 1, "allChars": 25, "textChars": 4, "titleChars": 7, - "textWordChars": 4, "titleWordChars": 7 + "allWordChars": 24, "textWordChars": 4, "titleWordChars": 7 } # Scene w/Keyword @@ -1406,7 +1406,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 1, "paragraphCount": 1, "allWords": 6, "textWords": 1, "titleWords": 1, "allChars": 30, "textChars": 4, "titleChars": 7, - "textWordChars": 4, "titleWordChars": 7 + "allWordChars": 27, "textWordChars": 4, "titleWordChars": 7 } # Long Text @@ -1466,7 +1466,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): "titleCount": 4, "paragraphCount": 5, "allWords": 596, "textWords": 528, "titleWords": 12, "allChars": 3859, "textChars": 3513, "titleChars": 46, - "textWordChars": 2990, "titleWordChars": 38 + "allWordChars": 3289, "textWordChars": 2990, "titleWordChars": 38 } # END Test testCoreToken_CountStats