Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add all words char count to tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-02-29 22:40:14 +01:00
parent dccc442852
commit cca309d7d7
2 changed files with 32 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tokenizer.py
+23 -6
View File
@@ -772,6 +772,8 @@ class Tokenizer(ABC):
allChars = self._counts.get("allChars", 0) allChars = self._counts.get("allChars", 0)
textChars = self._counts.get("textChars", 0) textChars = self._counts.get("textChars", 0)
titleChars = self._counts.get("titleChars", 0) titleChars = self._counts.get("titleChars", 0)
allWordChars = self._counts.get("allWordChars", 0)
textWordChars = self._counts.get("textWordChars", 0) textWordChars = self._counts.get("textWordChars", 0)
titleWordChars = self._counts.get("titleWordChars", 0) titleWordChars = self._counts.get("titleWordChars", 0)
@@ -783,6 +785,7 @@ class Tokenizer(ABC):
tWords = tText.split() tWords = tText.split()
nWords = len(tWords) nWords = len(tWords)
nChars = len(tText) nChars = len(tText)
nWChars = len("".join(tWords))
if tType == self.T_EMPTY: if tType == self.T_EMPTY:
if len(para) > 0: if len(para) > 0:
@@ -790,13 +793,15 @@ class Tokenizer(ABC):
tPWords = tTemp.split() tPWords = tTemp.split()
nPWords = len(tPWords) nPWords = len(tPWords)
nPChars = len(tTemp) nPChars = len(tTemp)
nPWChars = len("".join(tPWords))
paragraphCount += 1 paragraphCount += 1
allWords += nPWords allWords += nPWords
textWords += nPWords textWords += nPWords
allChars += nPChars allChars += nPChars
textChars += nPChars textChars += nPChars
textWordChars += len("".join(tPWords)) allWordChars += nPWChars
textWordChars += nPWChars
para = [] para = []
elif tType in self.L_HEADINGS: elif tType in self.L_HEADINGS:
@@ -804,38 +809,48 @@ class Tokenizer(ABC):
allWords += nWords allWords += nWords
titleWords += nWords titleWords += nWords
allChars += nChars allChars += nChars
allWordChars += nWChars
titleChars += nChars titleChars += nChars
titleWordChars += len("".join(tWords)) titleWordChars += nWChars
elif tType == self.T_SEP: elif tType == self.T_SEP:
allWords += nWords allWords += nWords
allChars += nChars allChars += nChars
allWordChars += nWChars
elif tType == self.T_TEXT: elif tType == self.T_TEXT:
para.append(tText.rstrip()) para.append(tText.rstrip())
elif tType == self.T_SYNOPSIS and self._doSynopsis: elif tType == self.T_SYNOPSIS and self._doSynopsis:
text = "{0}: {1}".format(self._localLookup("Synopsis"), tText) text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
allWords += len(text.split()) words = text.split()
allWords += len(words)
allChars += len(text) allChars += len(text)
allWordChars += len("".join(words))
elif tType == self.T_SHORT and self._doSynopsis: elif tType == self.T_SHORT and self._doSynopsis:
text = "{0}: {1}".format(self._localLookup("Short Description"), tText) text = "{0}: {1}".format(self._localLookup("Short Description"), tText)
allWords += len(text.split()) words = text.split()
allWords += len(words)
allChars += len(text) allChars += len(text)
allWordChars += len("".join(words))
elif tType == self.T_COMMENT and self._doComments: elif tType == self.T_COMMENT and self._doComments:
text = "{0}: {1}".format(self._localLookup("Comment"), tText) text = "{0}: {1}".format(self._localLookup("Comment"), tText)
allWords += len(text.split()) words = text.split()
allWords += len(words)
allChars += len(text) allChars += len(text)
allWordChars += len("".join(words))
elif tType == self.T_KEYWORD and self._doKeywords: elif tType == self.T_KEYWORD and self._doKeywords:
valid, bits, _ = self._project.index.scanThis("@"+tText) valid, bits, _ = self._project.index.scanThis("@"+tText)
if valid and bits: if valid and bits:
key = self._localLookup(nwLabels.KEY_NAME[bits[0]]) key = self._localLookup(nwLabels.KEY_NAME[bits[0]])
text = "{0}: {1}".format(key, ", ".join(bits[1:])) text = "{0}: {1}".format(key, ", ".join(bits[1:]))
allWords += len(text.split()) words = text.split()
allWords += len(words)
allChars += len(text) allChars += len(text)
allWordChars += len("".join(words))
self._counts["titleCount"] = titleCount self._counts["titleCount"] = titleCount
self._counts["paragraphCount"] = paragraphCount self._counts["paragraphCount"] = paragraphCount
@@ -847,6 +862,8 @@ class Tokenizer(ABC):
self._counts["allChars"] = allChars self._counts["allChars"] = allChars
self._counts["textChars"] = textChars self._counts["textChars"] = textChars
self._counts["titleChars"] = titleChars self._counts["titleChars"] = titleChars
self._counts["allWordChars"] = allWordChars
self._counts["textWordChars"] = textWordChars self._counts["textWordChars"] = textWordChars
self._counts["titleWordChars"] = titleWordChars self._counts["titleWordChars"] = titleWordChars
tests/test_core/test_core_tokenizer.py
+9 -9
View File
@@ -1278,7 +1278,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 0, "titleCount": 1, "paragraphCount": 0,
"allWords": 3, "textWords": 0, "titleWords": 3, "allWords": 3, "textWords": 0, "titleWords": 3,
"allChars": 15, "textChars": 0, "titleChars": 15, "allChars": 15, "textChars": 0, "titleChars": 15,
"textWordChars": 0, "titleWordChars": 13 "allWordChars": 13, "textWordChars": 0, "titleWordChars": 13
} }
# Header w/Format # Header w/Format
@@ -1294,7 +1294,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 0, "titleCount": 1, "paragraphCount": 0,
"allWords": 5, "textWords": 0, "titleWords": 5, "allWords": 5, "textWords": 0, "titleWords": 5,
"allChars": 20, "textChars": 0, "titleChars": 20, "allChars": 20, "textChars": 0, "titleChars": 20,
"textWordChars": 0, "titleWordChars": 16 "allWordChars": 16, "textWordChars": 0, "titleWordChars": 16
} }
# Two Paragraphs # Two Paragraphs
@@ -1308,7 +1308,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 0, "paragraphCount": 2, "titleCount": 0, "paragraphCount": 2,
"allWords": 9, "textWords": 9, "titleWords": 0, "allWords": 9, "textWords": 9, "titleWords": 0,
"allChars": 47, "textChars": 47, "titleChars": 0, "allChars": 47, "textChars": 47, "titleChars": 0,
"textWordChars": 40, "titleWordChars": 0 "allWordChars": 40, "textWordChars": 40, "titleWordChars": 0
} }
# Two Scenes w/Separator # Two Scenes w/Separator
@@ -1326,7 +1326,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 2, "titleCount": 1, "paragraphCount": 2,
"allWords": 6, "textWords": 2, "titleWords": 1, "allWords": 6, "textWords": 2, "titleWords": 1,
"allChars": 20, "textChars": 8, "titleChars": 7, "allChars": 20, "textChars": 8, "titleChars": 7,
"textWordChars": 8, "titleWordChars": 7 "allWordChars": 18, "textWordChars": 8, "titleWordChars": 7
} }
# Scene w/Synopsis # Scene w/Synopsis
@@ -1346,7 +1346,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1, "allWords": 4, "textWords": 1, "titleWords": 1,
"allChars": 26, "textChars": 4, "titleChars": 7, "allChars": 26, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7 "allWordChars": 25, "textWordChars": 4, "titleWordChars": 7
} }
# Scene w/Short # Scene w/Short
@@ -1366,7 +1366,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 5, "textWords": 1, "titleWords": 1, "allWords": 5, "textWords": 1, "titleWords": 1,
"allChars": 35, "textChars": 4, "titleChars": 7, "allChars": 35, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7 "allWordChars": 33, "textWordChars": 4, "titleWordChars": 7
} }
# Scene w/Comment # Scene w/Comment
@@ -1386,7 +1386,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1, "allWords": 4, "textWords": 1, "titleWords": 1,
"allChars": 25, "textChars": 4, "titleChars": 7, "allChars": 25, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7 "allWordChars": 24, "textWordChars": 4, "titleWordChars": 7
} }
# Scene w/Keyword # Scene w/Keyword
@@ -1406,7 +1406,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 6, "textWords": 1, "titleWords": 1, "allWords": 6, "textWords": 1, "titleWords": 1,
"allChars": 30, "textChars": 4, "titleChars": 7, "allChars": 30, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7 "allWordChars": 27, "textWordChars": 4, "titleWordChars": 7
} }
# Long Text # Long Text
@@ -1466,7 +1466,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 4, "paragraphCount": 5, "titleCount": 4, "paragraphCount": 5,
"allWords": 596, "textWords": 528, "titleWords": 12, "allWords": 596, "textWords": 528, "titleWords": 12,
"allChars": 3859, "textChars": 3513, "titleChars": 46, "allChars": 3859, "textChars": 3513, "titleChars": 46,
"textWordChars": 2990, "titleWordChars": 38 "allWordChars": 3289, "textWordChars": 2990, "titleWordChars": 38
} }
# END Test testCoreToken_CountStats # END Test testCoreToken_CountStats