Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add all words char count to tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-02-29 22:40:14 +01:00
parent dccc442852
commit cca309d7d7
2 changed files with 32 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tokenizer.py
+23 -6
View File
@@ -772,6 +772,8 @@ class Tokenizer(ABC):
allChars = self._counts.get("allChars", 0)
textChars = self._counts.get("textChars", 0)
titleChars = self._counts.get("titleChars", 0)
allWordChars = self._counts.get("allWordChars", 0)
textWordChars = self._counts.get("textWordChars", 0)
titleWordChars = self._counts.get("titleWordChars", 0)
@@ -783,6 +785,7 @@ class Tokenizer(ABC):
tWords = tText.split()
nWords = len(tWords)
nChars = len(tText)
nWChars = len("".join(tWords))
if tType == self.T_EMPTY:
if len(para) > 0:
@@ -790,13 +793,15 @@ class Tokenizer(ABC):
tPWords = tTemp.split()
nPWords = len(tPWords)
nPChars = len(tTemp)
nPWChars = len("".join(tPWords))
paragraphCount += 1
allWords += nPWords
textWords += nPWords
allChars += nPChars
textChars += nPChars
textWordChars += len("".join(tPWords))
allWordChars += nPWChars
textWordChars += nPWChars
para = []
elif tType in self.L_HEADINGS:
@@ -804,38 +809,48 @@ class Tokenizer(ABC):
allWords += nWords
titleWords += nWords
allChars += nChars
allWordChars += nWChars
titleChars += nChars
titleWordChars += len("".join(tWords))
titleWordChars += nWChars
elif tType == self.T_SEP:
allWords += nWords
allChars += nChars
allWordChars += nWChars
elif tType == self.T_TEXT:
para.append(tText.rstrip())
elif tType == self.T_SYNOPSIS and self._doSynopsis:
text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
allWords += len(text.split())
words = text.split()
allWords += len(words)
allChars += len(text)
allWordChars += len("".join(words))
elif tType == self.T_SHORT and self._doSynopsis:
text = "{0}: {1}".format(self._localLookup("Short Description"), tText)
allWords += len(text.split())
words = text.split()
allWords += len(words)
allChars += len(text)
allWordChars += len("".join(words))
elif tType == self.T_COMMENT and self._doComments:
text = "{0}: {1}".format(self._localLookup("Comment"), tText)
allWords += len(text.split())
words = text.split()
allWords += len(words)
allChars += len(text)
allWordChars += len("".join(words))
elif tType == self.T_KEYWORD and self._doKeywords:
valid, bits, _ = self._project.index.scanThis("@"+tText)
if valid and bits:
key = self._localLookup(nwLabels.KEY_NAME[bits[0]])
text = "{0}: {1}".format(key, ", ".join(bits[1:]))
allWords += len(text.split())
words = text.split()
allWords += len(words)
allChars += len(text)
allWordChars += len("".join(words))
self._counts["titleCount"] = titleCount
self._counts["paragraphCount"] = paragraphCount
@@ -847,6 +862,8 @@ class Tokenizer(ABC):
self._counts["allChars"] = allChars
self._counts["textChars"] = textChars
self._counts["titleChars"] = titleChars
self._counts["allWordChars"] = allWordChars
self._counts["textWordChars"] = textWordChars
self._counts["titleWordChars"] = titleWordChars
tests/test_core/test_core_tokenizer.py
+9 -9
View File
@@ -1278,7 +1278,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 0,
"allWords": 3, "textWords": 0, "titleWords": 3,
"allChars": 15, "textChars": 0, "titleChars": 15,
"textWordChars": 0, "titleWordChars": 13
"allWordChars": 13, "textWordChars": 0, "titleWordChars": 13
}
# Header w/Format
@@ -1294,7 +1294,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 0,
"allWords": 5, "textWords": 0, "titleWords": 5,
"allChars": 20, "textChars": 0, "titleChars": 20,
"textWordChars": 0, "titleWordChars": 16
"allWordChars": 16, "textWordChars": 0, "titleWordChars": 16
}
# Two Paragraphs
@@ -1308,7 +1308,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 0, "paragraphCount": 2,
"allWords": 9, "textWords": 9, "titleWords": 0,
"allChars": 47, "textChars": 47, "titleChars": 0,
"textWordChars": 40, "titleWordChars": 0
"allWordChars": 40, "textWordChars": 40, "titleWordChars": 0
}
# Two Scenes w/Separator
@@ -1326,7 +1326,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 2,
"allWords": 6, "textWords": 2, "titleWords": 1,
"allChars": 20, "textChars": 8, "titleChars": 7,
"textWordChars": 8, "titleWordChars": 7
"allWordChars": 18, "textWordChars": 8, "titleWordChars": 7
}
# Scene w/Synopsis
@@ -1346,7 +1346,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
"allChars": 26, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7
"allWordChars": 25, "textWordChars": 4, "titleWordChars": 7
}
# Scene w/Short
@@ -1366,7 +1366,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1,
"allWords": 5, "textWords": 1, "titleWords": 1,
"allChars": 35, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7
"allWordChars": 33, "textWordChars": 4, "titleWordChars": 7
}
# Scene w/Comment
@@ -1386,7 +1386,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
"allChars": 25, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7
"allWordChars": 24, "textWordChars": 4, "titleWordChars": 7
}
# Scene w/Keyword
@@ -1406,7 +1406,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 1, "paragraphCount": 1,
"allWords": 6, "textWords": 1, "titleWords": 1,
"allChars": 30, "textChars": 4, "titleChars": 7,
"textWordChars": 4, "titleWordChars": 7
"allWordChars": 27, "textWordChars": 4, "titleWordChars": 7
}
# Long Text
@@ -1466,7 +1466,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
"titleCount": 4, "paragraphCount": 5,
"allWords": 596, "textWords": 528, "titleWords": 12,
"allChars": 3859, "textChars": 3513, "titleChars": 46,
"textWordChars": 2990, "titleWordChars": 38
"allWordChars": 3289, "textWordChars": 2990, "titleWordChars": 38
}
# END Test testCoreToken_CountStats