Move text blocks first also in stats counter as an optimisation
This commit is contained in:
@@ -926,21 +926,7 @@ class Tokenizer(ABC):
|
|||||||
nChars = len(tText)
|
nChars = len(tText)
|
||||||
nWChars = len("".join(tWords))
|
nWChars = len("".join(tWords))
|
||||||
|
|
||||||
if tType in self.L_HEADINGS:
|
if tType == self.T_TEXT:
|
||||||
titleCount += 1
|
|
||||||
allWords += nWords
|
|
||||||
titleWords += nWords
|
|
||||||
allChars += nChars
|
|
||||||
allWordChars += nWChars
|
|
||||||
titleChars += nChars
|
|
||||||
titleWordChars += nWChars
|
|
||||||
|
|
||||||
elif tType == self.T_SEP:
|
|
||||||
allWords += nWords
|
|
||||||
allChars += nChars
|
|
||||||
allWordChars += nWChars
|
|
||||||
|
|
||||||
elif tType == self.T_TEXT:
|
|
||||||
tPWords = tText.split()
|
tPWords = tText.split()
|
||||||
nPWords = len(tPWords)
|
nPWords = len(tPWords)
|
||||||
nPChars = len(tText)
|
nPChars = len(tText)
|
||||||
@@ -954,6 +940,20 @@ class Tokenizer(ABC):
|
|||||||
allWordChars += nPWChars
|
allWordChars += nPWChars
|
||||||
textWordChars += nPWChars
|
textWordChars += nPWChars
|
||||||
|
|
||||||
|
elif tType in self.L_HEADINGS:
|
||||||
|
titleCount += 1
|
||||||
|
allWords += nWords
|
||||||
|
titleWords += nWords
|
||||||
|
allChars += nChars
|
||||||
|
allWordChars += nWChars
|
||||||
|
titleChars += nChars
|
||||||
|
titleWordChars += nWChars
|
||||||
|
|
||||||
|
elif tType == self.T_SEP:
|
||||||
|
allWords += nWords
|
||||||
|
allChars += nChars
|
||||||
|
allWordChars += nWChars
|
||||||
|
|
||||||
elif tType == self.T_SYNOPSIS and self._doSynopsis:
|
elif tType == self.T_SYNOPSIS and self._doSynopsis:
|
||||||
text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
|
text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
|
||||||
words = text.split()
|
words = text.split()
|
||||||
|
|||||||
Reference in New Issue
Block a user