Add test coverage of tokenizer stats counter

2024-02-28 19:10:05 +01:00
parent 9f7fc90f2d
commit dccc442852
2 changed files with 222 additions and 0 deletions
@@ -777,6 +777,9 @@ class Tokenizer(ABC):

        para = []
        for tType, _, tText, _, _ in self._tokens:
+            tText = tText.replace(nwUnicode.U_ENDASH, " ")
+            tText = tText.replace(nwUnicode.U_EMDASH, " ")
+
            tWords = tText.split()
            nWords = len(tWords)
            nChars = len(tText)