Add stats counting to tokenizer class

novelwriter/core/docbuild.py

@@ -52,14 +52,15 @@ class NWBuildDocument:
     manuscript, based on a build definition object (BuildSettings).
     """
 
-    __slots__ = ("_project", "_build", "_queue", "_error", "_cache")
+    __slots__ = ("_project", "_build", "_queue", "_error", "_cache", "_count")
 
-    def __init__(self, project: NWProject, build: BuildSettings) -> None:
+    def __init__(self, project: NWProject, build: BuildSettings, doCount: bool = False) -> None:
         self._project = project
         self._build = build
         self._queue = []
         self._error = None
         self._cache = None
+        self._count = doCount
         return
 
     ##
@@ -319,6 +320,8 @@ class NWBuildDocument:
                     bldObj.doPreProcessing()
                     bldObj.tokenizeText()
                     bldObj.doHeaders()
+                    if self._count:
+                        bldObj.countStats()
                     if convert:
                         bldObj.doConvert()
                 else:

novelwriter/core/tokenizer.py

@@ -110,6 +110,9 @@ class Tokenizer(ABC):
     A_IND_L    = 0x0100  # Left indentation
     A_IND_R    = 0x0200  # Right indentation
 
+    # Lookups
+    L_HEADINGS = [T_TITLE, T_UNNUM, T_HEAD1, T_HEAD2, T_HEAD3, T_HEAD4]
+
     def __init__(self, project: NWProject) -> None:
 
         self._project = project
@@ -118,6 +121,7 @@ class Tokenizer(ABC):
         self._text   = ""    # The raw text to be tokenized
         self._nwItem = None  # The NWItem currently being processed
         self._result = ""    # The result of the last document
+        self._counts = {}    # Counter data
 
         self._keepMarkdown = False  # Whether to keep the markdown text
         self._allMarkdown  = []     # The result novelWriter markdown of all documents
@@ -210,6 +214,11 @@ class Tokenizer(ABC):
         """The combined novelWriter Markdown text."""
         return self._allMarkdown
 
+    @property
+    def textStats(self) -> dict[str, int]:
+        """The collected stats about the text."""
+        return self._counts
+
     @property
     def errData(self) -> list:
         """The error data."""
@@ -751,6 +760,96 @@ class Tokenizer(ABC):
 
         return True
 
+    def countStats(self) -> dict[str, int]:
+        """Count stats on the tokenized text."""
+        titleCount = self._counts.get("titleCount", 0)
+        paragraphCount = self._counts.get("paragraphCount", 0)
+
+        allWords = self._counts.get("allWords", 0)
+        textWords = self._counts.get("textWords", 0)
+        titleWords = self._counts.get("titleWords", 0)
+
+        allChars = self._counts.get("allChars", 0)
+        textChars = self._counts.get("textChars", 0)
+        titleChars = self._counts.get("titleChars", 0)
+        textWordChars = self._counts.get("textWordChars", 0)
+        titleWordChars = self._counts.get("titleWordChars", 0)
+
+        para = []
+        for tType, _, tText, _, _ in self._tokens:
+            tWords = tText.split()
+            nWords = len(tWords)
+            nChars = len(tText)
+
+            if tType == self.T_EMPTY:
+                if len(para) > 0:
+                    tTemp = "\n".join(para)
+                    tPWords = tTemp.split()
+                    nPWords = len(tPWords)
+                    nPChars = len(tTemp)
+
+                    paragraphCount += 1
+                    allWords += nPWords
+                    textWords += nPWords
+                    allChars += nPChars
+                    textChars += nPChars
+                    textWordChars += len("".join(tPWords))
+                para = []
+
+            elif tType in self.L_HEADINGS:
+                titleCount += 1
+                allWords += nWords
+                titleWords += nWords
+                allChars += nChars
+                titleChars += nChars
+                titleWordChars += len("".join(tWords))
+
+            elif tType == self.T_SEP:
+                allWords += nWords
+                allChars += nChars
+
+            elif tType == self.T_TEXT:
+                para.append(tText.rstrip())
+
+            elif tType == self.T_SYNOPSIS and self._doSynopsis:
+                text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
+                allWords += len(text.split())
+                allChars += len(text)
+
+            elif tType == self.T_SHORT and self._doSynopsis:
+                text = "{0}: {1}".format(self._localLookup("Short Description"), tText)
+                allWords += len(text.split())
+                allChars += len(text)
+
+            elif tType == self.T_COMMENT and self._doComments:
+                text = "{0}: {1}".format(self._localLookup("Comment"), tText)
+                allWords += len(text.split())
+                allChars += len(text)
+
+            elif tType == self.T_KEYWORD and self._doKeywords:
+                valid, bits, _ = self._project.index.scanThis("@"+tText)
+                if valid and bits:
+                    text = "{0}: {1}".format(
+                        self._localLookup(nwLabels.KEY_NAME[bits[0]]), ", ".join(bits[1:])
+                    )
+                    allWords += len(text.split())
+                    allChars += len(text)
+
+        self._counts["titleCount"] = titleCount
+        self._counts["paragraphCount"] = paragraphCount
+
+        self._counts["allWords"] = allWords
+        self._counts["textWords"] = textWords
+        self._counts["titleWords"] = titleWords
+
+        self._counts["allChars"] = allChars
+        self._counts["textChars"] = textChars
+        self._counts["titleChars"] = titleChars
+        self._counts["textWordChars"] = textWordChars
+        self._counts["titleWordChars"] = titleWordChars
+
+        return {}
+
     def saveRawMarkdown(self, path: str | Path) -> None:
         """Save the raw text to a plain text file."""
         with open(path, mode="w", encoding="utf-8") as outFile:

novelwriter/tools/manuscript.py

@@ -245,7 +245,7 @@ class GuiManuscript(QDialog):
                 if isinstance(build, BuildSettings):
                     self._updatePreview(data, build)
             except Exception:
-                logger.error("Failed to save build cache")
+                logger.error("Failed to load build cache")
                 logException()
                 return
 
@@ -327,7 +327,7 @@ class GuiManuscript(QDialog):
         if build is None:
             return
 
-        docBuild = NWBuildDocument(SHARED.project, build)
+        docBuild = NWBuildDocument(SHARED.project, build, doCount=True)
         docBuild.queueAll()
 
         self.docPreview.beginNewBuild(len(docBuild))
@@ -340,6 +340,7 @@ class GuiManuscript(QDialog):
         result = {
             "uuid": build.buildID,
             "time": int(time()),
+            "stats": buildObj.textStats,
             "styles": buildObj.getStyleSheet(),
             "html": buildObj.fullHTML,
         }