Move outline generation to Tokenizer class instead

novelwriter/core/tokenizer.py

@@ -35,13 +35,13 @@ from functools import partial
 
 from PyQt5.QtCore import QCoreApplication, QRegularExpression
 
-from novelwriter.enum import nwComment, nwItemLayout
 from novelwriter.common import formatTimeStamp, numberToRoman, checkInt
 from novelwriter.constants import (
     nwHeadFmt, nwKeyWords, nwLabels, nwRegEx, nwShortcode, nwUnicode, trConst
 )
 from novelwriter.core.index import processComment
 from novelwriter.core.project import NWProject
+from novelwriter.enum import nwComment, nwItemLayout
 
 logger = logging.getLogger(__name__)
 
@@ -120,13 +120,14 @@ class Tokenizer(ABC):
         self._text   = ""    # The raw text to be tokenized
         self._handle = None  # The item handle currently being processed
         self._result = ""    # The result of the last document
-        self._counts = {}    # Counter data
 
         self._keepMarkdown = False  # Whether to keep the markdown text
         self._allMarkdown  = []     # The result novelWriter markdown of all documents
 
-        # Processed Tokens
+        # Processed Tokens and Meta Data
         self._tokens: list[tuple[int, int, str, list[tuple[int, int]], int]] = []
+        self._counts: dict[str, int] = {}
+        self._outline: dict[str, str] = {}
 
         # User Settings
         self._textFont     = "Serif"  # Output text font
@@ -226,6 +227,11 @@ class Tokenizer(ABC):
         """The collected stats about the text."""
         return self._counts
 
+    @property
+    def textOutline(self) -> dict[str, str]:
+        """The generated outline of the text."""
+        return self._outline
+
     @property
     def errData(self) -> list[str]:
         """The error data."""
@@ -392,43 +398,42 @@ class Tokenizer(ABC):
     def doConvert(self) -> None:
         raise NotImplementedError
 
-    def addRootHeading(self, tHandle: str) -> bool:
+    def addRootHeading(self, tHandle: str) -> None:
         """Add a heading at the start of a new root folder."""
-        tItem = self._project.tree[tHandle]
-        if not tItem or not tItem.isRootType():
-            return False
+        self._text = ""
+        self._handle = None
 
-        if self._isFirst:
-            textAlign = self.A_CENTRE
-            self._isFirst = False
-        else:
-            textAlign = self.A_PBB | self.A_CENTRE
+        if (tItem := self._project.tree[tHandle]) and tItem.isRootType():
+            self._handle = tHandle
+            if self._isFirst:
+                textAlign = self.A_CENTRE
+                self._isFirst = False
+            else:
+                textAlign = self.A_PBB | self.A_CENTRE
 
-        trNotes = self._localLookup("Notes")
-        title = f"{trNotes}: {tItem.itemName}"
-        self._tokens = []
-        self._tokens.append((
-            self.T_TITLE, 0, title, [], textAlign
-        ))
-        if self._keepMarkdown:
-            self._allMarkdown.append(f"# {title}\n\n")
+            trNotes = self._localLookup("Notes")
+            title = f"{trNotes}: {tItem.itemName}"
+            self._tokens = []
+            self._tokens.append((
+                self.T_TITLE, 1, title, [], textAlign
+            ))
+            if self._keepMarkdown:
+                self._allMarkdown.append(f"#! {title}\n\n")
 
-        return True
+        return
 
     def setText(self, tHandle: str, text: str | None = None) -> None:
         """Set the text for the tokenizer from a handle. If text is not
-        set, its is loaded from the file.
+        set, it's is loaded from the file.
         """
         self._text = ""
         self._handle = None
         if nwItem := self._project.tree[tHandle]:
             if text is None:
                 text = self._project.storage.getDocument(tHandle).readDocument() or ""
-
             self._text = text
             self._handle = tHandle
             self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT
-
         return
 
     def doPreProcessing(self) -> None:
@@ -798,7 +803,29 @@ class Tokenizer(ABC):
 
         return
 
-    def countStats(self) -> dict[str, int]:
+    def buildOutline(self) -> None:
+        """Build an outline of the text up to level 3 headings."""
+        tHandle = self._handle or ""
+        isNovel = self._isNovel
+        for tType, nHead, tText, _, _ in self._tokens:
+            if tType == self.T_TITLE:
+                prefix = "TT"
+            elif tType == self.T_HEAD1:
+                prefix = "PT" if isNovel else "H1"
+            elif tType == self.T_HEAD2:
+                prefix = "CH" if isNovel else "H2"
+            elif tType == self.T_HEAD3:
+                prefix = "SC" if isNovel else "H3"
+            else:
+                continue
+
+            key = f"{tHandle}:T{nHead:04d}"
+            text = tText.replace(nwHeadFmt.BR, " ").replace("&", "&")
+            self._outline[key] = f"{prefix}|{text}"
+
+        return
+
+    def countStats(self) -> None:
         """Count stats on the tokenized text."""
         titleCount = self._counts.get("titleCount", 0)
         paragraphCount = self._counts.get("paragraphCount", 0)
@@ -905,7 +932,7 @@ class Tokenizer(ABC):
         self._counts["textWordChars"] = textWordChars
         self._counts["titleWordChars"] = titleWordChars
 
-        return {}
+        return
 
     def saveRawMarkdown(self, path: str | Path) -> None:
         """Save the raw text to a plain text file."""