Move outline generation to Tokenizer class instead

novelwriter/core/docbuild.py

@@ -52,7 +52,10 @@ class NWBuildDocument:
     manuscript, based on a build definition object (BuildSettings).
     """
 
-    __slots__ = ("_project", "_build", "_queue", "_error", "_cache", "_count", "_preview")
+    __slots__ = (
+        "_project", "_build", "_queue", "_error", "_cache", "_count",
+        "_outline", "_preview"
+    )
 
     def __init__(self, project: NWProject, build: BuildSettings) -> None:
         self._project = project
@@ -61,6 +64,7 @@ class NWBuildDocument:
         self._error = None
         self._cache = None
         self._count = False
+        self._outline = False
         self._preview = False
         return
 
@@ -86,13 +90,21 @@ class NWBuildDocument:
     ##
 
     def setCountEnabled(self, state: bool) -> None:
-        """Turn on/off stats counting for builds."""
+        """Turn on/off stats for builds."""
         self._count = state
         return
 
+    def setBuildOutline(self, state: bool) -> None:
+        """Turn on/off outline for builds."""
+        self._outline = state
+        return
+
     def setPreviewMode(self, state: bool) -> None:
-        """Set the preview mode of the build. Implies count mode."""
+        """Set the preview mode of the build. This also enables stats
+        count and outline mode.
+        """
         self._preview = state
+        self._outline = state
         self._count = state
         return
 
@@ -364,12 +376,16 @@ class NWBuildDocument:
                         bldObj.doConvert()
                     if self._count:
                         bldObj.countStats()
+                    if self._outline:
+                        bldObj.buildOutline()
                 elif tItem.isFileType():
                     bldObj.setText(tHandle)
                     bldObj.doPreProcessing()
                     bldObj.tokenizeText()
                     if self._count:
                         bldObj.countStats()
+                    if self._outline:
+                        bldObj.buildOutline()
                     if convert:
                         bldObj.doConvert()
                 else:

novelwriter/core/index.py

@@ -33,7 +33,7 @@ import logging
 from time import time
 from typing import TYPE_CHECKING
 from pathlib import Path
-from collections.abc import Generator, ItemsView, Iterable
+from collections.abc import ItemsView, Iterable
 
 from novelwriter import SHARED
 from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
@@ -523,7 +523,7 @@ class NWIndex:
             return tItem[sTitle]
         return None
 
-    def iterItemHeadings(self, tHandle: str) -> Generator[str, IndexHeading]:
+    def iterItemHeadings(self, tHandle: str) -> Iterable[tuple[str, IndexHeading]]:
         """Get all headings for a specific item."""
         if tItem := self._itemIndex[tHandle]:
             yield from tItem.items()
@@ -531,7 +531,7 @@ class NWIndex:
 
     def novelStructure(
         self, rootHandle: str | None = None, activeOnly: bool = True
-    ) -> Generator[tuple[str, str, str, IndexHeading]]:
+    ) -> Iterable[tuple[str, str, str, IndexHeading]]:
         """Iterate over all titles in the novel, in the correct order as
         they appear in the tree view and in the respective document
         files, but skipping all note files.
@@ -673,7 +673,7 @@ class NWIndex:
 
     def getTagsData(
         self, activeOnly: bool = True
-    ) -> Generator[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
+    ) -> Iterable[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
         """Return all known tags."""
         for tag, data in self._tagsIndex.items():
             iItem = self._itemIndex[data.get("handle")]

novelwriter/core/project.py

@@ -31,7 +31,7 @@ from time import time
 from typing import TYPE_CHECKING
 from pathlib import Path
 from functools import partial
-from collections.abc import Generator
+from collections.abc import Iterable
 
 from PyQt5.QtCore import QCoreApplication
 
@@ -517,7 +517,7 @@ class NWProject:
     #  Class Methods
     ##
 
-    def iterProjectItems(self) -> Generator[NWItem]:
+    def iterProjectItems(self) -> Iterable[NWItem]:
         """This function ensures that the item tree loaded is sent to
         the GUI tree view in such a way that the tree can be built. That
         is, the parent item must be sent before its child. In principle,

novelwriter/core/sessions.py

@@ -29,7 +29,7 @@ import logging
 from time import time
 from typing import TYPE_CHECKING
 from pathlib import Path
-from collections.abc import Generator
+from collections.abc import Iterable
 
 from novelwriter.error import logException
 from novelwriter.common import formatTimeStamp
@@ -110,7 +110,7 @@ class NWSessionLog:
 
         return True
 
-    def iterRecords(self) -> Generator[dict]:
+    def iterRecords(self) -> Iterable[dict]:
         """Iterate through all records in the log."""
         sessFile = self._project.storage.getMetaFile(nwFiles.SESS_FILE)
         if isinstance(sessFile, Path) and sessFile.is_file():

novelwriter/core/status.py

@@ -28,10 +28,10 @@ import random
 import logging
 
 from typing import TYPE_CHECKING, Literal
-from collections.abc import Generator, ItemsView, Iterator, KeysView, ValuesView
+from collections.abc import ItemsView, Iterable, Iterator, KeysView, ValuesView
 
 from PyQt5.QtGui import QIcon, QPainter, QPainterPath, QPixmap, QColor
-from PyQt5.QtCore import QRectF, Qt
+from PyQt5.QtCore import QRectF
 
 from novelwriter import CONFIG
 from novelwriter.common import minmax, simplified
@@ -193,7 +193,7 @@ class NWStatus:
             self._store[key]["count"] += 1
         return
 
-    def pack(self) -> Generator[tuple[str, dict]]:
+    def pack(self) -> Iterable[tuple[str, dict]]:
         """Pack the status entries into a dictionary."""
         for key, data in self._store.items():
             yield (data["name"], {
@@ -248,7 +248,7 @@ class NWStatus:
     def _createIcon(self, red: int, green: int, blue: int) -> QIcon:
         """Generate an icon for a status label."""
         pixmap = QPixmap(self._iPX, self._iPX)
-        pixmap.fill(Qt.transparent)
+        pixmap.fill(QColor(0, 0, 0, 0))
 
         painter = QPainter(pixmap)
         painter.setRenderHint(QPainter.Antialiasing)

novelwriter/core/tohtml.py

@@ -55,7 +55,6 @@ class ToHtml(Tokenizer):
         self._genMode = self.M_EXPORT
         self._cssStyles = True
         self._fullHTML: list[str] = []
-        self._navMap: dict[str, str] = {}
 
         # Internals
         self._trMap = {}
@@ -71,10 +70,6 @@ class ToHtml(Tokenizer):
     def fullHTML(self) -> list[str]:
         return self._fullHTML
 
-    @property
-    def navigationMap(self) -> dict[str, str]:
-        return self._navMap
-
     ##
     #  Setters
     ##
@@ -174,6 +169,8 @@ class ToHtml(Tokenizer):
         pStyle = None
         lines = []
 
+        tHandle = self._handle
+
         for tType, nHead, tText, tFormat, tStyle in self._tokens:
 
             # Replace < and > with HTML entities
@@ -232,11 +229,9 @@ class ToHtml(Tokenizer):
             else:
                 hStyle = ""
 
-            if self._linkHeadings and self._handle:
+            if self._linkHeadings and tHandle:
-                tHH = f"{self._handle}:T{nHead:04d}"
+                aNm = f"<a name='{tHandle}:T{nHead:04d}'></a>"
-                aNm = f"<a name='{tHH}'></a>"
             else:
-                tHH = ""
                 aNm = ""
 
             # Process Text Type
@@ -256,32 +251,22 @@ class ToHtml(Tokenizer):
             elif tType == self.T_TITLE:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<h1 class='title'{hStyle}>{aNm}{tHead}</h1>\n")
-                if tHH:
-                    self._navMap[tHH] = f"TT:{tHead}"
 
             elif tType == self.T_HEAD1:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h1}{h1Cl}{hStyle}>{aNm}{tHead}</{h1}>\n")
-                if tHH:
-                    self._navMap[tHH] = f"H1:{tHead}"
 
             elif tType == self.T_HEAD2:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
-                if tHH:
-                    self._navMap[tHH] = f"H2:{tHead}"
 
             elif tType == self.T_HEAD3:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h3}{hStyle}>{aNm}{tHead}</{h3}>\n")
-                if tHH:
-                    self._navMap[tHH] = f"H3:{tHead}"
 
             elif tType == self.T_HEAD4:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h4}{hStyle}>{aNm}{tHead}</{h4}>\n")
-                if tHH:
-                    self._navMap[tHH] = f"H4:{tHead}"
 
             elif tType == self.T_SEP:
                 lines.append(f"<p class='sep'{hStyle}>{tText}</p>\n")

novelwriter/core/tokenizer.py

@@ -35,13 +35,13 @@ from functools import partial
 
 from PyQt5.QtCore import QCoreApplication, QRegularExpression
 
-from novelwriter.enum import nwComment, nwItemLayout
 from novelwriter.common import formatTimeStamp, numberToRoman, checkInt
 from novelwriter.constants import (
     nwHeadFmt, nwKeyWords, nwLabels, nwRegEx, nwShortcode, nwUnicode, trConst
 )
 from novelwriter.core.index import processComment
 from novelwriter.core.project import NWProject
+from novelwriter.enum import nwComment, nwItemLayout
 
 logger = logging.getLogger(__name__)
 
@@ -120,13 +120,14 @@ class Tokenizer(ABC):
         self._text   = ""    # The raw text to be tokenized
         self._handle = None  # The item handle currently being processed
         self._result = ""    # The result of the last document
-        self._counts = {}    # Counter data
 
         self._keepMarkdown = False  # Whether to keep the markdown text
         self._allMarkdown  = []     # The result novelWriter markdown of all documents
 
-        # Processed Tokens
+        # Processed Tokens and Meta Data
         self._tokens: list[tuple[int, int, str, list[tuple[int, int]], int]] = []
+        self._counts: dict[str, int] = {}
+        self._outline: dict[str, str] = {}
 
         # User Settings
         self._textFont     = "Serif"  # Output text font
@@ -226,6 +227,11 @@ class Tokenizer(ABC):
         """The collected stats about the text."""
         return self._counts
 
+    @property
+    def textOutline(self) -> dict[str, str]:
+        """The generated outline of the text."""
+        return self._outline
+
     @property
     def errData(self) -> list[str]:
         """The error data."""
@@ -392,43 +398,42 @@ class Tokenizer(ABC):
     def doConvert(self) -> None:
         raise NotImplementedError
 
-    def addRootHeading(self, tHandle: str) -> bool:
+    def addRootHeading(self, tHandle: str) -> None:
         """Add a heading at the start of a new root folder."""
-        tItem = self._project.tree[tHandle]
+        self._text = ""
-        if not tItem or not tItem.isRootType():
+        self._handle = None
-            return False
 
-        if self._isFirst:
+        if (tItem := self._project.tree[tHandle]) and tItem.isRootType():
-            textAlign = self.A_CENTRE
+            self._handle = tHandle
-            self._isFirst = False
+            if self._isFirst:
-        else:
+                textAlign = self.A_CENTRE
-            textAlign = self.A_PBB | self.A_CENTRE
+                self._isFirst = False
+            else:
+                textAlign = self.A_PBB | self.A_CENTRE
 
-        trNotes = self._localLookup("Notes")
+            trNotes = self._localLookup("Notes")
-        title = f"{trNotes}: {tItem.itemName}"
+            title = f"{trNotes}: {tItem.itemName}"
-        self._tokens = []
+            self._tokens = []
-        self._tokens.append((
+            self._tokens.append((
-            self.T_TITLE, 0, title, [], textAlign
+                self.T_TITLE, 1, title, [], textAlign
-        ))
+            ))
-        if self._keepMarkdown:
+            if self._keepMarkdown:
-            self._allMarkdown.append(f"# {title}\n\n")
+                self._allMarkdown.append(f"#! {title}\n\n")
 
-        return True
+        return
 
     def setText(self, tHandle: str, text: str | None = None) -> None:
         """Set the text for the tokenizer from a handle. If text is not
-        set, its is loaded from the file.
+        set, it's is loaded from the file.
         """
         self._text = ""
         self._handle = None
         if nwItem := self._project.tree[tHandle]:
             if text is None:
                 text = self._project.storage.getDocument(tHandle).readDocument() or ""
-
             self._text = text
             self._handle = tHandle
             self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT
-
         return
 
     def doPreProcessing(self) -> None:
@@ -798,7 +803,29 @@ class Tokenizer(ABC):
 
         return
 
-    def countStats(self) -> dict[str, int]:
+    def buildOutline(self) -> None:
+        """Build an outline of the text up to level 3 headings."""
+        tHandle = self._handle or ""
+        isNovel = self._isNovel
+        for tType, nHead, tText, _, _ in self._tokens:
+            if tType == self.T_TITLE:
+                prefix = "TT"
+            elif tType == self.T_HEAD1:
+                prefix = "PT" if isNovel else "H1"
+            elif tType == self.T_HEAD2:
+                prefix = "CH" if isNovel else "H2"
+            elif tType == self.T_HEAD3:
+                prefix = "SC" if isNovel else "H3"
+            else:
+                continue
+
+            key = f"{tHandle}:T{nHead:04d}"
+            text = tText.replace(nwHeadFmt.BR, " ").replace("&", "&")
+            self._outline[key] = f"{prefix}|{text}"
+
+        return
+
+    def countStats(self) -> None:
         """Count stats on the tokenized text."""
         titleCount = self._counts.get("titleCount", 0)
         paragraphCount = self._counts.get("paragraphCount", 0)
@@ -905,7 +932,7 @@ class Tokenizer(ABC):
         self._counts["textWordChars"] = textWordChars
         self._counts["titleWordChars"] = titleWordChars
 
-        return {}
+        return
 
     def saveRawMarkdown(self, path: str | Path) -> None:
         """Save the raw text to a plain text file."""

novelwriter/core/tree.py

@@ -28,7 +28,7 @@ import logging
 
 from typing import TYPE_CHECKING, Literal, overload
 from pathlib import Path
-from collections.abc import Generator, Iterator
+from collections.abc import Iterable, Iterator
 
 from novelwriter.enum import nwItemClass, nwItemLayout, nwItemType
 from novelwriter.error import logException
@@ -387,7 +387,7 @@ class NWTree:
             rootClasses.add(nwItem.itemClass)
         return rootClasses
 
-    def iterRoots(self, itemClass: nwItemClass | None) -> Generator[tuple[str, NWItem]]:
+    def iterRoots(self, itemClass: nwItemClass | None) -> Iterable[tuple[str, NWItem]]:
         """Iterate over all root items of a given class in order."""
         for tHandle in self._order:
             nwItem = self.__getitem__(tHandle)

novelwriter/gui/docviewer.py

@@ -41,13 +41,13 @@ from PyQt5.QtWidgets import (
 )
 
 from novelwriter import CONFIG, SHARED
+from novelwriter.constants import nwHeaders, nwUnicode
+from novelwriter.core.tohtml import ToHtml
 from novelwriter.enum import nwItemType, nwDocAction, nwDocMode
 from novelwriter.error import logException
-from novelwriter.constants import nwHeaders, nwUnicode
+from novelwriter.extensions.eventfilters import WheelEventFilter
 from novelwriter.extensions.modified import NIconToolButton
 from novelwriter.gui.theme import STYLES_MIN_TOOLBUTTON
-from novelwriter.core.tohtml import ToHtml
-from novelwriter.extensions.eventfilters import WheelEventFilter
 
 logger = logging.getLogger(__name__)
 
@@ -633,7 +633,7 @@ class GuiDocViewHeader(QWidget):
 
         # Internal Variables
         self._docHandle = None
-        self._docOutline: dict[int, tuple[str, int]] = {}
+        self._docOutline: dict[str, tuple[str, int]] = {}
 
         iPx = SHARED.theme.baseIconSize
         mPx = CONFIG.pxInt(4)
@@ -730,7 +730,7 @@ class GuiDocViewHeader(QWidget):
         self.refreshButton.setVisible(False)
         return
 
-    def setOutline(self, data: dict[int, tuple[str, int]]) -> None:
+    def setOutline(self, data: dict[str, tuple[str, int]]) -> None:
         """Set the document outline dataset."""
         tHandle = self._docHandle
         if data != self._docOutline and tHandle:

novelwriter/gui/editordocument.py

@@ -23,10 +23,10 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
 from __future__ import annotations
 
-from collections.abc import Generator
 import logging
 
 from time import time
+from collections.abc import Iterable
 
 from PyQt5.QtGui import QTextBlock, QTextCursor, QTextDocument
 from PyQt5.QtCore import QObject, pyqtSlot
@@ -114,7 +114,7 @@ class GuiTextDocument(QTextDocument):
                         return word, cPos, cLen, SHARED.spelling.suggestWords(word)
         return "", -1, -1, []
 
-    def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Generator[QTextBlock]:
+    def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Iterable[QTextBlock]:
         """Iterate over all text blocks of a given type."""
         count = 0
         for i in range(self.blockCount()):

novelwriter/tools/manuscript.py

@@ -58,14 +58,6 @@ if TYPE_CHECKING:  # pragma: no cover
 
 logger = logging.getLogger(__name__)
 
-OUTLINE_MAP = {
-    "TT": 0,
-    "H1": 1,
-    "H2": 2,
-    "H3": 3,
-    "H4": 4,
-}
-
 
 class GuiManuscript(QDialog):
     """GUI Tools: Manuscript Tool
@@ -368,7 +360,7 @@ class GuiManuscript(QDialog):
             "uuid": build.buildID,
             "time": int(time()),
             "stats": buildObj.textStats,
-            "outline": buildObj.navigationMap,
+            "outline": buildObj.textOutline,
             "styles": buildObj.getStyleSheet(),
             "html": buildObj.fullHTML,
         }
@@ -708,22 +700,24 @@ class _OutlineWidget(QWidget):
             root = self.listView.invisibleRootItem()
             parent = root
             indent = False
-            for anchor, text in data.items():
+            for anchor, entry in data.items():
-                level = OUTLINE_MAP.get(text[:2], -1)
+                prefix, _, text = entry.partition("|")
-                text = text[3:]
+                if prefix in ("TT", "PT", "CH", "SC", "H1", "H2"):
-                if 0 <= level < 4:
                     item = QTreeWidgetItem([text])
                     item.setData(0, self.D_LINE, anchor)
-                    if level == 0:
+                    if prefix == "TT":
                         item.setFont(0, tFont)
                         item.setForeground(0, tBrush)
-                    elif level == 1:
+                        root.addChild(item)
+                        parent = root
+                    elif prefix == "PT":
                         item.setFont(0, hFont)
-
+                        root.addChild(item)
-                    if level < 3:
+                        parent = root
+                    elif prefix in ("CH", "H1"):
                         root.addChild(item)
                         parent = item
-                    elif parent:
+                    elif prefix in ("SC", "H2"):
                         parent.addChild(item)
                         indent = True