From 144ba66d9d7f55946d29b1e1c531dfe78588f3bd Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Wed, 20 Mar 2024 23:44:25 +0100
Subject: [PATCH] Remove redundant variables in tokenizer

---
 novelwriter/core/tohtml.py             | 14 ++++----
 novelwriter/core/tokenizer.py          | 31 +++++++----------
 novelwriter/tools/manuscript.py        |  3 ++
 tests/test_core/test_core_tohtml.py    | 22 ++++++------
 tests/test_core/test_core_tokenizer.py | 47 +++-----------------------
 tests/test_core/test_core_tomd.py      |  3 --
 6 files changed, 38 insertions(+), 82 deletions(-)

diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py
index ed791223..ad548aa6 100644
--- a/novelwriter/core/tohtml.py
+++ b/novelwriter/core/tohtml.py
@@ -232,8 +232,8 @@ class ToHtml(Tokenizer):
             else:
                 hStyle = ""
 
-            if self._linkHeadings and self._nwItem:
-                tHH = f"{self._nwItem.itemHandle}:T{nHead:04d}"
+            if self._linkHeadings and self._handle:
+                tHH = f"{self._handle}:T{nHead:04d}"
                 aNm = f"<a name='{tHH}'></a>"
             else:
                 tHH = ""
@@ -257,31 +257,31 @@ class ToHtml(Tokenizer):
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<h1 class='title'{hStyle}>{aNm}{tHead}</h1>\n")
                 if tHH:
-                    self._navMap[tHH] = f"TT {tHead}"
+                    self._navMap[tHH] = f"TT:{tHead}"
 
             elif tType == self.T_HEAD1:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h1}{h1Cl}{hStyle}>{aNm}{tHead}</{h1}>\n")
                 if tHH:
-                    self._navMap[tHH] = f"H1 {tHead}"
+                    self._navMap[tHH] = f"H1:{tHead}"
 
             elif tType == self.T_HEAD2:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
                 if tHH:
-                    self._navMap[tHH] = f"H2 {tHead}"
+                    self._navMap[tHH] = f"H2:{tHead}"
 
             elif tType == self.T_HEAD3:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h3}{hStyle}>{aNm}{tHead}</{h3}>\n")
                 if tHH:
-                    self._navMap[tHH] = f"H3 {tHead}"
+                    self._navMap[tHH] = f"H3:{tHead}"
 
             elif tType == self.T_HEAD4:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
                 lines.append(f"<{h4}{hStyle}>{aNm}{tHead}</{h4}>\n")
                 if tHH:
-                    self._navMap[tHH] = f"H4 {tHead}"
+                    self._navMap[tHH] = f"H4:{tHead}"
 
             elif tType == self.T_SEP:
                 lines.append(f"<p class='sep'{hStyle}>{tText}</p>\n")
diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py
index 15de37be..dd30cfc6 100644
--- a/novelwriter/core/tokenizer.py
+++ b/novelwriter/core/tokenizer.py
@@ -118,7 +118,7 @@ class Tokenizer(ABC):
 
         # Data Variables
         self._text   = ""    # The raw text to be tokenized
-        self._nwItem = None  # The NWItem currently being processed
+        self._handle = None  # The item handle currently being processed
         self._result = ""    # The result of the last document
         self._counts = {}    # Counter data
 
@@ -176,9 +176,7 @@ class Tokenizer(ABC):
         self._noSep      = True  # Flag to indicate that we don't want a scene separator
 
         # This File
-        self._isNone  = False  # Document has unknown layout
         self._isNovel = False  # Document is a novel document
-        self._isNote  = False  # Document is a project note
         self._isFirst = True   # Document is the first in a set
 
         # Error Handling
@@ -417,24 +415,21 @@ class Tokenizer(ABC):
 
         return True
 
-    def setText(self, tHandle: str, text: str | None = None) -> bool:
+    def setText(self, tHandle: str, text: str | None = None) -> None:
         """Set the text for the tokenizer from a handle. If text is not
-        set, load it from the file.
+        set, its is loaded from the file.
         """
-        self._nwItem = self._project.tree[tHandle]
-        if self._nwItem is None:
-            return False
+        self._text = ""
+        self._handle = None
+        if nwItem := self._project.tree[tHandle]:
+            if text is None:
+                text = self._project.storage.getDocument(tHandle).readDocument() or ""
 
-        if text is None:
-            text = self._project.storage.getDocument(tHandle).readDocument() or ""
+            self._text = text
+            self._handle = tHandle
+            self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT
 
-        self._text = text
-
-        self._isNone  = self._nwItem.itemLayout == nwItemLayout.NO_LAYOUT
-        self._isNovel = self._nwItem.itemLayout == nwItemLayout.DOCUMENT
-        self._isNote  = self._nwItem.itemLayout == nwItemLayout.NOTE
-
-        return True
+        return
 
     def doPreProcessing(self) -> None:
         """Run trough the various replace dictionaries."""
@@ -470,7 +465,7 @@ class Tokenizer(ABC):
         """
         self._tokens = []
         if self._isNovel:
-            self._hFormatter.setHandle(self._nwItem.itemHandle if self._nwItem else None)
+            self._hFormatter.setHandle(self._handle)
 
         nHead = 0
         breakNext = False
diff --git a/novelwriter/tools/manuscript.py b/novelwriter/tools/manuscript.py
index b213acb2..906b37d5 100644
--- a/novelwriter/tools/manuscript.py
+++ b/novelwriter/tools/manuscript.py
@@ -707,6 +707,7 @@ class _OutlineWidget(QWidget):
 
             root = self.listView.invisibleRootItem()
             parent = root
+            indent = False
             for anchor, text in data.items():
                 level = OUTLINE_MAP.get(text[:2], -1)
                 text = text[3:]
@@ -724,7 +725,9 @@ class _OutlineWidget(QWidget):
                         parent = item
                     elif parent:
                         parent.addChild(item)
+                        indent = True
 
+            self.listView.setIndentation(SHARED.theme.baseIconSize if indent else CONFIG.pxInt(4))
             self._outline = data
 
         return
diff --git a/tests/test_core/test_core_tohtml.py b/tests/test_core/test_core_tohtml.py
index 9f656ca4..30aefd0a 100644
--- a/tests/test_core/test_core_tohtml.py
+++ b/tests/test_core/test_core_tohtml.py
@@ -38,7 +38,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # ===================
 
     html._isNovel = True
-    html._isNote = False
     html._isFirst = True
 
     # Header 1
@@ -87,33 +86,33 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # ==================
 
     html._isNovel = False
-    html._isNote = True
     html._isFirst = True
+    html._handle = "0000000000000"
     html.setLinkHeadings(True)
 
     # Header 1
     html._text = "# Heading One\n"
     html.tokenizeText()
     html.doConvert()
-    assert html.result == "<h1><a name='T0001'></a>Heading One</h1>\n"
+    assert html.result == "<h1><a name='0000000000000:T0001'></a>Heading One</h1>\n"
 
     # Header 2
     html._text = "## Heading Two\n"
     html.tokenizeText()
     html.doConvert()
-    assert html.result == "<h2><a name='T0001'></a>Heading Two</h2>\n"
+    assert html.result == "<h2><a name='0000000000000:T0001'></a>Heading Two</h2>\n"
 
     # Header 3
     html._text = "### Heading Three\n"
     html.tokenizeText()
     html.doConvert()
-    assert html.result == "<h3><a name='T0001'></a>Heading Three</h3>\n"
+    assert html.result == "<h3><a name='0000000000000:T0001'></a>Heading Three</h3>\n"
 
     # Header 4
     html._text = "#### Heading Four\n"
     html.tokenizeText()
     html.doConvert()
-    assert html.result == "<h4><a name='T0001'></a>Heading Four</h4>\n"
+    assert html.result == "<h4><a name='0000000000000:T0001'></a>Heading Four</h4>\n"
 
     # Title
     html._text = "#! Heading One\n"
@@ -121,14 +120,14 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     html.doConvert()
     assert html.result == (
         "<h1 class='title' style='text-align: center; page-break-before: always;'>"
-        "<a name='T0001'></a>Heading One</h1>\n"
+        "<a name='0000000000000:T0001'></a>Heading One</h1>\n"
     )
 
     # Unnumbered
     html._text = "##! Heading Two\n"
     html.tokenizeText()
     html.doConvert()
-    assert html.result == "<h2><a name='T0001'></a>Heading Two</h2>\n"
+    assert html.result == "<h2><a name='0000000000000:T0001'></a>Heading Two</h2>\n"
 
 # END Test testCoreToHtml_ConvertHeaders
 
@@ -140,7 +139,6 @@ def testCoreToHtml_ConvertParagraphs(mockGUI):
     html = ToHtml(project)
 
     html._isNovel = True
-    html._isNote = False
     html._isFirst = True
 
     # Paragraphs
@@ -269,7 +267,7 @@ def testCoreToHtml_ConvertDirect(mockGUI):
     html = ToHtml(project)
 
     html._isNovel = True
-    html._isNote = False
+    html._handle = "0000000000000"
     html.setLinkHeadings(True)
 
     # Special Titles
@@ -283,7 +281,7 @@ def testCoreToHtml_ConvertDirect(mockGUI):
     html.doConvert()
     assert html.result == (
         "<h1 class='title' style='text-align: center; page-break-before: always;'>"
-        "<a name='T0001'></a>A Title</h1>\n"
+        "<a name='0000000000000:T0001'></a>A Title</h1>\n"
     )
 
     # Unnumbered
@@ -294,7 +292,7 @@ def testCoreToHtml_ConvertDirect(mockGUI):
     html.doConvert()
     assert html.result == (
         "<h1 style='page-break-before: always;'>"
-        "<a name='T0001'></a>Prologue</h1>\n"
+        "<a name='0000000000000:T0001'></a>Prologue</h1>\n"
     )
 
     # Separators
diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py
index 30558c9c..68ecfdf7 100644
--- a/tests/test_core/test_core_tokenizer.py
+++ b/tests/test_core/test_core_tokenizer.py
@@ -199,16 +199,16 @@ def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
     )
 
     # Set Text
-    assert tokens.setText("stuff") is False
-    assert tokens.setText(C.hSceneDoc) is True
+    tokens.setText("stuff")
+    assert tokens._text == ""
+
+    tokens.setText(C.hSceneDoc)
     assert tokens._text == docText
 
-    assert tokens.setText(C.hSceneDoc, docText) is True
+    tokens.setText(C.hSceneDoc, docText)
     assert tokens._text == docText
 
-    assert tokens._isNone is False
     assert tokens._isNovel is True
-    assert tokens._isNote is False
 
     # Pre Processing
     tokens.doPreProcessing()
@@ -259,7 +259,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Story File
     tokens._isNovel = True
-    tokens._isNote  = False
     tokens._isFirst = True
     tokens._text = "#! Novel Title\n"
 
@@ -272,7 +271,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Note File
     tokens._isNovel = False
-    tokens._isNote  = True
     tokens._isFirst = True
     tokens._text = "#! Note Title\n"
 
@@ -288,7 +286,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Story File
     tokens._isNovel = True
-    tokens._isNote  = False
     tokens._isFirst = True
     tokens._text = "# Novel Title\n"
 
@@ -301,7 +298,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Note File
     tokens._isNovel = False
-    tokens._isNote  = True
     tokens._isFirst = True
     tokens._text = "# Note Title\n"
 
@@ -317,7 +313,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Story File
     tokens._isNovel = True
-    tokens._isNote  = False
     tokens._noBreak = False
     tokens._text = "## Chapter One\n"
 
@@ -330,7 +325,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Note File
     tokens._isNovel = False
-    tokens._isNote  = True
     tokens._text = "## Heading 2\n"
 
     tokens.tokenizeText()
@@ -345,7 +339,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Story File
     tokens._isNovel = True
-    tokens._isNote  = False
     tokens._text = "### Scene One\n"
 
     tokens.tokenizeText()
@@ -357,7 +350,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Note File
     tokens._isNovel = False
-    tokens._isNote  = True
     tokens._text = "### Heading 3\n"
 
     tokens.tokenizeText()
@@ -372,7 +364,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Story File
     tokens._isNovel = True
-    tokens._isNote  = False
     tokens._text = "#### A Section\n"
 
     tokens.tokenizeText()
@@ -384,7 +375,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Note File
     tokens._isNovel = False
-    tokens._isNote  = True
     tokens._text = "#### Heading 4\n"
 
     tokens.tokenizeText()
@@ -399,7 +389,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Story File
     tokens._isNovel = True
-    tokens._isNote  = False
     tokens._isFirst = False
     tokens._text = "#! Title\n"
 
@@ -412,7 +401,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Note File
     tokens._isNovel = False
-    tokens._isNote  = True
     tokens._isFirst = False
     tokens._text = "#! Title\n"
 
@@ -428,7 +416,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Story File
     tokens._isNovel = True
-    tokens._isNote  = False
     tokens._text = "##! Prologue\n"
 
     tokens.tokenizeText()
@@ -440,7 +427,6 @@ def testCoreToken_HeaderFormat(mockGUI):
 
     # Note File
     tokens._isNovel = False
-    tokens._isNote  = True
     tokens._text = "##! Prologue\n"
 
     tokens.tokenizeText()
@@ -479,7 +465,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Novel Docs
     tokens._isNovel = True
-    tokens._isNote  = False
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_NONE
@@ -499,7 +484,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Note Docs
     tokens._isNovel = False
-    tokens._isNote  = True
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_NONE
@@ -530,7 +514,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Novel Docs
     tokens._isNovel = True
-    tokens._isNote  = False
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_CENTRE
@@ -550,7 +533,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Note Docs
     tokens._isNovel = False
-    tokens._isNote  = True
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_NONE
@@ -581,7 +563,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Novel Docs
     tokens._isNovel = True
-    tokens._isNote  = False
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_PBB
@@ -601,7 +582,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Note Docs
     tokens._isNovel = False
-    tokens._isNote  = True
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_NONE
@@ -632,7 +612,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Novel Docs
     tokens._isNovel = True
-    tokens._isNote  = False
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_CENTRE | Tokenizer.A_PBB
@@ -652,7 +631,6 @@ def testCoreToken_HeaderStyle(mockGUI):
 
     # Note Docs
     tokens._isNovel = False
-    tokens._isNote  = True
 
     # First Document is False
     assert processStyle("# Title\n", False) == Tokenizer.A_NONE
@@ -673,7 +651,6 @@ def testCoreToken_HeaderStyle(mockGUI):
     # Check Separation
     # ================
     tokens._isNovel = True
-    tokens._isNote  = False
 
     # Title Styles
     tokens.setTitleStyle(True, True)
@@ -1310,8 +1287,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     #  Story Files
     ##
 
-    tokens._isNone  = False
-    tokens._isNote  = False
     tokens._isNovel = True
 
     # Titles
@@ -1530,8 +1505,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
     tokens = BareTokenizer(project)
-    tokens._isNone  = False
-    tokens._isNote  = False
     tokens._isNovel = True
 
     # Short Text
@@ -1740,8 +1713,6 @@ def testCoreToken_SceneSeparators(mockGUI):
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
     md = ToMarkdown(project)
-    md._isNone = False
-    md._isNote = False
     md._isNovel = True
 
     # Separator Handling, Titles
@@ -1858,8 +1829,6 @@ def testCoreToken_SceneSeparators(mockGUI):
     # Requires a fresh builder class
     md = ToMarkdown(project)
     md.setExtendedMarkdown()
-    md._isNone = False
-    md._isNote = False
     md._isNovel = True
 
     md._text = (
@@ -1919,8 +1888,6 @@ def testCoreToken_HeaderVisibility(mockGUI):
     # Novel Files
     # ===========
 
-    md._isNone = False
-    md._isNote = False
     md._isNovel = True
 
     # Show All
@@ -1973,8 +1940,6 @@ def testCoreToken_HeaderVisibility(mockGUI):
     # Note Files
     # ==========
 
-    md._isNone = False
-    md._isNote = True
     md._isNovel = False
 
     # Hide All
@@ -2015,8 +1980,6 @@ def testCoreToken_CounterHandling(mockGUI):
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
     md = ToMarkdown(project)
-    md._isNone = False
-    md._isNote = False
     md._isNovel = True
 
     # Counter Handling, Novel Titles
diff --git a/tests/test_core/test_core_tomd.py b/tests/test_core/test_core_tomd.py
index c67caf63..2192517f 100644
--- a/tests/test_core/test_core_tomd.py
+++ b/tests/test_core/test_core_tomd.py
@@ -35,7 +35,6 @@ def testCoreToMarkdown_ConvertHeaders(mockGUI):
     toMD = ToMarkdown(project)
 
     toMD._isNovel = True
-    toMD._isNote = False
     toMD._isFirst = True
 
     # Header 1
@@ -84,7 +83,6 @@ def testCoreToMarkdown_ConvertParagraphs(mockGUI):
     toMD = ToMarkdown(project)
 
     toMD._isNovel = True
-    toMD._isNote = False
     toMD._isFirst = True
 
     # Text for Extended Markdown
@@ -200,7 +198,6 @@ def testCoreToMarkdown_ConvertDirect(mockGUI):
     toMD = ToMarkdown(project)
 
     toMD._isNovel = True
-    toMD._isNote = False
 
     # Special Titles
     # ==============