Remove the header processing function and add comments to the tokenizer

novelwriter/core/docbuild.py

@@ -352,7 +352,6 @@ class NWBuildDocument:
                     bldObj.setText(tHandle)
                     bldObj.doPreProcessing()
                     bldObj.tokenizeText()
-                    bldObj.doHeaders()
                     if self._count:
                         bldObj.countStats()
                     if convert:

novelwriter/core/tokenizer.py

@@ -169,7 +169,7 @@ class Tokenizer(ABC):
 
         # Instance Variables
         self._hFormatter = HeadingFormatter(self._project)
-        self._skipSep = False  # Flag to indicate that we skip the scene separator
+        self._noSep = False  # Flag to indicate that we don't want a scene separator
 
         # This File
         self._isNone  = False  # Document has unknown layout
@@ -486,10 +486,11 @@ class Tokenizer(ABC):
             # =================
 
             if aLine[0] == "[":
-                # Parse special formatting line
-                # This must be a separate if statement, as it may not
-                # reach a continue statement and must therefore proceed
-                # to check other formats.
+                # Special Formats
+                # ===============
+                # Parse special formatting line. This must be a separate if
+                # statement, as it may not reach a continue statement and must
+                # therefore proceed to check other formats.
 
                 if sLine in ("[newpage]", "[new page]"):
                     breakNext = True
@@ -514,8 +515,12 @@ class Tokenizer(ABC):
                     continue
 
             if aLine[0] == "%":
+                # Comments
+                # ========
+                # All style comments are processed and the exact type exact
+                # style extracted. Ignored comments on the '%~' format are
+                # skipped completely.
                 if aLine[1] == "~":
-                    # Completely ignore the paragraph
                     continue
 
                 cStyle, cText, _ = processComment(aLine)
@@ -539,6 +544,11 @@ class Tokenizer(ABC):
                         tmpMarkdown.append("%s\n" % aLine)
 
             elif aLine[0] == "@":
+                # Keywords
+                # ========
+                # Only valid keyword lines are parsed, and any ignored keywords
+                # are automatically skipped.
+
                 valid, bits, _ = self._project.index.scanThis(aLine)
                 if valid and bits and bits[0] not in self._skipKeywords:
                     self._tokens.append((
@@ -548,13 +558,19 @@ class Tokenizer(ABC):
                         tmpMarkdown.append("%s\n" % aLine)
 
             elif aLine[:2] == "# ":
+                # Partition Headers
+                # =================
+                # Partition headers are only formatted in novel documents, and
+                # otherwise unchanged. Scene separators are disabled
+                # immediately after partitions, and scene numbers are reset.
+
                 nHead += 1
                 tText = aLine[2:].strip()
                 tStyle = self.A_NONE
                 if self._isNovel:
                     tText = self._hFormatter.apply(self._fmtTitle, tText, nHead)
                     tStyle = self._titleStyle
-                    self._skipSep = True
+                    self._noSep = True
                     self._hFormatter.resetScene()
 
                 self._tokens.append((
@@ -564,6 +580,14 @@ class Tokenizer(ABC):
                     tmpMarkdown.append("%s\n" % aLine)
 
             elif aLine[:3] == "## ":
+                # Chapter Headers
+                # ===============
+                # Chapter headers are only formatted in novel documents, and
+                # otherwise unchanged. Chapter numbers are bumped before the
+                # heading is formatted. Scene separators are disabled
+                # immediately after chapter headers, and scene numbers are
+                # reset.
+
                 nHead += 1
                 tText = aLine[3:].strip()
                 tStyle = self.A_NONE
@@ -571,7 +595,7 @@ class Tokenizer(ABC):
                     self._hFormatter.incChapter()
                     tText = self._hFormatter.apply(self._fmtChapter, tText, nHead)
                     tStyle = self._chapterStyle
-                    self._skipSep = True
+                    self._noSep = True
                     self._hFormatter.resetScene()
 
                 self._tokens.append((
@@ -581,6 +605,17 @@ class Tokenizer(ABC):
                     tmpMarkdown.append("%s\n" % aLine)
 
             elif aLine[:4] == "### ":
+                # Scene Headers
+                # =============
+                # Scene headers in novel documents are treated as centred
+                # separators if the formatting does not change the text. If the
+                # format is empty, the scene can be hidden or a blank paragraph
+                # (skip). When the scene title has static text or no text, it
+                # is always ignored if the noSep flag is set. This prevents
+                # separators immediately after other titles. Scene numbers are
+                # always incremented before formatting. For notes, the header
+                # is unchanged.
+
                 nHead += 1
                 tText = aLine[4:].strip()
                 tType = self.T_HEAD3
@@ -590,13 +625,13 @@ class Tokenizer(ABC):
                     tText = self._hFormatter.apply(self._fmtScene, tText, nHead)
                     tStyle = self._sceneStyle
                     if tText == "":
-                        tType = self.T_EMPTY if self._skipSep or self._hideScene else self.T_SKIP
+                        tType = self.T_EMPTY if self._noSep or self._hideScene else self.T_SKIP
                         tStyle = self.A_NONE
                     elif tText == self._fmtScene:
-                        tText = "" if self._skipSep else tText
-                        tType = self.T_EMPTY if self._skipSep else self.T_SEP
-                        tStyle = self.A_NONE if self._skipSep else self.A_CENTRE
-                    self._skipSep = False
+                        tText = "" if self._noSep else tText
+                        tType = self.T_EMPTY if self._noSep else self.T_SEP
+                        tStyle = self.A_NONE if self._noSep else self.A_CENTRE
+                    self._noSep = False
 
                 self._tokens.append((
                     tType, nHead, tText, [], tStyle
@@ -605,6 +640,13 @@ class Tokenizer(ABC):
                     tmpMarkdown.append("%s\n" % aLine)
 
             elif aLine[:5] == "#### ":
+                # Section Headers
+                # ===============
+                # Section headers in novel docs are treated as centred
+                # separators if the formatting does not change the text. If the
+                # format is empty, the section can be hidden or a blank
+                # paragraph (skip). For notes, the header is unchanged.
+
                 nHead += 1
                 tText = aLine[5:].strip()
                 tType = self.T_HEAD4
@@ -624,6 +666,12 @@ class Tokenizer(ABC):
                     tmpMarkdown.append("%s\n" % aLine)
 
             elif aLine[:3] == "#! ":
+                # Main Title
+                # ==========
+                # Main titles are allowed in any document, and they are always
+                # centred and start on a new page. For novel documents, we also
+                # reset all counters when such a title is encountered.
+
                 nHead += 1
                 self._tokens.append((
                     self.T_TITLE, nHead, aLine[3:].strip(), [], self.A_PBB | self.A_CENTRE
@@ -631,11 +679,15 @@ class Tokenizer(ABC):
                 if self._keepMarkdown:
                     tmpMarkdown.append("%s\n" % aLine)
                 if self._isNovel:
-                    # For new titles, we reset all counters
-                    self._skipSep = True
+                    self._noSep = True
                     self._hFormatter.resetAll()
 
             elif aLine[:4] == "##! ":
+                # Unnumbered Chapter Header
+                # =========================
+                # Unnumbered chapters are only meaningful in Novel docs, so if
+                # we're in a note, we convert them to a plain level 2 header.
+
                 nHead += 1
                 tText = aLine[4:].strip()
                 tType = self.T_HEAD2
@@ -644,10 +696,9 @@ class Tokenizer(ABC):
                     tText = self._hFormatter.apply(self._fmtUnNum, tText, nHead)
                     tType = self.T_UNNUM
                     tStyle = self._chapterStyle
-                    self._skipSep = True
+                    self._noSep = True
                     self._hFormatter.resetScene()
 
-                # If we're not in a novel section, we just treat this as a regular H2
                 self._tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
@@ -655,6 +706,10 @@ class Tokenizer(ABC):
                     tmpMarkdown.append("%s\n" % aLine)
 
             else:
+                # Text Lines
+                # ==========
+                # Anything remaining at this point is body text. If body text
+                # is not disabled, we proceed to process text formatting.
                 if not self._doBodyText:
                     # Skip all body text
                     continue
@@ -745,14 +800,6 @@ class Tokenizer(ABC):
 
         return
 
-    def doHeaders(self) -> bool:
-        """Apply formatting to the text headers for novel files. This
-        also applies chapter and scene numbering.
-        """
-        if not self._isNovel:
-            return False
-        return True
-
     def countStats(self) -> dict[str, int]:
         """Count stats on the tokenized text."""
         titleCount = self._counts.get("titleCount", 0)

novelwriter/text/counting.py

@@ -31,7 +31,6 @@ from novelwriter.constants import nwRegEx, nwUnicode
 
 RX_SC = re.compile(nwRegEx.FMT_SC)
 RX_LO = re.compile(r"(?i)(?<!\\)(\[(?:vspace|newpage|new page)(:\d+)?)(?<!\\)(\])")
-RX_IN = re.compile(r"^>{1,2}\s*|\s*<{1,2}$")
 
 
 def preProcessText(text: str, keepHeaders: bool = True) -> list[str]:

tests/test_core/test_core_tohtml.py

@@ -44,7 +44,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # Header 1
     html._text = "# Partition\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == (
         "<h1 class='title' style='text-align: center;'>Partition</h1>\n"
@@ -53,7 +52,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # Header 2
     html._text = "## Chapter Title\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == (
         "<h1 style='page-break-before: always;'>Chapter Title</h1>\n"
@@ -62,21 +60,18 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # Header 3
     html._text = "### Scene Title\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h2>Scene Title</h2>\n"
 
     # Header 4
     html._text = "#### Section Title\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h3>Section Title</h3>\n"
 
     # Title
     html._text = "#! Title\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == (
         "<h1 class='title' style='text-align: center; page-break-before: always;'>Title</h1>\n"
@@ -85,7 +80,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # Unnumbered
     html._text = "##! Prologue\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h1 style='page-break-before: always;'>Prologue</h1>\n"
 
@@ -100,35 +94,30 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # Header 1
     html._text = "# Heading One\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h1><a name='T0001'></a>Heading One</h1>\n"
 
     # Header 2
     html._text = "## Heading Two\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h2><a name='T0001'></a>Heading Two</h2>\n"
 
     # Header 3
     html._text = "### Heading Three\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h3><a name='T0001'></a>Heading Three</h3>\n"
 
     # Header 4
     html._text = "#### Heading Four\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h4><a name='T0001'></a>Heading Four</h4>\n"
 
     # Title
     html._text = "#! Heading One\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == (
         "<h1 class='title' style='text-align: center; page-break-before: always;'>"
@@ -138,7 +127,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
     # Unnumbered
     html._text = "##! Heading Two\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == "<h2><a name='T0001'></a>Heading Two</h2>\n"
 
@@ -245,7 +233,6 @@ def testCoreToHtml_ConvertParagraphs(mockGUI):
     html.setKeywords(True)
     html._text = "## Chapter\n\n@pov: Bod\n@plot: Main\n@location: Europe\n\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == (
         "<h1 style='page-break-before: always;'>Chapter</h1>\n"
@@ -477,7 +464,6 @@ def testCoreToHtml_SpecialCases(mockGUI):
     html._noBreak = False
     html._text = "## Heading <1>\n"
     html.tokenizeText()
-    html.doHeaders()
     html.doConvert()
     assert html.result == (
         "<h1 style='page-break-before: always;'>Heading &lt;1&gt;</h1>\n"
@@ -551,7 +537,6 @@ def testCoreToHtml_Complex(mockGUI, fncPath):
         html._text = docText[i]
         html.doPreProcessing()
         html.tokenizeText()
-        html.doHeaders()
         html.doConvert()
         assert html.result == resText[i]
 

tests/test_core/test_core_tokenizer.py

@@ -276,7 +276,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "# Novel Title\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Novel Title", [], Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -290,7 +289,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "# Note Title\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -307,7 +305,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "## Chapter One\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -320,7 +317,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "## Heading 2\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD2, 1, "Heading 2", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -336,7 +332,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "### Scene One\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD3, 1, "Scene One", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -349,7 +344,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "### Heading 3\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD3, 1, "Heading 3", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -365,7 +359,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "#### A Section\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD4, 1, "A Section", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -378,7 +371,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "#### Heading 4\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD4, 1, "Heading 4", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -395,7 +387,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "#! Title\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -409,7 +400,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "#! Title\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -425,7 +415,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "##! Prologue\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_UNNUM, 1, "Prologue", [], Tokenizer.A_PBB),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -438,7 +427,6 @@ def testCoreToken_HeaderFormat(mockGUI):
     tokens._text = "##! Prologue\n"
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -459,7 +447,6 @@ def testCoreToken_HeaderStyle(mockGUI):
         tokens._isFirst = first
         tokens._noBreak = first
         tokens.tokenizeText()
-        tokens.doHeaders()
         return tokens._tokens[0][4]
 
     # No Styles
@@ -1120,7 +1107,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "# Title Two\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == correctResp
 
     # Command w/Space
@@ -1131,7 +1117,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "# Title Two\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == correctResp
 
     # Trailing Spaces
@@ -1142,7 +1127,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "# Title Two\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == correctResp
 
     # Single Empty Paragraph
@@ -1154,7 +1138,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1175,7 +1158,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1193,7 +1175,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1213,7 +1194,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1230,7 +1210,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1247,7 +1226,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1268,7 +1246,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1288,7 +1265,6 @@ def testCoreToken_SpecialFormat(mockGUI):
         "Some text to go here ...\n\n"
     )
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1313,17 +1289,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     project._loadProjectLocalisation()
     tokens = BareTokenizer(project)
 
-    # Nothing
-    tokens._text = "Some text ...\n"
-    assert tokens.doHeaders() is False
-    tokens._isNone = True
-    assert tokens.doHeaders() is False
-    tokens._isNone = False
-    assert tokens.doHeaders() is False
-    tokens._isNote = True
-    assert tokens.doHeaders() is False
-    tokens._isNote = False
-
     ##
     #  Story Files
     ##
@@ -1340,7 +1305,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "# Part One\n"
     tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1351,7 +1315,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "# Part One\n"
     tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1364,7 +1327,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "## Chapter One\n"
     tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD2, 1, "C: Chapter One", [], Tokenizer.A_PBB),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1374,7 +1336,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "##! Prologue\n"
     tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_UNNUM, 1, "U: Prologue", [], Tokenizer.A_PBB),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1385,7 +1346,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}")
     tokens._hFormatter._chCount = 0
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1395,7 +1355,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "## Chapter\n"
     tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD2, 1, "Chapter II", [], Tokenizer.A_PBB),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1405,7 +1364,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "## Chapter\n"
     tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD2, 1, "Chapter iii", [], Tokenizer.A_PBB),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1418,7 +1376,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD3, 1, "S: Scene One", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1428,7 +1385,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", True)
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1437,9 +1393,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._skipSep = True
+    tokens._noSep = True
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1448,9 +1403,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._skipSep = False
+    tokens._noSep = False
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1459,9 +1413,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._skipSep = True
+    tokens._noSep = True
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1470,9 +1423,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._skipSep = False
+    tokens._noSep = False
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1484,7 +1436,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._hFormatter._scAbsCount = 0
     tokens._hFormatter._scChCount = 0
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD3, 1, "Scene 1", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1496,7 +1447,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._hFormatter._scAbsCount = 0
     tokens._hFormatter._scChCount = 1
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD3, 1, "Scene 3.2", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1509,7 +1459,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "#### A Section\n"
     tokens.setSectionFormat("", True)
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1519,7 +1468,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "#### A Section\n"
     tokens.setSectionFormat("", False)
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1529,7 +1477,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "#### A Section\n"
     tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_HEAD4, 1, "X: A Section", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1539,25 +1486,22 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._text = "#### A Section\n"
     tokens.setSectionFormat("* * *", False)
     tokens.tokenizeText()
-    tokens.doHeaders()
     assert tokens._tokens == [
         (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
     ]
 
     # Check the first scene detector, plain text
-    tokens._skipSep = True
+    tokens._noSep = True
     tokens._text = "Some text ...\n"
     tokens.tokenizeText()
-    tokens.doHeaders()
-    assert tokens._skipSep is True
+    assert tokens._noSep is True
 
     # Check the first scene detector, text plus scene
-    tokens._skipSep = True
+    tokens._noSep = True
     tokens._text = "Some text ...\n\n### Scene\n\nText"
     tokens.tokenizeText()
-    tokens.doHeaders()
-    assert tokens._skipSep is False
+    assert tokens._noSep is False
 
 # END Test testCoreToken_ProcessHeaders
 
@@ -1580,7 +1524,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens._text = "## A Chapter Title\n\n"
     tokens._counts = {}
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert tokens._tokens[0][2] == "A Chapter Title"
     assert tokens.textStats == {
@@ -1596,7 +1539,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}")
     tokens._hFormatter.resetAll()
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert tokens._tokens[0][2] == "C 1: A Chapter Title"
     assert tokens.textStats == {
@@ -1611,7 +1553,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens._text = "Some text\non two lines.\n\nWith a second paragraph.\n\n"
     tokens._counts = {}
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert tokens.textStats == {
         "titleCount": 0, "paragraphCount": 2,
@@ -1626,7 +1567,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens.setChapterFormat(nwHeadFmt.TITLE)
     tokens.setSceneFormat("* * *", False)
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert [t[2] for t in tokens._tokens] == [
         "Chapter", "", "", "", "Text", "", "* * *", "", "Text", ""
@@ -1646,7 +1586,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens.setSceneFormat("* * *", False)
     tokens.setSynopsis(True)
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert [t[2] for t in tokens._tokens] == [
         "Chapter", "", "", "", "Stuff", "", "Text", ""
@@ -1666,7 +1605,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens.setSceneFormat("* * *", False)
     tokens.setSynopsis(True)
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert [t[2] for t in tokens._tokens] == [
         "Chapter", "", "", "", "Stuff", "", "Text", ""
@@ -1686,7 +1624,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens.setSceneFormat("* * *", False)
     tokens.setComments(True)
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert [t[2] for t in tokens._tokens] == [
         "Chapter", "", "", "", "Stuff", "", "Text", ""
@@ -1706,7 +1643,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens.setSceneFormat("* * *", False)
     tokens.setKeywords(True)
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert [t[2] for t in tokens._tokens] == [
         "Chapter", "", "", "", "pov: Jane", "", "Text", ""
@@ -1769,7 +1705,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
     tokens.setKeywords(True)
 
     tokens.tokenizeText()
-    tokens.doHeaders()
     tokens.countStats()
     assert tokens.textStats == {
         "titleCount": 4, "paragraphCount": 5,
@@ -1816,7 +1751,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
     # Static Separator
     md.setSceneFormat("* * *", False)
     md.tokenizeText()
-    md.doHeaders()
     md.doConvert()
     assert md.result == (
         "# T: Title One\n\n"
@@ -1832,7 +1766,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
     # Scene Title Formatted
     md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
     md.tokenizeText()
-    md.doHeaders()
     md.doConvert()
     assert md.result == (
         "# T: Title One\n\n"
@@ -1870,7 +1803,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
     # Static Separator
     md.setSceneFormat("* * *", False)
     md.tokenizeText()
-    md.doHeaders()
     md.doConvert()
     assert md.result == (
         "# T: Title One\n\n"
@@ -1887,7 +1819,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
     # Scene Title Formatted
     md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
     md.tokenizeText()
-    md.doHeaders()
     md.doConvert()
     assert md.result == (
         "# T: Title One\n\n"
@@ -1938,7 +1869,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
 
     # Two Novel Format
     md.tokenizeText()
-    md.doHeaders()
     md.doConvert()
     assert md.result == (
         "# Novel One\n\n"

tests/test_core/test_core_toodt.py

@@ -246,7 +246,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
     # Header 1
     odt._text = "# Title\n"
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -260,7 +259,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
     # Header 2
     odt._text = "## Chapter\n"
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -274,7 +272,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
     # Header 3
     odt._text = "### Scene\n"
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -288,7 +285,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
     # Header 4
     odt._text = "#### Section\n"
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -303,7 +299,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
     odt._isFirst = True
     odt._text = "#! Title\n"
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -317,7 +312,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
     # Unnumbered chapter
     odt._text = "##! Prologue\n"
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -507,7 +501,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
     odt._text = "### Scene One\n\nText\n\n### Scene Two\n\nText"
     odt.setSceneFormat("* * *", False)
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -525,7 +518,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
     odt._text = "### Scene One\n\nText\n\n### Scene Two\n\nText"
     odt.setSceneFormat("", False)
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -613,7 +605,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
         "Text\n\n"
     )
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -738,7 +729,6 @@ def testCoreToOdt_SaveFlat(mockGUI, fncPath, tstPaths):
         "Text\n\n"
     )
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()
@@ -778,7 +768,6 @@ def testCoreToOdt_SaveFull(mockGUI, fncPath, tstPaths):
         "Text\n\n"
     )
     odt.tokenizeText()
-    odt.doHeaders()
     odt.initDocument()
     odt.doConvert()
     odt.closeDocument()