From b687ffe6968595cdc997f99e2eb00571cb33266c Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Fri, 4 Dec 2020 00:46:29 +0100
Subject: [PATCH] New test for Tokenizer class

---
 nw/core/tokenizer.py         |  15 +-
 tests/README.md              |   3 +-
 tests/lipsum/ToC.json        |  77 -----
 tests/minimal/ToC.json       |  17 -
 tests/test_core_tokenizer.py | 645 +++++++++++++++++++++++++++++++++++
 5 files changed, 655 insertions(+), 102 deletions(-)
 delete mode 100644 tests/lipsum/ToC.json
 delete mode 100644 tests/minimal/ToC.json
 create mode 100644 tests/test_core_tokenizer.py

diff --git a/nw/core/tokenizer.py b/nw/core/tokenizer.py
index 7870493c..6039121d 100644
--- a/nw/core/tokenizer.py
+++ b/nw/core/tokenizer.py
@@ -180,7 +180,7 @@ class Tokenizer():
     ##
 
     def addRootHeading(self, theHandle):
-        """Add a heading at the start if a new root folder.
+        """Add a heading at the start of a new root folder.
         """
         theItem = self.theProject.projTree[theHandle]
         if theItem is None:
@@ -205,7 +205,7 @@ class Tokenizer():
         self.theHandle = theHandle
         self.theItem   = self.theProject.projTree[theHandle]
         if self.theItem is None:
-            return
+            return False
 
         if theText is not None:
             # If the text is set, just use that
@@ -234,7 +234,7 @@ class Tokenizer():
         self.isNote  = self.theItem.itemLayout == nwItemLayout.NOTE
         self.isNovel = self.isBook or self.isUnNum or self.isChap or self.isScene
 
-        return
+        return True
 
     def getResult(self):
         """Return the result from the conversion.
@@ -244,6 +244,8 @@ class Tokenizer():
     def getResultSize(self):
         """Return the size of the result from the conversion.
         """
+        if self.theResult is None:
+            return 0
         return len(self.theResult)
 
     def getFilteredMarkdown(self):
@@ -445,7 +447,7 @@ class Tokenizer():
         """
         # No special header formatting for notes and no-layout files
         if self.isNone or self.isNote:
-            return
+            return False
 
         # For novel files, we need to handle chapter numbering, scene
         # numbering, and scene breaks
@@ -479,8 +481,7 @@ class Tokenizer():
                     if self.isUnNum:
                         tTemp = self._formatHeading(self.fmtUnNum, tToken[2])
                     elif tToken[2].startswith("*"):
-                        tTemp = self._formatHeading(self.fmtUnNum, tToken[2])
-                        tTemp = tTemp[1:].lstrip()
+                        tTemp = self._formatHeading(self.fmtUnNum, tToken[2][1:].lstrip())
                     else:
                         self.numChapter += 1
                         tTemp = self._formatHeading(self.fmtChapter, tToken[2])
@@ -663,7 +664,7 @@ class Tokenizer():
                         self.A_LEFT
                     )
 
-        return
+        return True
 
     ##
     #  Internal Functions
diff --git a/tests/README.md b/tests/README.md
index 797fa2ea..b9c221f9 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -52,7 +52,7 @@ pytest-3 -v -m core
 
 Available markers are:
 
-* '`core`' for test covering the classes in the `nw/core` folder
+* '`core`' for unit tests covering the classes in the `nw/core` folder
 
 ## Tests
 
@@ -69,3 +69,4 @@ The commands for the respective test categories are listed below.
 | Unit | NWStatus class     | nw/core/status.py     | `-m core` | `-k testCoreStatus`   |
 | Unit | NWTree class       | nw/core/tree.py       | `-m core` | `-k testCoreTree`     |
 | Unit | OptionsState class | nw/core/options.py    | `-m core` | `-k testCoreOptions`  |
+| Unit | Tokenizer class    | nw/core/tokenizer.py  | `-m core` | `-k testCoreToken`    |
diff --git a/tests/lipsum/ToC.json b/tests/lipsum/ToC.json
deleted file mode 100644
index 4d22a21c..00000000
--- a/tests/lipsum/ToC.json
+++ /dev/null
@@ -1,77 +0,0 @@
-[
-  [
-    "content/04468803b92e1.nwd",
-    "WORLD",
-    "Ancient Europe"
-  ],
-  [
-    "content/2426c6f0ca922.nwd",
-    "PLOT",
-    "Main"
-  ],
-  [
-    "content/441420a886d82.nwd",
-    "NOVEL",
-    "Chapter Two"
-  ],
-  [
-    "content/47666c91c7ccf.nwd",
-    "NOVEL",
-    "Scene Five"
-  ],
-  [
-    "content/4c4f28287af27.nwd",
-    "CHARACTER",
-    "Mr. Nobody"
-  ],
-  [
-    "content/7a992350f3eb6.nwd",
-    "NOVEL",
-    "Lorem Ipsum"
-  ],
-  [
-    "content/846352075de7d.nwd",
-    "NOVEL",
-    "Interlude"
-  ],
-  [
-    "content/88243afbe5ed8.nwd",
-    "NOVEL",
-    "Scene One"
-  ],
-  [
-    "content/88d59a277361b.nwd",
-    "NOVEL",
-    "Prologue"
-  ],
-  [
-    "content/8c58a65414c23.nwd",
-    "NOVEL",
-    "Front Matter"
-  ],
-  [
-    "content/db7e733775d4d.nwd",
-    "NOVEL",
-    "Act One"
-  ],
-  [
-    "content/eb103bc70c90c.nwd",
-    "NOVEL",
-    "Scene Three"
-  ],
-  [
-    "content/f8c0562e50f1b.nwd",
-    "NOVEL",
-    "Scene Four"
-  ],
-  [
-    "content/f96ec11c6a3da.nwd",
-    "NOVEL",
-    "Scene Two"
-  ],
-  [
-    "content/fb609cd8319dc.nwd",
-    "NOVEL",
-    "Chapter One"
-  ]
-]
\ No newline at end of file
diff --git a/tests/minimal/ToC.json b/tests/minimal/ToC.json
deleted file mode 100644
index 5881c392..00000000
--- a/tests/minimal/ToC.json
+++ /dev/null
@@ -1,17 +0,0 @@
-[
-  [
-    "content/8c659a11cd429.nwd",
-    "NOVEL",
-    "New Scene"
-  ],
-  [
-    "content/a35baf2e93843.nwd",
-    "NOVEL",
-    "Title Page"
-  ],
-  [
-    "content/f5ab3e30151e1.nwd",
-    "NOVEL",
-    "New Chapter"
-  ]
-]
\ No newline at end of file
diff --git a/tests/test_core_tokenizer.py b/tests/test_core_tokenizer.py
new file mode 100644
index 00000000..559f3a1e
--- /dev/null
+++ b/tests/test_core_tokenizer.py
@@ -0,0 +1,645 @@
+# -*- coding: utf-8 -*-
+"""novelWriter Tokenizer Class Tester
+"""
+
+import pytest
+
+from nw.core import NWProject, NWDoc
+from nw.core.tokenizer import Tokenizer
+
+@pytest.mark.core
+def testCoreToken_Setters(dummyGUI):
+    """Test all the setters for the Tokenizer class.
+    """
+    theProject = NWProject(dummyGUI)
+    theToken = Tokenizer(dummyGUI, theProject)
+
+    # Verify defaults
+    assert theToken.fmtTitle == "%title%"
+    assert theToken.fmtChapter == "%title%"
+    assert theToken.fmtUnNum == "%title%"
+    assert theToken.fmtScene == "%title%"
+    assert theToken.fmtSection == "%title%"
+    assert theToken.hideScene is False
+    assert theToken.hideSection is False
+    assert theToken.linkHeaders is False
+    assert theToken.doBodyText is True
+    assert theToken.doSynopsis is False
+    assert theToken.doComments is False
+    assert theToken.doKeywords is False
+    assert theToken.doJustify is False
+
+    # Set new values
+    theToken.setTitleFormat("T: %title%")
+    theToken.setChapterFormat("C: %title%")
+    theToken.setUnNumberedFormat("U: %title%")
+    theToken.setSceneFormat("S: %title%", True)
+    theToken.setSectionFormat("X: %title%", True)
+    theToken.setLinkHeaders(True)
+    theToken.setBodyText(False)
+    theToken.setSynopsis(True)
+    theToken.setComments(True)
+    theToken.setKeywords(True)
+    theToken.setJustify(True)
+
+    # Check new values
+    assert theToken.fmtTitle == "T: %title%"
+    assert theToken.fmtChapter == "C: %title%"
+    assert theToken.fmtUnNum == "U: %title%"
+    assert theToken.fmtScene == "S: %title%"
+    assert theToken.fmtSection == "X: %title%"
+    assert theToken.hideScene is True
+    assert theToken.hideSection is True
+    assert theToken.linkHeaders is True
+    assert theToken.doBodyText is False
+    assert theToken.doSynopsis is True
+    assert theToken.doComments is True
+    assert theToken.doKeywords is True
+    assert theToken.doJustify is True
+
+# END Test testCoreToken_Setters
+
+@pytest.mark.core
+def testCoreToken_TextOps(monkeypatch, nwMinimal, dummyGUI):
+    """Test handling files and text in the Tokenizer class.
+    """
+    theProject = NWProject(dummyGUI)
+    theProject.projTree.setSeed(42)
+    theToken = Tokenizer(theProject, dummyGUI)
+
+    assert theProject.openProject(nwMinimal)
+    sHandle = "8c659a11cd429"
+
+    # Set some content to work with
+
+    docText = (
+        "### Scene Six\n\n"
+        "This is text with _italic text_, some **bold text**, some ~~deleted text~~, "
+        "and some **_mixed text_** and **some _nested_ text**.\n\n"
+        "#### Replace\n\n"
+        "Also, replace <A> and <B>.\n\n"
+    )
+    docTextR = docText.replace("<A>", "this").replace("<B>", "that")
+
+    nDoc = NWDoc(theProject, dummyGUI)
+    nDoc.openDocument(sHandle)
+    nDoc.saveDocument(docText)
+    nDoc.clearDocument()
+
+    theProject.setAutoReplace({"A": "this", "B": "that"})
+
+    assert theProject.saveProject()
+
+    # Root heading
+    assert theToken.addRootHeading("dummy") is False
+    assert theToken.addRootHeading(sHandle) is False
+    assert theToken.addRootHeading("7695ce551d265") is True
+    assert theToken.theMarkdown == "# Notes: Plot\n\n"
+
+    # Set text
+    assert theToken.setText("dummy") is False
+    assert theToken.setText(sHandle) is True
+    assert theToken.theText == docText
+
+    monkeypatch.setattr("nw.constants.nwConst.MAX_DOCSIZE", 100)
+    assert theToken.setText(sHandle, docText) is True
+    assert theToken.theText == (
+        "# ERROR\n\n"
+        "Document 'New Scene' is too big (0.00 MB). Skipping.\n\n"
+    )
+    monkeypatch.undo()
+
+    assert theToken.setText(sHandle, docText) is True
+    assert theToken.theText == docText
+
+    assert theToken.isNone is False
+    assert theToken.isTitle is False
+    assert theToken.isBook is False
+    assert theToken.isPage is False
+    assert theToken.isPart is False
+    assert theToken.isUnNum is False
+    assert theToken.isChap is False
+    assert theToken.isScene is True
+    assert theToken.isNote is False
+    assert theToken.isNovel is True
+
+    # Auto replace
+    theToken.doAutoReplace()
+    assert theToken.theText == docTextR
+
+    # Access
+    assert theToken.getResult() is None
+    assert theToken.getResultSize() == 0
+    theToken.theResult = ""
+    assert theToken.getResultSize() == 0
+
+    # Post Processing
+    theToken.theResult = r"This is text with escapes: \** \~~ \__"
+    theToken.doPostProcessing()
+    assert theToken.theResult == "This is text with escapes: ** ~~ __"
+
+# END Test testCoreToken_TextOps
+
+@pytest.mark.core
+def testCoreToken_Tokenize(dummyGUI):
+    """Test the tokenization of the Tokenizer class.
+    """
+    theProject = NWProject(dummyGUI)
+    theToken = Tokenizer(theProject, dummyGUI)
+
+    # Header 1
+    theToken.theText = "# Novel Title\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD1, 1, "Novel Title", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "# Novel Title\n\n"
+
+    # Header 2
+    theToken.theText = "## Chapter One\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "## Chapter One\n\n"
+
+    # Header 3
+    theToken.theText = "### Scene One\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD3, 1, "Scene One", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "### Scene One\n\n"
+
+    # Header 4
+    theToken.theText = "#### A Section\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD4, 1, "A Section", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "#### A Section\n\n"
+
+    # Comment
+    theToken.theText = "% A comment\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_COMMENT, 1, "A comment", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "\n"
+
+    theToken.setComments(True)
+    theToken.tokenizeText()
+    assert theToken.theMarkdown == "% A comment\n\n"
+
+    # Symopsis
+    theToken.theText = "%synopsis: The synopsis\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_SYNOPSIS, 1, "The synopsis", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    theToken.theText = "% synopsis: The synopsis\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_SYNOPSIS, 1, "The synopsis", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "\n"
+
+    theToken.setSynopsis(True)
+    theToken.tokenizeText()
+    assert theToken.theMarkdown == "% synopsis: The synopsis\n\n"
+
+    # Keyword
+    theToken.theText = "@char: Bod\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_KEYWORD, 1, "char: Bod", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "\n"
+
+    theToken.setKeywords(True)
+    theToken.tokenizeText()
+    assert theToken.theMarkdown == "@char: Bod\n\n"
+
+    # Text
+    theToken.theText = "Some plain text\non two lines\n\n\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_TEXT, 1, "Some plain text", [], Tokenizer.A_NONE),
+        (Tokenizer.T_TEXT, 2, "on two lines", [], Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "Some plain text\non two lines\n\n\n\n"
+
+    theToken.setBodyText(False)
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "\n\n\n"
+    theToken.setBodyText(True)
+
+    # Text Emphasis
+    theToken.theText = "Some **bolded text** on this lines\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (
+            Tokenizer.T_TEXT, 1,
+            "Some **bolded text** on this lines",
+            [
+                [5,  2, Tokenizer.FMT_B_B],
+                [18, 2, Tokenizer.FMT_B_E],
+            ],
+            Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "Some **bolded text** on this lines\n\n"
+
+    theToken.theText = "Some _italic text_ on this lines\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (
+            Tokenizer.T_TEXT, 1,
+            "Some _italic text_ on this lines",
+            [
+                [5,  1, Tokenizer.FMT_I_B],
+                [17, 1, Tokenizer.FMT_I_E],
+            ],
+            Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "Some _italic text_ on this lines\n\n"
+
+    theToken.theText = "Some **_bold italic text_** on this lines\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (
+            Tokenizer.T_TEXT, 1,
+            "Some **_bold italic text_** on this lines",
+            [
+                [5,  2, Tokenizer.FMT_B_B],
+                [7,  1, Tokenizer.FMT_I_B],
+                [24, 1, Tokenizer.FMT_I_E],
+                [25, 2, Tokenizer.FMT_B_E],
+            ],
+            Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "Some **_bold italic text_** on this lines\n\n"
+
+    theToken.theText = "Some ~~strikethrough text~~ on this lines\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (
+            Tokenizer.T_TEXT, 1,
+            "Some ~~strikethrough text~~ on this lines",
+            [
+                [5,  2, Tokenizer.FMT_D_B],
+                [25, 2, Tokenizer.FMT_D_E],
+            ],
+            Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == "Some ~~strikethrough text~~ on this lines\n\n"
+
+    theToken.theText = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
+    theToken.tokenizeText()
+    assert theToken.theTokens == [
+        (
+            Tokenizer.T_TEXT, 1,
+            "Some **nested bold and _italic_ and ~~strikethrough~~ text** here",
+            [
+                [5,  2, Tokenizer.FMT_B_B],
+                [23, 1, Tokenizer.FMT_I_B],
+                [30, 1, Tokenizer.FMT_I_E],
+                [36, 2, Tokenizer.FMT_D_B],
+                [51, 2, Tokenizer.FMT_D_E],
+                [58, 2, Tokenizer.FMT_B_E],
+            ],
+            Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+    assert theToken.theMarkdown == (
+        "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
+    )
+
+    # Check the markdown function as well
+    assert theToken.getFilteredMarkdown() == (
+        "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
+    )
+
+# END Test testCoreToken_Tokenize
+
+@pytest.mark.core
+def testCoreToken_Headers(dummyGUI):
+    """Test the header and page parser of the Tokenizer class.
+    """
+    theProject = NWProject(dummyGUI)
+    theToken = Tokenizer(theProject, dummyGUI)
+
+    # Nothing
+    theToken.theText = "Some text ...\n"
+    assert theToken.doHeaders() is True
+    theToken.isNone = True
+    assert theToken.doHeaders() is False
+    theToken.isNone = False
+    assert theToken.doHeaders() is True
+    theToken.isNote = True
+    assert theToken.doHeaders() is False
+    theToken.isNote = False
+
+    ##
+    #  Novel
+    ##
+
+    theToken.isNovel = True
+
+    # Titles
+    # ======
+
+    # H1: Title
+    theToken.theText = "# Novel Title\n"
+    theToken.setTitleFormat(r"T: %title%")
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD1, 1, "T: Novel Title", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # Chapters
+    # ========
+
+    # H2: Chapter
+    theToken.theText = "## Chapter One\n"
+    theToken.setChapterFormat(r"C: %title%")
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD2, 1, "C: Chapter One", None, Tokenizer.A_PBB),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H2: Unnumbered Chapter
+    theToken.theText = "## Chapter One\n"
+    theToken.setUnNumberedFormat(r"U: %title%")
+    theToken.isUnNum = True
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD2, 1, "U: Chapter One", None, Tokenizer.A_PBB),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H2: Unnumbered Chapter with Star
+    theToken.theText = "## *Prologue\n"
+    theToken.setUnNumberedFormat(r"U: %title%")
+    theToken.isUnNum = False
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD2, 1, "U: Prologue", None, Tokenizer.A_PBB),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H2: Chapter Word Number
+    theToken.theText = "## Chapter\n"
+    theToken.setChapterFormat(r"Chapter %chw%")
+    theToken.numChapter = 0
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H2: Chapter Roman Number Upper Case
+    theToken.theText = "## Chapter\n"
+    theToken.setChapterFormat(r"Chapter %chI%")
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD2, 1, "Chapter II", None, Tokenizer.A_PBB),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H2: Chapter Roman Number Lower Case
+    theToken.theText = "## Chapter\n"
+    theToken.setChapterFormat(r"Chapter %chi%")
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD2, 1, "Chapter iii", None, Tokenizer.A_PBB),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # Scenes
+    # ======
+
+    # H3: Scene w/Title
+    theToken.theText = "### Scene One\n"
+    theToken.setSceneFormat(r"S: %title%", False)
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD3, 1, "S: Scene One", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H3: Scene Hidden wo/Format
+    theToken.theText = "### Scene One\n"
+    theToken.setSceneFormat(r"", True)
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H3: Scene wo/Format, first
+    theToken.theText = "### Scene One\n"
+    theToken.setSceneFormat(r"", False)
+    theToken.firstScene = True
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H3: Scene wo/Format, not first
+    theToken.theText = "### Scene One\n"
+    theToken.setSceneFormat(r"", False)
+    theToken.firstScene = False
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H3: Scene Separator, first
+    theToken.theText = "### Scene One\n"
+    theToken.setSceneFormat(r"* * *", False)
+    theToken.firstScene = True
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H3: Scene Separator, not first
+    theToken.theText = "### Scene One\n"
+    theToken.setSceneFormat(r"* * *", False)
+    theToken.firstScene = False
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H3: Scene w/Absolute Number
+    theToken.theText = "### A Scene\n"
+    theToken.setSceneFormat(r"Scene %sca%", False)
+    theToken.numAbsScene = 0
+    theToken.numChScene = 0
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD3, 1, "Scene 1", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H3: Scene w/Chapter Number
+    theToken.theText = "### A Scene\n"
+    theToken.setSceneFormat(r"Scene %ch%.%sc%", False)
+    theToken.numAbsScene = 0
+    theToken.numChScene = 1
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD3, 1, "Scene 3.2", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # Sections
+    # ========
+
+    # H4: Section Hidden wo/Format
+    theToken.theText = "#### A Section\n"
+    theToken.setSectionFormat(r"", True)
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H4: Section Visible wo/Format
+    theToken.theText = "#### A Section\n"
+    theToken.setSectionFormat(r"", False)
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H4: Section w/Format
+    theToken.theText = "#### A Section\n"
+    theToken.setSectionFormat(r"X: %title%", False)
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD4, 1, "X: A Section", None, Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # H4: Section Separator
+    theToken.theText = "#### A Section\n"
+    theToken.setSectionFormat(r"* * *", False)
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
+    ]
+
+    # Check the first scene detector
+    assert theToken.firstScene is False
+    theToken.firstScene = True
+    assert theToken.firstScene is True
+    theToken.theText = "Some text ...\n"
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.firstScene is False
+
+    ##
+    #  Title or Partition
+    ##
+
+    theToken.isNovel = False
+
+    # H1: Title
+    theToken.theText = "# Novel Title\n"
+    theToken.setTitleFormat(r"T: %title%")
+    theToken.tokenizeText()
+    theToken.isTitle = True
+    theToken.isPart = False
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_TITLE, 1, "Novel Title", None, Tokenizer.A_PBB_NO | Tokenizer.A_CENTRE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_PBA | Tokenizer.A_CENTRE),
+    ]
+
+    # H1: Partition
+    theToken.theText = "# Partition Title\n"
+    theToken.setTitleFormat(r"T: %title%")
+    theToken.tokenizeText()
+    theToken.isTitle = False
+    theToken.isPart = True
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_HEAD1, 1, "Partition Title", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
+        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_PBA | Tokenizer.A_CENTRE),
+    ]
+
+    ##
+    #  Page
+    ##
+
+    theToken.isNovel = False
+    theToken.isTitle = False
+    theToken.isPart = False
+    theToken.isPage = True
+
+    # Some Page Text
+    theToken.theText = "Page text\n\nMore text\n"
+    theToken.tokenizeText()
+    theToken.doHeaders()
+    assert theToken.theTokens == [
+        (Tokenizer.T_TEXT, 1, "Page text", [], Tokenizer.A_PBB | Tokenizer.A_LEFT),
+        (Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_LEFT),
+        (Tokenizer.T_TEXT, 3, "More text", [], Tokenizer.A_LEFT),
+        (Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_LEFT),
+    ]
+
+# END Test testCoreToken_Headers