Fix scene separators and chapter and scene counters (#1711)

novelwriter/core/tokenizer.py

@@ -89,7 +89,7 @@ class Tokenizer(ABC):
     T_UNNUM    = 7   # Unnumbered
     T_HEAD1    = 8   # Header 1
     T_HEAD2    = 9   # Header 2
-    T_HEAD3    = 10   # Header 3
+    T_HEAD3    = 10  # Header 3
     T_HEAD4    = 11  # Header 4
     T_TEXT     = 12  # Text line
     T_SEP      = 13  # Scene separator
@@ -158,7 +158,7 @@ class Tokenizer(ABC):
 
         # Instance Variables
         self._hFormatter = HeadingFormatter(self._project)
-        self._firstScene = False  # Flag to indicate that the first scene of the chapter
+        self._allowSeparator = False  # Flag to indicate that the first scene of the chapter
 
         # This File
         self._isNone  = False  # Document has unknown layout
@@ -615,9 +615,9 @@ class Tokenizer(ABC):
             # Make sure the token array doesn't start with a page break
             # on the very first page, adding a blank first page.
             if self._tokens[0][4] & self.A_PBB:
-                tToken = self._tokens[0]
+                token = self._tokens[0]
                 self._tokens[0] = (
-                    tToken[0], tToken[1], tToken[2], tToken[3], tToken[4] & ~self.A_PBB
+                    token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
                 )
 
         # Always add an empty line at the end of the file
@@ -637,22 +637,20 @@ class Tokenizer(ABC):
         pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
         nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
         tCount = len(self._tokens)
-        for n, tToken in enumerate(self._tokens):
+        for n, token in enumerate(self._tokens):
 
             if n > 0:
                 pToken = self._tokens[n-1]
             if n < tCount - 1:
                 nToken = self._tokens[n+1]
 
-            if tToken[0] == self.T_KEYWORD:
+            if token[0] == self.T_KEYWORD:
-                aStyle = tToken[4]
+                aStyle = token[4]
                 if pToken[0] == self.T_KEYWORD:
                     aStyle |= self.A_Z_TOPMRG
                 if nToken[0] == self.T_KEYWORD:
                     aStyle |= self.A_Z_BTMMRG
-                self._tokens[n] = (
+                self._tokens[n] = (token[0], token[1], token[2], token[3], aStyle)
-                    tToken[0], tToken[1], tToken[2], tToken[3], aStyle
-                )
 
         return
 
@@ -665,93 +663,91 @@ class Tokenizer(ABC):
 
         self._hFormatter.setHandle(self._nwItem.itemHandle if self._nwItem else None)
 
-        for n, tToken in enumerate(self._tokens):
+        for n, token in enumerate(self._tokens):
 
-            # In case we see text before a scene, we reset the flag
+            if token[0] == self.T_TEXT:
-            if tToken[0] == self.T_TEXT:
+                # If we see text before a scene, we consider it a "scene"
-                self._firstScene = False
+                self._allowSeparator = False
 
-            elif tToken[0] == self.T_HEAD1:
+            elif token[0] == self.T_TITLE:  # Title
-                # Partition
+                # For new titles, we reset all counters
+                self._allowSeparator = True
+                self._hFormatter.resetAll()
 
-                tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1])
+            elif token[0] == self.T_HEAD1:  # Partition
+
+                tTemp = self._hFormatter.apply(self._fmtTitle, token[2], token[1])
                 self._tokens[n] = (
-                    tToken[0], tToken[1], tTemp, [], tToken[4]
+                    token[0], token[1], tTemp, [], token[4]
-                )
-
-            elif tToken[0] in (self.T_HEAD2, self.T_UNNUM):
-                # Chapter
-
-                # Numbered or Unnumbered
-                if tToken[0] == self.T_UNNUM:
-                    tTemp = self._hFormatter.apply(self._fmtUnNum, tToken[2], tToken[1])
-                else:
-                    self._hFormatter.incChapter()
-                    tTemp = self._hFormatter.apply(self._fmtChapter, tToken[2], tToken[1])
-
-                # Format the chapter header
-                self._tokens[n] = (
-                    tToken[0], tToken[1], tTemp, [], tToken[4]
                 )
 
                 # Set scene variables
-                self._firstScene = True
+                self._allowSeparator = True
                 self._hFormatter.resetScene()
 
-            elif tToken[0] == self.T_HEAD3:
+            elif token[0] in (self.T_HEAD2, self.T_UNNUM):  # Chapter
-                # Scene
+
+                # Numbered or Unnumbered
+                if token[0] == self.T_UNNUM:
+                    tTemp = self._hFormatter.apply(self._fmtUnNum, token[2], token[1])
+                else:
+                    self._hFormatter.incChapter()
+                    tTemp = self._hFormatter.apply(self._fmtChapter, token[2], token[1])
+
+                # Format the chapter header
+                self._tokens[n] = (
+                    token[0], token[1], tTemp, [], token[4]
+                )
+
+                # Set scene variables
+                self._allowSeparator = True
+                self._hFormatter.resetScene()
+
+            elif token[0] == self.T_HEAD3:  # Scene
 
                 self._hFormatter.incScene()
 
-                tTemp = self._hFormatter.apply(self._fmtScene, tToken[2], tToken[1])
+                tTemp = self._hFormatter.apply(self._fmtScene, token[2], token[1])
                 if tTemp == "" and self._hideScene:
                     self._tokens[n] = (
-                        self.T_EMPTY, tToken[1], "", [], self.A_NONE
+                        self.T_EMPTY, token[1], "", [], self.A_NONE
                     )
                 elif tTemp == "" and not self._hideScene:
-                    if self._firstScene:
+                    self._tokens[n] = (
-                        self._tokens[n] = (
+                        self.T_EMPTY if self._allowSeparator else self.T_SKIP, token[1],
-                            self.T_EMPTY, tToken[1], "", [], self.A_NONE
+                        "", [], self.A_NONE if self._allowSeparator else token[4]
-                        )
+                    )
-                    else:
-                        self._tokens[n] = (
-                            self.T_SKIP, tToken[1], "", [], tToken[4]
-                        )
                 elif tTemp == self._fmtScene:
-                    if self._firstScene:
+                    self._tokens[n] = (
-                        self._tokens[n] = (
+                        self.T_EMPTY if self._allowSeparator else self.T_SEP, token[1],
-                            self.T_EMPTY, tToken[1], "", [], self.A_NONE
+                        "" if self._allowSeparator else tTemp, [],
-                        )
+                        self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE)
-                    else:
+                    )
-                        self._tokens[n] = (
-                            self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE
-                        )
                 else:
                     self._tokens[n] = (
-                        tToken[0], tToken[1], tTemp, [], tToken[4]
+                        token[0], token[1], tTemp, [], token[4]
                     )
 
-                self._firstScene = False
+                self._allowSeparator = False
 
-            elif tToken[0] == self.T_HEAD4:
+            elif token[0] == self.T_HEAD4:  # Section
-                # Section
 
-                tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1])
+                tTemp = self._hFormatter.apply(self._fmtSection, token[2], token[1])
                 if tTemp == "" and self._hideSection:
                     self._tokens[n] = (
-                        self.T_EMPTY, tToken[1], "", [], self.A_NONE
+                        self.T_EMPTY, token[1], "", [], self.A_NONE
                     )
                 elif tTemp == "" and not self._hideSection:
                     self._tokens[n] = (
-                        self.T_SKIP, tToken[1], "", [], tToken[4]
+                        self.T_SKIP, token[1], "", [], token[4]
                     )
                 elif tTemp == self._fmtSection:
                     self._tokens[n] = (
-                        self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE
+                        self.T_SEP, token[1], tTemp, [], token[4] | self.A_CENTRE
                     )
                 else:
                     self._tokens[n] = (
-                        tToken[0], tToken[1], tTemp, [], tToken[4]
+                        token[0], token[1], tTemp, [], token[4]
                     )
 
         return True
@@ -849,6 +845,13 @@ class HeadingFormatter:
         self._scAbsCount += 1
         return
 
+    def resetAll(self) -> None:
+        """Reset all counters."""
+        self._chCount = 0
+        self._scChCount = 0
+        self._scAbsCount = 0
+        return
+
     def resetScene(self) -> None:
         """Reset the chapter scene counter."""
         self._scChCount = 0

tests/test_core/test_core_tokenizer.py

@@ -26,6 +26,7 @@ import pytest
 from tools import C, buildTestProject, readFile
 
 from novelwriter.constants import nwHeadFmt
+from novelwriter.core.tomd import ToMarkdown
 from novelwriter.core.project import NWProject
 from novelwriter.core.tokenizer import HeadingFormatter, Tokenizer, stripEscape
 
@@ -1010,7 +1011,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     tokens._isNote = False
 
     ##
-    #  Story FIles
+    #  Story Files
     ##
 
     tokens._isNone  = False
@@ -1122,7 +1123,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._firstScene = True
+    tokens._allowSeparator = True
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1133,7 +1134,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._firstScene = False
+    tokens._allowSeparator = False
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1144,7 +1145,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._firstScene = True
+    tokens._allowSeparator = True
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1155,7 +1156,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._firstScene = False
+    tokens._allowSeparator = False
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1192,7 +1193,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
 
     # H4: Section Hidden wo/Format
     tokens._text = "#### A Section\n"
-    tokens.setSectionFormat(r"", True)
+    tokens.setSectionFormat("", True)
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1231,16 +1232,203 @@ def testCoreToken_ProcessHeaders(mockGUI):
     ]
 
     # Check the first scene detector
-    assert tokens._firstScene is False
+    assert tokens._allowSeparator is False
-    tokens._firstScene = True
+    tokens._allowSeparator = True
     tokens._text = "Some text ...\n"
     tokens.tokenizeText()
     tokens.doHeaders()
-    assert tokens._firstScene is False
+    assert tokens._allowSeparator is False
 
 # END Test testCoreToken_ProcessHeaders
 
 
+@pytest.mark.core
+def testCoreToken_HeaderCounterAndVisibility(mockGUI):
+    """Test the header counter and visibility of the Tokenizer class.
+    This is a special test to cover issue #1704.
+    """
+    project = NWProject()
+    project.data.setLanguage("en")
+    project._loadProjectLocalisation()
+    md = ToMarkdown(project)
+    md._isNone = False
+    md._isNote = False
+    md._isNovel = True
+
+    # Separator Handling, Titles
+    # ==========================
+
+    md._text = (
+        "# Title One\n\n"
+        "### Scene One\n\n"
+        "Text\n\n"
+        "### Scene Two\n\n"
+        "Text\n\n"
+        "# Title Two\n\n"
+        "### Scene Three\n\n"
+        "Text\n\n"
+        "### Scene Four\n\n"
+        "Text\n\n"
+    )
+    md.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
+    md.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
+    md.setSectionFormat("", True)
+
+    # Static Separator
+    md.setSceneFormat("* * *", False)
+    md.tokenizeText()
+    md.doHeaders()
+    md.doConvert()
+    assert md.result == (
+        "# T: Title One\n\n"
+        "Text\n\n"
+        "* * *\n\n"
+        "Text\n\n"
+        "# T: Title Two\n\n"
+        "Text\n\n"
+        "* * *\n\n"
+        "Text\n\n"
+    )
+
+    # Scene Title Formatted
+    md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
+    md.tokenizeText()
+    md.doHeaders()
+    md.doConvert()
+    assert md.result == (
+        "# T: Title One\n\n"
+        "### S: Scene One\n\n"
+        "Text\n\n"
+        "### S: Scene Two\n\n"
+        "Text\n\n"
+        "# T: Title Two\n\n"
+        "### S: Scene Three\n\n"
+        "Text\n\n"
+        "### S: Scene Four\n\n"
+        "Text\n\n"
+    )
+
+    # Separator Handling, Chapters
+    # ============================
+
+    md._text = (
+        "# Title One\n\n"
+        "## Chapter One\n\n"
+        "### Scene One\n\n"
+        "Text\n\n"
+        "### Scene Two\n\n"
+        "Text\n\n"
+        "## Chapter Two\n\n"
+        "### Scene Three\n\n"
+        "Text\n\n"
+        "### Scene Four\n\n"
+        "Text\n\n"
+    )
+    md.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
+    md.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
+    md.setSectionFormat("", True)
+
+    # Static Separator
+    md.setSceneFormat("* * *", False)
+    md.tokenizeText()
+    md.doHeaders()
+    md.doConvert()
+    assert md.result == (
+        "# T: Title One\n\n"
+        "## C: Chapter One\n\n"
+        "Text\n\n"
+        "* * *\n\n"
+        "Text\n\n"
+        "## C: Chapter Two\n\n"
+        "Text\n\n"
+        "* * *\n\n"
+        "Text\n\n"
+    )
+
+    # Scene Title Formatted
+    md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
+    md.tokenizeText()
+    md.doHeaders()
+    md.doConvert()
+    assert md.result == (
+        "# T: Title One\n\n"
+        "## C: Chapter One\n\n"
+        "### S: Scene One\n\n"
+        "Text\n\n"
+        "### S: Scene Two\n\n"
+        "Text\n\n"
+        "## C: Chapter Two\n\n"
+        "### S: Scene Three\n\n"
+        "Text\n\n"
+        "### S: Scene Four\n\n"
+        "Text\n\n"
+    )
+
+    # Counter Handling, Novel Titles
+    # ==============================
+
+    md._text = (
+        "#! Novel One\n\n"
+        "## Chapter One\n\n"
+        "### Scene One\n\n"
+        "Text\n\n"
+        "### Scene Two\n\n"
+        "Text\n\n"
+        "## Chapter Two\n\n"
+        "### Scene Three\n\n"
+        "Text\n\n"
+        "### Scene Four\n\n"
+        "Text\n\n"
+        "#! Novel Two\n\n"
+        "## Chapter One\n\n"
+        "### Scene One\n\n"
+        "Text\n\n"
+        "### Scene Two\n\n"
+        "Text\n\n"
+        "## Chapter Two\n\n"
+        "### Scene Three\n\n"
+        "Text\n\n"
+        "### Scene Four\n\n"
+        "Text\n\n"
+    )
+    md.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
+    md.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}")
+    md.setSceneFormat(f"S {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM} ({nwHeadFmt.SC_ABS}): "
+                      f"{nwHeadFmt.TITLE}", False)
+    md.setSectionFormat("", True)
+
+    # Two Novel Format
+    md.tokenizeText()
+    md.doHeaders()
+    md.doConvert()
+    assert md.result == (
+        "# Novel One\n\n"
+        "## C 1: Chapter One\n\n"
+        "### S 1.1 (1): Scene One\n\n"
+        "Text\n\n"
+        "### S 1.2 (2): Scene Two\n\n"
+        "Text\n\n"
+        "## C 2: Chapter Two\n\n"
+        "### S 2.1 (3): Scene Three\n\n"
+        "Text\n\n"
+        "### S 2.2 (4): Scene Four\n\n"
+        "Text\n\n"
+        "# Novel Two\n\n"
+        "## C 1: Chapter One\n\n"
+        "### S 1.1 (1): Scene One\n\n"
+        "Text\n\n"
+        "### S 1.2 (2): Scene Two\n\n"
+        "Text\n\n"
+        "## C 2: Chapter Two\n\n"
+        "### S 2.1 (3): Scene Three\n\n"
+        "Text\n\n"
+        "### S 2.2 (4): Scene Four\n\n"
+        "Text\n\n"
+    )
+
+# END Test testCoreToken_HeaderCounterAndVisibility
+
+
 @pytest.mark.core
 def testCoreIndex_HeadingFormatter(fncPath, mockRnd):
     """Check the HeadingFormatter class."""
@@ -1306,6 +1494,12 @@ def testCoreIndex_HeadingFormatter(fncPath, mockRnd):
     formatter.incScene()
     assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 2.1 - Scene 5 - Hi Bob"
 
+    # New Main Title
+    formatter.resetAll()
+    formatter.incChapter()
+    formatter.incScene()
+    assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.1 - Scene 1 - Hi Bob"
+
     # Special Formats
     # ===============
     formatter._chCount = 2