Make sure scene and chapter info is properly reset on headers in the tokenizer

novelwriter/core/tokenizer.py

@@ -158,7 +158,7 @@ class Tokenizer(ABC):
 
         # Instance Variables
         self._hFormatter = HeadingFormatter(self._project)
-        self._firstScene = False  # Flag to indicate that the first scene of the chapter
+        self._allowSeparator = False  # Flag to indicate that the first scene of the chapter
 
         # This File
         self._isNone  = False  # Document has unknown layout
@@ -667,9 +667,14 @@ class Tokenizer(ABC):
 
         for n, tToken in enumerate(self._tokens):
 
-            # In case we see text before a scene, we reset the flag
             if tToken[0] == self.T_TEXT:
-                self._firstScene = False
+                # If we see text before a scene, we consider it a "scene"
+                self._allowSeparator = False
+
+            elif tToken[0] == self.T_TITLE:  # Title
+                # For titles, we reset all counters
+                self._allowSeparator = True
+                self._hFormatter.resetAll()
 
             elif tToken[0] == self.T_HEAD1:  # Partition
 
@@ -679,8 +684,8 @@ class Tokenizer(ABC):
                 )
 
                 # Set scene variables
-                # self._firstScene = True
-                # self._hFormatter.resetScene()
+                self._allowSeparator = True
+                self._hFormatter.resetScene()
 
             elif tToken[0] in (self.T_HEAD2, self.T_UNNUM):  # Chapter
 
@@ -697,7 +702,7 @@ class Tokenizer(ABC):
                 )
 
                 # Set scene variables
-                self._firstScene = True
+                self._allowSeparator = True
                 self._hFormatter.resetScene()
 
             elif tToken[0] == self.T_HEAD3:  # Scene
@@ -710,20 +715,20 @@ class Tokenizer(ABC):
                         self.T_EMPTY, tToken[1], "", [], self.A_NONE
                     )
                 elif tTemp == "" and not self._hideScene:
-                    t1 = self.T_EMPTY if self._firstScene else self.T_SKIP
-                    t4 = self.A_NONE if self._firstScene else tToken[4]
+                    t1 = self.T_EMPTY if self._allowSeparator else self.T_SKIP
+                    t4 = self.A_NONE if self._allowSeparator else tToken[4]
                     self._tokens[n] = (t1, tToken[1], "", [], t4)
                 elif tTemp == self._fmtScene:
-                    t1 = self.T_EMPTY if self._firstScene else self.T_SEP
-                    t2 = "" if self._firstScene else tTemp
-                    t4 = self.A_NONE if self._firstScene else (tToken[4] | self.A_CENTRE)
+                    t1 = self.T_EMPTY if self._allowSeparator else self.T_SEP
+                    t2 = "" if self._allowSeparator else tTemp
+                    t4 = self.A_NONE if self._allowSeparator else (tToken[4] | self.A_CENTRE)
                     self._tokens[n] = (t1, tToken[1], t2, [], t4)
                 else:
                     self._tokens[n] = (
                         tToken[0], tToken[1], tTemp, [], tToken[4]
                     )
 
-                self._firstScene = False
+                self._allowSeparator = False
 
             elif tToken[0] == self.T_HEAD4:  # Section
 
@@ -840,6 +845,13 @@ class HeadingFormatter:
         self._scAbsCount += 1
         return
 
+    def resetAll(self) -> None:
+        """Reset all counters."""
+        self._chCount = 0
+        self._scChCount = 0
+        self._scAbsCount = 0
+        return
+
     def resetScene(self) -> None:
         """Reset the chapter scene counter."""
         self._scChCount = 0

tests/test_core/test_core_tokenizer.py

@@ -1122,7 +1122,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._firstScene = True
+    tokens._allowSeparator = True
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1133,7 +1133,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._firstScene = False
+    tokens._allowSeparator = False
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1144,7 +1144,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._firstScene = True
+    tokens._allowSeparator = True
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1155,7 +1155,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._firstScene = False
+    tokens._allowSeparator = False
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1231,12 +1231,12 @@ def testCoreToken_ProcessHeaders(mockGUI):
     ]
 
     # Check the first scene detector
-    assert tokens._firstScene is False
-    tokens._firstScene = True
+    assert tokens._allowSeparator is False
+    tokens._allowSeparator = True
     tokens._text = "Some text ...\n"
     tokens.tokenizeText()
     tokens.doHeaders()
-    assert tokens._firstScene is False
+    assert tokens._allowSeparator is False
 
 # END Test testCoreToken_ProcessHeaders