Merge branch 'dev' into mark_text

novelwriter/core/tokenizer.py

@@ -160,7 +160,7 @@ class Tokenizer(ABC):
 
         # Instance Variables
         self._hFormatter = HeadingFormatter(self._project)
-        self._allowSeparator = False  # Flag to indicate that the first scene of the chapter
+        self._skipSeparator = False  # Flag to indicate that we skip the scene separator
 
         # This File
         self._isNone  = False  # Document has unknown layout
@@ -668,13 +668,9 @@ class Tokenizer(ABC):
 
         for n, token in enumerate(self._tokens):
 
-            if token[0] == self.T_TEXT:
+            if token[0] == self.T_TITLE:  # Title
-                # If we see text before a scene, we consider it a "scene"
-                self._allowSeparator = False
-
-            elif token[0] == self.T_TITLE:  # Title
                 # For new titles, we reset all counters
-                self._allowSeparator = True
+                self._skipSeparator = True
                 self._hFormatter.resetAll()
 
             elif token[0] == self.T_HEAD1:  # Partition
@@ -685,10 +681,10 @@ class Tokenizer(ABC):
                 )
 
                 # Set scene variables
-                self._allowSeparator = True
+                self._skipSeparator = True
                 self._hFormatter.resetScene()
 
-            elif token[0] in (self.T_HEAD2, self.T_UNNUM):  # Chapter
+            elif token[0] in (self.T_HEAD2, self.T_UNNUM):  # Chapter, Unnumbered
 
                 # Numbered or Unnumbered
                 if token[0] == self.T_UNNUM:
@@ -703,7 +699,7 @@ class Tokenizer(ABC):
                 )
 
                 # Set scene variables
-                self._allowSeparator = True
+                self._skipSeparator = True
                 self._hFormatter.resetScene()
 
             elif token[0] == self.T_HEAD3:  # Scene
@@ -717,21 +713,21 @@ class Tokenizer(ABC):
                     )
                 elif tTemp == "" and not self._hideScene:
                     self._tokens[n] = (
-                        self.T_EMPTY if self._allowSeparator else self.T_SKIP, token[1],
+                        self.T_EMPTY if self._skipSeparator else self.T_SKIP, token[1],
-                        "", [], self.A_NONE if self._allowSeparator else token[4]
+                        "", [], self.A_NONE if self._skipSeparator else token[4]
                     )
                 elif tTemp == self._fmtScene:
                     self._tokens[n] = (
-                        self.T_EMPTY if self._allowSeparator else self.T_SEP, token[1],
+                        self.T_EMPTY if self._skipSeparator else self.T_SEP, token[1],
-                        "" if self._allowSeparator else tTemp, [],
+                        "" if self._skipSeparator else tTemp, [],
-                        self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE)
+                        self.A_NONE if self._skipSeparator else (token[4] | self.A_CENTRE)
                     )
                 else:
                     self._tokens[n] = (
                         token[0], token[1], tTemp, [], token[4]
                     )
 
-                self._allowSeparator = False
+                self._skipSeparator = False
 
             elif token[0] == self.T_HEAD4:  # Section
 

tests/test_core/test_core_tokenizer.py

@@ -1123,7 +1123,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._allowSeparator = True
+    tokens._skipSeparator = True
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1134,7 +1134,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene wo/Format, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("", False)
-    tokens._allowSeparator = False
+    tokens._skipSeparator = False
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1145,7 +1145,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._allowSeparator = True
+    tokens._skipSeparator = True
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1156,7 +1156,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
     # H3: Scene Separator, not first
     tokens._text = "### Scene One\n"
     tokens.setSceneFormat("* * *", False)
-    tokens._allowSeparator = False
+    tokens._skipSeparator = False
     tokens.tokenizeText()
     tokens.doHeaders()
     assert tokens._tokens == [
@@ -1231,13 +1231,19 @@ def testCoreToken_ProcessHeaders(mockGUI):
         (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
     ]
 
-    # Check the first scene detector
+    # Check the first scene detector, plain text
-    assert tokens._allowSeparator is False
+    tokens._skipSeparator = True
-    tokens._allowSeparator = True
     tokens._text = "Some text ...\n"
     tokens.tokenizeText()
     tokens.doHeaders()
-    assert tokens._allowSeparator is False
+    assert tokens._skipSeparator is True
+
+    # Check the first scene detector, text plus scene
+    tokens._skipSeparator = True
+    tokens._text = "Some text ...\n\n### Scene\n\nText"
+    tokens.tokenizeText()
+    tokens.doHeaders()
+    assert tokens._skipSeparator is False
 
 # END Test testCoreToken_ProcessHeaders
 

tests/test_tools/test_tools_manuscript.py

@@ -47,7 +47,7 @@ def testManuscript_Init(monkeypatch, qtbot: QtBot, nwGUI: GuiMain, projPath: Pat
     buildTestProject(nwGUI, projPath)
     nwGUI.openProject(projPath)
     SHARED.project.storage.getDocument(C.hChapterDoc).writeDocument("## A Chapter\n\n\t\tHi")
-    allText = "New Novel\nBy Jane Doe\nA Chapter\n\t\tHi\n* * *"
+    allText = "New Novel\nBy Jane Doe\nA Chapter\n\t\tHi"
 
     nwGUI.mainMenu.aBuildManuscript.activate(QAction.Trigger)
     qtbot.waitUntil(lambda: SHARED.findTopLevelWidget(GuiManuscript) is not None, timeout=1000)