diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 22bf0e85..dd9ff0a6 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -160,7 +160,7 @@ class Tokenizer(ABC): # Instance Variables self._hFormatter = HeadingFormatter(self._project) - self._allowSeparator = False # Flag to indicate that the first scene of the chapter + self._skipSeparator = False # Flag to indicate that we skip the scene separator # This File self._isNone = False # Document has unknown layout @@ -668,13 +668,9 @@ class Tokenizer(ABC): for n, token in enumerate(self._tokens): - if token[0] == self.T_TEXT: - # If we see text before a scene, we consider it a "scene" - self._allowSeparator = False - - elif token[0] == self.T_TITLE: # Title + if token[0] == self.T_TITLE: # Title # For new titles, we reset all counters - self._allowSeparator = True + self._skipSeparator = True self._hFormatter.resetAll() elif token[0] == self.T_HEAD1: # Partition @@ -685,10 +681,10 @@ class Tokenizer(ABC): ) # Set scene variables - self._allowSeparator = True + self._skipSeparator = True self._hFormatter.resetScene() - elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter + elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter, Unnumbered # Numbered or Unnumbered if token[0] == self.T_UNNUM: @@ -703,7 +699,7 @@ class Tokenizer(ABC): ) # Set scene variables - self._allowSeparator = True + self._skipSeparator = True self._hFormatter.resetScene() elif token[0] == self.T_HEAD3: # Scene @@ -717,21 +713,21 @@ class Tokenizer(ABC): ) elif tTemp == "" and not self._hideScene: self._tokens[n] = ( - self.T_EMPTY if self._allowSeparator else self.T_SKIP, token[1], - "", [], self.A_NONE if self._allowSeparator else token[4] + self.T_EMPTY if self._skipSeparator else self.T_SKIP, token[1], + "", [], self.A_NONE if self._skipSeparator else token[4] ) elif tTemp == self._fmtScene: self._tokens[n] = ( - self.T_EMPTY if self._allowSeparator else self.T_SEP, token[1], - "" if self._allowSeparator else tTemp, [], - self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE) + self.T_EMPTY if self._skipSeparator else self.T_SEP, token[1], + "" if self._skipSeparator else tTemp, [], + self.A_NONE if self._skipSeparator else (token[4] | self.A_CENTRE) ) else: self._tokens[n] = ( token[0], token[1], tTemp, [], token[4] ) - self._allowSeparator = False + self._skipSeparator = False elif token[0] == self.T_HEAD4: # Section diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index d07ab411..a74e567c 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -1123,7 +1123,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._allowSeparator = True + tokens._skipSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1134,7 +1134,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._allowSeparator = False + tokens._skipSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1145,7 +1145,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._allowSeparator = True + tokens._skipSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1156,7 +1156,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._allowSeparator = False + tokens._skipSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1231,13 +1231,19 @@ def testCoreToken_ProcessHeaders(mockGUI): (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] - # Check the first scene detector - assert tokens._allowSeparator is False - tokens._allowSeparator = True + # Check the first scene detector, plain text + tokens._skipSeparator = True tokens._text = "Some text ...\n" tokens.tokenizeText() tokens.doHeaders() - assert tokens._allowSeparator is False + assert tokens._skipSeparator is True + + # Check the first scene detector, text plus scene + tokens._skipSeparator = True + tokens._text = "Some text ...\n\n### Scene\n\nText" + tokens.tokenizeText() + tokens.doHeaders() + assert tokens._skipSeparator is False # END Test testCoreToken_ProcessHeaders diff --git a/tests/test_tools/test_tools_manuscript.py b/tests/test_tools/test_tools_manuscript.py index e1615737..734642d4 100644 --- a/tests/test_tools/test_tools_manuscript.py +++ b/tests/test_tools/test_tools_manuscript.py @@ -47,7 +47,7 @@ def testManuscript_Init(monkeypatch, qtbot: QtBot, nwGUI: GuiMain, projPath: Pat buildTestProject(nwGUI, projPath) nwGUI.openProject(projPath) SHARED.project.storage.getDocument(C.hChapterDoc).writeDocument("## A Chapter\n\n\t\tHi") - allText = "New Novel\nBy Jane Doe\nA Chapter\n\t\tHi\n* * *" + allText = "New Novel\nBy Jane Doe\nA Chapter\n\t\tHi" nwGUI.mainMenu.aBuildManuscript.activate(QAction.Trigger) qtbot.waitUntil(lambda: SHARED.findTopLevelWidget(GuiManuscript) is not None, timeout=1000)