From de16d2d6a19935ebefe798745feab3c35ab79aa2 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 25 Feb 2024 22:49:12 +0100 Subject: [PATCH 1/2] Make plain text not affect scene separator insertion --- novelwriter/core/tokenizer.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 495d3e07..ecd33414 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -158,7 +158,7 @@ class Tokenizer(ABC): # Instance Variables self._hFormatter = HeadingFormatter(self._project) - self._allowSeparator = False # Flag to indicate that the first scene of the chapter + self._skipSeparator = False # Flag to indicate that we skip the scene separator # This File self._isNone = False # Document has unknown layout @@ -665,13 +665,9 @@ class Tokenizer(ABC): for n, token in enumerate(self._tokens): - if token[0] == self.T_TEXT: - # If we see text before a scene, we consider it a "scene" - self._allowSeparator = False - - elif token[0] == self.T_TITLE: # Title + if token[0] == self.T_TITLE: # Title # For new titles, we reset all counters - self._allowSeparator = True + self._skipSeparator = True self._hFormatter.resetAll() elif token[0] == self.T_HEAD1: # Partition @@ -682,10 +678,10 @@ class Tokenizer(ABC): ) # Set scene variables - self._allowSeparator = True + self._skipSeparator = True self._hFormatter.resetScene() - elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter + elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter, Unnumbered # Numbered or Unnumbered if token[0] == self.T_UNNUM: @@ -700,7 +696,7 @@ class Tokenizer(ABC): ) # Set scene variables - self._allowSeparator = True + self._skipSeparator = True self._hFormatter.resetScene() elif token[0] == self.T_HEAD3: # Scene @@ -714,21 +710,21 @@ class Tokenizer(ABC): ) elif tTemp == "" and not self._hideScene: self._tokens[n] = ( - self.T_EMPTY if self._allowSeparator else self.T_SKIP, token[1], - "", [], self.A_NONE if self._allowSeparator else token[4] + self.T_EMPTY if self._skipSeparator else self.T_SKIP, token[1], + "", [], self.A_NONE if self._skipSeparator else token[4] ) elif tTemp == self._fmtScene: self._tokens[n] = ( - self.T_EMPTY if self._allowSeparator else self.T_SEP, token[1], - "" if self._allowSeparator else tTemp, [], - self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE) + self.T_EMPTY if self._skipSeparator else self.T_SEP, token[1], + "" if self._skipSeparator else tTemp, [], + self.A_NONE if self._skipSeparator else (token[4] | self.A_CENTRE) ) else: self._tokens[n] = ( token[0], token[1], tTemp, [], token[4] ) - self._allowSeparator = False + self._skipSeparator = False elif token[0] == self.T_HEAD4: # Section From 87a1bb050e2f9d3cd29fa370f391f0bcc533aec7 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 25 Feb 2024 22:49:27 +0100 Subject: [PATCH 2/2] Update tests --- tests/test_core/test_core_tokenizer.py | 22 ++++++++++++++-------- tests/test_tools/test_tools_manuscript.py | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index d07ab411..a74e567c 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -1123,7 +1123,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._allowSeparator = True + tokens._skipSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1134,7 +1134,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._allowSeparator = False + tokens._skipSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1145,7 +1145,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._allowSeparator = True + tokens._skipSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1156,7 +1156,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._allowSeparator = False + tokens._skipSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1231,13 +1231,19 @@ def testCoreToken_ProcessHeaders(mockGUI): (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] - # Check the first scene detector - assert tokens._allowSeparator is False - tokens._allowSeparator = True + # Check the first scene detector, plain text + tokens._skipSeparator = True tokens._text = "Some text ...\n" tokens.tokenizeText() tokens.doHeaders() - assert tokens._allowSeparator is False + assert tokens._skipSeparator is True + + # Check the first scene detector, text plus scene + tokens._skipSeparator = True + tokens._text = "Some text ...\n\n### Scene\n\nText" + tokens.tokenizeText() + tokens.doHeaders() + assert tokens._skipSeparator is False # END Test testCoreToken_ProcessHeaders diff --git a/tests/test_tools/test_tools_manuscript.py b/tests/test_tools/test_tools_manuscript.py index e1615737..734642d4 100644 --- a/tests/test_tools/test_tools_manuscript.py +++ b/tests/test_tools/test_tools_manuscript.py @@ -47,7 +47,7 @@ def testManuscript_Init(monkeypatch, qtbot: QtBot, nwGUI: GuiMain, projPath: Pat buildTestProject(nwGUI, projPath) nwGUI.openProject(projPath) SHARED.project.storage.getDocument(C.hChapterDoc).writeDocument("## A Chapter\n\n\t\tHi") - allText = "New Novel\nBy Jane Doe\nA Chapter\n\t\tHi\n* * *" + allText = "New Novel\nBy Jane Doe\nA Chapter\n\t\tHi" nwGUI.mainMenu.aBuildManuscript.activate(QAction.Trigger) qtbot.waitUntil(lambda: SHARED.findTopLevelWidget(GuiManuscript) is not None, timeout=1000)