From eb105951ac3ffb0f55c4e64065f303e5959add52 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:28:12 +0100 Subject: [PATCH 1/5] Clean up the tokenizer code a little --- novelwriter/core/tokenizer.py | 39 ++++++++++++++--------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 90f771f8..d9f5de4f 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -671,16 +671,18 @@ class Tokenizer(ABC): if tToken[0] == self.T_TEXT: self._firstScene = False - elif tToken[0] == self.T_HEAD1: - # Partition + elif tToken[0] == self.T_HEAD1: # Partition tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1]) self._tokens[n] = ( tToken[0], tToken[1], tTemp, [], tToken[4] ) - elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): - # Chapter + # Set scene variables + # self._firstScene = True + # self._hFormatter.resetScene() + + elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter # Numbered or Unnumbered if tToken[0] == self.T_UNNUM: @@ -698,8 +700,7 @@ class Tokenizer(ABC): self._firstScene = True self._hFormatter.resetScene() - elif tToken[0] == self.T_HEAD3: - # Scene + elif tToken[0] == self.T_HEAD3: # Scene self._hFormatter.incScene() @@ -709,23 +710,14 @@ class Tokenizer(ABC): self.T_EMPTY, tToken[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideScene: - if self._firstScene: - self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE - ) - else: - self._tokens[n] = ( - self.T_SKIP, tToken[1], "", [], tToken[4] - ) + t1 = self.T_EMPTY if self._firstScene else self.T_SKIP + t4 = self.A_NONE if self._firstScene else tToken[4] + self._tokens[n] = (t1, tToken[1], "", [], t4) elif tTemp == self._fmtScene: - if self._firstScene: - self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE - ) - else: - self._tokens[n] = ( - self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE - ) + t1 = self.T_EMPTY if self._firstScene else self.T_SEP + t2 = "" if self._firstScene else tTemp + t4 = self.A_NONE if self._firstScene else (tToken[4] | self.A_CENTRE) + self._tokens[n] = (t1, tToken[1], t2, [], t4) else: self._tokens[n] = ( tToken[0], tToken[1], tTemp, [], tToken[4] @@ -733,8 +725,7 @@ class Tokenizer(ABC): self._firstScene = False - elif tToken[0] == self.T_HEAD4: - # Section + elif tToken[0] == self.T_HEAD4: # Section tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1]) if tTemp == "" and self._hideSection: From ef845881457519064f4cc074c4be0ff867fe1ae6 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:29:13 +0100 Subject: [PATCH 2/5] Make sure scene and chapter info is properly reset on headers in the tokenizer --- novelwriter/core/tokenizer.py | 36 +++++++++++++++++--------- tests/test_core/test_core_tokenizer.py | 14 +++++----- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index d9f5de4f..fb758fbc 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -158,7 +158,7 @@ class Tokenizer(ABC): # Instance Variables self._hFormatter = HeadingFormatter(self._project) - self._firstScene = False # Flag to indicate that the first scene of the chapter + self._allowSeparator = False # Flag to indicate that the first scene of the chapter # This File self._isNone = False # Document has unknown layout @@ -667,9 +667,14 @@ class Tokenizer(ABC): for n, tToken in enumerate(self._tokens): - # In case we see text before a scene, we reset the flag if tToken[0] == self.T_TEXT: - self._firstScene = False + # If we see text before a scene, we consider it a "scene" + self._allowSeparator = False + + elif tToken[0] == self.T_TITLE: # Title + # For titles, we reset all counters + self._allowSeparator = True + self._hFormatter.resetAll() elif tToken[0] == self.T_HEAD1: # Partition @@ -679,8 +684,8 @@ class Tokenizer(ABC): ) # Set scene variables - # self._firstScene = True - # self._hFormatter.resetScene() + self._allowSeparator = True + self._hFormatter.resetScene() elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter @@ -697,7 +702,7 @@ class Tokenizer(ABC): ) # Set scene variables - self._firstScene = True + self._allowSeparator = True self._hFormatter.resetScene() elif tToken[0] == self.T_HEAD3: # Scene @@ -710,20 +715,20 @@ class Tokenizer(ABC): self.T_EMPTY, tToken[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideScene: - t1 = self.T_EMPTY if self._firstScene else self.T_SKIP - t4 = self.A_NONE if self._firstScene else tToken[4] + t1 = self.T_EMPTY if self._allowSeparator else self.T_SKIP + t4 = self.A_NONE if self._allowSeparator else tToken[4] self._tokens[n] = (t1, tToken[1], "", [], t4) elif tTemp == self._fmtScene: - t1 = self.T_EMPTY if self._firstScene else self.T_SEP - t2 = "" if self._firstScene else tTemp - t4 = self.A_NONE if self._firstScene else (tToken[4] | self.A_CENTRE) + t1 = self.T_EMPTY if self._allowSeparator else self.T_SEP + t2 = "" if self._allowSeparator else tTemp + t4 = self.A_NONE if self._allowSeparator else (tToken[4] | self.A_CENTRE) self._tokens[n] = (t1, tToken[1], t2, [], t4) else: self._tokens[n] = ( tToken[0], tToken[1], tTemp, [], tToken[4] ) - self._firstScene = False + self._allowSeparator = False elif tToken[0] == self.T_HEAD4: # Section @@ -840,6 +845,13 @@ class HeadingFormatter: self._scAbsCount += 1 return + def resetAll(self) -> None: + """Reset all counters.""" + self._chCount = 0 + self._scChCount = 0 + self._scAbsCount = 0 + return + def resetScene(self) -> None: """Reset the chapter scene counter.""" self._scChCount = 0 diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 163cd151..9bf12b1a 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -1122,7 +1122,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._firstScene = True + tokens._allowSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1133,7 +1133,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._firstScene = False + tokens._allowSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1144,7 +1144,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._firstScene = True + tokens._allowSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1155,7 +1155,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._firstScene = False + tokens._allowSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1231,12 +1231,12 @@ def testCoreToken_ProcessHeaders(mockGUI): ] # Check the first scene detector - assert tokens._firstScene is False - tokens._firstScene = True + assert tokens._allowSeparator is False + tokens._allowSeparator = True tokens._text = "Some text ...\n" tokens.tokenizeText() tokens.doHeaders() - assert tokens._firstScene is False + assert tokens._allowSeparator is False # END Test testCoreToken_ProcessHeaders From 404b102a54bb351ab95771e1ddb4c1984ec6a286 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:44:11 +0100 Subject: [PATCH 3/5] Rename some variables in the tokenizer --- novelwriter/core/tokenizer.py | 70 ++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index fb758fbc..9941fb11 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -615,9 +615,9 @@ class Tokenizer(ABC): # Make sure the token array doesn't start with a page break # on the very first page, adding a blank first page. if self._tokens[0][4] & self.A_PBB: - tToken = self._tokens[0] + token = self._tokens[0] self._tokens[0] = ( - tToken[0], tToken[1], tToken[2], tToken[3], tToken[4] & ~self.A_PBB + token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB ) # Always add an empty line at the end of the file @@ -637,22 +637,20 @@ class Tokenizer(ABC): pToken = (self.T_EMPTY, 0, "", [], self.A_NONE) nToken = (self.T_EMPTY, 0, "", [], self.A_NONE) tCount = len(self._tokens) - for n, tToken in enumerate(self._tokens): + for n, token in enumerate(self._tokens): if n > 0: pToken = self._tokens[n-1] if n < tCount - 1: nToken = self._tokens[n+1] - if tToken[0] == self.T_KEYWORD: - aStyle = tToken[4] + if token[0] == self.T_KEYWORD: + aStyle = token[4] if pToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_TOPMRG if nToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_BTMMRG - self._tokens[n] = ( - tToken[0], tToken[1], tToken[2], tToken[3], aStyle - ) + self._tokens[n] = (token[0], token[1], token[2], token[3], aStyle) return @@ -665,89 +663,93 @@ class Tokenizer(ABC): self._hFormatter.setHandle(self._nwItem.itemHandle if self._nwItem else None) - for n, tToken in enumerate(self._tokens): + for n, token in enumerate(self._tokens): - if tToken[0] == self.T_TEXT: + if token[0] == self.T_TEXT: # If we see text before a scene, we consider it a "scene" self._allowSeparator = False - elif tToken[0] == self.T_TITLE: # Title - # For titles, we reset all counters + elif token[0] == self.T_TITLE: # Title + # For new titles, we reset all counters self._allowSeparator = True self._hFormatter.resetAll() - elif tToken[0] == self.T_HEAD1: # Partition + elif token[0] == self.T_HEAD1: # Partition - tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1]) + tTemp = self._hFormatter.apply(self._fmtTitle, token[2], token[1]) self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] + token[0], token[1], tTemp, [], token[4] ) # Set scene variables self._allowSeparator = True self._hFormatter.resetScene() - elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter + elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter # Numbered or Unnumbered - if tToken[0] == self.T_UNNUM: - tTemp = self._hFormatter.apply(self._fmtUnNum, tToken[2], tToken[1]) + if token[0] == self.T_UNNUM: + tTemp = self._hFormatter.apply(self._fmtUnNum, token[2], token[1]) else: self._hFormatter.incChapter() - tTemp = self._hFormatter.apply(self._fmtChapter, tToken[2], tToken[1]) + tTemp = self._hFormatter.apply(self._fmtChapter, token[2], token[1]) # Format the chapter header self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] + token[0], token[1], tTemp, [], token[4] ) # Set scene variables self._allowSeparator = True self._hFormatter.resetScene() - elif tToken[0] == self.T_HEAD3: # Scene + elif token[0] == self.T_HEAD3: # Scene self._hFormatter.incScene() - tTemp = self._hFormatter.apply(self._fmtScene, tToken[2], tToken[1]) + tTemp = self._hFormatter.apply(self._fmtScene, token[2], token[1]) if tTemp == "" and self._hideScene: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE + self.T_EMPTY, token[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideScene: t1 = self.T_EMPTY if self._allowSeparator else self.T_SKIP - t4 = self.A_NONE if self._allowSeparator else tToken[4] - self._tokens[n] = (t1, tToken[1], "", [], t4) + t4 = self.A_NONE if self._allowSeparator else token[4] + self._tokens[n] = ( + t1, token[1], "", [], t4 + ) elif tTemp == self._fmtScene: t1 = self.T_EMPTY if self._allowSeparator else self.T_SEP t2 = "" if self._allowSeparator else tTemp - t4 = self.A_NONE if self._allowSeparator else (tToken[4] | self.A_CENTRE) - self._tokens[n] = (t1, tToken[1], t2, [], t4) + t4 = self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE) + self._tokens[n] = ( + t1, token[1], t2, [], t4 + ) else: self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] + token[0], token[1], tTemp, [], token[4] ) self._allowSeparator = False - elif tToken[0] == self.T_HEAD4: # Section + elif token[0] == self.T_HEAD4: # Section - tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1]) + tTemp = self._hFormatter.apply(self._fmtSection, token[2], token[1]) if tTemp == "" and self._hideSection: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE + self.T_EMPTY, token[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideSection: self._tokens[n] = ( - self.T_SKIP, tToken[1], "", [], tToken[4] + self.T_SKIP, token[1], "", [], token[4] ) elif tTemp == self._fmtSection: self._tokens[n] = ( - self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE + self.T_SEP, token[1], tTemp, [], token[4] | self.A_CENTRE ) else: self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] + token[0], token[1], tTemp, [], token[4] ) return True From 6db7f049bcbc5eaafc0b869db95ebae9a63d9905 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 24 Feb 2024 15:24:31 +0100 Subject: [PATCH 4/5] Add test coverage --- novelwriter/core/tokenizer.py | 2 +- tests/test_core/test_core_tokenizer.py | 198 ++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 3 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 9941fb11..4555fc22 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -89,7 +89,7 @@ class Tokenizer(ABC): T_UNNUM = 7 # Unnumbered T_HEAD1 = 8 # Header 1 T_HEAD2 = 9 # Header 2 - T_HEAD3 = 10 # Header 3 + T_HEAD3 = 10 # Header 3 T_HEAD4 = 11 # Header 4 T_TEXT = 12 # Text line T_SEP = 13 # Scene separator diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 9bf12b1a..d07ab411 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -26,6 +26,7 @@ import pytest from tools import C, buildTestProject, readFile from novelwriter.constants import nwHeadFmt +from novelwriter.core.tomd import ToMarkdown from novelwriter.core.project import NWProject from novelwriter.core.tokenizer import HeadingFormatter, Tokenizer, stripEscape @@ -1010,7 +1011,7 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._isNote = False ## - # Story FIles + # Story Files ## tokens._isNone = False @@ -1192,7 +1193,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H4: Section Hidden wo/Format tokens._text = "#### A Section\n" - tokens.setSectionFormat(r"", True) + tokens.setSectionFormat("", True) tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1241,6 +1242,193 @@ def testCoreToken_ProcessHeaders(mockGUI): # END Test testCoreToken_ProcessHeaders +@pytest.mark.core +def testCoreToken_HeaderCounterAndVisibility(mockGUI): + """Test the header counter and visibility of the Tokenizer class. + This is a special test to cover issue #1704. + """ + project = NWProject() + project.data.setLanguage("en") + project._loadProjectLocalisation() + md = ToMarkdown(project) + md._isNone = False + md._isNote = False + md._isNovel = True + + # Separator Handling, Titles + # ========================== + + md._text = ( + "# Title One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "# Title Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + ) + md.setTitleFormat(f"T: {nwHeadFmt.TITLE}") + md.setChapterFormat(f"C: {nwHeadFmt.TITLE}") + md.setSectionFormat("", True) + + # Static Separator + md.setSceneFormat("* * *", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + "# T: Title Two\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + ) + + # Scene Title Formatted + md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "### S: Scene One\n\n" + "Text\n\n" + "### S: Scene Two\n\n" + "Text\n\n" + "# T: Title Two\n\n" + "### S: Scene Three\n\n" + "Text\n\n" + "### S: Scene Four\n\n" + "Text\n\n" + ) + + # Separator Handling, Chapters + # ============================ + + md._text = ( + "# Title One\n\n" + "## Chapter One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "## Chapter Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + ) + md.setTitleFormat(f"T: {nwHeadFmt.TITLE}") + md.setChapterFormat(f"C: {nwHeadFmt.TITLE}") + md.setSectionFormat("", True) + + # Static Separator + md.setSceneFormat("* * *", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "## C: Chapter One\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + "## C: Chapter Two\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + ) + + # Scene Title Formatted + md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "## C: Chapter One\n\n" + "### S: Scene One\n\n" + "Text\n\n" + "### S: Scene Two\n\n" + "Text\n\n" + "## C: Chapter Two\n\n" + "### S: Scene Three\n\n" + "Text\n\n" + "### S: Scene Four\n\n" + "Text\n\n" + ) + + # Counter Handling, Novel Titles + # ============================== + + md._text = ( + "#! Novel One\n\n" + "## Chapter One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "## Chapter Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + "#! Novel Two\n\n" + "## Chapter One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "## Chapter Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + ) + md.setTitleFormat(f"T: {nwHeadFmt.TITLE}") + md.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}") + md.setSceneFormat(f"S {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM} ({nwHeadFmt.SC_ABS}): " + f"{nwHeadFmt.TITLE}", False) + md.setSectionFormat("", True) + + # Two Novel Format + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# Novel One\n\n" + "## C 1: Chapter One\n\n" + "### S 1.1 (1): Scene One\n\n" + "Text\n\n" + "### S 1.2 (2): Scene Two\n\n" + "Text\n\n" + "## C 2: Chapter Two\n\n" + "### S 2.1 (3): Scene Three\n\n" + "Text\n\n" + "### S 2.2 (4): Scene Four\n\n" + "Text\n\n" + "# Novel Two\n\n" + "## C 1: Chapter One\n\n" + "### S 1.1 (1): Scene One\n\n" + "Text\n\n" + "### S 1.2 (2): Scene Two\n\n" + "Text\n\n" + "## C 2: Chapter Two\n\n" + "### S 2.1 (3): Scene Three\n\n" + "Text\n\n" + "### S 2.2 (4): Scene Four\n\n" + "Text\n\n" + ) + +# END Test testCoreToken_HeaderCounterAndVisibility + + @pytest.mark.core def testCoreIndex_HeadingFormatter(fncPath, mockRnd): """Check the HeadingFormatter class.""" @@ -1306,6 +1494,12 @@ def testCoreIndex_HeadingFormatter(fncPath, mockRnd): formatter.incScene() assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 2.1 - Scene 5 - Hi Bob" + # New Main Title + formatter.resetAll() + formatter.incChapter() + formatter.incScene() + assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.1 - Scene 1 - Hi Bob" + # Special Formats # =============== formatter._chCount = 2 From 6ef646acb24a100399edf22405ee4036c793e110 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 24 Feb 2024 15:29:58 +0100 Subject: [PATCH 5/5] Simplify the scene separator handler --- novelwriter/core/tokenizer.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 4555fc22..495d3e07 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -713,17 +713,15 @@ class Tokenizer(ABC): self.T_EMPTY, token[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideScene: - t1 = self.T_EMPTY if self._allowSeparator else self.T_SKIP - t4 = self.A_NONE if self._allowSeparator else token[4] self._tokens[n] = ( - t1, token[1], "", [], t4 + self.T_EMPTY if self._allowSeparator else self.T_SKIP, token[1], + "", [], self.A_NONE if self._allowSeparator else token[4] ) elif tTemp == self._fmtScene: - t1 = self.T_EMPTY if self._allowSeparator else self.T_SEP - t2 = "" if self._allowSeparator else tTemp - t4 = self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE) self._tokens[n] = ( - t1, token[1], t2, [], t4 + self.T_EMPTY if self._allowSeparator else self.T_SEP, token[1], + "" if self._allowSeparator else tTemp, [], + self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE) ) else: self._tokens[n] = (