diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 90f771f8..495d3e07 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -89,7 +89,7 @@ class Tokenizer(ABC): T_UNNUM = 7 # Unnumbered T_HEAD1 = 8 # Header 1 T_HEAD2 = 9 # Header 2 - T_HEAD3 = 10 # Header 3 + T_HEAD3 = 10 # Header 3 T_HEAD4 = 11 # Header 4 T_TEXT = 12 # Text line T_SEP = 13 # Scene separator @@ -158,7 +158,7 @@ class Tokenizer(ABC): # Instance Variables self._hFormatter = HeadingFormatter(self._project) - self._firstScene = False # Flag to indicate that the first scene of the chapter + self._allowSeparator = False # Flag to indicate that the first scene of the chapter # This File self._isNone = False # Document has unknown layout @@ -615,9 +615,9 @@ class Tokenizer(ABC): # Make sure the token array doesn't start with a page break # on the very first page, adding a blank first page. if self._tokens[0][4] & self.A_PBB: - tToken = self._tokens[0] + token = self._tokens[0] self._tokens[0] = ( - tToken[0], tToken[1], tToken[2], tToken[3], tToken[4] & ~self.A_PBB + token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB ) # Always add an empty line at the end of the file @@ -637,22 +637,20 @@ class Tokenizer(ABC): pToken = (self.T_EMPTY, 0, "", [], self.A_NONE) nToken = (self.T_EMPTY, 0, "", [], self.A_NONE) tCount = len(self._tokens) - for n, tToken in enumerate(self._tokens): + for n, token in enumerate(self._tokens): if n > 0: pToken = self._tokens[n-1] if n < tCount - 1: nToken = self._tokens[n+1] - if tToken[0] == self.T_KEYWORD: - aStyle = tToken[4] + if token[0] == self.T_KEYWORD: + aStyle = token[4] if pToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_TOPMRG if nToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_BTMMRG - self._tokens[n] = ( - tToken[0], tToken[1], tToken[2], tToken[3], aStyle - ) + self._tokens[n] = (token[0], token[1], token[2], token[3], aStyle) return @@ -665,93 +663,91 @@ class Tokenizer(ABC): self._hFormatter.setHandle(self._nwItem.itemHandle if self._nwItem else None) - for n, tToken in enumerate(self._tokens): + for n, token in enumerate(self._tokens): - # In case we see text before a scene, we reset the flag - if tToken[0] == self.T_TEXT: - self._firstScene = False + if token[0] == self.T_TEXT: + # If we see text before a scene, we consider it a "scene" + self._allowSeparator = False - elif tToken[0] == self.T_HEAD1: - # Partition + elif token[0] == self.T_TITLE: # Title + # For new titles, we reset all counters + self._allowSeparator = True + self._hFormatter.resetAll() - tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1]) + elif token[0] == self.T_HEAD1: # Partition + + tTemp = self._hFormatter.apply(self._fmtTitle, token[2], token[1]) self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] - ) - - elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): - # Chapter - - # Numbered or Unnumbered - if tToken[0] == self.T_UNNUM: - tTemp = self._hFormatter.apply(self._fmtUnNum, tToken[2], tToken[1]) - else: - self._hFormatter.incChapter() - tTemp = self._hFormatter.apply(self._fmtChapter, tToken[2], tToken[1]) - - # Format the chapter header - self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] + token[0], token[1], tTemp, [], token[4] ) # Set scene variables - self._firstScene = True + self._allowSeparator = True self._hFormatter.resetScene() - elif tToken[0] == self.T_HEAD3: - # Scene + elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter + + # Numbered or Unnumbered + if token[0] == self.T_UNNUM: + tTemp = self._hFormatter.apply(self._fmtUnNum, token[2], token[1]) + else: + self._hFormatter.incChapter() + tTemp = self._hFormatter.apply(self._fmtChapter, token[2], token[1]) + + # Format the chapter header + self._tokens[n] = ( + token[0], token[1], tTemp, [], token[4] + ) + + # Set scene variables + self._allowSeparator = True + self._hFormatter.resetScene() + + elif token[0] == self.T_HEAD3: # Scene self._hFormatter.incScene() - tTemp = self._hFormatter.apply(self._fmtScene, tToken[2], tToken[1]) + tTemp = self._hFormatter.apply(self._fmtScene, token[2], token[1]) if tTemp == "" and self._hideScene: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE + self.T_EMPTY, token[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideScene: - if self._firstScene: - self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE - ) - else: - self._tokens[n] = ( - self.T_SKIP, tToken[1], "", [], tToken[4] - ) + self._tokens[n] = ( + self.T_EMPTY if self._allowSeparator else self.T_SKIP, token[1], + "", [], self.A_NONE if self._allowSeparator else token[4] + ) elif tTemp == self._fmtScene: - if self._firstScene: - self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE - ) - else: - self._tokens[n] = ( - self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE - ) + self._tokens[n] = ( + self.T_EMPTY if self._allowSeparator else self.T_SEP, token[1], + "" if self._allowSeparator else tTemp, [], + self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE) + ) else: self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] + token[0], token[1], tTemp, [], token[4] ) - self._firstScene = False + self._allowSeparator = False - elif tToken[0] == self.T_HEAD4: - # Section + elif token[0] == self.T_HEAD4: # Section - tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1]) + tTemp = self._hFormatter.apply(self._fmtSection, token[2], token[1]) if tTemp == "" and self._hideSection: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", [], self.A_NONE + self.T_EMPTY, token[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideSection: self._tokens[n] = ( - self.T_SKIP, tToken[1], "", [], tToken[4] + self.T_SKIP, token[1], "", [], token[4] ) elif tTemp == self._fmtSection: self._tokens[n] = ( - self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE + self.T_SEP, token[1], tTemp, [], token[4] | self.A_CENTRE ) else: self._tokens[n] = ( - tToken[0], tToken[1], tTemp, [], tToken[4] + token[0], token[1], tTemp, [], token[4] ) return True @@ -849,6 +845,13 @@ class HeadingFormatter: self._scAbsCount += 1 return + def resetAll(self) -> None: + """Reset all counters.""" + self._chCount = 0 + self._scChCount = 0 + self._scAbsCount = 0 + return + def resetScene(self) -> None: """Reset the chapter scene counter.""" self._scChCount = 0 diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 163cd151..d07ab411 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -26,6 +26,7 @@ import pytest from tools import C, buildTestProject, readFile from novelwriter.constants import nwHeadFmt +from novelwriter.core.tomd import ToMarkdown from novelwriter.core.project import NWProject from novelwriter.core.tokenizer import HeadingFormatter, Tokenizer, stripEscape @@ -1010,7 +1011,7 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._isNote = False ## - # Story FIles + # Story Files ## tokens._isNone = False @@ -1122,7 +1123,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._firstScene = True + tokens._allowSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1133,7 +1134,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._firstScene = False + tokens._allowSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1144,7 +1145,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._firstScene = True + tokens._allowSeparator = True tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1155,7 +1156,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._firstScene = False + tokens._allowSeparator = False tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1192,7 +1193,7 @@ def testCoreToken_ProcessHeaders(mockGUI): # H4: Section Hidden wo/Format tokens._text = "#### A Section\n" - tokens.setSectionFormat(r"", True) + tokens.setSectionFormat("", True) tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ @@ -1231,16 +1232,203 @@ def testCoreToken_ProcessHeaders(mockGUI): ] # Check the first scene detector - assert tokens._firstScene is False - tokens._firstScene = True + assert tokens._allowSeparator is False + tokens._allowSeparator = True tokens._text = "Some text ...\n" tokens.tokenizeText() tokens.doHeaders() - assert tokens._firstScene is False + assert tokens._allowSeparator is False # END Test testCoreToken_ProcessHeaders +@pytest.mark.core +def testCoreToken_HeaderCounterAndVisibility(mockGUI): + """Test the header counter and visibility of the Tokenizer class. + This is a special test to cover issue #1704. + """ + project = NWProject() + project.data.setLanguage("en") + project._loadProjectLocalisation() + md = ToMarkdown(project) + md._isNone = False + md._isNote = False + md._isNovel = True + + # Separator Handling, Titles + # ========================== + + md._text = ( + "# Title One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "# Title Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + ) + md.setTitleFormat(f"T: {nwHeadFmt.TITLE}") + md.setChapterFormat(f"C: {nwHeadFmt.TITLE}") + md.setSectionFormat("", True) + + # Static Separator + md.setSceneFormat("* * *", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + "# T: Title Two\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + ) + + # Scene Title Formatted + md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "### S: Scene One\n\n" + "Text\n\n" + "### S: Scene Two\n\n" + "Text\n\n" + "# T: Title Two\n\n" + "### S: Scene Three\n\n" + "Text\n\n" + "### S: Scene Four\n\n" + "Text\n\n" + ) + + # Separator Handling, Chapters + # ============================ + + md._text = ( + "# Title One\n\n" + "## Chapter One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "## Chapter Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + ) + md.setTitleFormat(f"T: {nwHeadFmt.TITLE}") + md.setChapterFormat(f"C: {nwHeadFmt.TITLE}") + md.setSectionFormat("", True) + + # Static Separator + md.setSceneFormat("* * *", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "## C: Chapter One\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + "## C: Chapter Two\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + ) + + # Scene Title Formatted + md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# T: Title One\n\n" + "## C: Chapter One\n\n" + "### S: Scene One\n\n" + "Text\n\n" + "### S: Scene Two\n\n" + "Text\n\n" + "## C: Chapter Two\n\n" + "### S: Scene Three\n\n" + "Text\n\n" + "### S: Scene Four\n\n" + "Text\n\n" + ) + + # Counter Handling, Novel Titles + # ============================== + + md._text = ( + "#! Novel One\n\n" + "## Chapter One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "## Chapter Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + "#! Novel Two\n\n" + "## Chapter One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "## Chapter Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + ) + md.setTitleFormat(f"T: {nwHeadFmt.TITLE}") + md.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}") + md.setSceneFormat(f"S {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM} ({nwHeadFmt.SC_ABS}): " + f"{nwHeadFmt.TITLE}", False) + md.setSectionFormat("", True) + + # Two Novel Format + md.tokenizeText() + md.doHeaders() + md.doConvert() + assert md.result == ( + "# Novel One\n\n" + "## C 1: Chapter One\n\n" + "### S 1.1 (1): Scene One\n\n" + "Text\n\n" + "### S 1.2 (2): Scene Two\n\n" + "Text\n\n" + "## C 2: Chapter Two\n\n" + "### S 2.1 (3): Scene Three\n\n" + "Text\n\n" + "### S 2.2 (4): Scene Four\n\n" + "Text\n\n" + "# Novel Two\n\n" + "## C 1: Chapter One\n\n" + "### S 1.1 (1): Scene One\n\n" + "Text\n\n" + "### S 1.2 (2): Scene Two\n\n" + "Text\n\n" + "## C 2: Chapter Two\n\n" + "### S 2.1 (3): Scene Three\n\n" + "Text\n\n" + "### S 2.2 (4): Scene Four\n\n" + "Text\n\n" + ) + +# END Test testCoreToken_HeaderCounterAndVisibility + + @pytest.mark.core def testCoreIndex_HeadingFormatter(fncPath, mockRnd): """Check the HeadingFormatter class.""" @@ -1306,6 +1494,12 @@ def testCoreIndex_HeadingFormatter(fncPath, mockRnd): formatter.incScene() assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 2.1 - Scene 5 - Hi Bob" + # New Main Title + formatter.resetAll() + formatter.incChapter() + formatter.incScene() + assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.1 - Scene 1 - Hi Bob" + # Special Formats # =============== formatter._chCount = 2