From afe4a29fd19974237fca2e9624b002a6963c067d Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Tue, 21 May 2024 20:53:45 +0200 Subject: [PATCH] Strip consecutive empty paragraphs when tokenizing --- novelwriter/core/tokenizer.py | 69 ++++++++++++++--------- tests/test_core/test_core_tokenizer.py | 76 ++++++++++---------------- 2 files changed, 72 insertions(+), 73 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 15cd8046..700f679f 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -152,6 +152,7 @@ class Tokenizer(ABC): self._doComments = False # Also process comments self._doKeywords = False # Also process keywords like tags and references self._skipKeywords = set() # Keywords to ignore + self._keepBreaks = True # Keep line breaks in paragraphs # Margins self._marginTitle = (1.000, 0.500) @@ -409,6 +410,11 @@ class Tokenizer(ABC): self._skipKeywords = set(x.lower().strip() for x in keywords.split(",")) return + def setKeepLineBreaks(self, state: bool) -> None: + """Keep line breaks in paragraphs.""" + self._keepBreaks = state + return + def setKeepMarkdown(self, state: bool) -> None: """Keep original markdown during build.""" self._keepMD = state @@ -490,7 +496,6 @@ class Tokenizer(ABC): 4: The internal formatting map of the text, self.FMT_* 5: The style of the block, self.A_* """ - self._tokens = [] if self._isNovel: self._hFormatter.setHandle(self._handle) @@ -498,12 +503,13 @@ class Tokenizer(ABC): breakNext = False tmpMarkdown = [] tHandle = self._handle or "" + tokens = [] for aLine in self._text.splitlines(): sLine = aLine.strip().lower() # Check for blank lines if len(sLine) == 0: - self._tokens.append(( + tokens.append(( self.T_EMPTY, nHead, "", [], self.A_NONE )) if self._keepMD: @@ -532,7 +538,7 @@ class Tokenizer(ABC): continue elif sLine == "[vspace]": - self._tokens.append( + tokens.append( (self.T_SKIP, nHead, "", [], sAlign) ) continue @@ -540,11 +546,11 @@ class Tokenizer(ABC): elif sLine.startswith("[vspace:") and sLine.endswith("]"): nSkip = checkInt(sLine[8:-1], 0) if nSkip >= 1: - self._tokens.append( + tokens.append( (self.T_SKIP, nHead, "", [], sAlign) ) if nSkip > 1: - self._tokens += (nSkip - 1) * [ + tokens += (nSkip - 1) * [ (self.T_SKIP, nHead, "", [], self.A_NONE) ] continue @@ -561,14 +567,14 @@ class Tokenizer(ABC): cStyle, cKey, cText, _, _ = processComment(aLine) if cStyle == nwComment.SYNOPSIS: tLine, tFmt = self._extractFormats(cText) - self._tokens.append(( + tokens.append(( self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign )) if self._doSynopsis and self._keepMD: tmpMarkdown.append(f"{aLine}\n") elif cStyle == nwComment.SHORT: tLine, tFmt = self._extractFormats(cText) - self._tokens.append(( + tokens.append(( self.T_SHORT, nHead, tLine, tFmt, sAlign )) if self._doSynopsis and self._keepMD: @@ -580,7 +586,7 @@ class Tokenizer(ABC): tmpMarkdown.append(f"{aLine}\n") else: tLine, tFmt = self._extractFormats(cText) - self._tokens.append(( + tokens.append(( self.T_COMMENT, nHead, tLine, tFmt, sAlign )) if self._doComments and self._keepMD: @@ -594,7 +600,7 @@ class Tokenizer(ABC): valid, bits, _ = self._project.index.scanThis(aLine) if valid and bits and bits[0] not in self._skipKeywords: - self._tokens.append(( + tokens.append(( self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign )) if self._doKeywords and self._keepMD: @@ -630,7 +636,7 @@ class Tokenizer(ABC): self._hFormatter.resetAll() self._noSep = True - self._tokens.append(( + tokens.append(( tType, nHead, tText, [], tStyle )) if self._keepMD: @@ -665,7 +671,7 @@ class Tokenizer(ABC): self._hFormatter.resetScene() self._noSep = True - self._tokens.append(( + tokens.append(( tType, nHead, tText, [], tStyle )) if self._keepMD: @@ -706,7 +712,7 @@ class Tokenizer(ABC): tStyle = self.A_NONE if self._noSep else self.A_CENTRE self._noSep = False - self._tokens.append(( + tokens.append(( tType, nHead, tText, [], tStyle )) if self._keepMD: @@ -736,7 +742,7 @@ class Tokenizer(ABC): tType = self.T_SEP tStyle = self.A_CENTRE - self._tokens.append(( + tokens.append(( tType, nHead, tText, [], tStyle )) if self._keepMD: @@ -784,26 +790,26 @@ class Tokenizer(ABC): # Process formats tLine, tFmt = self._extractFormats(aLine) - self._tokens.append(( + tokens.append(( self.T_TEXT, nHead, tLine, tFmt, sAlign )) if self._keepMD: tmpMarkdown.append(f"{aLine}\n") # If we have content, turn off the first page flag - if self._isFirst and self._tokens: + if self._isFirst and tokens: self._isFirst = False # First document has been processed # Make sure the token array doesn't start with a page break # on the very first page, adding a blank first page. - if self._tokens[0][4] & self.A_PBB: - token = self._tokens[0] - self._tokens[0] = ( + if tokens[0][4] & self.A_PBB: + token = tokens[0] + tokens[0] = ( token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB ) # Always add an empty line at the end of the file - self._tokens.append(( + tokens.append(( self.T_EMPTY, nHead, "", [], self.A_NONE )) if self._keepMD: @@ -814,25 +820,36 @@ class Tokenizer(ABC): # =========== # Some items need a second pass + self._tokens = [] pToken = (self.T_EMPTY, 0, "", [], self.A_NONE) nToken = (self.T_EMPTY, 0, "", [], self.A_NONE) - tCount = len(self._tokens) - for n, token in enumerate(self._tokens): + + tCount = len(tokens) + for n, token in enumerate(tokens): if n > 0: - pToken = self._tokens[n-1] + pToken = tokens[n-1] # Look behind if n < tCount - 1: - nToken = self._tokens[n+1] + nToken = tokens[n+1] # Look ahead - if token[0] == self.T_KEYWORD: + if token[0] == self.T_EMPTY: + # Strip multiple empty + if pToken[0] != self.T_EMPTY: + self._tokens.append(token) + + elif token[0] == self.T_KEYWORD: + # Adjust margins for lines in a list of keyword lines aStyle = token[4] if pToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_TOPMRG if nToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_BTMMRG - self._tokens[n] = ( + self._tokens.append(( token[0], token[1], token[2], token[3], aStyle - ) + )) + + else: + self._tokens.append(token) return diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index b29ada81..6fd04bdd 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -717,9 +717,7 @@ def testCoreToken_MetaFormat(mockGUI): # Ignore Text tokens._text = "%~ Some text\n" tokens.tokenizeText() - assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), - ] + assert tokens._tokens == [] assert tokens.allMarkdown[-1] == "\n" # Synopsis @@ -829,7 +827,6 @@ def testCoreToken_MarginFormat(mockGUI): (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Right-indent, right-aligned", [], rIndAlign), (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == ( "Some regular text\n\n" @@ -950,6 +947,24 @@ def testCoreToken_ExtractFormats(mockGUI): ] +@pytest.mark.core +def testCoreToken_Paragraphs(mockGUI): + """Test the splitting of paragraphs.""" + project = NWProject() + tokens = BareTokenizer(project) + tokens.setKeepMarkdown(True) + + # Collapse empty lines + tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n" + tokens.tokenizeText() + assert tokens._tokens == [ + (Tokenizer.T_TEXT, 0, "First paragraph", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + (Tokenizer.T_TEXT, 0, "Second paragraph", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + ] + + @pytest.mark.core def testCoreToken_TextFormat(mockGUI): """Test the tokenization of text formats in the Tokenizer class.""" @@ -964,18 +979,12 @@ def testCoreToken_TextFormat(mockGUI): (Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n" tokens.setBodyText(False) tokens.tokenizeText() - assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), - ] + assert tokens._tokens == [] assert tokens.allMarkdown[-1] == "\n\n\n" tokens.setBodyText(True) @@ -1083,10 +1092,8 @@ def testCoreToken_SpecialFormat(mockGUI): correctResp = [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_HEAD1, 2, "Title Two", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), ] # Command wo/Space @@ -1135,7 +1142,6 @@ def testCoreToken_SpecialFormat(mockGUI): (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Multiple Empty Paragraphs @@ -1155,7 +1161,6 @@ def testCoreToken_SpecialFormat(mockGUI): (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Three Skips @@ -1174,7 +1179,6 @@ def testCoreToken_SpecialFormat(mockGUI): (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 1 @@ -1187,10 +1191,8 @@ def testCoreToken_SpecialFormat(mockGUI): assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 2 @@ -1203,10 +1205,8 @@ def testCoreToken_SpecialFormat(mockGUI): assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 3 @@ -1219,10 +1219,8 @@ def testCoreToken_SpecialFormat(mockGUI): assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Empty Paragraph and Page Break @@ -1239,12 +1237,10 @@ def testCoreToken_SpecialFormat(mockGUI): assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Multiple Skip @@ -1258,14 +1254,12 @@ def testCoreToken_SpecialFormat(mockGUI): assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] @@ -1371,20 +1365,14 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "### Scene One\n" tokens.setSceneFormat("", True) tokens.tokenizeText() - assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - ] + assert tokens._tokens == [] # H3: Scene wo/Format, first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) tokens._noSep = True tokens.tokenizeText() - assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - ] + assert tokens._tokens == [] # H3: Scene wo/Format, not first tokens._text = "### Scene One\n" @@ -1401,10 +1389,7 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.setSceneFormat("* * *", False) tokens._noSep = True tokens.tokenizeText() - assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - ] + assert tokens._tokens == [] # H3: Scene Separator, not first tokens._text = "### Scene One\n" @@ -1445,10 +1430,7 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat("", True) tokens.tokenizeText() - assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), - ] + assert tokens._tokens == [] # H4: Section Visible wo/Format tokens._text = "#### A Section\n" @@ -1615,7 +1597,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.tokenizeText() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "", "", "Text", "", "* * *", "", "Text", "" + "Chapter", "", "Text", "", "* * *", "", "Text", "" ] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 2, @@ -1634,7 +1616,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.tokenizeText() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "", "", "Stuff", "", "Text", "" + "Chapter", "", "Stuff", "", "Text", "" ] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, @@ -1653,7 +1635,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.tokenizeText() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "", "", "Stuff", "", "Text", "" + "Chapter", "", "Stuff", "", "Text", "" ] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, @@ -1672,7 +1654,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.tokenizeText() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "", "", "Stuff", "", "Text", "" + "Chapter", "", "Stuff", "", "Text", "" ] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, @@ -1691,7 +1673,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.tokenizeText() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "", "", "pov: Jane", "", "Text", "" + "Chapter", "", "pov: Jane", "", "Text", "" ] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1,