From d8bb2ad3aeb23eefe393cee44e42c0bb9a69de9e Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Tue, 21 May 2024 21:58:34 +0200 Subject: [PATCH] Merge lines of the same paragraph already in the tokenizer --- novelwriter/core/tokenizer.py | 54 ++++++++++++++++------ tests/test_core/test_core_tokenizer.py | 64 +++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 700f679f..44417c67 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -50,6 +50,7 @@ RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.D T_Formats = list[tuple[int, int, str]] T_Comment = tuple[str, T_Formats] +T_Token = tuple[int, int, str, T_Formats, int] def stripEscape(text: str) -> str: @@ -129,7 +130,7 @@ class Tokenizer(ABC): self._keepMD = False # Whether to keep the markdown text # Tokens and Meta Data (Per Document) - self._tokens: list[tuple[int, int, str, T_Formats, int]] = [] + self._tokens: list[T_Token] = [] self._footnotes: dict[str, T_Comment] = {} # Tokens and Meta Data (Per Instance) @@ -503,7 +504,7 @@ class Tokenizer(ABC): breakNext = False tmpMarkdown = [] tHandle = self._handle or "" - tokens = [] + tokens: list[T_Token] = [] for aLine in self._text.splitlines(): sLine = aLine.strip().lower() @@ -803,9 +804,9 @@ class Tokenizer(ABC): # Make sure the token array doesn't start with a page break # on the very first page, adding a blank first page. if tokens[0][4] & self.A_PBB: - token = tokens[0] + cToken = tokens[0] tokens[0] = ( - token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB + cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~self.A_PBB ) # Always add an empty line at the end of the file @@ -818,38 +819,63 @@ class Tokenizer(ABC): # Second Pass # =========== - # Some items need a second pass + # This second pass strips away consecutive blank lines, and + # combines consecutive text lines into the same paragraph. + # It also ensures that there isn't paragraph spacing between + # meta data lines for formats that has spacing. self._tokens = [] - pToken = (self.T_EMPTY, 0, "", [], self.A_NONE) - nToken = (self.T_EMPTY, 0, "", [], self.A_NONE) + pToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE) + nToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE) + + lineSep = "\n" if self._keepBreaks else " " + pLines: list[T_Token] = [] tCount = len(tokens) - for n, token in enumerate(tokens): + for n, cToken in enumerate(tokens): if n > 0: pToken = tokens[n-1] # Look behind if n < tCount - 1: nToken = tokens[n+1] # Look ahead - if token[0] == self.T_EMPTY: + if cToken[0] == self.T_EMPTY: # Strip multiple empty if pToken[0] != self.T_EMPTY: - self._tokens.append(token) + self._tokens.append(cToken) - elif token[0] == self.T_KEYWORD: + elif cToken[0] == self.T_KEYWORD: # Adjust margins for lines in a list of keyword lines - aStyle = token[4] + aStyle = cToken[4] if pToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_TOPMRG if nToken[0] == self.T_KEYWORD: aStyle |= self.A_Z_BTMMRG self._tokens.append(( - token[0], token[1], token[2], token[3], aStyle + cToken[0], cToken[1], cToken[2], cToken[3], aStyle )) + elif cToken[0] == self.T_TEXT: + # Combine lines from the same paragraph + pLines.append(cToken) + if nToken[0] != self.T_TEXT: + nLines = len(pLines) + if nLines == 1: + self._tokens.append(pLines[0]) + elif nLines > 1: + tTxt = "" + tFmt: T_Formats = [] + for aToken in pLines: + tLen = len(tTxt) + tTxt += f"{aToken[2]}{lineSep}" + tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3]) + self._tokens.append(( + self.T_TEXT, pLines[0][1], tTxt[:-1], tFmt, pLines[0][4] + )) + pLines = [] + else: - self._tokens.append(token) + self._tokens.append(cToken) return diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 6fd04bdd..3122f182 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -964,6 +964,67 @@ def testCoreToken_Paragraphs(mockGUI): (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] + # Combine multi-line paragraphs, keep breaks + tokens._text = "This is text\nspanning multiple\nlines" + tokens.setKeepLineBreaks(True) + tokens.tokenizeText() + assert tokens._tokens == [ + (Tokenizer.T_TEXT, 0, "This is text\nspanning multiple\nlines", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + ] + + # Combine multi-line paragraphs, remove breaks + tokens._text = "This is text\nspanning multiple\nlines" + tokens.setKeepLineBreaks(False) + tokens.tokenizeText() + assert tokens._tokens == [ + (Tokenizer.T_TEXT, 0, "This is text spanning multiple lines", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + ] + + # Combine multi-line paragraphs, remove breaks, with formatting + tokens._text = "This **is text**\nspanning _multiple_\nlines" + tokens.setKeepLineBreaks(False) + tokens.tokenizeText() + assert tokens._tokens == [ + ( + Tokenizer.T_TEXT, 0, + "This is text spanning multiple lines", + [ + (5, Tokenizer.FMT_B_B, ""), + (12, Tokenizer.FMT_B_E, ""), + (22, Tokenizer.FMT_I_B, ""), + (30, Tokenizer.FMT_I_E, ""), + ], + Tokenizer.A_NONE + ), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + ] + + # Make sure titles break a paragraph + tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n" + tokens.setKeepLineBreaks(False) + tokens.tokenizeText() + assert tokens._tokens == [ + (Tokenizer.T_HEAD1, 1, "Title", [], Tokenizer.A_NONE), + ( + Tokenizer.T_TEXT, 1, "Text on two lines.", [ + (5, Tokenizer.FMT_I_B, ""), + (7, Tokenizer.FMT_I_E, ""), + ], Tokenizer.A_NONE + ), + (Tokenizer.T_HEAD2, 2, "Chapter", [], Tokenizer.A_NONE), + ( + Tokenizer.T_TEXT, 2, "More text here.", [ + (5, Tokenizer.FMT_B_B, ""), + (9, Tokenizer.FMT_B_E, ""), + (10, Tokenizer.FMT_I_B, ""), + (14, Tokenizer.FMT_I_E, ""), + ], Tokenizer.A_NONE + ), + (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), + ] + @pytest.mark.core def testCoreToken_TextFormat(mockGUI): @@ -976,8 +1037,7 @@ def testCoreToken_TextFormat(mockGUI): tokens._text = "Some plain text\non two lines\n\n\n" tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE), - (Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE), + (Tokenizer.T_TEXT, 0, "Some plain text\non two lines", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"