Merge lines of the same paragraph already in the tokenizer

novelwriter/core/tokenizer.py

@@ -50,6 +50,7 @@ RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.D
 
 T_Formats = list[tuple[int, int, str]]
 T_Comment = tuple[str, T_Formats]
+T_Token = tuple[int, int, str, T_Formats, int]
 
 
 def stripEscape(text: str) -> str:
@@ -129,7 +130,7 @@ class Tokenizer(ABC):
         self._keepMD = False  # Whether to keep the markdown text
 
         # Tokens and Meta Data (Per Document)
-        self._tokens: list[tuple[int, int, str, T_Formats, int]] = []
+        self._tokens: list[T_Token] = []
         self._footnotes: dict[str, T_Comment] = {}
 
         # Tokens and Meta Data (Per Instance)
@@ -503,7 +504,7 @@ class Tokenizer(ABC):
         breakNext = False
         tmpMarkdown = []
         tHandle = self._handle or ""
-        tokens = []
+        tokens: list[T_Token] = []
         for aLine in self._text.splitlines():
             sLine = aLine.strip().lower()
 
@@ -803,9 +804,9 @@ class Tokenizer(ABC):
             # Make sure the token array doesn't start with a page break
             # on the very first page, adding a blank first page.
             if tokens[0][4] & self.A_PBB:
-                token = tokens[0]
+                cToken = tokens[0]
                 tokens[0] = (
-                    token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
+                    cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~self.A_PBB
                 )
 
         # Always add an empty line at the end of the file
@@ -818,38 +819,63 @@ class Tokenizer(ABC):
 
         # Second Pass
         # ===========
-        # Some items need a second pass
+        # This second pass strips away consecutive blank lines, and
+        # combines consecutive text lines into the same paragraph.
+        # It also ensures that there isn't paragraph spacing between
+        # meta data lines for formats that has spacing.
 
         self._tokens = []
-        pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
+        pToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE)
-        nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
+        nToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE)
+
+        lineSep = "\n" if self._keepBreaks else " "
+        pLines: list[T_Token] = []
 
         tCount = len(tokens)
-        for n, token in enumerate(tokens):
+        for n, cToken in enumerate(tokens):
 
             if n > 0:
                 pToken = tokens[n-1]  # Look behind
             if n < tCount - 1:
                 nToken = tokens[n+1]  # Look ahead
 
-            if token[0] == self.T_EMPTY:
+            if cToken[0] == self.T_EMPTY:
                 # Strip multiple empty
                 if pToken[0] != self.T_EMPTY:
-                    self._tokens.append(token)
+                    self._tokens.append(cToken)
 
-            elif token[0] == self.T_KEYWORD:
+            elif cToken[0] == self.T_KEYWORD:
                 # Adjust margins for lines in a list of keyword lines
-                aStyle = token[4]
+                aStyle = cToken[4]
                 if pToken[0] == self.T_KEYWORD:
                     aStyle |= self.A_Z_TOPMRG
                 if nToken[0] == self.T_KEYWORD:
                     aStyle |= self.A_Z_BTMMRG
                 self._tokens.append((
-                    token[0], token[1], token[2], token[3], aStyle
+                    cToken[0], cToken[1], cToken[2], cToken[3], aStyle
                 ))
 
+            elif cToken[0] == self.T_TEXT:
+                # Combine lines from the same paragraph
+                pLines.append(cToken)
+                if nToken[0] != self.T_TEXT:
+                    nLines = len(pLines)
+                    if nLines == 1:
+                        self._tokens.append(pLines[0])
+                    elif nLines > 1:
+                        tTxt = ""
+                        tFmt: T_Formats = []
+                        for aToken in pLines:
+                            tLen = len(tTxt)
+                            tTxt += f"{aToken[2]}{lineSep}"
+                            tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3])
+                        self._tokens.append((
+                            self.T_TEXT, pLines[0][1], tTxt[:-1], tFmt, pLines[0][4]
+                        ))
+                    pLines = []
+
             else:
-                self._tokens.append(token)
+                self._tokens.append(cToken)
 
         return
 

tests/test_core/test_core_tokenizer.py

@@ -964,6 +964,67 @@ def testCoreToken_Paragraphs(mockGUI):
         (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
     ]
 
+    # Combine multi-line paragraphs, keep breaks
+    tokens._text = "This is text\nspanning multiple\nlines"
+    tokens.setKeepLineBreaks(True)
+    tokens.tokenizeText()
+    assert tokens._tokens == [
+        (Tokenizer.T_TEXT, 0, "This is text\nspanning multiple\nlines", [], Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
+    ]
+
+    # Combine multi-line paragraphs, remove breaks
+    tokens._text = "This is text\nspanning multiple\nlines"
+    tokens.setKeepLineBreaks(False)
+    tokens.tokenizeText()
+    assert tokens._tokens == [
+        (Tokenizer.T_TEXT, 0, "This is text spanning multiple lines", [], Tokenizer.A_NONE),
+        (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
+    ]
+
+    # Combine multi-line paragraphs, remove breaks, with formatting
+    tokens._text = "This **is text**\nspanning _multiple_\nlines"
+    tokens.setKeepLineBreaks(False)
+    tokens.tokenizeText()
+    assert tokens._tokens == [
+        (
+            Tokenizer.T_TEXT, 0,
+            "This is text spanning multiple lines",
+            [
+                (5,  Tokenizer.FMT_B_B, ""),
+                (12, Tokenizer.FMT_B_E, ""),
+                (22, Tokenizer.FMT_I_B, ""),
+                (30, Tokenizer.FMT_I_E, ""),
+            ],
+            Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
+    ]
+
+    # Make sure titles break a paragraph
+    tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n"
+    tokens.setKeepLineBreaks(False)
+    tokens.tokenizeText()
+    assert tokens._tokens == [
+        (Tokenizer.T_HEAD1, 1, "Title", [], Tokenizer.A_NONE),
+        (
+            Tokenizer.T_TEXT, 1, "Text on two lines.", [
+                (5, Tokenizer.FMT_I_B, ""),
+                (7, Tokenizer.FMT_I_E, ""),
+            ], Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_HEAD2, 2, "Chapter", [], Tokenizer.A_NONE),
+        (
+            Tokenizer.T_TEXT, 2, "More text here.", [
+                (5,  Tokenizer.FMT_B_B, ""),
+                (9,  Tokenizer.FMT_B_E, ""),
+                (10, Tokenizer.FMT_I_B, ""),
+                (14, Tokenizer.FMT_I_E, ""),
+            ], Tokenizer.A_NONE
+        ),
+        (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE),
+    ]
+
 
 @pytest.mark.core
 def testCoreToken_TextFormat(mockGUI):
@@ -976,8 +1037,7 @@ def testCoreToken_TextFormat(mockGUI):
     tokens._text = "Some plain text\non two lines\n\n\n"
     tokens.tokenizeText()
     assert tokens._tokens == [
-        (Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE),
+        (Tokenizer.T_TEXT, 0, "Some plain text\non two lines", [], Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE),
         (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
     ]
     assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"