Check indent and justify after lines are combined in tokenizer, not before (#2426)

novelwriter/formats/tokenizer.py

@@ -880,31 +880,21 @@ class Tokenizer(ABC):
                 if nBlock[0] != BlockTyp.TEXT:
                     # Next block is not text, so we add the buffer to blocks
                     nLines = len(pLines)
-                    cStyle = pLines[0][4]
-                    if firstIndent and not (self._noIndent or cStyle & BlockFmt.ALIGNED):
-                        # If paragraph indentation is enabled, not temporarily
-                        # turned off, and the block is not aligned, we add the
-                        # text indentation flag
-                        cStyle |= BlockFmt.IND_T
+                    tFmt: T_Formats = []
+                    pTxt = ""
+                    cStyle = BlockFmt.NONE
 
                     if nLines == 1:
-                        # The paragraph contains a single line, so we just save
-                        # that directly to the blocks list. If justify is
-                        # enabled, and there is no alignment, we apply it.
-                        if doJustify and not cStyle & BlockFmt.ALIGNED:
-                            cStyle |= BlockFmt.JUSTIFY
-
+                        # The paragraph contains a single line
+                        tFmt = pLines[0][3]
                         pTxt = pLines[0][2].translate(transMapB)
-                        sBlocks.append((
-                            BlockTyp.TEXT, pLines[0][1], pTxt, pLines[0][3], cStyle
-                        ))
+                        cStyle = pLines[0][4]
 
                     elif nLines > 1:
                         # The paragraph contains multiple lines, so we need to
                         # join them according to the line break policy, and
                         # recompute all the formatting markers
                         tTxt = ""
-                        tFmt: T_Formats = []
                         for aBlock in pLines:
                             tLen = len(tTxt)
                             tTxt += f"{aBlock[2]}{lineSep}"
@@ -912,6 +902,18 @@ class Tokenizer(ABC):
                             cStyle |= aBlock[4]
 
                         pTxt = tTxt[:-1].translate(transMapB)
+
+                    if nLines:
+                        isAligned = cStyle & BlockFmt.ALIGNED
+                        if firstIndent and not (self._noIndent or isAligned):
+                            # If paragraph indentation is enabled, not temporarily
+                            # turned off, and the block is not aligned, we add the
+                            # text indentation flag
+                            cStyle |= BlockFmt.IND_T
+
+                        if doJustify and not isAligned:
+                            cStyle |= BlockFmt.JUSTIFY
+
                         sBlocks.append((
                             BlockTyp.TEXT, pLines[0][1], pTxt, tFmt, cStyle
                         ))

tests/test_formats/test_fmt_tokenizer.py

@@ -1074,6 +1074,80 @@ def testFmtToken_Paragraphs(mockGUI):
     ]
 
 
+@pytest.mark.core
+def testFmtToken_BreakAlignIndent(mockGUI):
+    """Test the splitting of paragraphs with alignment."""
+    project = NWProject()
+    tokens = BareTokenizer(project)
+    tokens._handle = TMH
+
+    for text in [
+        "This is text <<\nspanning multiple\nlines",
+        "This is text\nspanning multiple <<\nlines",
+        "This is text\nspanning multiple\nlines <<",
+    ]:
+        # Preserve Breaks
+        tokens.setKeepLineBreaks(True)
+        tokens._text = text
+        tokens.tokenizeText()
+        assert tokens._blocks == [
+            (BlockTyp.TEXT, "", "This is text\nspanning multiple\nlines", [], BlockFmt.LEFT),
+        ]
+
+        # Don't Preserve Breaks
+        tokens.setKeepLineBreaks(False)
+        tokens._text = text
+        tokens.tokenizeText()
+        assert tokens._blocks == [
+            (BlockTyp.TEXT, "", "This is text spanning multiple lines", [], BlockFmt.LEFT),
+        ]
+
+        # With Justify
+        # This should disable justify
+        tokens.setKeepLineBreaks(True)
+        tokens.setJustify(True)
+        tokens._text = text
+        tokens.tokenizeText()
+        assert tokens._blocks == [
+            (BlockTyp.TEXT, "", "This is text\nspanning multiple\nlines", [], BlockFmt.LEFT),
+        ]
+
+        # With Indent
+        # This should disable indent
+        tokens.setKeepLineBreaks(True)
+        tokens.setFirstLineIndent(True, 1.0, False)
+        tokens._text = text
+        tokens.tokenizeText()
+        assert tokens._blocks == [
+            (BlockTyp.TEXT, "", "This is text\nspanning multiple\nlines", [], BlockFmt.LEFT),
+        ]
+
+
+@pytest.mark.core
+def testFmtToken_BreakJustify(mockGUI):
+    """Test the of processing of justify with breaks."""
+    project = NWProject()
+    tokens = BareTokenizer(project)
+    tokens._handle = TMH
+    tokens.setJustify(True)
+
+    # Applied to all lines when breaks are preserved
+    tokens._text = "This is text\nspanning multiple\nlines"
+    tokens.setKeepLineBreaks(True)
+    tokens.tokenizeText()
+    assert tokens._blocks == [
+        (BlockTyp.TEXT, "", "This is text\nspanning multiple\nlines", [], BlockFmt.JUSTIFY),
+    ]
+
+    # Turning off breaks should make no difference (see issue #2426)
+    tokens._text = "This is text\nspanning multiple\nlines"
+    tokens.setKeepLineBreaks(False)
+    tokens.tokenizeText()
+    assert tokens._blocks == [
+        (BlockTyp.TEXT, "", "This is text spanning multiple lines", [], BlockFmt.JUSTIFY),
+    ]
+
+
 @pytest.mark.core
 def testFmtToken_TextFormat(mockGUI):
     """Test the tokenization of text formats in the Tokenizer class."""

tests/test_formats/test_fmt_toodt.py

@@ -733,7 +733,7 @@ def testFmtToOdt_ConvertParagraphs(mockGUI):
         '<office:text>'
         '<text:h text:style-name="Heading_20_2" text:outline-level="2">Scene</text:h>'
         '<text:p text:style-name="P7">Regular paragraph</text:p>'
-        '<text:p text:style-name="Text_20_body">with<text:line-break />break</text:p>'
+        '<text:p text:style-name="P7">with<text:line-break />break</text:p>'
         '<text:p text:style-name="Text_20_body">Left Align</text:p>'
         '</office:text>'
     )