diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py index 2fa617ca..69a422c8 100644 --- a/novelwriter/formats/tokenizer.py +++ b/novelwriter/formats/tokenizer.py @@ -526,6 +526,7 @@ class Tokenizer(ABC): if self._isNovel: self._hFormatter.setHandle(self._handle) + # Replace all instances of [br] with a placeholder character text = REGEX_PATTERNS.lineBreak.sub("\uffff", self._text) nHead = 0 @@ -534,7 +535,7 @@ class Tokenizer(ABC): tHandle = self._handle or "" tBlocks: list[T_Block] = [B_EMPTY] for bLine in text.splitlines(): - aLine = bLine.replace("\uffff", "\n") + aLine = bLine.replace("\uffff", "") # Remove placeholder characters sLine = aLine.strip().lower() # Check for blank lines diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py index 5549c14a..115203e3 100644 --- a/tests/test_formats/test_fmt_tokenizer.py +++ b/tests/test_formats/test_fmt_tokenizer.py @@ -1056,72 +1056,121 @@ def testFmtToken_TextFormat(mockGUI): tokens._text = "Some **bolded text** on this lines\n" tokens.tokenizeText() assert tokens._blocks == [( - BlockTyp.TEXT, "", "Some bolded text on this lines", - [ + BlockTyp.TEXT, "", "Some bolded text on this lines", [ (5, TextFmt.B_B, ""), (16, TextFmt.B_E, ""), - ], - BlockFmt.NONE + ], BlockFmt.NONE )] assert tokens._raw[-1] == "Some **bolded text** on this lines\n\n" tokens._text = "Some _italic text_ on this lines\n" tokens.tokenizeText() assert tokens._blocks == [( - BlockTyp.TEXT, "", "Some italic text on this lines", - [ + BlockTyp.TEXT, "", "Some italic text on this lines", [ (5, TextFmt.I_B, ""), (16, TextFmt.I_E, ""), - ], - BlockFmt.NONE + ], BlockFmt.NONE )] assert tokens._raw[-1] == "Some _italic text_ on this lines\n\n" tokens._text = "Some **_bold italic text_** on this lines\n" tokens.tokenizeText() assert tokens._blocks == [( - BlockTyp.TEXT, "", "Some bold italic text on this lines", - [ + BlockTyp.TEXT, "", "Some bold italic text on this lines", [ (5, TextFmt.B_B, ""), (5, TextFmt.I_B, ""), (21, TextFmt.I_E, ""), (21, TextFmt.B_E, ""), - ], - BlockFmt.NONE + ], BlockFmt.NONE )] assert tokens._raw[-1] == "Some **_bold italic text_** on this lines\n\n" tokens._text = "Some ~~strikethrough text~~ on this lines\n" tokens.tokenizeText() assert tokens._blocks == [( - BlockTyp.TEXT, "", "Some strikethrough text on this lines", - [ + BlockTyp.TEXT, "", "Some strikethrough text on this lines", [ (5, TextFmt.D_B, ""), (23, TextFmt.D_E, ""), - ], - BlockFmt.NONE + ], BlockFmt.NONE )] assert tokens._raw[-1] == "Some ~~strikethrough text~~ on this lines\n\n" tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n" tokens.tokenizeText() assert tokens._blocks == [( - BlockTyp.TEXT, "", "Some nested bold and italic and strikethrough text here", - [ + BlockTyp.TEXT, "", "Some nested bold and italic and strikethrough text here", [ (5, TextFmt.B_B, ""), (21, TextFmt.I_B, ""), (27, TextFmt.I_E, ""), (32, TextFmt.D_B, ""), (45, TextFmt.D_E, ""), (50, TextFmt.B_E, ""), - ], - BlockFmt.NONE + ], BlockFmt.NONE )] assert tokens._raw[-1] == ( "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n" ) +@pytest.mark.core +def testFmtToken_LineBreak(mockGUI): + """Test processing of forced line breaks in the Tokenizer class.""" + project = NWProject() + tokens = BareTokenizer(project) + tokens._handle = TMH + tokens.setComments(True) + + # They are stripped in headers + tokens._text = "## Hello[br] World" + tokens.tokenizeText() + assert tokens._blocks == [ + (BlockTyp.HEAD2, TM1, "Hello World", [], BlockFmt.NONE) + ] + + # They are stripped in comments + tokens._text = "% Hello[br] World" + tokens.tokenizeText() + assert tokens._blocks == [( + BlockTyp.COMMENT, "", "Comment: Hello World", [ + (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"), + (8, TextFmt.COL_E, ""), (8, TextFmt.B_E, ""), + (9, TextFmt.COL_B, "comment"), (20, TextFmt.COL_E, ""), + ], BlockFmt.NONE + )] + + # They are used in text, with breaks enabled + tokens.setKeepLineBreaks(True) + tokens._text = "Hello[br]\nWorld" + tokens.tokenizeText() + assert tokens._blocks == [ + (BlockTyp.TEXT, "", "Hello\nWorld", [], BlockFmt.NONE) + ] + + # They are used in text, with breaks disabled + tokens.setKeepLineBreaks(False) + tokens._text = "Hello[br]\nWorld" + tokens.tokenizeText() + assert tokens._blocks == [ + (BlockTyp.TEXT, "", "Hello\nWorld", [], BlockFmt.NONE) + ] + + # Without forced breaks, they are preserved with breaks enabled + tokens.setKeepLineBreaks(True) + tokens._text = "Hello\nWorld" + tokens.tokenizeText() + assert tokens._blocks == [ + (BlockTyp.TEXT, "", "Hello\nWorld", [], BlockFmt.NONE) + ] + + # Without forced breaks, they are not preserved with breaks disabled + tokens.setKeepLineBreaks(False) + tokens._text = "Hello\nWorld" + tokens.tokenizeText() + assert tokens._blocks == [ + (BlockTyp.TEXT, "", "Hello World", [], BlockFmt.NONE) + ] + + @pytest.mark.core def testFmtToken_Dialogue(mockGUI): """Test the tokenization of dialogue in the Tokenizer class."""