Add working forced line break implementation

novelwriter/formats/tokenizer.py

@@ -519,18 +519,22 @@ class Tokenizer(ABC):
         isNovel = self._isNovel
         keepRaw = self._keepRaw
         doJustify = self._doJustify
+        keepBreaks = self._keepBreaks
         indentFirst = self._indentFirst
         firstIndent = self._firstIndent
 
         if self._isNovel:
             self._hFormatter.setHandle(self._handle)
 
+        text = REGEX_PATTERNS.lineBreak.sub("\uffff", self._text)
+
         nHead = 0
         breakNext = False
         tmpMarkdown = []
         tHandle = self._handle or ""
         tBlocks: list[T_Block] = [B_EMPTY]
-        for aLine in self._text.splitlines():
+        for bLine in text.splitlines():
+            aLine = bLine.replace("\uffff", "\n")
             sLine = aLine.strip().lower()
 
             # Check for blank lines
@@ -782,19 +786,19 @@ class Tokenizer(ABC):
                 alnRight = False
                 indLeft = False
                 indRight = False
-                if aLine.startswith(">>"):
+                if bLine.startswith(">>"):
                     alnRight = True
-                    aLine = aLine[2:].lstrip(" ")
+                    bLine = bLine[2:].lstrip(" ")
-                elif aLine.startswith(">"):
+                elif bLine.startswith(">"):
                     indLeft = True
-                    aLine = aLine[1:].lstrip(" ")
+                    bLine = bLine[1:].lstrip(" ")
 
-                if aLine.endswith("<<"):
+                if bLine.endswith("<<"):
                     alnLeft = True
-                    aLine = aLine[:-2].rstrip(" ")
+                    bLine = bLine[:-2].rstrip(" ")
-                elif aLine.endswith("<"):
+                elif bLine.endswith("<"):
                     indRight = True
-                    aLine = aLine[:-1].rstrip(" ")
+                    bLine = bLine[:-1].rstrip(" ")
 
                 if alnLeft and alnRight:
                     sAlign |= BlockFmt.CENTRE
@@ -809,7 +813,7 @@ class Tokenizer(ABC):
                     sAlign |= BlockFmt.IND_R
 
                 # Process formats
-                tLine, tFmt = self._extractFormats(aLine, hDialog=isNovel)
+                tLine, tFmt = self._extractFormats(bLine, hDialog=isNovel)
                 tBlocks.append((
                     BlockTyp.TEXT, "", tLine, tFmt, sAlign
                 ))
@@ -840,7 +844,7 @@ class Tokenizer(ABC):
         # It also ensures that there isn't paragraph spacing between
         # meta data lines for formats that have spacing.
 
-        lineSep = "\n" if self._keepBreaks else " "
+        lineSep = "\n" if keepBreaks else " "
 
         pLines: list[T_Block] = []
         sBlocks: list[T_Block] = []
@@ -890,9 +894,12 @@ class Tokenizer(ABC):
                         # enabled, and there is no alignment, we apply it.
                         if doJustify and not cStyle & BlockFmt.ALIGNED:
                             cStyle |= BlockFmt.JUSTIFY
+
+                        pTxt = pLines[0][2].replace("\uffff", "\n")
                         sBlocks.append((
-                            BlockTyp.TEXT, pLines[0][1], pLines[0][2], pLines[0][3], cStyle
+                            BlockTyp.TEXT, pLines[0][1], pTxt, pLines[0][3], cStyle
                         ))
+
                     elif nLines > 1:
                         # The paragraph contains multiple lines, so we need to
                         # join them according to the line break policy, and
@@ -903,8 +910,11 @@ class Tokenizer(ABC):
                             tLen = len(tTxt)
                             tTxt += f"{aBlock[2]}{lineSep}"
                             tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3])
+                            cStyle |= aBlock[4]
+
+                        pTxt = tTxt[:-1].replace("\uffff", "\n")
                         sBlocks.append((
-                            BlockTyp.TEXT, pLines[0][1], tTxt[:-1], tFmt, cStyle
+                            BlockTyp.TEXT, pLines[0][1], pTxt, tFmt, cStyle
                         ))
 
                     # Reset buffer and make sure text indent is on for next pass

sample/content/ba8a28a246524.nwd

@@ -1,24 +1,24 @@
 %%~name: Interlude
 %%~path: 7031beac91f75/ba8a28a246524
 %%~kind: NOVEL/DOCUMENT
-%%~hash: 721d3d15e0233186354ac6fa61f27db10f38e6bc
+%%~hash: 5c8f68d48573b576dcaacf6d6928c496ec361b9d
-%%~date: Unknown/2024-03-14 22:55:04
+%%~date: Unknown/2024-10-24 01:22:15
 ##! Interlude
 
 % Notice that this document has a title with a ‘!’ in it in addition to the two hash symbols. This means it is an unnumbered chapter. Unnumbered chapters can be treated separately from numbered chapters during export. Perfect for when you want to add an interlude, or for a prologue or epilogue.
 
-I am the very model of a modern Major-General
+I am the very model of a modern Major-General[br]
-I've information vegetable, animal, and mineral
+I've information vegetable, animal, and mineral[br]
-I know the kings of England, and I quote the fights historical
+I know the kings of England, and I quote the fights historical[br]
-From Marathon to Waterloo, in order categorical
+From Marathon to Waterloo, in order categorical <<
 
-I'm very well acquainted, too, with matters mathematical <<
+I'm very well acquainted, too, with matters mathematical[br]
-I understand equations, both the simple and quadratical
+I understand equations, both the simple and quadratical[br]
-About binomial theorem I'm teeming with a lot o’ news
+About binomial theorem I'm teeming with a lot o’ news[br]
-With many cheerful facts about the square of the hypotenuse
+With many cheerful facts about the square of the hypotenuse <<
 
-	With many cheerful facts about the square of the hypotenuse <<
+	With many cheerful facts about the square of the hypotenuse[br]
-	With many cheerful facts about the square of the hypotenuse
+	With many cheerful facts about the square of the hypotenuse[br]
-	With many cheerful facts about the square of the hypotepotenuse
+	With many cheerful facts about the square of the hypotepotenuse <<
 
 % Notice that the lines in the verse end in a single line break. Single line breaks do not create a new paragraph but instead insert a break within the paragraph. Press Ctrl+R to see what this renders like.

tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json

@@ -2,19 +2,19 @@
   "meta": {
     "projectName": "Lorem Ipsum",
     "novelAuthor": "lipsum.com",
-    "buildTime": 1729029144,
+    "buildTime": 1729725334,
-    "buildTimeStr": "2024-10-15 23:52:24"
+    "buildTimeStr": "2024-10-24 01:15:34"
   },
   "text": {
     "nwd": [
       [
         "#! Lorem Ipsum",
         "",
-        "**By lipsum.com**",
+        ">> **By lipsum.com** <<",
         "",
-        "\u201cNeque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit\u2026\u201d",
+        ">> \u201cNeque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit\u2026\u201d <<",
         "",
-        "\u201cThere is no one who loves pain itself, who seeks after it and wants to have it, simply because it is pain\u2026\u201d"
+        ">> \u201cThere is no one who loves pain itself, who seeks after it and wants to have it, simply because it is pain\u2026\u201d <<"
       ],
       [
         "",
@@ -36,7 +36,7 @@
       [
         "# Act One",
         "",
-        "\u201cFusce maximus felis libero\u201d"
+        ">> \u201cFusce maximus felis libero\u201d <<"
       ],
       [
         "## Chapter One",

tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.txt

@@ -1,10 +1,10 @@
 #! Lorem Ipsum
 
-**By lipsum.com**
+>> **By lipsum.com** <<
 
-“Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit…”
+>> “Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit…” <<
 
-“There is no one who loves pain itself, who seeks after it and wants to have it, simply because it is pain…”
+>> “There is no one who loves pain itself, who seeks after it and wants to have it, simply because it is pain…” <<
 
 
 % Exctracted from the lipsum.com website.
@@ -23,7 +23,7 @@ _Lorem Ipsum_ is simply dummy text[footnote:f9kgf] of the printing and typesetti
 
 # Act One
 
-“Fusce maximus felis libero”
+>> “Fusce maximus felis libero” <<
 
 ## Chapter One
 

tests/test_core/test_core_docbuild.py

@@ -412,7 +412,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
     assert error == []
     assert docFile.read_text(encoding="utf-8") == (
         "#! New Novel\n\n"
-        "By Jane Doe\n\n"
+        ">> By Jane Doe <<\n\n"
         "## New Chapter\n\n\n"
         "### New Scene\n\n\n"
     )
@@ -442,7 +442,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
     assert error == []
     assert docFile.read_text(encoding="utf-8") == (
         "#! New Novel\n\n"
-        "By Jane Doe\n\n"
+        ">> By Jane Doe <<\n\n"
         "## New Chapter\n\n\n"
         "### New Scene\n\n\n"
     )
@@ -566,7 +566,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
     assert isinstance(docBuild.lastBuild, ToRaw)
     assert docFile.read_text(encoding="utf-8") == (
         "#! New Novel\n\n"
-        "By Jane Doe\n\n"
+        ">> By Jane Doe <<\n\n"
         "## New Chapter\n\n\n"
         "### New Scene\n\n\n"
         "#! Notes: Plot\n\n"

tests/test_formats/test_fmt_tokenizer.py

@@ -842,13 +842,13 @@ def testFmtToken_MarginFormat(mockGUI):
     ]
     assert tokens._raw[-1] == (
         "Some regular text\n\n"
-        "Some left-aligned text\n\n"
+        "Some left-aligned text <<\n\n"
-        "Some right-aligned text\n\n"
+        ">> Some right-aligned text\n\n"
-        "Some centered text\n\n"
+        ">> Some centered text <<\n\n"
-        "Left-indented block\n\n"
+        "> Left-indented block\n\n"
-        "Right-indented block\n\n"
+        "Right-indented block <\n\n"
-        "Double-indented block\n\n"
+        "> Double-indented block <\n\n"
-        "Right-indent, right-aligned\n\n\n"
+        ">> Right-indent, right-aligned <\n\n\n"
     )