Simplify JSON and raw text writing

novelwriter/core/docbuild.py

@@ -38,10 +38,11 @@ from novelwriter.core.project import NWProject
 from novelwriter.enum import nwBuildFmt
 from novelwriter.error import formatException, logException
 from novelwriter.formats.tohtml import ToHtml
-from novelwriter.formats.tokenizer import Tokenizer, ToRaw
+from novelwriter.formats.tokenizer import Tokenizer
 from novelwriter.formats.tomarkdown import ToMarkdown
 from novelwriter.formats.toodt import ToOdt
 from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
+from novelwriter.formats.toraw import ToRaw
 
 logger = logging.getLogger(__name__)
 
@@ -146,12 +147,10 @@ class NWBuildDocument:
 
         return
 
-    def iterBuild(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
+    def iterBuildDocument(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
         """Wrapper for builders based on format."""
-        asJson = False
-
         if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
-            makeObj = ToOdt(self._project, isFlat=(bFormat == nwBuildFmt.FODT))
+            makeObj = ToOdt(self._project, bFormat == nwBuildFmt.FODT)
             filtered = self._setupBuild(makeObj)
             makeObj.initDocument()
             yield from self._iterBuild(makeObj, filtered)
@@ -164,7 +163,6 @@ class NWBuildDocument:
             makeObj.appendFootnotes()
             if not self._build.getBool("html.preserveTabs"):
                 makeObj.replaceTabs()
-            asJson = (bFormat == nwBuildFmt.J_HTML)
 
         elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
             makeObj = ToMarkdown(self._project)
@@ -177,20 +175,16 @@ class NWBuildDocument:
 
         elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
             makeObj = ToRaw(self._project)
-            makeObj.setKeepMarkdown(True)
             filtered = self._setupBuild(makeObj)
             yield from self._iterBuild(makeObj, filtered)
-
+            if self._build.getBool("format.replaceTabs"):
-            asJson = (bFormat == nwBuildFmt.J_NWD)
+                makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
 
         self._error = None
         self._cache = makeObj
 
         try:
-            if isinstance(makeObj, ToHtml | ToRaw):
+            makeObj.saveDocument(path)
-                makeObj.saveDocument(path, asJson=asJson)
-            else:
-                makeObj.saveDocument(path)
         except Exception as exc:
             logException()
             self._error = formatException(exc)

novelwriter/formats/tohtml.py

@@ -290,9 +290,9 @@ class ToHtml(Tokenizer):
 
         return
 
-    def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Save the data to an HTML file."""
-        if asJson:
+        if path.suffix.lower() == ".json":
             ts = time()
             data = {
                 "meta": {

novelwriter/formats/tokenizer.py

@@ -134,10 +134,10 @@ class Tokenizer(ABC):
         self._project = project
 
         # Data Variables
-        self._text   = ""     # The raw text to be tokenized
+        self._text    = ""     # The raw text to be tokenized
-        self._handle = None   # The item handle currently being processed
+        self._handle  = None   # The item handle currently being processed
-        self._result = ""     # The result of the last document
+        self._result  = ""     # The result of the last document
-        self._keepMD = False  # Whether to keep the markdown text
+        self._keepRaw = False  # Whether to keep the raw text, used by ToRaw
 
         # Tokens and Meta Data (Per Document)
         self._tokens: list[T_Token] = []
@@ -473,11 +473,6 @@ class Tokenizer(ABC):
         self._keepBreaks = state
         return
 
-    def setKeepMarkdown(self, state: bool) -> None:
-        """Keep original markdown during build."""
-        self._keepMD = state
-        return
-
     ##
     #  Class Methods
     ##
@@ -487,7 +482,7 @@ class Tokenizer(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         raise NotImplementedError
 
     def addRootHeading(self, tHandle: str) -> None:
@@ -509,7 +504,7 @@ class Tokenizer(ABC):
             self._tokens.append((
                 self.T_TITLE, 1, title, [], textAlign
             ))
-            if self._keepMD:
+            if self._keepRaw:
                 self._markdown.append(f"#! {title}\n\n")
 
         return
@@ -574,7 +569,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     self.T_EMPTY, nHead, "", [], self.A_NONE
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append("\n")
 
                 continue
@@ -632,26 +627,26 @@ class Tokenizer(ABC):
                     tokens.append((
                         self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doSynopsis and self._keepMD:
+                    if self._doSynopsis and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
                 elif cStyle == nwComment.SHORT:
                     tLine, tFmt = self._extractFormats(cText)
                     tokens.append((
                         self.T_SHORT, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doSynopsis and self._keepMD:
+                    if self._doSynopsis and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
                 elif cStyle == nwComment.FOOTNOTE:
                     tLine, tFmt = self._extractFormats(cText, skip=self.FMT_FNOTE)
                     self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt)
-                    if self._keepMD:
+                    if self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
                 else:
                     tLine, tFmt = self._extractFormats(cText)
                     tokens.append((
                         self.T_COMMENT, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doComments and self._keepMD:
+                    if self._doComments and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith("@"):
@@ -668,7 +663,7 @@ class Tokenizer(ABC):
                     tokens.append((
                         self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
                     ))
-                    if self._doKeywords and self._keepMD:
+                    if self._doKeywords and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("# ", "#! ")):
@@ -704,7 +699,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("## ", "##! ")):
@@ -739,7 +734,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("### ", "###! ")):
@@ -780,7 +775,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith("#### "):
@@ -810,7 +805,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             else:
@@ -858,7 +853,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     self.T_TEXT, nHead, tLine, tFmt, sAlign
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
         # If we have content, turn off the first page flag
@@ -877,7 +872,7 @@ class Tokenizer(ABC):
         tokens.append((
             self.T_EMPTY, nHead, "", [], self.A_NONE
         ))
-        if self._keepMD:
+        if self._keepRaw:
             tmpMarkdown.append("\n")
             self._markdown.append("".join(tmpMarkdown))
 
@@ -1245,41 +1240,3 @@ class HeadingFormatter:
             hFormat = hFormat.replace(nwHeadFmt.CHAR_FOCUS, fText)
 
         return hFormat
-
-
-class ToRaw(Tokenizer):
-
-    def __init__(self, project: NWProject) -> None:
-        super().__init__(project)
-        self._keepMD = True
-        return
-
-    def doConvert(self) -> None:
-        return
-
-    def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
-        """Save the raw text to a plain text file."""
-        if asJson:
-            ts = time()
-            data = {
-                "meta": {
-                    "projectName": self._project.data.name,
-                    "novelAuthor": self._project.data.author,
-                    "buildTime": int(ts),
-                    "buildTimeStr": formatTimeStamp(ts),
-                },
-                "text": {
-                    "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
-                }
-            }
-            with open(path, mode="w", encoding="utf-8") as fObj:
-                json.dump(data, fObj, indent=2)
-
-        else:
-            with open(path, mode="w", encoding="utf-8") as outFile:
-                for nwdPage in self._markdown:
-                    outFile.write(nwdPage)
-
-        logger.info("Wrote file: %s", path)
-
-        return

novelwriter/formats/tomarkdown.py

@@ -199,7 +199,7 @@ class ToMarkdown(Tokenizer):
 
         return
 
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Save the data to a plain text file."""
         with open(path, mode="w", encoding="utf-8") as outFile:
             outFile.write("".join(self._fullMD))
@@ -210,8 +210,6 @@ class ToMarkdown(Tokenizer):
         """Replace tabs with spaces."""
         spaces = spaceChar*nSpaces
         self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
-        if self._keepMD:
-            self._markdown = [p.replace("\t", spaces) for p in self._markdown]
         return
 
     ##

novelwriter/formats/toodt.py

@@ -543,7 +543,7 @@ class ToOdt(Tokenizer):
             self._xText.insert(0, xFields)
         return
 
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Save the data to an .fodt or .odt file."""
         if self._isFlat:
             with open(path, mode="wb") as fObj:

novelwriter/formats/toqdoc.py

@@ -275,7 +275,7 @@ class ToQTextDocument(Tokenizer):
 
         return
 
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Not implemented."""
         return
 

novelwriter/formats/toraw.py

@@ -0,0 +1,86 @@
+"""
+novelWriter – Raw NW Text Format
+================================
+
+File History:
+Created: 2024-10-15 [2.6b1] ToRaw
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+from __future__ import annotations
+
+import json
+import logging
+
+from pathlib import Path
+from time import time
+
+from novelwriter.common import formatTimeStamp
+from novelwriter.core.project import NWProject
+from novelwriter.formats.tokenizer import Tokenizer
+
+logger = logging.getLogger(__name__)
+
+
+class ToRaw(Tokenizer):
+    """Core: Raw novelWriter Text Writer
+
+    A class that will collect the minimally altered original source text
+    and write it to either a text or JSON file.
+    """
+
+    def __init__(self, project: NWProject) -> None:
+        super().__init__(project)
+        self._keepRaw = True
+        return
+
+    def doConvert(self) -> None:
+        """No conversion to perform."""
+        return
+
+    def saveDocument(self, path: Path) -> None:
+        """Save the raw text to a plain text file."""
+        if path.suffix.lower() == ".json":
+            ts = time()
+            data = {
+                "meta": {
+                    "projectName": self._project.data.name,
+                    "novelAuthor": self._project.data.author,
+                    "buildTime": int(ts),
+                    "buildTimeStr": formatTimeStamp(ts),
+                },
+                "text": {
+                    "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
+                }
+            }
+            with open(path, mode="w", encoding="utf-8") as fObj:
+                json.dump(data, fObj, indent=2)
+
+        else:
+            with open(path, mode="w", encoding="utf-8") as outFile:
+                for nwdPage in self._markdown:
+                    outFile.write(nwdPage)
+
+        logger.info("Wrote file: %s", path)
+
+        return
+
+    def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
+        """Replace tabs with spaces."""
+        spaces = spaceChar*nSpaces
+        self._markdown = [p.replace("\t", spaces) for p in self._markdown]
+        return

novelwriter/tools/manusbuild.py

@@ -333,7 +333,7 @@ class GuiManuscriptBuild(NDialog):
         docBuild.queueAll()
 
         self.buildProgress.setMaximum(len(docBuild))
-        for i, _ in docBuild.iterBuild(buildPath, bFormat):
+        for i, _ in docBuild.iterBuildDocument(buildPath, bFormat):
             self.buildProgress.setValue(i+1)
 
         self._build.setLastBuildPath(bPath)

tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json

@@ -2,8 +2,8 @@
   "meta": {
     "projectName": "Lorem Ipsum",
     "novelAuthor": "lipsum.com",
-    "buildTime": 1729027593,
+    "buildTime": 1729029144,
-    "buildTimeStr": "2024-10-15 23:26:33"
+    "buildTimeStr": "2024-10-15 23:52:24"
   },
   "text": {
     "nwd": [
@@ -96,11 +96,11 @@
         "",
         "% Exctracted from the lipsum.com website.",
         "",
-        "\tIt is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
+        "    It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
         "",
-        "\tThe point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
+        "    The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
         "",
-        "\tMany desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
+        "    Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
       ],
       [
         "## Chapter Two",

tests/test_core/test_core_docbuild.py

@@ -32,9 +32,9 @@ from novelwriter.core.docbuild import NWBuildDocument
 from novelwriter.core.project import NWProject
 from novelwriter.enum import nwBuildFmt
 from novelwriter.formats.tohtml import ToHtml
-from novelwriter.formats.tokenizer import ToRaw
 from novelwriter.formats.tomarkdown import ToMarkdown
 from novelwriter.formats.toodt import ToOdt
+from novelwriter.formats.toraw import ToRaw
 
 from tests.mocked import causeException, causeOSError
 from tests.tools import ODT_IGNORE, C, buildTestProject, cmpFiles
@@ -102,7 +102,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -120,7 +120,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.ODT):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -137,7 +137,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.fodt"
-        for _ in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -153,7 +153,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
         count = 0
         error = []
         docFile = fncPath / "Lorem Ipsum Err.fodt"
-        for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
+        for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
             count += 1 if success else 0
             if not success and docBuild.error:
                 error.append(docBuild.error)
@@ -204,7 +204,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -224,7 +224,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_HTML):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -242,7 +242,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.htm"
-        for _ in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -272,7 +272,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -292,7 +292,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -310,7 +310,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.md"
-        for _ in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -340,7 +340,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -360,7 +360,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_NWD):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -378,7 +378,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.md"
-        for _ in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -402,7 +402,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
     count = 0
     error = []
     docFile = fncPath / "Minimal.txt"
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -432,7 +432,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
     count = 0
     error = []
     docFile = fncPath / "Minimal.txt"
-    for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -474,7 +474,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # ODT Format
     docFile = fncPath / "Minimal.odt"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.ODT)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -484,7 +484,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # FODT Format
     docFile = fncPath / "Minimal.fodt"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.FODT)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -494,7 +494,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # HTML Format
     docFile = fncPath / "Minimal.html"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.HTML)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -504,7 +504,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # JSON HTML Format
     docFile = fncPath / "Minimal.json"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_HTML)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -516,7 +516,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # Standard Markdown Format
     docFile = fncPath / "Minimal.md"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.STD_MD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -537,7 +537,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # Extended Markdown Format
     docFile = fncPath / "Minimal.md"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -558,7 +558,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # NWD Format
     docFile = fncPath / "Minimal.txt"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.NWD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -579,7 +579,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # JSON NWD Format
     docFile = fncPath / "Minimal.json"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_NWD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]

tests/test_formats/test_fmt_tohtml.py

@@ -644,12 +644,12 @@ def testFmtToHtml_Save(mockGUI, fncPath):
     )
 
     saveFile = fncPath / "outFile.htm"
-    html.saveDocument(saveFile, asJson=False)
+    html.saveDocument(saveFile)
     assert saveFile.read_text(encoding="utf-8") == htmlDoc
 
     # JSON + HTML
     saveFile = fncPath / "outFile.json"
-    html.saveDocument(saveFile, asJson=True)
+    html.saveDocument(saveFile)
     data = json.loads(saveFile.read_text(encoding="utf-8"))
     assert data["meta"]["projectName"] == ""
     assert data["meta"]["novelAuthor"] == ""
@@ -664,7 +664,6 @@ def testFmtToHtml_Methods(mockGUI):
     """Test all the other methods of the ToHtml class."""
     project = NWProject()
     html = ToHtml(project)
-    html.setKeepMarkdown(True)
 
     # Auto-Replace, keep Unicode
     docText = "Text with <brackets> & short–dash, long—dash …\n"

tests/test_formats/test_fmt_tokenizer.py

@@ -31,6 +31,7 @@ from novelwriter.constants import nwHeadFmt
 from novelwriter.core.project import NWProject
 from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer, stripEscape
 from novelwriter.formats.tomarkdown import ToMarkdown
+from novelwriter.formats.toraw import ToRaw
 
 from tests.tools import C, buildTestProject, readFile
 
@@ -43,11 +44,24 @@ class BareTokenizer(Tokenizer):
         super().saveDocument(path)  # type: ignore (deliberate check)
 
 
+@pytest.mark.core
+def testFmtToken_Abstracts(mockGUI, tstPaths):
+    """Test all the abstract methods of the Tokenizer class."""
+    project = NWProject()
+    tokens = BareTokenizer(project)
+
+    with pytest.raises(NotImplementedError):
+        tokens.doConvert()
+
+    with pytest.raises(NotImplementedError):
+        tokens.saveDocument(tstPaths)
+
+
 @pytest.mark.core
 def testFmtToken_Setters(mockGUI):
     """Test all the setters for the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
 
     # Verify defaults
     assert tokens._fmtPart == nwHeadFmt.TITLE
@@ -164,8 +178,7 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
 
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Set some content to work with
     docText = (
@@ -235,13 +248,6 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
         ]
     }
 
-    # Check abstract methods
-    with pytest.raises(NotImplementedError):
-        tokens.doConvert()
-
-    with pytest.raises(NotImplementedError):
-        tokens.saveDocument(fncPath)
-
 
 @pytest.mark.core
 def testFmtToken_StripEscape():
@@ -256,8 +262,7 @@ def testFmtToken_StripEscape():
 def testFmtToken_HeaderFormat(mockGUI):
     """Test the tokenization of header formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Title
     # =====
@@ -431,7 +436,7 @@ def testFmtToken_HeaderFormat(mockGUI):
 def testFmtToken_HeaderStyle(mockGUI):
     """Test the styling of headers in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
 
     def processStyle(text: str, first: bool) -> int:
         tokens._text = text
@@ -692,8 +697,7 @@ def testFmtToken_HeaderStyle(mockGUI):
 def testFmtToken_MetaFormat(mockGUI):
     """Test the tokenization of meta formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Comment
     tokens._text = "% A comment\n"
@@ -780,8 +784,7 @@ def testFmtToken_MetaFormat(mockGUI):
 def testFmtToken_MarginFormat(mockGUI):
     """Test the tokenization of margin formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Alignment and Indentation
     dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
@@ -823,8 +826,7 @@ def testFmtToken_MarginFormat(mockGUI):
 def testFmtToken_ExtractFormats(mockGUI):
     """Test the extraction of formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Markdown
     # ========
@@ -930,8 +932,7 @@ def testFmtToken_ExtractFormats(mockGUI):
 def testFmtToken_Paragraphs(mockGUI):
     """Test the splitting of paragraphs."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Collapse empty lines
     tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
@@ -1003,8 +1004,7 @@ def testFmtToken_Paragraphs(mockGUI):
 def testFmtToken_TextFormat(mockGUI):
     """Test the tokenization of text formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Text
     tokens._text = "Some plain text\non two lines\n\n\n"
@@ -1104,7 +1104,7 @@ def testFmtToken_Dialogue(mockGUI):
     CONFIG.narratorBreak  = "\u2013"
 
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
     tokens.setDialogueHighlight(True)
     tokens._isNovel = True
 
@@ -1191,7 +1191,7 @@ def testFmtToken_Dialogue(mockGUI):
 def testFmtToken_SpecialFormat(mockGUI):
     """Test the tokenization of special formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
 
     tokens._isNovel = True
 
@@ -1353,7 +1353,7 @@ def testFmtToken_SpecialFormat(mockGUI):
 def testFmtToken_TextIndent(mockGUI):
     """Test the handling of text indent in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
 
     # No First Indent
     tokens.setFirstLineIndent(True, 1.0, False)
@@ -1438,7 +1438,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
     project = NWProject()
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
     tokens._isNovel = True
 
     # Titles
@@ -1627,7 +1627,7 @@ def testFmtToken_BuildOutline(mockGUI, ipsumText):
     project = NWProject()
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
 
     # Novel
     tokens._isNovel = True
@@ -1691,7 +1691,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
     project = NWProject()
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
     tokens._isNovel = True
 
     # Short Text

tests/test_formats/test_fmt_tomarkdown.py

@@ -250,7 +250,6 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
     """Test the save method of the ToMarkdown class."""
     project = NWProject()
     toMD = ToMarkdown(project)
-    toMD.setKeepMarkdown(True)
     toMD._isNovel = True
 
     # Build Project
@@ -287,7 +286,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
 
     toMD.replaceTabs(nSpaces=4, spaceChar=" ")
     resText[6] = "#### A Section\n\n    More text in scene two.\n\n"
-    assert toMD.allMarkdown == resText
+    assert toMD.fullMD == resText
 
     # Check File
     # ==========