Refactor document builder (#2047)

novelwriter/core/docbuild.py

@@ -42,6 +42,7 @@ from novelwriter.formats.tokenizer import Tokenizer
 from novelwriter.formats.tomarkdown import ToMarkdown
 from novelwriter.formats.toodt import ToOdt
 from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
+from novelwriter.formats.toraw import ToRaw
 
 logger = logging.getLogger(__name__)
 
@@ -146,24 +147,67 @@ class NWBuildDocument:
 
         return
 
-    def iterBuild(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
+    def iterBuildDocument(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
         """Wrapper for builders based on format."""
+        if bFormat in (nwBuildFmt.J_HTML, nwBuildFmt.J_NWD):
+            # Ensure that JSON output has the correct extension
+            path = path.with_suffix(".json")
+
         if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
-            yield from self.iterBuildOpenDocument(path, bFormat == nwBuildFmt.FODT)
+            makeObj = ToOdt(self._project, bFormat == nwBuildFmt.FODT)
+            filtered = self._setupBuild(makeObj)
+            makeObj.initDocument()
+
+            yield from self._iterBuild(makeObj, filtered)
+
+            makeObj.closeDocument()
+
         elif bFormat in (nwBuildFmt.HTML, nwBuildFmt.J_HTML):
-            yield from self.iterBuildHTML(path, asJson=bFormat == nwBuildFmt.J_HTML)
+            makeObj = ToHtml(self._project)
+            filtered = self._setupBuild(makeObj)
+
+            yield from self._iterBuild(makeObj, filtered)
+
+            makeObj.appendFootnotes()
+            if not self._build.getBool("html.preserveTabs"):
+                makeObj.replaceTabs()
+
         elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
-            yield from self.iterBuildMarkdown(path, bFormat == nwBuildFmt.EXT_MD)
+            makeObj = ToMarkdown(self._project, bFormat == nwBuildFmt.EXT_MD)
+            filtered = self._setupBuild(makeObj)
+
+            yield from self._iterBuild(makeObj, filtered)
+
+            makeObj.appendFootnotes()
+            if self._build.getBool("format.replaceTabs"):
+                makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
+
         elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
-            yield from self.iterBuildNWD(path, asJson=bFormat == nwBuildFmt.J_NWD)
+            makeObj = ToRaw(self._project)
+            filtered = self._setupBuild(makeObj)
+
+            yield from self._iterBuild(makeObj, filtered)
+
+            if self._build.getBool("format.replaceTabs"):
+                makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
+
+        self._error = None
+        self._cache = makeObj
+
+        try:
+            makeObj.saveDocument(path)
+        except Exception as exc:
+            logException()
+            self._error = formatException(exc)
+
         return
 
-    def iterBuildOpenDocument(self, path: Path, isFlat: bool) -> Iterable[tuple[int, bool]]:
+    ##
-        """Build an Open Document file."""
+    #  Internal Functions
-        makeObj = ToOdt(self._project, isFlat=isFlat)
+    ##
-        filtered = self._setupBuild(makeObj)
-        makeObj.initDocument()
 
+    def _iterBuild(self, makeObj: Tokenizer, filtered: dict) -> Iterable[tuple[int, bool]]:
+        """Iterate over buildable documents."""
         self._count = True
         for i, tHandle in enumerate(self._queue):
             self._error = None
@@ -171,116 +215,8 @@ class NWBuildDocument:
                 yield i, self._doBuild(makeObj, tHandle)
             else:
                 yield i, False
-
-        makeObj.closeDocument()
-
-        self._error = None
-        self._cache = makeObj
-
-        try:
-            makeObj.saveDocument(path)
-        except Exception as exc:
-            logException()
-            self._error = formatException(exc)
-
         return
 
-    def iterBuildHTML(self, path: Path | None, asJson: bool = False) -> Iterable[tuple[int, bool]]:
-        """Build an HTML file. If path is None, no file is saved. This
-        is used for generating build previews.
-        """
-        makeObj = ToHtml(self._project)
-        filtered = self._setupBuild(makeObj)
-
-        self._count = False
-        for i, tHandle in enumerate(self._queue):
-            self._error = None
-            if filtered.get(tHandle, (False, 0))[0]:
-                yield i, self._doBuild(makeObj, tHandle)
-            else:
-                yield i, False
-
-        makeObj.appendFootnotes()
-
-        if not self._build.getBool("html.preserveTabs"):
-            makeObj.replaceTabs()
-
-        self._error = None
-        self._cache = makeObj
-
-        if isinstance(path, Path):
-            try:
-                makeObj.saveDocument(path, asJson=asJson)
-            except Exception as exc:
-                logException()
-                self._error = formatException(exc)
-
-        return
-
-    def iterBuildMarkdown(self, path: Path, extendedMd: bool) -> Iterable[tuple[int, bool]]:
-        """Build a Markdown file."""
-        makeObj = ToMarkdown(self._project)
-        filtered = self._setupBuild(makeObj)
-
-        makeObj.setExtendedMarkdown(extendedMd)
-        if self._build.getBool("format.replaceTabs"):
-            makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
-
-        self._count = False
-        for i, tHandle in enumerate(self._queue):
-            self._error = None
-            if filtered.get(tHandle, (False, 0))[0]:
-                yield i, self._doBuild(makeObj, tHandle)
-            else:
-                yield i, False
-
-        makeObj.appendFootnotes()
-
-        self._error = None
-        self._cache = makeObj
-
-        try:
-            makeObj.saveDocument(path)
-        except Exception as exc:
-            logException()
-            self._error = formatException(exc)
-
-        return
-
-    def iterBuildNWD(self, path: Path | None, asJson: bool = False) -> Iterable[tuple[int, bool]]:
-        """Build a novelWriter Markdown file."""
-        makeObj = ToMarkdown(self._project)
-        filtered = self._setupBuild(makeObj)
-
-        makeObj.setKeepMarkdown(True)
-
-        self._count = False
-        for i, tHandle in enumerate(self._queue):
-            self._error = None
-            if filtered.get(tHandle, (False, 0))[0]:
-                yield i, self._doBuild(makeObj, tHandle, convert=False)
-            else:
-                yield i, False
-
-        if self._build.getBool("format.replaceTabs"):
-            makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
-
-        self._error = None
-        self._cache = makeObj
-
-        if isinstance(path, Path):
-            try:
-                makeObj.saveRawDocument(path, asJson=asJson)
-            except Exception as exc:
-                logException()
-                self._error = formatException(exc)
-
-        return
-
-    ##
-    #  Internal Functions
-    ##
-
     def _setupBuild(self, bldObj: Tokenizer) -> dict:
         """Configure the build object."""
         # Get Settings

novelwriter/formats/tohtml.py

@@ -290,9 +290,9 @@ class ToHtml(Tokenizer):
 
         return
 
-    def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Save the data to an HTML file."""
-        if asJson:
+        if path.suffix.lower() == ".json":
             ts = time()
             data = {
                 "meta": {

novelwriter/formats/tokenizer.py

@@ -134,10 +134,10 @@ class Tokenizer(ABC):
         self._project = project
 
         # Data Variables
-        self._text   = ""     # The raw text to be tokenized
+        self._text    = ""     # The raw text to be tokenized
-        self._handle = None   # The item handle currently being processed
+        self._handle  = None   # The item handle currently being processed
-        self._result = ""     # The result of the last document
+        self._result  = ""     # The result of the last document
-        self._keepMD = False  # Whether to keep the markdown text
+        self._keepRaw = False  # Whether to keep the raw text, used by ToRaw
 
         # Tokens and Meta Data (Per Document)
         self._tokens: list[T_Token] = []
@@ -473,11 +473,6 @@ class Tokenizer(ABC):
         self._keepBreaks = state
         return
 
-    def setKeepMarkdown(self, state: bool) -> None:
-        """Keep original markdown during build."""
-        self._keepMD = state
-        return
-
     ##
     #  Class Methods
     ##
@@ -487,7 +482,7 @@ class Tokenizer(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         raise NotImplementedError
 
     def addRootHeading(self, tHandle: str) -> None:
@@ -509,7 +504,7 @@ class Tokenizer(ABC):
             self._tokens.append((
                 self.T_TITLE, 1, title, [], textAlign
             ))
-            if self._keepMD:
+            if self._keepRaw:
                 self._markdown.append(f"#! {title}\n\n")
 
         return
@@ -574,7 +569,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     self.T_EMPTY, nHead, "", [], self.A_NONE
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append("\n")
 
                 continue
@@ -632,26 +627,26 @@ class Tokenizer(ABC):
                     tokens.append((
                         self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doSynopsis and self._keepMD:
+                    if self._doSynopsis and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
                 elif cStyle == nwComment.SHORT:
                     tLine, tFmt = self._extractFormats(cText)
                     tokens.append((
                         self.T_SHORT, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doSynopsis and self._keepMD:
+                    if self._doSynopsis and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
                 elif cStyle == nwComment.FOOTNOTE:
                     tLine, tFmt = self._extractFormats(cText, skip=self.FMT_FNOTE)
                     self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt)
-                    if self._keepMD:
+                    if self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
                 else:
                     tLine, tFmt = self._extractFormats(cText)
                     tokens.append((
                         self.T_COMMENT, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doComments and self._keepMD:
+                    if self._doComments and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith("@"):
@@ -668,7 +663,7 @@ class Tokenizer(ABC):
                     tokens.append((
                         self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
                     ))
-                    if self._doKeywords and self._keepMD:
+                    if self._doKeywords and self._keepRaw:
                         tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("# ", "#! ")):
@@ -704,7 +699,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("## ", "##! ")):
@@ -739,7 +734,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("### ", "###! ")):
@@ -780,7 +775,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith("#### "):
@@ -810,7 +805,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
             else:
@@ -858,7 +853,7 @@ class Tokenizer(ABC):
                 tokens.append((
                     self.T_TEXT, nHead, tLine, tFmt, sAlign
                 ))
-                if self._keepMD:
+                if self._keepRaw:
                     tmpMarkdown.append(f"{aLine}\n")
 
         # If we have content, turn off the first page flag
@@ -877,7 +872,7 @@ class Tokenizer(ABC):
         tokens.append((
             self.T_EMPTY, nHead, "", [], self.A_NONE
         ))
-        if self._keepMD:
+        if self._keepRaw:
             tmpMarkdown.append("\n")
             self._markdown.append("".join(tmpMarkdown))
 

novelwriter/formats/tomarkdown.py

@@ -81,11 +81,11 @@ class ToMarkdown(Tokenizer):
     supports concatenating novelWriter markup files.
     """
 
-    def __init__(self, project: NWProject) -> None:
+    def __init__(self, project: NWProject, extended: bool) -> None:
         super().__init__(project)
         self._fullMD: list[str] = []
         self._usedNotes: dict[str, int] = {}
-        self._extended = True
+        self._extended = extended
         return
 
     ##
@@ -97,15 +97,6 @@ class ToMarkdown(Tokenizer):
         """Return the markdown as a list."""
         return self._fullMD
 
-    ##
-    #  Setters
-    ##
-
-    def setExtendedMarkdown(self, state: bool) -> None:
-        """Set the converter to use Extended Markdown formatting."""
-        self._extended = state
-        return
-
     ##
     #  Class Methods
     ##
@@ -199,7 +190,7 @@ class ToMarkdown(Tokenizer):
 
         return
 
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Save the data to a plain text file."""
         with open(path, mode="w", encoding="utf-8") as outFile:
             outFile.write("".join(self._fullMD))
@@ -210,8 +201,6 @@ class ToMarkdown(Tokenizer):
         """Replace tabs with spaces."""
         spaces = spaceChar*nSpaces
         self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
-        if self._keepMD:
-            self._markdown = [p.replace("\t", spaces) for p in self._markdown]
         return
 
     ##

novelwriter/formats/toodt.py

@@ -543,7 +543,7 @@ class ToOdt(Tokenizer):
             self._xText.insert(0, xFields)
         return
 
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Save the data to an .fodt or .odt file."""
         if self._isFlat:
             with open(path, mode="wb") as fObj:

novelwriter/formats/toqdoc.py

@@ -275,7 +275,7 @@ class ToQTextDocument(Tokenizer):
 
         return
 
-    def saveDocument(self, path: str | Path) -> None:
+    def saveDocument(self, path: Path) -> None:
         """Not implemented."""
         return
 

novelwriter/formats/toraw.py

@@ -0,0 +1,86 @@
+"""
+novelWriter – Raw NW Text Format
+================================
+
+File History:
+Created: 2024-10-15 [2.6b1] ToRaw
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+from __future__ import annotations
+
+import json
+import logging
+
+from pathlib import Path
+from time import time
+
+from novelwriter.common import formatTimeStamp
+from novelwriter.core.project import NWProject
+from novelwriter.formats.tokenizer import Tokenizer
+
+logger = logging.getLogger(__name__)
+
+
+class ToRaw(Tokenizer):
+    """Core: Raw novelWriter Text Writer
+
+    A class that will collect the minimally altered original source text
+    and write it to either a text or JSON file.
+    """
+
+    def __init__(self, project: NWProject) -> None:
+        super().__init__(project)
+        self._keepRaw = True
+        return
+
+    def doConvert(self) -> None:
+        """No conversion to perform."""
+        return
+
+    def saveDocument(self, path: Path) -> None:
+        """Save the raw text to a plain text file."""
+        if path.suffix.lower() == ".json":
+            ts = time()
+            data = {
+                "meta": {
+                    "projectName": self._project.data.name,
+                    "novelAuthor": self._project.data.author,
+                    "buildTime": int(ts),
+                    "buildTimeStr": formatTimeStamp(ts),
+                },
+                "text": {
+                    "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
+                }
+            }
+            with open(path, mode="w", encoding="utf-8") as fObj:
+                json.dump(data, fObj, indent=2)
+
+        else:
+            with open(path, mode="w", encoding="utf-8") as outFile:
+                for nwdPage in self._markdown:
+                    outFile.write(nwdPage)
+
+        logger.info("Wrote file: %s", path)
+
+        return
+
+    def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
+        """Replace tabs with spaces."""
+        spaces = spaceChar*nSpaces
+        self._markdown = [p.replace("\t", spaces) for p in self._markdown]
+        return

novelwriter/tools/manusbuild.py

@@ -333,7 +333,7 @@ class GuiManuscriptBuild(NDialog):
         docBuild.queueAll()
 
         self.buildProgress.setMaximum(len(docBuild))
-        for i, _ in docBuild.iterBuild(buildPath, bFormat):
+        for i, _ in docBuild.iterBuildDocument(buildPath, bFormat):
             self.buildProgress.setValue(i+1)
 
         self._build.setLastBuildPath(bPath)

tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json

@@ -2,8 +2,8 @@
   "meta": {
     "projectName": "Lorem Ipsum",
     "novelAuthor": "lipsum.com",
-    "buildTime": 1714229171,
+    "buildTime": 1729029144,
-    "buildTimeStr": "2024-04-27 16:46:11"
+    "buildTimeStr": "2024-10-15 23:52:24"
   },
   "text": {
     "nwd": [

tests/test_core/test_core_docbuild.py

@@ -34,6 +34,7 @@ from novelwriter.enum import nwBuildFmt
 from novelwriter.formats.tohtml import ToHtml
 from novelwriter.formats.tomarkdown import ToMarkdown
 from novelwriter.formats.toodt import ToOdt
+from novelwriter.formats.toraw import ToRaw
 
 from tests.mocked import causeException, causeOSError
 from tests.tools import ODT_IGNORE, C, buildTestProject, cmpFiles
@@ -101,7 +102,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildOpenDocument(docFile, True):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -119,7 +120,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildOpenDocument(docFile, False):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -136,7 +137,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.fodt"
-        for _ in docBuild.iterBuildOpenDocument(docFile, True):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -152,7 +153,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
         count = 0
         error = []
         docFile = fncPath / "Lorem Ipsum Err.fodt"
-        for _, success in docBuild.iterBuildOpenDocument(docFile, True):
+        for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
             count += 1 if success else 0
             if not success and docBuild.error:
                 error.append(docBuild.error)
@@ -203,7 +204,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildHTML(docFile):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -223,7 +224,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildHTML(docFile, asJson=True):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -241,7 +242,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.htm"
-        for _ in docBuild.iterBuildHTML(docFile):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -271,7 +272,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildMarkdown(docFile, False):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -291,7 +292,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildMarkdown(docFile, True):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -309,7 +310,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.md"
-        for _ in docBuild.iterBuildMarkdown(docFile, False):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -339,7 +340,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -359,7 +360,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
 
     count = 0
     error = []
-    for _, success in docBuild.iterBuildNWD(docFile, asJson=True):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -377,7 +378,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
         mp.setattr("builtins.open", causeOSError)
 
         docFile = fncPath / "Lorem Ipsum Err.md"
-        for _ in docBuild.iterBuildNWD(docFile):
+        for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
             pass
 
         assert docBuild.error == "OSError: Mock OSError"
@@ -401,7 +402,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
     count = 0
     error = []
     docFile = fncPath / "Minimal.txt"
-    for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -431,7 +432,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
     count = 0
     error = []
     docFile = fncPath / "Minimal.txt"
-    for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
+    for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
         count += 1 if success else 0
         if docBuild.error:
             error.append(docBuild.error)
@@ -473,7 +474,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # ODT Format
     docFile = fncPath / "Minimal.odt"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.ODT)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -483,7 +484,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # FODT Format
     docFile = fncPath / "Minimal.fodt"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.FODT)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -493,7 +494,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # HTML Format
     docFile = fncPath / "Minimal.html"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.HTML)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -503,7 +504,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # JSON HTML Format
     docFile = fncPath / "Minimal.json"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_HTML)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -515,7 +516,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # Standard Markdown Format
     docFile = fncPath / "Minimal.md"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.STD_MD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -536,7 +537,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # Extended Markdown Format
     docFile = fncPath / "Minimal.md"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
@@ -557,11 +558,11 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # NWD Format
     docFile = fncPath / "Minimal.txt"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.NWD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
-    assert isinstance(docBuild.lastBuild, ToMarkdown)
+    assert isinstance(docBuild.lastBuild, ToRaw)
     assert docFile.read_text(encoding="utf-8") == (
         "#! New Novel\n\n"
         "By Jane Doe\n\n"
@@ -578,11 +579,11 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
 
     # JSON NWD Format
     docFile = fncPath / "Minimal.json"
-    assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_NWD)) == [
+    assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD)) == [
         (0, True), (1, True), (2, False), (3, True), (4, True),
         (5, True), (6, True), (7, True), (8, True), (9, False),
     ]
-    assert isinstance(docBuild.lastBuild, ToMarkdown)
+    assert isinstance(docBuild.lastBuild, ToRaw)
     data = json.loads(docFile.read_text(encoding="utf-8"))
     assert "meta" in data
     assert "text" in data

tests/test_formats/test_fmt_tohtml.py

@@ -644,12 +644,12 @@ def testFmtToHtml_Save(mockGUI, fncPath):
     )
 
     saveFile = fncPath / "outFile.htm"
-    html.saveDocument(saveFile, asJson=False)
+    html.saveDocument(saveFile)
     assert saveFile.read_text(encoding="utf-8") == htmlDoc
 
     # JSON + HTML
     saveFile = fncPath / "outFile.json"
-    html.saveDocument(saveFile, asJson=True)
+    html.saveDocument(saveFile)
     data = json.loads(saveFile.read_text(encoding="utf-8"))
     assert data["meta"]["projectName"] == ""
     assert data["meta"]["novelAuthor"] == ""
@@ -664,7 +664,6 @@ def testFmtToHtml_Methods(mockGUI):
     """Test all the other methods of the ToHtml class."""
     project = NWProject()
     html = ToHtml(project)
-    html.setKeepMarkdown(True)
 
     # Auto-Replace, keep Unicode
     docText = "Text with <brackets> & short–dash, long—dash …\n"

tests/test_formats/test_fmt_tokenizer.py

@@ -31,6 +31,7 @@ from novelwriter.constants import nwHeadFmt
 from novelwriter.core.project import NWProject
 from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer, stripEscape
 from novelwriter.formats.tomarkdown import ToMarkdown
+from novelwriter.formats.toraw import ToRaw
 
 from tests.tools import C, buildTestProject, readFile
 
@@ -43,6 +44,19 @@ class BareTokenizer(Tokenizer):
         super().saveDocument(path)  # type: ignore (deliberate check)
 
 
+@pytest.mark.core
+def testFmtToken_Abstracts(mockGUI, tstPaths):
+    """Test all the abstract methods of the Tokenizer class."""
+    project = NWProject()
+    tokens = BareTokenizer(project)
+
+    with pytest.raises(NotImplementedError):
+        tokens.doConvert()
+
+    with pytest.raises(NotImplementedError):
+        tokens.saveDocument(tstPaths)
+
+
 @pytest.mark.core
 def testFmtToken_Setters(mockGUI):
     """Test all the setters for the Tokenizer class."""
@@ -164,8 +178,7 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
 
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Set some content to work with
     docText = (
@@ -235,13 +248,6 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
         ]
     }
 
-    # Check abstract methods
-    with pytest.raises(NotImplementedError):
-        tokens.doConvert()
-
-    with pytest.raises(NotImplementedError):
-        tokens.saveDocument(fncPath)
-
 
 @pytest.mark.core
 def testFmtToken_StripEscape():
@@ -256,8 +262,7 @@ def testFmtToken_StripEscape():
 def testFmtToken_HeaderFormat(mockGUI):
     """Test the tokenization of header formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Title
     # =====
@@ -692,8 +697,7 @@ def testFmtToken_HeaderStyle(mockGUI):
 def testFmtToken_MetaFormat(mockGUI):
     """Test the tokenization of meta formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Comment
     tokens._text = "% A comment\n"
@@ -780,8 +784,7 @@ def testFmtToken_MetaFormat(mockGUI):
 def testFmtToken_MarginFormat(mockGUI):
     """Test the tokenization of margin formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Alignment and Indentation
     dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
@@ -824,7 +827,6 @@ def testFmtToken_ExtractFormats(mockGUI):
     """Test the extraction of formats in the Tokenizer class."""
     project = NWProject()
     tokens = BareTokenizer(project)
-    tokens.setKeepMarkdown(True)
 
     # Markdown
     # ========
@@ -931,7 +933,6 @@ def testFmtToken_Paragraphs(mockGUI):
     """Test the splitting of paragraphs."""
     project = NWProject()
     tokens = BareTokenizer(project)
-    tokens.setKeepMarkdown(True)
 
     # Collapse empty lines
     tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
@@ -1003,8 +1004,7 @@ def testFmtToken_Paragraphs(mockGUI):
 def testFmtToken_TextFormat(mockGUI):
     """Test the tokenization of text formats in the Tokenizer class."""
     project = NWProject()
-    tokens = BareTokenizer(project)
+    tokens = ToRaw(project)
-    tokens.setKeepMarkdown(True)
 
     # Text
     tokens._text = "Some plain text\non two lines\n\n\n"
@@ -1887,7 +1887,7 @@ def testFmtToken_SceneSeparators(mockGUI):
     project = NWProject()
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
-    md = ToMarkdown(project)
+    md = ToMarkdown(project, False)
     md._isNovel = True
 
     # Separator Handling, Titles
@@ -2002,8 +2002,7 @@ def testFmtToken_SceneSeparators(mockGUI):
     # Separators with Scenes Only
     # ===========================
     # Requires a fresh builder class
-    md = ToMarkdown(project)
+    md = ToMarkdown(project, True)
-    md.setExtendedMarkdown(True)
     md._isNovel = True
 
     md._text = (
@@ -2038,7 +2037,7 @@ def testFmtToken_HeaderVisibility(mockGUI):
     project = NWProject()
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
-    md = ToMarkdown(project)
+    md = ToMarkdown(project, False)
 
     md._text = (
         "#! Novel\n\n"
@@ -2150,7 +2149,7 @@ def testFmtToken_CounterHandling(mockGUI):
     project = NWProject()
     project.data.setLanguage("en")
     project._loadProjectLocalisation()
-    md = ToMarkdown(project)
+    md = ToMarkdown(project, False)
     md._isNovel = True
 
     # Counter Handling, Novel Titles

tests/test_formats/test_fmt_tomarkdown.py

@@ -30,7 +30,7 @@ from novelwriter.formats.tomarkdown import ToMarkdown
 def testFmtToMarkdown_ConvertHeaders(mockGUI):
     """Test header formats in the ToMarkdown class."""
     project = NWProject()
-    toMD = ToMarkdown(project)
+    toMD = ToMarkdown(project, False)
 
     toMD._isNovel = True
     toMD._isFirst = True
@@ -76,13 +76,13 @@ def testFmtToMarkdown_ConvertHeaders(mockGUI):
 def testFmtToMarkdown_ConvertParagraphs(mockGUI):
     """Test paragraph formats in the ToMarkdown class."""
     project = NWProject()
-    toMD = ToMarkdown(project)
+    toMD = ToMarkdown(project, False)
 
     toMD._isNovel = True
     toMD._isFirst = True
 
     # Text for Extended Markdown
-    toMD.setExtendedMarkdown(True)
+    toMD._extended = True
     toMD._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
     toMD.tokenizeText()
     toMD.doConvert()
@@ -91,7 +91,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
     )
 
     # Text for Standard Markdown
-    toMD.setExtendedMarkdown(False)
+    toMD._extended = False
     toMD._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
     toMD.tokenizeText()
     toMD.doConvert()
@@ -100,7 +100,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
     )
 
     # Shortcodes for Extended Markdown
-    toMD.setExtendedMarkdown(True)
+    toMD._extended = True
     toMD._text = (
         "Some [b]bold[/b], [i]italic[/i], [s]strike[/s], [u]underline[/u], [m]mark[/m], "
         "super[sup]script[/sup], sub[sub]script[/sub] here\n"
@@ -113,7 +113,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
     )
 
     # Shortcodes for Standard Markdown
-    toMD.setExtendedMarkdown(False)
+    toMD._extended = False
     toMD._text = (
         "Some [b]bold[/b], [i]italic[/i], [s]strike[/s], [u]underline[/u], [m]mark[/m], "
         "super[sup]script[/sup], sub[sub]script[/sub] here\n"
@@ -212,10 +212,8 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
 def testFmtToMarkdown_ConvertDirect(mockGUI):
     """Test the converter directly using the ToMarkdown class."""
     project = NWProject()
-    toMD = ToMarkdown(project)
+    toMD = ToMarkdown(project, False)
-
     toMD._isNovel = True
-    toMD.setExtendedMarkdown(False)
 
     # Special Titles
     # ==============
@@ -249,8 +247,7 @@ def testFmtToMarkdown_ConvertDirect(mockGUI):
 def testFmtToMarkdown_Save(mockGUI, fncPath):
     """Test the save method of the ToMarkdown class."""
     project = NWProject()
-    toMD = ToMarkdown(project)
+    toMD = ToMarkdown(project, False)
-    toMD.setKeepMarkdown(True)
     toMD._isNovel = True
 
     # Build Project
@@ -287,7 +284,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
 
     toMD.replaceTabs(nSpaces=4, spaceChar=" ")
     resText[6] = "#### A Section\n\n    More text in scene two.\n\n"
-    assert toMD.allMarkdown == resText
+    assert toMD.fullMD == resText
 
     # Check File
     # ==========
@@ -301,7 +298,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
 def testFmtToMarkdown_Format(mockGUI):
     """Test all the formatters for the ToMarkdown class."""
     project = NWProject()
-    toMD = ToMarkdown(project)
+    toMD = ToMarkdown(project, False)
 
     assert toMD._formatKeywords("", toMD.A_NONE) == ""
     assert toMD._formatKeywords("tag: Jane", toMD.A_NONE) == "**Tag:** Jane\n\n"