From 34696ba3a8fd5801dec1c064500a3c52d8b641fe Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Tue, 15 Oct 2024 23:29:11 +0200
Subject: [PATCH 1/4] Merge doc build functions
---
novelwriter/core/docbuild.py | 167 +++++-------------
novelwriter/formats/tokenizer.py | 38 ++++
.../mBuildDocBuild_NWD_Lorem_Ipsum.json | 10 +-
tests/test_core/test_core_docbuild.py | 35 ++--
4 files changed, 110 insertions(+), 140 deletions(-)
diff --git a/novelwriter/core/docbuild.py b/novelwriter/core/docbuild.py
index 7fce8e92..7bd43c51 100644
--- a/novelwriter/core/docbuild.py
+++ b/novelwriter/core/docbuild.py
@@ -38,7 +38,7 @@ from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.error import formatException, logException
from novelwriter.formats.tohtml import ToHtml
-from novelwriter.formats.tokenizer import Tokenizer
+from novelwriter.formats.tokenizer import Tokenizer, ToRaw
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
@@ -148,22 +148,61 @@ class NWBuildDocument:
def iterBuild(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
"""Wrapper for builders based on format."""
+ asJson = False
+
if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
- yield from self.iterBuildOpenDocument(path, bFormat == nwBuildFmt.FODT)
+ makeObj = ToOdt(self._project, isFlat=(bFormat == nwBuildFmt.FODT))
+ filtered = self._setupBuild(makeObj)
+ makeObj.initDocument()
+ yield from self._iterBuild(makeObj, filtered)
+ makeObj.closeDocument()
+
elif bFormat in (nwBuildFmt.HTML, nwBuildFmt.J_HTML):
- yield from self.iterBuildHTML(path, asJson=bFormat == nwBuildFmt.J_HTML)
+ makeObj = ToHtml(self._project)
+ filtered = self._setupBuild(makeObj)
+ yield from self._iterBuild(makeObj, filtered)
+ makeObj.appendFootnotes()
+ if not self._build.getBool("html.preserveTabs"):
+ makeObj.replaceTabs()
+ asJson = (bFormat == nwBuildFmt.J_HTML)
+
elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
- yield from self.iterBuildMarkdown(path, bFormat == nwBuildFmt.EXT_MD)
+ makeObj = ToMarkdown(self._project)
+ makeObj.setExtendedMarkdown(bFormat == nwBuildFmt.EXT_MD)
+ filtered = self._setupBuild(makeObj)
+ yield from self._iterBuild(makeObj, filtered)
+ makeObj.appendFootnotes()
+ if self._build.getBool("format.replaceTabs"):
+ makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
+
elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
- yield from self.iterBuildNWD(path, asJson=bFormat == nwBuildFmt.J_NWD)
+ makeObj = ToRaw(self._project)
+ makeObj.setKeepMarkdown(True)
+ filtered = self._setupBuild(makeObj)
+ yield from self._iterBuild(makeObj, filtered)
+
+ asJson = (bFormat == nwBuildFmt.J_NWD)
+
+ self._error = None
+ self._cache = makeObj
+
+ try:
+ if isinstance(makeObj, ToHtml | ToRaw):
+ makeObj.saveDocument(path, asJson=asJson)
+ else:
+ makeObj.saveDocument(path)
+ except Exception as exc:
+ logException()
+ self._error = formatException(exc)
+
return
- def iterBuildOpenDocument(self, path: Path, isFlat: bool) -> Iterable[tuple[int, bool]]:
- """Build an Open Document file."""
- makeObj = ToOdt(self._project, isFlat=isFlat)
- filtered = self._setupBuild(makeObj)
- makeObj.initDocument()
+ ##
+ # Internal Functions
+ ##
+ def _iterBuild(self, makeObj: Tokenizer, filtered: dict) -> Iterable[tuple[int, bool]]:
+ """Iterate over buildable documents."""
self._count = True
for i, tHandle in enumerate(self._queue):
self._error = None
@@ -171,116 +210,8 @@ class NWBuildDocument:
yield i, self._doBuild(makeObj, tHandle)
else:
yield i, False
-
- makeObj.closeDocument()
-
- self._error = None
- self._cache = makeObj
-
- try:
- makeObj.saveDocument(path)
- except Exception as exc:
- logException()
- self._error = formatException(exc)
-
return
- def iterBuildHTML(self, path: Path | None, asJson: bool = False) -> Iterable[tuple[int, bool]]:
- """Build an HTML file. If path is None, no file is saved. This
- is used for generating build previews.
- """
- makeObj = ToHtml(self._project)
- filtered = self._setupBuild(makeObj)
-
- self._count = False
- for i, tHandle in enumerate(self._queue):
- self._error = None
- if filtered.get(tHandle, (False, 0))[0]:
- yield i, self._doBuild(makeObj, tHandle)
- else:
- yield i, False
-
- makeObj.appendFootnotes()
-
- if not self._build.getBool("html.preserveTabs"):
- makeObj.replaceTabs()
-
- self._error = None
- self._cache = makeObj
-
- if isinstance(path, Path):
- try:
- makeObj.saveDocument(path, asJson=asJson)
- except Exception as exc:
- logException()
- self._error = formatException(exc)
-
- return
-
- def iterBuildMarkdown(self, path: Path, extendedMd: bool) -> Iterable[tuple[int, bool]]:
- """Build a Markdown file."""
- makeObj = ToMarkdown(self._project)
- filtered = self._setupBuild(makeObj)
-
- makeObj.setExtendedMarkdown(extendedMd)
- if self._build.getBool("format.replaceTabs"):
- makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
-
- self._count = False
- for i, tHandle in enumerate(self._queue):
- self._error = None
- if filtered.get(tHandle, (False, 0))[0]:
- yield i, self._doBuild(makeObj, tHandle)
- else:
- yield i, False
-
- makeObj.appendFootnotes()
-
- self._error = None
- self._cache = makeObj
-
- try:
- makeObj.saveDocument(path)
- except Exception as exc:
- logException()
- self._error = formatException(exc)
-
- return
-
- def iterBuildNWD(self, path: Path | None, asJson: bool = False) -> Iterable[tuple[int, bool]]:
- """Build a novelWriter Markdown file."""
- makeObj = ToMarkdown(self._project)
- filtered = self._setupBuild(makeObj)
-
- makeObj.setKeepMarkdown(True)
-
- self._count = False
- for i, tHandle in enumerate(self._queue):
- self._error = None
- if filtered.get(tHandle, (False, 0))[0]:
- yield i, self._doBuild(makeObj, tHandle, convert=False)
- else:
- yield i, False
-
- if self._build.getBool("format.replaceTabs"):
- makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
-
- self._error = None
- self._cache = makeObj
-
- if isinstance(path, Path):
- try:
- makeObj.saveRawDocument(path, asJson=asJson)
- except Exception as exc:
- logException()
- self._error = formatException(exc)
-
- return
-
- ##
- # Internal Functions
- ##
-
def _setupBuild(self, bldObj: Tokenizer) -> dict:
"""Configure the build object."""
# Get Settings
diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py
index ea003b4c..faeeca9d 100644
--- a/novelwriter/formats/tokenizer.py
+++ b/novelwriter/formats/tokenizer.py
@@ -1245,3 +1245,41 @@ class HeadingFormatter:
hFormat = hFormat.replace(nwHeadFmt.CHAR_FOCUS, fText)
return hFormat
+
+
+class ToRaw(Tokenizer):
+
+ def __init__(self, project: NWProject) -> None:
+ super().__init__(project)
+ self._keepMD = True
+ return
+
+ def doConvert(self) -> None:
+ return
+
+ def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
+ """Save the raw text to a plain text file."""
+ if asJson:
+ ts = time()
+ data = {
+ "meta": {
+ "projectName": self._project.data.name,
+ "novelAuthor": self._project.data.author,
+ "buildTime": int(ts),
+ "buildTimeStr": formatTimeStamp(ts),
+ },
+ "text": {
+ "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
+ }
+ }
+ with open(path, mode="w", encoding="utf-8") as fObj:
+ json.dump(data, fObj, indent=2)
+
+ else:
+ with open(path, mode="w", encoding="utf-8") as outFile:
+ for nwdPage in self._markdown:
+ outFile.write(nwdPage)
+
+ logger.info("Wrote file: %s", path)
+
+ return
diff --git a/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json b/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
index 333a90e1..5f8a5bf5 100644
--- a/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
+++ b/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
@@ -2,8 +2,8 @@
"meta": {
"projectName": "Lorem Ipsum",
"novelAuthor": "lipsum.com",
- "buildTime": 1714229171,
- "buildTimeStr": "2024-04-27 16:46:11"
+ "buildTime": 1729027593,
+ "buildTimeStr": "2024-10-15 23:26:33"
},
"text": {
"nwd": [
@@ -96,11 +96,11 @@
"",
"% Exctracted from the lipsum.com website.",
"",
- " It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
+ "\tIt is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
"",
- " The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
+ "\tThe point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
"",
- " Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
+ "\tMany desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
],
[
"## Chapter Two",
diff --git a/tests/test_core/test_core_docbuild.py b/tests/test_core/test_core_docbuild.py
index 391cc2b6..5c4e35e0 100644
--- a/tests/test_core/test_core_docbuild.py
+++ b/tests/test_core/test_core_docbuild.py
@@ -32,6 +32,7 @@ from novelwriter.core.docbuild import NWBuildDocument
from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.formats.tohtml import ToHtml
+from novelwriter.formats.tokenizer import ToRaw
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
@@ -101,7 +102,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
- for _, success in docBuild.iterBuildOpenDocument(docFile, True):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -119,7 +120,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
- for _, success in docBuild.iterBuildOpenDocument(docFile, False):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.ODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -136,7 +137,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.fodt"
- for _ in docBuild.iterBuildOpenDocument(docFile, True):
+ for _ in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -152,7 +153,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
docFile = fncPath / "Lorem Ipsum Err.fodt"
- for _, success in docBuild.iterBuildOpenDocument(docFile, True):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if not success and docBuild.error:
error.append(docBuild.error)
@@ -203,7 +204,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuildHTML(docFile):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -223,7 +224,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuildHTML(docFile, asJson=True):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -241,7 +242,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.htm"
- for _ in docBuild.iterBuildHTML(docFile):
+ for _ in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -271,7 +272,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
- for _, success in docBuild.iterBuildMarkdown(docFile, False):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -291,7 +292,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
- for _, success in docBuild.iterBuildMarkdown(docFile, True):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -309,7 +310,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
- for _ in docBuild.iterBuildMarkdown(docFile, False):
+ for _ in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -339,7 +340,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -359,7 +360,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuildNWD(docFile, asJson=True):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -377,7 +378,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
- for _ in docBuild.iterBuildNWD(docFile):
+ for _ in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -401,7 +402,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
- for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -431,7 +432,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
- for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
+ for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -561,7 +562,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
- assert isinstance(docBuild.lastBuild, ToMarkdown)
+ assert isinstance(docBuild.lastBuild, ToRaw)
assert docFile.read_text(encoding="utf-8") == (
"#! New Novel\n\n"
"By Jane Doe\n\n"
@@ -582,7 +583,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
- assert isinstance(docBuild.lastBuild, ToMarkdown)
+ assert isinstance(docBuild.lastBuild, ToRaw)
data = json.loads(docFile.read_text(encoding="utf-8"))
assert "meta" in data
assert "text" in data
From c633a56bfb473e31a27e8c474015b4ceba384c37 Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Tue, 15 Oct 2024 23:59:25 +0200
Subject: [PATCH 2/4] Simplify JSON and raw text writing
---
novelwriter/core/docbuild.py | 20 ++---
novelwriter/formats/tohtml.py | 4 +-
novelwriter/formats/tokenizer.py | 79 ++++-------------
novelwriter/formats/tomarkdown.py | 4 +-
novelwriter/formats/toodt.py | 2 +-
novelwriter/formats/toqdoc.py | 2 +-
novelwriter/formats/toraw.py | 86 +++++++++++++++++++
novelwriter/tools/manusbuild.py | 2 +-
.../mBuildDocBuild_NWD_Lorem_Ipsum.json | 10 +--
tests/test_core/test_core_docbuild.py | 48 +++++------
tests/test_formats/test_fmt_tohtml.py | 5 +-
tests/test_formats/test_fmt_tokenizer.py | 58 ++++++-------
tests/test_formats/test_fmt_tomarkdown.py | 3 +-
13 files changed, 178 insertions(+), 145 deletions(-)
create mode 100644 novelwriter/formats/toraw.py
diff --git a/novelwriter/core/docbuild.py b/novelwriter/core/docbuild.py
index 7bd43c51..f88c209d 100644
--- a/novelwriter/core/docbuild.py
+++ b/novelwriter/core/docbuild.py
@@ -38,10 +38,11 @@ from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.error import formatException, logException
from novelwriter.formats.tohtml import ToHtml
-from novelwriter.formats.tokenizer import Tokenizer, ToRaw
+from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
+from novelwriter.formats.toraw import ToRaw
logger = logging.getLogger(__name__)
@@ -146,12 +147,10 @@ class NWBuildDocument:
return
- def iterBuild(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
+ def iterBuildDocument(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
"""Wrapper for builders based on format."""
- asJson = False
-
if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
- makeObj = ToOdt(self._project, isFlat=(bFormat == nwBuildFmt.FODT))
+ makeObj = ToOdt(self._project, bFormat == nwBuildFmt.FODT)
filtered = self._setupBuild(makeObj)
makeObj.initDocument()
yield from self._iterBuild(makeObj, filtered)
@@ -164,7 +163,6 @@ class NWBuildDocument:
makeObj.appendFootnotes()
if not self._build.getBool("html.preserveTabs"):
makeObj.replaceTabs()
- asJson = (bFormat == nwBuildFmt.J_HTML)
elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
makeObj = ToMarkdown(self._project)
@@ -177,20 +175,16 @@ class NWBuildDocument:
elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
makeObj = ToRaw(self._project)
- makeObj.setKeepMarkdown(True)
filtered = self._setupBuild(makeObj)
yield from self._iterBuild(makeObj, filtered)
-
- asJson = (bFormat == nwBuildFmt.J_NWD)
+ if self._build.getBool("format.replaceTabs"):
+ makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
self._error = None
self._cache = makeObj
try:
- if isinstance(makeObj, ToHtml | ToRaw):
- makeObj.saveDocument(path, asJson=asJson)
- else:
- makeObj.saveDocument(path)
+ makeObj.saveDocument(path)
except Exception as exc:
logException()
self._error = formatException(exc)
diff --git a/novelwriter/formats/tohtml.py b/novelwriter/formats/tohtml.py
index 78f359b9..7eb4b33d 100644
--- a/novelwriter/formats/tohtml.py
+++ b/novelwriter/formats/tohtml.py
@@ -290,9 +290,9 @@ class ToHtml(Tokenizer):
return
- def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
+ def saveDocument(self, path: Path) -> None:
"""Save the data to an HTML file."""
- if asJson:
+ if path.suffix.lower() == ".json":
ts = time()
data = {
"meta": {
diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py
index faeeca9d..a2db6cce 100644
--- a/novelwriter/formats/tokenizer.py
+++ b/novelwriter/formats/tokenizer.py
@@ -134,10 +134,10 @@ class Tokenizer(ABC):
self._project = project
# Data Variables
- self._text = "" # The raw text to be tokenized
- self._handle = None # The item handle currently being processed
- self._result = "" # The result of the last document
- self._keepMD = False # Whether to keep the markdown text
+ self._text = "" # The raw text to be tokenized
+ self._handle = None # The item handle currently being processed
+ self._result = "" # The result of the last document
+ self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Tokens and Meta Data (Per Document)
self._tokens: list[T_Token] = []
@@ -473,11 +473,6 @@ class Tokenizer(ABC):
self._keepBreaks = state
return
- def setKeepMarkdown(self, state: bool) -> None:
- """Keep original markdown during build."""
- self._keepMD = state
- return
-
##
# Class Methods
##
@@ -487,7 +482,7 @@ class Tokenizer(ABC):
raise NotImplementedError
@abstractmethod
- def saveDocument(self, path: str | Path) -> None:
+ def saveDocument(self, path: Path) -> None:
raise NotImplementedError
def addRootHeading(self, tHandle: str) -> None:
@@ -509,7 +504,7 @@ class Tokenizer(ABC):
self._tokens.append((
self.T_TITLE, 1, title, [], textAlign
))
- if self._keepMD:
+ if self._keepRaw:
self._markdown.append(f"#! {title}\n\n")
return
@@ -574,7 +569,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append("\n")
continue
@@ -632,26 +627,26 @@ class Tokenizer(ABC):
tokens.append((
self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
))
- if self._doSynopsis and self._keepMD:
+ if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_SHORT, nHead, tLine, tFmt, sAlign
))
- if self._doSynopsis and self._keepMD:
+ if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.FOOTNOTE:
tLine, tFmt = self._extractFormats(cText, skip=self.FMT_FNOTE)
self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt)
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_COMMENT, nHead, tLine, tFmt, sAlign
))
- if self._doComments and self._keepMD:
+ if self._doComments and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("@"):
@@ -668,7 +663,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
- if self._doKeywords and self._keepMD:
+ if self._doKeywords and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("# ", "#! ")):
@@ -704,7 +699,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("## ", "##! ")):
@@ -739,7 +734,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("### ", "###! ")):
@@ -780,7 +775,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("#### "):
@@ -810,7 +805,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
@@ -858,7 +853,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_TEXT, nHead, tLine, tFmt, sAlign
))
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag
@@ -877,7 +872,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
- if self._keepMD:
+ if self._keepRaw:
tmpMarkdown.append("\n")
self._markdown.append("".join(tmpMarkdown))
@@ -1245,41 +1240,3 @@ class HeadingFormatter:
hFormat = hFormat.replace(nwHeadFmt.CHAR_FOCUS, fText)
return hFormat
-
-
-class ToRaw(Tokenizer):
-
- def __init__(self, project: NWProject) -> None:
- super().__init__(project)
- self._keepMD = True
- return
-
- def doConvert(self) -> None:
- return
-
- def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
- """Save the raw text to a plain text file."""
- if asJson:
- ts = time()
- data = {
- "meta": {
- "projectName": self._project.data.name,
- "novelAuthor": self._project.data.author,
- "buildTime": int(ts),
- "buildTimeStr": formatTimeStamp(ts),
- },
- "text": {
- "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
- }
- }
- with open(path, mode="w", encoding="utf-8") as fObj:
- json.dump(data, fObj, indent=2)
-
- else:
- with open(path, mode="w", encoding="utf-8") as outFile:
- for nwdPage in self._markdown:
- outFile.write(nwdPage)
-
- logger.info("Wrote file: %s", path)
-
- return
diff --git a/novelwriter/formats/tomarkdown.py b/novelwriter/formats/tomarkdown.py
index a738513c..27075856 100644
--- a/novelwriter/formats/tomarkdown.py
+++ b/novelwriter/formats/tomarkdown.py
@@ -199,7 +199,7 @@ class ToMarkdown(Tokenizer):
return
- def saveDocument(self, path: str | Path) -> None:
+ def saveDocument(self, path: Path) -> None:
"""Save the data to a plain text file."""
with open(path, mode="w", encoding="utf-8") as outFile:
outFile.write("".join(self._fullMD))
@@ -210,8 +210,6 @@ class ToMarkdown(Tokenizer):
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
- if self._keepMD:
- self._markdown = [p.replace("\t", spaces) for p in self._markdown]
return
##
diff --git a/novelwriter/formats/toodt.py b/novelwriter/formats/toodt.py
index 8d3de780..bfb9d80e 100644
--- a/novelwriter/formats/toodt.py
+++ b/novelwriter/formats/toodt.py
@@ -543,7 +543,7 @@ class ToOdt(Tokenizer):
self._xText.insert(0, xFields)
return
- def saveDocument(self, path: str | Path) -> None:
+ def saveDocument(self, path: Path) -> None:
"""Save the data to an .fodt or .odt file."""
if self._isFlat:
with open(path, mode="wb") as fObj:
diff --git a/novelwriter/formats/toqdoc.py b/novelwriter/formats/toqdoc.py
index 43f10b69..07c54e45 100644
--- a/novelwriter/formats/toqdoc.py
+++ b/novelwriter/formats/toqdoc.py
@@ -275,7 +275,7 @@ class ToQTextDocument(Tokenizer):
return
- def saveDocument(self, path: str | Path) -> None:
+ def saveDocument(self, path: Path) -> None:
"""Not implemented."""
return
diff --git a/novelwriter/formats/toraw.py b/novelwriter/formats/toraw.py
new file mode 100644
index 00000000..d1c701b1
--- /dev/null
+++ b/novelwriter/formats/toraw.py
@@ -0,0 +1,86 @@
+"""
+novelWriter – Raw NW Text Format
+================================
+
+File History:
+Created: 2024-10-15 [2.6b1] ToRaw
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+"""
+from __future__ import annotations
+
+import json
+import logging
+
+from pathlib import Path
+from time import time
+
+from novelwriter.common import formatTimeStamp
+from novelwriter.core.project import NWProject
+from novelwriter.formats.tokenizer import Tokenizer
+
+logger = logging.getLogger(__name__)
+
+
+class ToRaw(Tokenizer):
+ """Core: Raw novelWriter Text Writer
+
+ A class that will collect the minimally altered original source text
+ and write it to either a text or JSON file.
+ """
+
+ def __init__(self, project: NWProject) -> None:
+ super().__init__(project)
+ self._keepRaw = True
+ return
+
+ def doConvert(self) -> None:
+ """No conversion to perform."""
+ return
+
+ def saveDocument(self, path: Path) -> None:
+ """Save the raw text to a plain text file."""
+ if path.suffix.lower() == ".json":
+ ts = time()
+ data = {
+ "meta": {
+ "projectName": self._project.data.name,
+ "novelAuthor": self._project.data.author,
+ "buildTime": int(ts),
+ "buildTimeStr": formatTimeStamp(ts),
+ },
+ "text": {
+ "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
+ }
+ }
+ with open(path, mode="w", encoding="utf-8") as fObj:
+ json.dump(data, fObj, indent=2)
+
+ else:
+ with open(path, mode="w", encoding="utf-8") as outFile:
+ for nwdPage in self._markdown:
+ outFile.write(nwdPage)
+
+ logger.info("Wrote file: %s", path)
+
+ return
+
+ def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
+ """Replace tabs with spaces."""
+ spaces = spaceChar*nSpaces
+ self._markdown = [p.replace("\t", spaces) for p in self._markdown]
+ return
diff --git a/novelwriter/tools/manusbuild.py b/novelwriter/tools/manusbuild.py
index cf18822b..e37bfca5 100644
--- a/novelwriter/tools/manusbuild.py
+++ b/novelwriter/tools/manusbuild.py
@@ -333,7 +333,7 @@ class GuiManuscriptBuild(NDialog):
docBuild.queueAll()
self.buildProgress.setMaximum(len(docBuild))
- for i, _ in docBuild.iterBuild(buildPath, bFormat):
+ for i, _ in docBuild.iterBuildDocument(buildPath, bFormat):
self.buildProgress.setValue(i+1)
self._build.setLastBuildPath(bPath)
diff --git a/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json b/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
index 5f8a5bf5..630d9592 100644
--- a/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
+++ b/tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
@@ -2,8 +2,8 @@
"meta": {
"projectName": "Lorem Ipsum",
"novelAuthor": "lipsum.com",
- "buildTime": 1729027593,
- "buildTimeStr": "2024-10-15 23:26:33"
+ "buildTime": 1729029144,
+ "buildTimeStr": "2024-10-15 23:52:24"
},
"text": {
"nwd": [
@@ -96,11 +96,11 @@
"",
"% Exctracted from the lipsum.com website.",
"",
- "\tIt is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
+ " It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
"",
- "\tThe point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
+ " The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
"",
- "\tMany desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
+ " Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
],
[
"## Chapter Two",
diff --git a/tests/test_core/test_core_docbuild.py b/tests/test_core/test_core_docbuild.py
index 5c4e35e0..23feea8b 100644
--- a/tests/test_core/test_core_docbuild.py
+++ b/tests/test_core/test_core_docbuild.py
@@ -32,9 +32,9 @@ from novelwriter.core.docbuild import NWBuildDocument
from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.formats.tohtml import ToHtml
-from novelwriter.formats.tokenizer import ToRaw
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
+from novelwriter.formats.toraw import ToRaw
from tests.mocked import causeException, causeOSError
from tests.tools import ODT_IGNORE, C, buildTestProject, cmpFiles
@@ -102,7 +102,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -120,7 +120,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.ODT):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -137,7 +137,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.fodt"
- for _ in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
+ for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -153,7 +153,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
docFile = fncPath / "Lorem Ipsum Err.fodt"
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if not success and docBuild.error:
error.append(docBuild.error)
@@ -204,7 +204,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -224,7 +224,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_HTML):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -242,7 +242,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.htm"
- for _ in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
+ for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -272,7 +272,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -292,7 +292,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -310,7 +310,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
- for _ in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
+ for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -340,7 +340,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -360,7 +360,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_NWD):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -378,7 +378,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
- for _ in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+ for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -402,7 +402,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -432,7 +432,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
- for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
+ for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -474,7 +474,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# ODT Format
docFile = fncPath / "Minimal.odt"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.ODT)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -484,7 +484,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# FODT Format
docFile = fncPath / "Minimal.fodt"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.FODT)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -494,7 +494,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# HTML Format
docFile = fncPath / "Minimal.html"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.HTML)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -504,7 +504,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON HTML Format
docFile = fncPath / "Minimal.json"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_HTML)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -516,7 +516,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Standard Markdown Format
docFile = fncPath / "Minimal.md"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.STD_MD)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -537,7 +537,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Extended Markdown Format
docFile = fncPath / "Minimal.md"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -558,7 +558,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# NWD Format
docFile = fncPath / "Minimal.txt"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.NWD)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -579,7 +579,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON NWD Format
docFile = fncPath / "Minimal.json"
- assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_NWD)) == [
+ assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
diff --git a/tests/test_formats/test_fmt_tohtml.py b/tests/test_formats/test_fmt_tohtml.py
index 49e7bf63..c1467566 100644
--- a/tests/test_formats/test_fmt_tohtml.py
+++ b/tests/test_formats/test_fmt_tohtml.py
@@ -644,12 +644,12 @@ def testFmtToHtml_Save(mockGUI, fncPath):
)
saveFile = fncPath / "outFile.htm"
- html.saveDocument(saveFile, asJson=False)
+ html.saveDocument(saveFile)
assert saveFile.read_text(encoding="utf-8") == htmlDoc
# JSON + HTML
saveFile = fncPath / "outFile.json"
- html.saveDocument(saveFile, asJson=True)
+ html.saveDocument(saveFile)
data = json.loads(saveFile.read_text(encoding="utf-8"))
assert data["meta"]["projectName"] == ""
assert data["meta"]["novelAuthor"] == ""
@@ -664,7 +664,6 @@ def testFmtToHtml_Methods(mockGUI):
"""Test all the other methods of the ToHtml class."""
project = NWProject()
html = ToHtml(project)
- html.setKeepMarkdown(True)
# Auto-Replace, keep Unicode
docText = "Text with & short–dash, long—dash …\n"
diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py
index 33531f02..4a29cb86 100644
--- a/tests/test_formats/test_fmt_tokenizer.py
+++ b/tests/test_formats/test_fmt_tokenizer.py
@@ -31,6 +31,7 @@ from novelwriter.constants import nwHeadFmt
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer, stripEscape
from novelwriter.formats.tomarkdown import ToMarkdown
+from novelwriter.formats.toraw import ToRaw
from tests.tools import C, buildTestProject, readFile
@@ -43,11 +44,24 @@ class BareTokenizer(Tokenizer):
super().saveDocument(path) # type: ignore (deliberate check)
+@pytest.mark.core
+def testFmtToken_Abstracts(mockGUI, tstPaths):
+ """Test all the abstract methods of the Tokenizer class."""
+ project = NWProject()
+ tokens = BareTokenizer(project)
+
+ with pytest.raises(NotImplementedError):
+ tokens.doConvert()
+
+ with pytest.raises(NotImplementedError):
+ tokens.saveDocument(tstPaths)
+
+
@pytest.mark.core
def testFmtToken_Setters(mockGUI):
"""Test all the setters for the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
# Verify defaults
assert tokens._fmtPart == nwHeadFmt.TITLE
@@ -164,8 +178,7 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
project.data.setLanguage("en")
project._loadProjectLocalisation()
- tokens = BareTokenizer(project)
- tokens.setKeepMarkdown(True)
+ tokens = ToRaw(project)
# Set some content to work with
docText = (
@@ -235,13 +248,6 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
]
}
- # Check abstract methods
- with pytest.raises(NotImplementedError):
- tokens.doConvert()
-
- with pytest.raises(NotImplementedError):
- tokens.saveDocument(fncPath)
-
@pytest.mark.core
def testFmtToken_StripEscape():
@@ -256,8 +262,7 @@ def testFmtToken_StripEscape():
def testFmtToken_HeaderFormat(mockGUI):
"""Test the tokenization of header formats in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
- tokens.setKeepMarkdown(True)
+ tokens = ToRaw(project)
# Title
# =====
@@ -431,7 +436,7 @@ def testFmtToken_HeaderFormat(mockGUI):
def testFmtToken_HeaderStyle(mockGUI):
"""Test the styling of headers in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
def processStyle(text: str, first: bool) -> int:
tokens._text = text
@@ -692,8 +697,7 @@ def testFmtToken_HeaderStyle(mockGUI):
def testFmtToken_MetaFormat(mockGUI):
"""Test the tokenization of meta formats in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
- tokens.setKeepMarkdown(True)
+ tokens = ToRaw(project)
# Comment
tokens._text = "% A comment\n"
@@ -780,8 +784,7 @@ def testFmtToken_MetaFormat(mockGUI):
def testFmtToken_MarginFormat(mockGUI):
"""Test the tokenization of margin formats in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
- tokens.setKeepMarkdown(True)
+ tokens = ToRaw(project)
# Alignment and Indentation
dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
@@ -823,8 +826,7 @@ def testFmtToken_MarginFormat(mockGUI):
def testFmtToken_ExtractFormats(mockGUI):
"""Test the extraction of formats in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
- tokens.setKeepMarkdown(True)
+ tokens = ToRaw(project)
# Markdown
# ========
@@ -930,8 +932,7 @@ def testFmtToken_ExtractFormats(mockGUI):
def testFmtToken_Paragraphs(mockGUI):
"""Test the splitting of paragraphs."""
project = NWProject()
- tokens = BareTokenizer(project)
- tokens.setKeepMarkdown(True)
+ tokens = ToRaw(project)
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
@@ -1003,8 +1004,7 @@ def testFmtToken_Paragraphs(mockGUI):
def testFmtToken_TextFormat(mockGUI):
"""Test the tokenization of text formats in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
- tokens.setKeepMarkdown(True)
+ tokens = ToRaw(project)
# Text
tokens._text = "Some plain text\non two lines\n\n\n"
@@ -1104,7 +1104,7 @@ def testFmtToken_Dialogue(mockGUI):
CONFIG.narratorBreak = "\u2013"
project = NWProject()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
tokens.setDialogueHighlight(True)
tokens._isNovel = True
@@ -1191,7 +1191,7 @@ def testFmtToken_Dialogue(mockGUI):
def testFmtToken_SpecialFormat(mockGUI):
"""Test the tokenization of special formats in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
tokens._isNovel = True
@@ -1353,7 +1353,7 @@ def testFmtToken_SpecialFormat(mockGUI):
def testFmtToken_TextIndent(mockGUI):
"""Test the handling of text indent in the Tokenizer class."""
project = NWProject()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
# No First Indent
tokens.setFirstLineIndent(True, 1.0, False)
@@ -1438,7 +1438,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
tokens._isNovel = True
# Titles
@@ -1627,7 +1627,7 @@ def testFmtToken_BuildOutline(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
# Novel
tokens._isNovel = True
@@ -1691,7 +1691,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- tokens = BareTokenizer(project)
+ tokens = ToRaw(project)
tokens._isNovel = True
# Short Text
diff --git a/tests/test_formats/test_fmt_tomarkdown.py b/tests/test_formats/test_fmt_tomarkdown.py
index 5f446699..538023b2 100644
--- a/tests/test_formats/test_fmt_tomarkdown.py
+++ b/tests/test_formats/test_fmt_tomarkdown.py
@@ -250,7 +250,6 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
"""Test the save method of the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
- toMD.setKeepMarkdown(True)
toMD._isNovel = True
# Build Project
@@ -287,7 +286,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
toMD.replaceTabs(nSpaces=4, spaceChar=" ")
resText[6] = "#### A Section\n\n More text in scene two.\n\n"
- assert toMD.allMarkdown == resText
+ assert toMD.fullMD == resText
# Check File
# ==========
From d2ac0ed53fd24d07bd1b5e2e8685b588e2d69189 Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Wed, 16 Oct 2024 00:15:56 +0200
Subject: [PATCH 3/4] Pass the Markdown style to the constructor
---
novelwriter/core/docbuild.py | 15 +++++++++++++--
novelwriter/formats/tomarkdown.py | 13 ++-----------
tests/test_formats/test_fmt_tokenizer.py | 9 ++++-----
tests/test_formats/test_fmt_tomarkdown.py | 20 +++++++++-----------
4 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/novelwriter/core/docbuild.py b/novelwriter/core/docbuild.py
index f88c209d..360ae8ed 100644
--- a/novelwriter/core/docbuild.py
+++ b/novelwriter/core/docbuild.py
@@ -149,26 +149,35 @@ class NWBuildDocument:
def iterBuildDocument(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
"""Wrapper for builders based on format."""
+ if bFormat in (nwBuildFmt.J_HTML, nwBuildFmt.J_NWD):
+ # Ensure that JSON output has the correct extension
+ path = path.with_suffix(".json")
+
if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
makeObj = ToOdt(self._project, bFormat == nwBuildFmt.FODT)
filtered = self._setupBuild(makeObj)
makeObj.initDocument()
+
yield from self._iterBuild(makeObj, filtered)
+
makeObj.closeDocument()
elif bFormat in (nwBuildFmt.HTML, nwBuildFmt.J_HTML):
makeObj = ToHtml(self._project)
filtered = self._setupBuild(makeObj)
+
yield from self._iterBuild(makeObj, filtered)
+
makeObj.appendFootnotes()
if not self._build.getBool("html.preserveTabs"):
makeObj.replaceTabs()
elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
- makeObj = ToMarkdown(self._project)
- makeObj.setExtendedMarkdown(bFormat == nwBuildFmt.EXT_MD)
+ makeObj = ToMarkdown(self._project, bFormat == nwBuildFmt.EXT_MD)
filtered = self._setupBuild(makeObj)
+
yield from self._iterBuild(makeObj, filtered)
+
makeObj.appendFootnotes()
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
@@ -176,7 +185,9 @@ class NWBuildDocument:
elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
makeObj = ToRaw(self._project)
filtered = self._setupBuild(makeObj)
+
yield from self._iterBuild(makeObj, filtered)
+
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
diff --git a/novelwriter/formats/tomarkdown.py b/novelwriter/formats/tomarkdown.py
index 27075856..82526cd5 100644
--- a/novelwriter/formats/tomarkdown.py
+++ b/novelwriter/formats/tomarkdown.py
@@ -81,11 +81,11 @@ class ToMarkdown(Tokenizer):
supports concatenating novelWriter markup files.
"""
- def __init__(self, project: NWProject) -> None:
+ def __init__(self, project: NWProject, extended: bool) -> None:
super().__init__(project)
self._fullMD: list[str] = []
self._usedNotes: dict[str, int] = {}
- self._extended = True
+ self._extended = extended
return
##
@@ -97,15 +97,6 @@ class ToMarkdown(Tokenizer):
"""Return the markdown as a list."""
return self._fullMD
- ##
- # Setters
- ##
-
- def setExtendedMarkdown(self, state: bool) -> None:
- """Set the converter to use Extended Markdown formatting."""
- self._extended = state
- return
-
##
# Class Methods
##
diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py
index 4a29cb86..916c4593 100644
--- a/tests/test_formats/test_fmt_tokenizer.py
+++ b/tests/test_formats/test_fmt_tokenizer.py
@@ -1887,7 +1887,7 @@ def testFmtToken_SceneSeparators(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- md = ToMarkdown(project)
+ md = ToMarkdown(project, False)
md._isNovel = True
# Separator Handling, Titles
@@ -2002,8 +2002,7 @@ def testFmtToken_SceneSeparators(mockGUI):
# Separators with Scenes Only
# ===========================
# Requires a fresh builder class
- md = ToMarkdown(project)
- md.setExtendedMarkdown(True)
+ md = ToMarkdown(project, True)
md._isNovel = True
md._text = (
@@ -2038,7 +2037,7 @@ def testFmtToken_HeaderVisibility(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- md = ToMarkdown(project)
+ md = ToMarkdown(project, False)
md._text = (
"#! Novel\n\n"
@@ -2150,7 +2149,7 @@ def testFmtToken_CounterHandling(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- md = ToMarkdown(project)
+ md = ToMarkdown(project, False)
md._isNovel = True
# Counter Handling, Novel Titles
diff --git a/tests/test_formats/test_fmt_tomarkdown.py b/tests/test_formats/test_fmt_tomarkdown.py
index 538023b2..177a2b5b 100644
--- a/tests/test_formats/test_fmt_tomarkdown.py
+++ b/tests/test_formats/test_fmt_tomarkdown.py
@@ -30,7 +30,7 @@ from novelwriter.formats.tomarkdown import ToMarkdown
def testFmtToMarkdown_ConvertHeaders(mockGUI):
"""Test header formats in the ToMarkdown class."""
project = NWProject()
- toMD = ToMarkdown(project)
+ toMD = ToMarkdown(project, False)
toMD._isNovel = True
toMD._isFirst = True
@@ -76,13 +76,13 @@ def testFmtToMarkdown_ConvertHeaders(mockGUI):
def testFmtToMarkdown_ConvertParagraphs(mockGUI):
"""Test paragraph formats in the ToMarkdown class."""
project = NWProject()
- toMD = ToMarkdown(project)
+ toMD = ToMarkdown(project, False)
toMD._isNovel = True
toMD._isFirst = True
# Text for Extended Markdown
- toMD.setExtendedMarkdown(True)
+ toMD._extended = True
toMD._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
toMD.tokenizeText()
toMD.doConvert()
@@ -91,7 +91,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
)
# Text for Standard Markdown
- toMD.setExtendedMarkdown(False)
+ toMD._extended = False
toMD._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
toMD.tokenizeText()
toMD.doConvert()
@@ -100,7 +100,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
)
# Shortcodes for Extended Markdown
- toMD.setExtendedMarkdown(True)
+ toMD._extended = True
toMD._text = (
"Some [b]bold[/b], [i]italic[/i], [s]strike[/s], [u]underline[/u], [m]mark[/m], "
"super[sup]script[/sup], sub[sub]script[/sub] here\n"
@@ -113,7 +113,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
)
# Shortcodes for Standard Markdown
- toMD.setExtendedMarkdown(False)
+ toMD._extended = False
toMD._text = (
"Some [b]bold[/b], [i]italic[/i], [s]strike[/s], [u]underline[/u], [m]mark[/m], "
"super[sup]script[/sup], sub[sub]script[/sub] here\n"
@@ -212,10 +212,8 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
def testFmtToMarkdown_ConvertDirect(mockGUI):
"""Test the converter directly using the ToMarkdown class."""
project = NWProject()
- toMD = ToMarkdown(project)
-
+ toMD = ToMarkdown(project, False)
toMD._isNovel = True
- toMD.setExtendedMarkdown(False)
# Special Titles
# ==============
@@ -249,7 +247,7 @@ def testFmtToMarkdown_ConvertDirect(mockGUI):
def testFmtToMarkdown_Save(mockGUI, fncPath):
"""Test the save method of the ToMarkdown class."""
project = NWProject()
- toMD = ToMarkdown(project)
+ toMD = ToMarkdown(project, False)
toMD._isNovel = True
# Build Project
@@ -300,7 +298,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
def testFmtToMarkdown_Format(mockGUI):
"""Test all the formatters for the ToMarkdown class."""
project = NWProject()
- toMD = ToMarkdown(project)
+ toMD = ToMarkdown(project, False)
assert toMD._formatKeywords("", toMD.A_NONE) == ""
assert toMD._formatKeywords("tag: Jane", toMD.A_NONE) == "**Tag:** Jane\n\n"
From 7bb292404ab632414cd51d10a43f7b395d206e9c Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Wed, 16 Oct 2024 00:23:18 +0200
Subject: [PATCH 4/4] Update Tokenizer tests
---
tests/test_formats/test_fmt_tokenizer.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py
index 916c4593..8cbcdaf0 100644
--- a/tests/test_formats/test_fmt_tokenizer.py
+++ b/tests/test_formats/test_fmt_tokenizer.py
@@ -61,7 +61,7 @@ def testFmtToken_Abstracts(mockGUI, tstPaths):
def testFmtToken_Setters(mockGUI):
"""Test all the setters for the Tokenizer class."""
project = NWProject()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
# Verify defaults
assert tokens._fmtPart == nwHeadFmt.TITLE
@@ -436,7 +436,7 @@ def testFmtToken_HeaderFormat(mockGUI):
def testFmtToken_HeaderStyle(mockGUI):
"""Test the styling of headers in the Tokenizer class."""
project = NWProject()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
def processStyle(text: str, first: bool) -> int:
tokens._text = text
@@ -826,7 +826,7 @@ def testFmtToken_MarginFormat(mockGUI):
def testFmtToken_ExtractFormats(mockGUI):
"""Test the extraction of formats in the Tokenizer class."""
project = NWProject()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
# Markdown
# ========
@@ -932,7 +932,7 @@ def testFmtToken_ExtractFormats(mockGUI):
def testFmtToken_Paragraphs(mockGUI):
"""Test the splitting of paragraphs."""
project = NWProject()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
@@ -1104,7 +1104,7 @@ def testFmtToken_Dialogue(mockGUI):
CONFIG.narratorBreak = "\u2013"
project = NWProject()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
tokens.setDialogueHighlight(True)
tokens._isNovel = True
@@ -1191,7 +1191,7 @@ def testFmtToken_Dialogue(mockGUI):
def testFmtToken_SpecialFormat(mockGUI):
"""Test the tokenization of special formats in the Tokenizer class."""
project = NWProject()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
tokens._isNovel = True
@@ -1353,7 +1353,7 @@ def testFmtToken_SpecialFormat(mockGUI):
def testFmtToken_TextIndent(mockGUI):
"""Test the handling of text indent in the Tokenizer class."""
project = NWProject()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
# No First Indent
tokens.setFirstLineIndent(True, 1.0, False)
@@ -1438,7 +1438,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
tokens._isNovel = True
# Titles
@@ -1627,7 +1627,7 @@ def testFmtToken_BuildOutline(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
# Novel
tokens._isNovel = True
@@ -1691,7 +1691,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
- tokens = ToRaw(project)
+ tokens = BareTokenizer(project)
tokens._isNovel = True
# Short Text