Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify JSON and raw text writing

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-15 23:59:25 +02:00
parent 34696ba3a8
commit c633a56bfb
13 changed files with 178 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/docbuild.py
+7 -13
View File
@@ -38,10 +38,11 @@ from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.error import formatException, logException
from novelwriter.formats.tohtml import ToHtml
from novelwriter.formats.tokenizer import Tokenizer, ToRaw
from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
from novelwriter.formats.toraw import ToRaw
logger = logging.getLogger(__name__)
@@ -146,12 +147,10 @@ class NWBuildDocument:
return
def iterBuild(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
def iterBuildDocument(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
"""Wrapper for builders based on format."""
asJson = False
if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
makeObj = ToOdt(self._project, isFlat=(bFormat == nwBuildFmt.FODT))
makeObj = ToOdt(self._project, bFormat == nwBuildFmt.FODT)
filtered = self._setupBuild(makeObj)
makeObj.initDocument()
yield from self._iterBuild(makeObj, filtered)
@@ -164,7 +163,6 @@ class NWBuildDocument:
makeObj.appendFootnotes()
if not self._build.getBool("html.preserveTabs"):
makeObj.replaceTabs()
asJson = (bFormat == nwBuildFmt.J_HTML)
elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
makeObj = ToMarkdown(self._project)
@@ -177,20 +175,16 @@ class NWBuildDocument:
elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
makeObj = ToRaw(self._project)
makeObj.setKeepMarkdown(True)
filtered = self._setupBuild(makeObj)
yield from self._iterBuild(makeObj, filtered)
asJson = (bFormat == nwBuildFmt.J_NWD)
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
self._error = None
self._cache = makeObj
try:
if isinstance(makeObj, ToHtml | ToRaw):
makeObj.saveDocument(path, asJson=asJson)
else:
makeObj.saveDocument(path)
makeObj.saveDocument(path)
except Exception as exc:
logException()
self._error = formatException(exc)
novelwriter/formats/tohtml.py
+2 -2
View File
@@ -290,9 +290,9 @@ class ToHtml(Tokenizer):
return
def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to an HTML file."""
if asJson:
if path.suffix.lower() == ".json":
ts = time()
data = {
"meta": {
novelwriter/formats/tokenizer.py
+18 -61
View File
@@ -134,10 +134,10 @@ class Tokenizer(ABC):
self._project = project
# Data Variables
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._keepMD = False # Whether to keep the markdown text
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Tokens and Meta Data (Per Document)
self._tokens: list[T_Token] = []
@@ -473,11 +473,6 @@ class Tokenizer(ABC):
self._keepBreaks = state
return
def setKeepMarkdown(self, state: bool) -> None:
"""Keep original markdown during build."""
self._keepMD = state
return
##
# Class Methods
##
@@ -487,7 +482,7 @@ class Tokenizer(ABC):
raise NotImplementedError
@abstractmethod
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
raise NotImplementedError
def addRootHeading(self, tHandle: str) -> None:
@@ -509,7 +504,7 @@ class Tokenizer(ABC):
self._tokens.append((
self.T_TITLE, 1, title, [], textAlign
))
if self._keepMD:
if self._keepRaw:
self._markdown.append(f"#! {title}\n\n")
return
@@ -574,7 +569,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append("\n")
continue
@@ -632,26 +627,26 @@ class Tokenizer(ABC):
tokens.append((
self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepMD:
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_SHORT, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepMD:
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.FOOTNOTE:
tLine, tFmt = self._extractFormats(cText, skip=self.FMT_FNOTE)
self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt)
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_COMMENT, nHead, tLine, tFmt, sAlign
))
if self._doComments and self._keepMD:
if self._doComments and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("@"):
@@ -668,7 +663,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepMD:
if self._doKeywords and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("# ", "#! ")):
@@ -704,7 +699,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("## ", "##! ")):
@@ -739,7 +734,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("### ", "###! ")):
@@ -780,7 +775,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("#### "):
@@ -810,7 +805,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
@@ -858,7 +853,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_TEXT, nHead, tLine, tFmt, sAlign
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag
@@ -877,7 +872,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append("\n")
self._markdown.append("".join(tmpMarkdown))
@@ -1245,41 +1240,3 @@ class HeadingFormatter:
hFormat = hFormat.replace(nwHeadFmt.CHAR_FOCUS, fText)
return hFormat
class ToRaw(Tokenizer):
def __init__(self, project: NWProject) -> None:
super().__init__(project)
self._keepMD = True
return
def doConvert(self) -> None:
return
def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
"""Save the raw text to a plain text file."""
if asJson:
ts = time()
data = {
"meta": {
"projectName": self._project.data.name,
"novelAuthor": self._project.data.author,
"buildTime": int(ts),
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
"nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
json.dump(data, fObj, indent=2)
else:
with open(path, mode="w", encoding="utf-8") as outFile:
for nwdPage in self._markdown:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
return
novelwriter/formats/tomarkdown.py
+1 -3
View File
@@ -199,7 +199,7 @@ class ToMarkdown(Tokenizer):
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to a plain text file."""
with open(path, mode="w", encoding="utf-8") as outFile:
outFile.write("".join(self._fullMD))
@@ -210,8 +210,6 @@ class ToMarkdown(Tokenizer):
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
if self._keepMD:
self._markdown = [p.replace("\t", spaces) for p in self._markdown]
return
##
novelwriter/formats/toodt.py
+1 -1
View File
@@ -543,7 +543,7 @@ class ToOdt(Tokenizer):
self._xText.insert(0, xFields)
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to an .fodt or .odt file."""
if self._isFlat:
with open(path, mode="wb") as fObj:
novelwriter/formats/toqdoc.py
+1 -1
View File
@@ -275,7 +275,7 @@ class ToQTextDocument(Tokenizer):
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Not implemented."""
return
novelwriter/formats/toraw.py
+86
View File
@@ -0,0 +1,86 @@
"""
novelWriter – Raw NW Text Format
================================
File History:
Created: 2024-10-15 [2.6b1] ToRaw
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from time import time
from novelwriter.common import formatTimeStamp
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__)
class ToRaw(Tokenizer):
"""Core: Raw novelWriter Text Writer
A class that will collect the minimally altered original source text
and write it to either a text or JSON file.
"""
def __init__(self, project: NWProject) -> None:
super().__init__(project)
self._keepRaw = True
return
def doConvert(self) -> None:
"""No conversion to perform."""
return
def saveDocument(self, path: Path) -> None:
"""Save the raw text to a plain text file."""
if path.suffix.lower() == ".json":
ts = time()
data = {
"meta": {
"projectName": self._project.data.name,
"novelAuthor": self._project.data.author,
"buildTime": int(ts),
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
"nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
json.dump(data, fObj, indent=2)
else:
with open(path, mode="w", encoding="utf-8") as outFile:
for nwdPage in self._markdown:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
return
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._markdown = [p.replace("\t", spaces) for p in self._markdown]
return
novelwriter/tools/manusbuild.py
+1 -1
View File
@@ -333,7 +333,7 @@ class GuiManuscriptBuild(NDialog):
docBuild.queueAll()
self.buildProgress.setMaximum(len(docBuild))
for i, _ in docBuild.iterBuild(buildPath, bFormat):
for i, _ in docBuild.iterBuildDocument(buildPath, bFormat):
self.buildProgress.setValue(i+1)
self._build.setLastBuildPath(bPath)
tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
+5 -5
View File
@@ -2,8 +2,8 @@
"meta": {
"projectName": "Lorem Ipsum",
"novelAuthor": "lipsum.com",
"buildTime": 1729027593,
"buildTimeStr": "2024-10-15 23:26:33"
"buildTime": 1729029144,
"buildTimeStr": "2024-10-15 23:52:24"
},
"text": {
"nwd": [
@@ -96,11 +96,11 @@
"",
"% Exctracted from the lipsum.com website.",
"",
"\tIt is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
" It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
"",
"\tThe point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
" The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
"",
"\tMany desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
" Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
],
[
"## Chapter Two",
tests/test_core/test_core_docbuild.py
+24 -24
View File
@@ -32,9 +32,9 @@ from novelwriter.core.docbuild import NWBuildDocument
from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.formats.tohtml import ToHtml
from novelwriter.formats.tokenizer import ToRaw
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toraw import ToRaw
from tests.mocked import causeException, causeOSError
from tests.tools import ODT_IGNORE, C, buildTestProject, cmpFiles
@@ -102,7 +102,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -120,7 +120,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.ODT):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -137,7 +137,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.fodt"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -153,7 +153,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
docFile = fncPath / "Lorem Ipsum Err.fodt"
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if not success and docBuild.error:
error.append(docBuild.error)
@@ -204,7 +204,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -224,7 +224,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_HTML):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -242,7 +242,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.htm"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -272,7 +272,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -292,7 +292,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -310,7 +310,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -340,7 +340,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -360,7 +360,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -378,7 +378,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -402,7 +402,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -432,7 +432,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -474,7 +474,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# ODT Format
docFile = fncPath / "Minimal.odt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.ODT)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -484,7 +484,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# FODT Format
docFile = fncPath / "Minimal.fodt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.FODT)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -494,7 +494,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# HTML Format
docFile = fncPath / "Minimal.html"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.HTML)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -504,7 +504,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON HTML Format
docFile = fncPath / "Minimal.json"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_HTML)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -516,7 +516,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Standard Markdown Format
docFile = fncPath / "Minimal.md"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.STD_MD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -537,7 +537,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Extended Markdown Format
docFile = fncPath / "Minimal.md"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -558,7 +558,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# NWD Format
docFile = fncPath / "Minimal.txt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.NWD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -579,7 +579,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON NWD Format
docFile = fncPath / "Minimal.json"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_NWD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
tests/test_formats/test_fmt_tohtml.py
+2 -3
View File
@@ -644,12 +644,12 @@ def testFmtToHtml_Save(mockGUI, fncPath):
)
saveFile = fncPath / "outFile.htm"
html.saveDocument(saveFile, asJson=False)
html.saveDocument(saveFile)
assert saveFile.read_text(encoding="utf-8") == htmlDoc
# JSON + HTML
saveFile = fncPath / "outFile.json"
html.saveDocument(saveFile, asJson=True)
html.saveDocument(saveFile)
data = json.loads(saveFile.read_text(encoding="utf-8"))
assert data["meta"]["projectName"] == ""
assert data["meta"]["novelAuthor"] == ""
@@ -664,7 +664,6 @@ def testFmtToHtml_Methods(mockGUI):
"""Test all the other methods of the ToHtml class."""
project = NWProject()
html = ToHtml(project)
html.setKeepMarkdown(True)
# Auto-Replace, keep Unicode
docText = "Text with <brackets> & short–dash, long—dash …\n"
tests/test_formats/test_fmt_tokenizer.py
+29 -29
View File
@@ -31,6 +31,7 @@ from novelwriter.constants import nwHeadFmt
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer, stripEscape
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toraw import ToRaw
from tests.tools import C, buildTestProject, readFile
@@ -43,11 +44,24 @@ class BareTokenizer(Tokenizer):
super().saveDocument(path) # type: ignore (deliberate check)
@pytest.mark.core
def testFmtToken_Abstracts(mockGUI, tstPaths):
"""Test all the abstract methods of the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
with pytest.raises(NotImplementedError):
tokens.doConvert()
with pytest.raises(NotImplementedError):
tokens.saveDocument(tstPaths)
@pytest.mark.core
def testFmtToken_Setters(mockGUI):
"""Test all the setters for the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
# Verify defaults
assert tokens._fmtPart == nwHeadFmt.TITLE
@@ -164,8 +178,7 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Set some content to work with
docText = (
@@ -235,13 +248,6 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
]
}
# Check abstract methods
with pytest.raises(NotImplementedError):
tokens.doConvert()
with pytest.raises(NotImplementedError):
tokens.saveDocument(fncPath)
@pytest.mark.core
def testFmtToken_StripEscape():
@@ -256,8 +262,7 @@ def testFmtToken_StripEscape():
def testFmtToken_HeaderFormat(mockGUI):
"""Test the tokenization of header formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Title
# =====
@@ -431,7 +436,7 @@ def testFmtToken_HeaderFormat(mockGUI):
def testFmtToken_HeaderStyle(mockGUI):
"""Test the styling of headers in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
def processStyle(text: str, first: bool) -> int:
tokens._text = text
@@ -692,8 +697,7 @@ def testFmtToken_HeaderStyle(mockGUI):
def testFmtToken_MetaFormat(mockGUI):
"""Test the tokenization of meta formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Comment
tokens._text = "% A comment\n"
@@ -780,8 +784,7 @@ def testFmtToken_MetaFormat(mockGUI):
def testFmtToken_MarginFormat(mockGUI):
"""Test the tokenization of margin formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Alignment and Indentation
dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
@@ -823,8 +826,7 @@ def testFmtToken_MarginFormat(mockGUI):
def testFmtToken_ExtractFormats(mockGUI):
"""Test the extraction of formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Markdown
# ========
@@ -930,8 +932,7 @@ def testFmtToken_ExtractFormats(mockGUI):
def testFmtToken_Paragraphs(mockGUI):
"""Test the splitting of paragraphs."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
@@ -1003,8 +1004,7 @@ def testFmtToken_Paragraphs(mockGUI):
def testFmtToken_TextFormat(mockGUI):
"""Test the tokenization of text formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Text
tokens._text = "Some plain text\non two lines\n\n\n"
@@ -1104,7 +1104,7 @@ def testFmtToken_Dialogue(mockGUI):
CONFIG.narratorBreak = "\u2013"
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens.setDialogueHighlight(True)
tokens._isNovel = True
@@ -1191,7 +1191,7 @@ def testFmtToken_Dialogue(mockGUI):
def testFmtToken_SpecialFormat(mockGUI):
"""Test the tokenization of special formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens._isNovel = True
@@ -1353,7 +1353,7 @@ def testFmtToken_SpecialFormat(mockGUI):
def testFmtToken_TextIndent(mockGUI):
"""Test the handling of text indent in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
# No First Indent
tokens.setFirstLineIndent(True, 1.0, False)
@@ -1438,7 +1438,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens._isNovel = True
# Titles
@@ -1627,7 +1627,7 @@ def testFmtToken_BuildOutline(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
# Novel
tokens._isNovel = True
@@ -1691,7 +1691,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens._isNovel = True
# Short Text
tests/test_formats/test_fmt_tomarkdown.py
+1 -2
View File
@@ -250,7 +250,6 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
"""Test the save method of the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
toMD.setKeepMarkdown(True)
toMD._isNovel = True
# Build Project
@@ -287,7 +286,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
toMD.replaceTabs(nSpaces=4, spaceChar=" ")
resText[6] = "#### A Section\n\n More text in scene two.\n\n"
assert toMD.allMarkdown == resText
assert toMD.fullMD == resText
# Check File
# ==========