Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor document builder (#2047)

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-16 00:26:32 +02:00
committed by GitHub
parent 93c105fe86 7bb292404a
commit c8d7ccadb7
13 changed files with 229 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/docbuild.py
+54 -118
View File
@@ -42,6 +42,7 @@ from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
from novelwriter.formats.toraw import ToRaw
logger = logging.getLogger(__name__)
@@ -146,24 +147,67 @@ class NWBuildDocument:
return
def iterBuild(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
def iterBuildDocument(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
"""Wrapper for builders based on format."""
if bFormat in (nwBuildFmt.J_HTML, nwBuildFmt.J_NWD):
# Ensure that JSON output has the correct extension
path = path.with_suffix(".json")
if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
yield from self.iterBuildOpenDocument(path, bFormat == nwBuildFmt.FODT)
makeObj = ToOdt(self._project, bFormat == nwBuildFmt.FODT)
filtered = self._setupBuild(makeObj)
makeObj.initDocument()
yield from self._iterBuild(makeObj, filtered)
makeObj.closeDocument()
elif bFormat in (nwBuildFmt.HTML, nwBuildFmt.J_HTML):
yield from self.iterBuildHTML(path, asJson=bFormat == nwBuildFmt.J_HTML)
makeObj = ToHtml(self._project)
filtered = self._setupBuild(makeObj)
yield from self._iterBuild(makeObj, filtered)
makeObj.appendFootnotes()
if not self._build.getBool("html.preserveTabs"):
makeObj.replaceTabs()
elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
yield from self.iterBuildMarkdown(path, bFormat == nwBuildFmt.EXT_MD)
makeObj = ToMarkdown(self._project, bFormat == nwBuildFmt.EXT_MD)
filtered = self._setupBuild(makeObj)
yield from self._iterBuild(makeObj, filtered)
makeObj.appendFootnotes()
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
yield from self.iterBuildNWD(path, asJson=bFormat == nwBuildFmt.J_NWD)
makeObj = ToRaw(self._project)
filtered = self._setupBuild(makeObj)
yield from self._iterBuild(makeObj, filtered)
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
self._error = None
self._cache = makeObj
try:
makeObj.saveDocument(path)
except Exception as exc:
logException()
self._error = formatException(exc)
return
def iterBuildOpenDocument(self, path: Path, isFlat: bool) -> Iterable[tuple[int, bool]]:
"""Build an Open Document file."""
makeObj = ToOdt(self._project, isFlat=isFlat)
filtered = self._setupBuild(makeObj)
makeObj.initDocument()
##
# Internal Functions
##
def _iterBuild(self, makeObj: Tokenizer, filtered: dict) -> Iterable[tuple[int, bool]]:
"""Iterate over buildable documents."""
self._count = True
for i, tHandle in enumerate(self._queue):
self._error = None
@@ -171,116 +215,8 @@ class NWBuildDocument:
yield i, self._doBuild(makeObj, tHandle)
else:
yield i, False
makeObj.closeDocument()
self._error = None
self._cache = makeObj
try:
makeObj.saveDocument(path)
except Exception as exc:
logException()
self._error = formatException(exc)
return
def iterBuildHTML(self, path: Path | None, asJson: bool = False) -> Iterable[tuple[int, bool]]:
"""Build an HTML file. If path is None, no file is saved. This
is used for generating build previews.
"""
makeObj = ToHtml(self._project)
filtered = self._setupBuild(makeObj)
self._count = False
for i, tHandle in enumerate(self._queue):
self._error = None
if filtered.get(tHandle, (False, 0))[0]:
yield i, self._doBuild(makeObj, tHandle)
else:
yield i, False
makeObj.appendFootnotes()
if not self._build.getBool("html.preserveTabs"):
makeObj.replaceTabs()
self._error = None
self._cache = makeObj
if isinstance(path, Path):
try:
makeObj.saveDocument(path, asJson=asJson)
except Exception as exc:
logException()
self._error = formatException(exc)
return
def iterBuildMarkdown(self, path: Path, extendedMd: bool) -> Iterable[tuple[int, bool]]:
"""Build a Markdown file."""
makeObj = ToMarkdown(self._project)
filtered = self._setupBuild(makeObj)
makeObj.setExtendedMarkdown(extendedMd)
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
self._count = False
for i, tHandle in enumerate(self._queue):
self._error = None
if filtered.get(tHandle, (False, 0))[0]:
yield i, self._doBuild(makeObj, tHandle)
else:
yield i, False
makeObj.appendFootnotes()
self._error = None
self._cache = makeObj
try:
makeObj.saveDocument(path)
except Exception as exc:
logException()
self._error = formatException(exc)
return
def iterBuildNWD(self, path: Path | None, asJson: bool = False) -> Iterable[tuple[int, bool]]:
"""Build a novelWriter Markdown file."""
makeObj = ToMarkdown(self._project)
filtered = self._setupBuild(makeObj)
makeObj.setKeepMarkdown(True)
self._count = False
for i, tHandle in enumerate(self._queue):
self._error = None
if filtered.get(tHandle, (False, 0))[0]:
yield i, self._doBuild(makeObj, tHandle, convert=False)
else:
yield i, False
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
self._error = None
self._cache = makeObj
if isinstance(path, Path):
try:
makeObj.saveRawDocument(path, asJson=asJson)
except Exception as exc:
logException()
self._error = formatException(exc)
return
##
# Internal Functions
##
def _setupBuild(self, bldObj: Tokenizer) -> dict:
"""Configure the build object."""
# Get Settings
novelwriter/formats/tohtml.py
+2 -2
View File
@@ -290,9 +290,9 @@ class ToHtml(Tokenizer):
return
def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to an HTML file."""
if asJson:
if path.suffix.lower() == ".json":
ts = time()
data = {
"meta": {
novelwriter/formats/tokenizer.py
+18 -23
View File
@@ -134,10 +134,10 @@ class Tokenizer(ABC):
self._project = project
# Data Variables
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._keepMD = False # Whether to keep the markdown text
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Tokens and Meta Data (Per Document)
self._tokens: list[T_Token] = []
@@ -473,11 +473,6 @@ class Tokenizer(ABC):
self._keepBreaks = state
return
def setKeepMarkdown(self, state: bool) -> None:
"""Keep original markdown during build."""
self._keepMD = state
return
##
# Class Methods
##
@@ -487,7 +482,7 @@ class Tokenizer(ABC):
raise NotImplementedError
@abstractmethod
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
raise NotImplementedError
def addRootHeading(self, tHandle: str) -> None:
@@ -509,7 +504,7 @@ class Tokenizer(ABC):
self._tokens.append((
self.T_TITLE, 1, title, [], textAlign
))
if self._keepMD:
if self._keepRaw:
self._markdown.append(f"#! {title}\n\n")
return
@@ -574,7 +569,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append("\n")
continue
@@ -632,26 +627,26 @@ class Tokenizer(ABC):
tokens.append((
self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepMD:
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_SHORT, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepMD:
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.FOOTNOTE:
tLine, tFmt = self._extractFormats(cText, skip=self.FMT_FNOTE)
self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt)
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_COMMENT, nHead, tLine, tFmt, sAlign
))
if self._doComments and self._keepMD:
if self._doComments and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("@"):
@@ -668,7 +663,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepMD:
if self._doKeywords and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("# ", "#! ")):
@@ -704,7 +699,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("## ", "##! ")):
@@ -739,7 +734,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("### ", "###! ")):
@@ -780,7 +775,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("#### "):
@@ -810,7 +805,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
@@ -858,7 +853,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_TEXT, nHead, tLine, tFmt, sAlign
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag
@@ -877,7 +872,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append("\n")
self._markdown.append("".join(tmpMarkdown))
novelwriter/formats/tomarkdown.py
+3 -14
View File
@@ -81,11 +81,11 @@ class ToMarkdown(Tokenizer):
supports concatenating novelWriter markup files.
"""
def __init__(self, project: NWProject) -> None:
def __init__(self, project: NWProject, extended: bool) -> None:
super().__init__(project)
self._fullMD: list[str] = []
self._usedNotes: dict[str, int] = {}
self._extended = True
self._extended = extended
return
##
@@ -97,15 +97,6 @@ class ToMarkdown(Tokenizer):
"""Return the markdown as a list."""
return self._fullMD
##
# Setters
##
def setExtendedMarkdown(self, state: bool) -> None:
"""Set the converter to use Extended Markdown formatting."""
self._extended = state
return
##
# Class Methods
##
@@ -199,7 +190,7 @@ class ToMarkdown(Tokenizer):
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to a plain text file."""
with open(path, mode="w", encoding="utf-8") as outFile:
outFile.write("".join(self._fullMD))
@@ -210,8 +201,6 @@ class ToMarkdown(Tokenizer):
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
if self._keepMD:
self._markdown = [p.replace("\t", spaces) for p in self._markdown]
return
##
novelwriter/formats/toodt.py
+1 -1
View File
@@ -543,7 +543,7 @@ class ToOdt(Tokenizer):
self._xText.insert(0, xFields)
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to an .fodt or .odt file."""
if self._isFlat:
with open(path, mode="wb") as fObj:
novelwriter/formats/toqdoc.py
+1 -1
View File
@@ -275,7 +275,7 @@ class ToQTextDocument(Tokenizer):
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Not implemented."""
return
novelwriter/formats/toraw.py
+86
View File
@@ -0,0 +1,86 @@
"""
novelWriter – Raw NW Text Format
================================
File History:
Created: 2024-10-15 [2.6b1] ToRaw
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from time import time
from novelwriter.common import formatTimeStamp
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__)
class ToRaw(Tokenizer):
"""Core: Raw novelWriter Text Writer
A class that will collect the minimally altered original source text
and write it to either a text or JSON file.
"""
def __init__(self, project: NWProject) -> None:
super().__init__(project)
self._keepRaw = True
return
def doConvert(self) -> None:
"""No conversion to perform."""
return
def saveDocument(self, path: Path) -> None:
"""Save the raw text to a plain text file."""
if path.suffix.lower() == ".json":
ts = time()
data = {
"meta": {
"projectName": self._project.data.name,
"novelAuthor": self._project.data.author,
"buildTime": int(ts),
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
"nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
json.dump(data, fObj, indent=2)
else:
with open(path, mode="w", encoding="utf-8") as outFile:
for nwdPage in self._markdown:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
return
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._markdown = [p.replace("\t", spaces) for p in self._markdown]
return
novelwriter/tools/manusbuild.py
+1 -1
View File
@@ -333,7 +333,7 @@ class GuiManuscriptBuild(NDialog):
docBuild.queueAll()
self.buildProgress.setMaximum(len(docBuild))
for i, _ in docBuild.iterBuild(buildPath, bFormat):
for i, _ in docBuild.iterBuildDocument(buildPath, bFormat):
self.buildProgress.setValue(i+1)
self._build.setLastBuildPath(bPath)
tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
+2 -2
View File
@@ -2,8 +2,8 @@
"meta": {
"projectName": "Lorem Ipsum",
"novelAuthor": "lipsum.com",
"buildTime": 1714229171,
"buildTimeStr": "2024-04-27 16:46:11"
"buildTime": 1729029144,
"buildTimeStr": "2024-10-15 23:52:24"
},
"text": {
"nwd": [
tests/test_core/test_core_docbuild.py
+26 -25
View File
@@ -34,6 +34,7 @@ from novelwriter.enum import nwBuildFmt
from novelwriter.formats.tohtml import ToHtml
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toraw import ToRaw
from tests.mocked import causeException, causeOSError
from tests.tools import ODT_IGNORE, C, buildTestProject, cmpFiles
@@ -101,7 +102,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
for _, success in docBuild.iterBuildOpenDocument(docFile, True):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -119,7 +120,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
for _, success in docBuild.iterBuildOpenDocument(docFile, False):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -136,7 +137,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.fodt"
for _ in docBuild.iterBuildOpenDocument(docFile, True):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -152,7 +153,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
docFile = fncPath / "Lorem Ipsum Err.fodt"
for _, success in docBuild.iterBuildOpenDocument(docFile, True):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if not success and docBuild.error:
error.append(docBuild.error)
@@ -203,7 +204,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuildHTML(docFile):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -223,7 +224,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuildHTML(docFile, asJson=True):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -241,7 +242,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.htm"
for _ in docBuild.iterBuildHTML(docFile):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -271,7 +272,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
for _, success in docBuild.iterBuildMarkdown(docFile, False):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -291,7 +292,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
for _, success in docBuild.iterBuildMarkdown(docFile, True):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -309,7 +310,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
for _ in docBuild.iterBuildMarkdown(docFile, False):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -339,7 +340,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -359,7 +360,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuildNWD(docFile, asJson=True):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -377,7 +378,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
for _ in docBuild.iterBuildNWD(docFile):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -401,7 +402,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -431,7 +432,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
for _, success in docBuild.iterBuildNWD(docFile, asJson=False):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -473,7 +474,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# ODT Format
docFile = fncPath / "Minimal.odt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.ODT)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -483,7 +484,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# FODT Format
docFile = fncPath / "Minimal.fodt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.FODT)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -493,7 +494,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# HTML Format
docFile = fncPath / "Minimal.html"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.HTML)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -503,7 +504,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON HTML Format
docFile = fncPath / "Minimal.json"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_HTML)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -515,7 +516,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Standard Markdown Format
docFile = fncPath / "Minimal.md"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.STD_MD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -536,7 +537,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Extended Markdown Format
docFile = fncPath / "Minimal.md"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -557,11 +558,11 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# NWD Format
docFile = fncPath / "Minimal.txt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.NWD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
assert isinstance(docBuild.lastBuild, ToMarkdown)
assert isinstance(docBuild.lastBuild, ToRaw)
assert docFile.read_text(encoding="utf-8") == (
"#! New Novel\n\n"
"By Jane Doe\n\n"
@@ -578,11 +579,11 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON NWD Format
docFile = fncPath / "Minimal.json"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_NWD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
assert isinstance(docBuild.lastBuild, ToMarkdown)
assert isinstance(docBuild.lastBuild, ToRaw)
data = json.loads(docFile.read_text(encoding="utf-8"))
assert "meta" in data
assert "text" in data
tests/test_formats/test_fmt_tohtml.py
+2 -3
View File
@@ -644,12 +644,12 @@ def testFmtToHtml_Save(mockGUI, fncPath):
)
saveFile = fncPath / "outFile.htm"
html.saveDocument(saveFile, asJson=False)
html.saveDocument(saveFile)
assert saveFile.read_text(encoding="utf-8") == htmlDoc
# JSON + HTML
saveFile = fncPath / "outFile.json"
html.saveDocument(saveFile, asJson=True)
html.saveDocument(saveFile)
data = json.loads(saveFile.read_text(encoding="utf-8"))
assert data["meta"]["projectName"] == ""
assert data["meta"]["novelAuthor"] == ""
@@ -664,7 +664,6 @@ def testFmtToHtml_Methods(mockGUI):
"""Test all the other methods of the ToHtml class."""
project = NWProject()
html = ToHtml(project)
html.setKeepMarkdown(True)
# Auto-Replace, keep Unicode
docText = "Text with <brackets> & short–dash, long—dash …\n"
tests/test_formats/test_fmt_tokenizer.py
+23 -24
View File
@@ -31,6 +31,7 @@ from novelwriter.constants import nwHeadFmt
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer, stripEscape
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toraw import ToRaw
from tests.tools import C, buildTestProject, readFile
@@ -43,6 +44,19 @@ class BareTokenizer(Tokenizer):
super().saveDocument(path) # type: ignore (deliberate check)
@pytest.mark.core
def testFmtToken_Abstracts(mockGUI, tstPaths):
"""Test all the abstract methods of the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
with pytest.raises(NotImplementedError):
tokens.doConvert()
with pytest.raises(NotImplementedError):
tokens.saveDocument(tstPaths)
@pytest.mark.core
def testFmtToken_Setters(mockGUI):
"""Test all the setters for the Tokenizer class."""
@@ -164,8 +178,7 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Set some content to work with
docText = (
@@ -235,13 +248,6 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
]
}
# Check abstract methods
with pytest.raises(NotImplementedError):
tokens.doConvert()
with pytest.raises(NotImplementedError):
tokens.saveDocument(fncPath)
@pytest.mark.core
def testFmtToken_StripEscape():
@@ -256,8 +262,7 @@ def testFmtToken_StripEscape():
def testFmtToken_HeaderFormat(mockGUI):
"""Test the tokenization of header formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Title
# =====
@@ -692,8 +697,7 @@ def testFmtToken_HeaderStyle(mockGUI):
def testFmtToken_MetaFormat(mockGUI):
"""Test the tokenization of meta formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Comment
tokens._text = "% A comment\n"
@@ -780,8 +784,7 @@ def testFmtToken_MetaFormat(mockGUI):
def testFmtToken_MarginFormat(mockGUI):
"""Test the tokenization of margin formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Alignment and Indentation
dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
@@ -824,7 +827,6 @@ def testFmtToken_ExtractFormats(mockGUI):
"""Test the extraction of formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
# Markdown
# ========
@@ -931,7 +933,6 @@ def testFmtToken_Paragraphs(mockGUI):
"""Test the splitting of paragraphs."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
@@ -1003,8 +1004,7 @@ def testFmtToken_Paragraphs(mockGUI):
def testFmtToken_TextFormat(mockGUI):
"""Test the tokenization of text formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Text
tokens._text = "Some plain text\non two lines\n\n\n"
@@ -1887,7 +1887,7 @@ def testFmtToken_SceneSeparators(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
md = ToMarkdown(project)
md = ToMarkdown(project, False)
md._isNovel = True
# Separator Handling, Titles
@@ -2002,8 +2002,7 @@ def testFmtToken_SceneSeparators(mockGUI):
# Separators with Scenes Only
# ===========================
# Requires a fresh builder class
md = ToMarkdown(project)
md.setExtendedMarkdown(True)
md = ToMarkdown(project, True)
md._isNovel = True
md._text = (
@@ -2038,7 +2037,7 @@ def testFmtToken_HeaderVisibility(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
md = ToMarkdown(project)
md = ToMarkdown(project, False)
md._text = (
"#! Novel\n\n"
@@ -2150,7 +2149,7 @@ def testFmtToken_CounterHandling(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
md = ToMarkdown(project)
md = ToMarkdown(project, False)
md._isNovel = True
# Counter Handling, Novel Titles
tests/test_formats/test_fmt_tomarkdown.py
+10 -13
View File
@@ -30,7 +30,7 @@ from novelwriter.formats.tomarkdown import ToMarkdown
def testFmtToMarkdown_ConvertHeaders(mockGUI):
"""Test header formats in the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
toMD = ToMarkdown(project, False)
toMD._isNovel = True
toMD._isFirst = True
@@ -76,13 +76,13 @@ def testFmtToMarkdown_ConvertHeaders(mockGUI):
def testFmtToMarkdown_ConvertParagraphs(mockGUI):
"""Test paragraph formats in the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
toMD = ToMarkdown(project, False)
toMD._isNovel = True
toMD._isFirst = True
# Text for Extended Markdown
toMD.setExtendedMarkdown(True)
toMD._extended = True
toMD._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
toMD.tokenizeText()
toMD.doConvert()
@@ -91,7 +91,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
)
# Text for Standard Markdown
toMD.setExtendedMarkdown(False)
toMD._extended = False
toMD._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
toMD.tokenizeText()
toMD.doConvert()
@@ -100,7 +100,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
)
# Shortcodes for Extended Markdown
toMD.setExtendedMarkdown(True)
toMD._extended = True
toMD._text = (
"Some [b]bold[/b], [i]italic[/i], [s]strike[/s], [u]underline[/u], [m]mark[/m], "
"super[sup]script[/sup], sub[sub]script[/sub] here\n"
@@ -113,7 +113,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
)
# Shortcodes for Standard Markdown
toMD.setExtendedMarkdown(False)
toMD._extended = False
toMD._text = (
"Some [b]bold[/b], [i]italic[/i], [s]strike[/s], [u]underline[/u], [m]mark[/m], "
"super[sup]script[/sup], sub[sub]script[/sub] here\n"
@@ -212,10 +212,8 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
def testFmtToMarkdown_ConvertDirect(mockGUI):
"""Test the converter directly using the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
toMD = ToMarkdown(project, False)
toMD._isNovel = True
toMD.setExtendedMarkdown(False)
# Special Titles
# ==============
@@ -249,8 +247,7 @@ def testFmtToMarkdown_ConvertDirect(mockGUI):
def testFmtToMarkdown_Save(mockGUI, fncPath):
"""Test the save method of the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
toMD.setKeepMarkdown(True)
toMD = ToMarkdown(project, False)
toMD._isNovel = True
# Build Project
@@ -287,7 +284,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
toMD.replaceTabs(nSpaces=4, spaceChar=" ")
resText[6] = "#### A Section\n\n More text in scene two.\n\n"
assert toMD.allMarkdown == resText
assert toMD.fullMD == resText
# Check File
# ==========
@@ -301,7 +298,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
def testFmtToMarkdown_Format(mockGUI):
"""Test all the formatters for the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
toMD = ToMarkdown(project, False)
assert toMD._formatKeywords("", toMD.A_NONE) == ""
assert toMD._formatKeywords("tag: Jane", toMD.A_NONE) == "**Tag:** Jane\n\n"