Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify JSON and raw text writing

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-15 23:59:25 +02:00
parent 34696ba3a8
commit c633a56bfb
13 changed files with 178 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/reference/mBuildDocBuild_NWD_Lorem_Ipsum.json
+5 -5
View File
@@ -2,8 +2,8 @@
"meta": {
"projectName": "Lorem Ipsum",
"novelAuthor": "lipsum.com",
"buildTime": 1729027593,
"buildTimeStr": "2024-10-15 23:26:33"
"buildTime": 1729029144,
"buildTimeStr": "2024-10-15 23:52:24"
},
"text": {
"nwd": [
@@ -96,11 +96,11 @@
"",
"% Exctracted from the lipsum.com website.",
"",
"\tIt is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
" It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout.",
"",
"\tThe point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
" The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English.",
"",
"\tMany desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
" Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like)."
],
[
"## Chapter Two",
tests/test_core/test_core_docbuild.py
+24 -24
View File
@@ -32,9 +32,9 @@ from novelwriter.core.docbuild import NWBuildDocument
from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.formats.tohtml import ToHtml
from novelwriter.formats.tokenizer import ToRaw
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toraw import ToRaw
from tests.mocked import causeException, causeOSError
from tests.tools import ODT_IGNORE, C, buildTestProject, cmpFiles
@@ -102,7 +102,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -120,7 +120,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.ODT):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -137,7 +137,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.fodt"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -153,7 +153,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP
count = 0
error = []
docFile = fncPath / "Lorem Ipsum Err.fodt"
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.FODT):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT):
count += 1 if success else 0
if not success and docBuild.error:
error.append(docBuild.error)
@@ -204,7 +204,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -224,7 +224,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_HTML):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -242,7 +242,7 @@ def testCoreDocBuild_HTML(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.htm"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.HTML):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -272,7 +272,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -292,7 +292,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -310,7 +310,7 @@ def testCoreDocBuild_Markdown(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.STD_MD):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -340,7 +340,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -360,7 +360,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
count = 0
error = []
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.J_NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -378,7 +378,7 @@ def testCoreDocBuild_NWD(monkeypatch, mockGUI, prjLipsum, fncPath, tstPaths):
mp.setattr("builtins.open", causeOSError)
docFile = fncPath / "Lorem Ipsum Err.md"
for _ in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _ in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
pass
assert docBuild.error == "OSError: Mock OSError"
@@ -402,7 +402,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -432,7 +432,7 @@ def testCoreDocBuild_Custom(mockGUI, fncPath: Path):
count = 0
error = []
docFile = fncPath / "Minimal.txt"
for _, success in docBuild.iterBuild(docFile, nwBuildFmt.NWD):
for _, success in docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD):
count += 1 if success else 0
if docBuild.error:
error.append(docBuild.error)
@@ -474,7 +474,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# ODT Format
docFile = fncPath / "Minimal.odt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.ODT)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.ODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -484,7 +484,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# FODT Format
docFile = fncPath / "Minimal.fodt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.FODT)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.FODT)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -494,7 +494,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# HTML Format
docFile = fncPath / "Minimal.html"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.HTML)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -504,7 +504,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON HTML Format
docFile = fncPath / "Minimal.json"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_HTML)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_HTML)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -516,7 +516,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Standard Markdown Format
docFile = fncPath / "Minimal.md"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.STD_MD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.STD_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -537,7 +537,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# Extended Markdown Format
docFile = fncPath / "Minimal.md"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.EXT_MD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.EXT_MD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -558,7 +558,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# NWD Format
docFile = fncPath / "Minimal.txt"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.NWD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
@@ -579,7 +579,7 @@ def testCoreDocBuild_IterBuild(mockGUI, fncPath: Path, mockRnd):
# JSON NWD Format
docFile = fncPath / "Minimal.json"
assert list(docBuild.iterBuild(docFile, nwBuildFmt.J_NWD)) == [
assert list(docBuild.iterBuildDocument(docFile, nwBuildFmt.J_NWD)) == [
(0, True), (1, True), (2, False), (3, True), (4, True),
(5, True), (6, True), (7, True), (8, True), (9, False),
]
tests/test_formats/test_fmt_tohtml.py
+2 -3
View File
@@ -644,12 +644,12 @@ def testFmtToHtml_Save(mockGUI, fncPath):
)
saveFile = fncPath / "outFile.htm"
html.saveDocument(saveFile, asJson=False)
html.saveDocument(saveFile)
assert saveFile.read_text(encoding="utf-8") == htmlDoc
# JSON + HTML
saveFile = fncPath / "outFile.json"
html.saveDocument(saveFile, asJson=True)
html.saveDocument(saveFile)
data = json.loads(saveFile.read_text(encoding="utf-8"))
assert data["meta"]["projectName"] == ""
assert data["meta"]["novelAuthor"] == ""
@@ -664,7 +664,6 @@ def testFmtToHtml_Methods(mockGUI):
"""Test all the other methods of the ToHtml class."""
project = NWProject()
html = ToHtml(project)
html.setKeepMarkdown(True)
# Auto-Replace, keep Unicode
docText = "Text with <brackets> & short–dash, long—dash …\n"
tests/test_formats/test_fmt_tokenizer.py
+29 -29
View File
@@ -31,6 +31,7 @@ from novelwriter.constants import nwHeadFmt
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer, stripEscape
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toraw import ToRaw
from tests.tools import C, buildTestProject, readFile
@@ -43,11 +44,24 @@ class BareTokenizer(Tokenizer):
super().saveDocument(path) # type: ignore (deliberate check)
@pytest.mark.core
def testFmtToken_Abstracts(mockGUI, tstPaths):
"""Test all the abstract methods of the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
with pytest.raises(NotImplementedError):
tokens.doConvert()
with pytest.raises(NotImplementedError):
tokens.saveDocument(tstPaths)
@pytest.mark.core
def testFmtToken_Setters(mockGUI):
"""Test all the setters for the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
# Verify defaults
assert tokens._fmtPart == nwHeadFmt.TITLE
@@ -164,8 +178,7 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Set some content to work with
docText = (
@@ -235,13 +248,6 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
]
}
# Check abstract methods
with pytest.raises(NotImplementedError):
tokens.doConvert()
with pytest.raises(NotImplementedError):
tokens.saveDocument(fncPath)
@pytest.mark.core
def testFmtToken_StripEscape():
@@ -256,8 +262,7 @@ def testFmtToken_StripEscape():
def testFmtToken_HeaderFormat(mockGUI):
"""Test the tokenization of header formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Title
# =====
@@ -431,7 +436,7 @@ def testFmtToken_HeaderFormat(mockGUI):
def testFmtToken_HeaderStyle(mockGUI):
"""Test the styling of headers in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
def processStyle(text: str, first: bool) -> int:
tokens._text = text
@@ -692,8 +697,7 @@ def testFmtToken_HeaderStyle(mockGUI):
def testFmtToken_MetaFormat(mockGUI):
"""Test the tokenization of meta formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Comment
tokens._text = "% A comment\n"
@@ -780,8 +784,7 @@ def testFmtToken_MetaFormat(mockGUI):
def testFmtToken_MarginFormat(mockGUI):
"""Test the tokenization of margin formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Alignment and Indentation
dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
@@ -823,8 +826,7 @@ def testFmtToken_MarginFormat(mockGUI):
def testFmtToken_ExtractFormats(mockGUI):
"""Test the extraction of formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Markdown
# ========
@@ -930,8 +932,7 @@ def testFmtToken_ExtractFormats(mockGUI):
def testFmtToken_Paragraphs(mockGUI):
"""Test the splitting of paragraphs."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
@@ -1003,8 +1004,7 @@ def testFmtToken_Paragraphs(mockGUI):
def testFmtToken_TextFormat(mockGUI):
"""Test the tokenization of text formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
tokens = ToRaw(project)
# Text
tokens._text = "Some plain text\non two lines\n\n\n"
@@ -1104,7 +1104,7 @@ def testFmtToken_Dialogue(mockGUI):
CONFIG.narratorBreak = "\u2013"
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens.setDialogueHighlight(True)
tokens._isNovel = True
@@ -1191,7 +1191,7 @@ def testFmtToken_Dialogue(mockGUI):
def testFmtToken_SpecialFormat(mockGUI):
"""Test the tokenization of special formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens._isNovel = True
@@ -1353,7 +1353,7 @@ def testFmtToken_SpecialFormat(mockGUI):
def testFmtToken_TextIndent(mockGUI):
"""Test the handling of text indent in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
# No First Indent
tokens.setFirstLineIndent(True, 1.0, False)
@@ -1438,7 +1438,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens._isNovel = True
# Titles
@@ -1627,7 +1627,7 @@ def testFmtToken_BuildOutline(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
# Novel
tokens._isNovel = True
@@ -1691,7 +1691,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens = ToRaw(project)
tokens._isNovel = True
# Short Text
tests/test_formats/test_fmt_tomarkdown.py
+1 -2
View File
@@ -250,7 +250,6 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
"""Test the save method of the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project)
toMD.setKeepMarkdown(True)
toMD._isNovel = True
# Build Project
@@ -287,7 +286,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
toMD.replaceTabs(nSpaces=4, spaceChar=" ")
resText[6] = "#### A Section\n\n More text in scene two.\n\n"
assert toMD.allMarkdown == resText
assert toMD.fullMD == resText
# Check File
# ==========