From 3712b2e93f81d92967d5e48070984be34b5a0f9a Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Wed, 23 Oct 2024 17:12:13 +0200
Subject: [PATCH] Combine the different ways of storing generated text data in
formats
---
novelwriter/formats/todocx.py | 2 -
novelwriter/formats/tohtml.py | 39 +---
novelwriter/formats/tokenizer.py | 25 +--
novelwriter/formats/tomarkdown.py | 28 +--
novelwriter/formats/toraw.py | 6 +-
tests/test_formats/test_fmt_tohtml.py | 113 +++++-----
tests/test_formats/test_fmt_tokenizer.py | 88 ++++----
tests/test_formats/test_fmt_tomarkdown.py | 251 +++++++++++-----------
8 files changed, 252 insertions(+), 300 deletions(-)
diff --git a/novelwriter/formats/todocx.py b/novelwriter/formats/todocx.py
index 4f77f137..f02e9251 100644
--- a/novelwriter/formats/todocx.py
+++ b/novelwriter/formats/todocx.py
@@ -214,8 +214,6 @@ class ToDocX(Tokenizer):
def doConvert(self) -> None:
"""Convert the list of text tokens into XML elements."""
- self._result = "" # Not used, but cleared just in case
-
bIndent = self._fontSize * self._blockIndent
for tType, _, tText, tFormat, tStyle in self._blocks:
diff --git a/novelwriter/formats/tohtml.py b/novelwriter/formats/tohtml.py
index f0bac888..d129349f 100644
--- a/novelwriter/formats/tohtml.py
+++ b/novelwriter/formats/tohtml.py
@@ -79,25 +79,12 @@ class ToHtml(Tokenizer):
def __init__(self, project: NWProject) -> None:
super().__init__(project)
-
- self._cssStyles = True
- self._fullHTML: list[str] = []
-
- # Internals
self._trMap = {}
+ self._cssStyles = True
self._usedNotes: dict[str, int] = {}
self.setReplaceUnicode(False)
-
return
- ##
- # Properties
- ##
-
- @property
- def fullHTML(self) -> list[str]:
- return self._fullHTML
-
##
# Setters
##
@@ -128,7 +115,7 @@ class ToHtml(Tokenizer):
def getFullResultSize(self) -> int:
"""Return the size of the full HTML result."""
- return sum(len(x) for x in self._fullHTML)
+ return sum(len(x) for x in self._pages)
def doPreProcessing(self) -> None:
"""Extend the auto-replace to also properly encode some unicode
@@ -140,8 +127,6 @@ class ToHtml(Tokenizer):
def doConvert(self) -> None:
"""Convert the list of text tokens into an HTML document."""
- self._result = ""
-
if self._isNovel:
# For story files, we bump the titles one level up
h1Cl = " class='title'"
@@ -258,8 +243,7 @@ class ToHtml(Tokenizer):
tClass = f"meta meta-{tMeta}"
lines.append(f"
{self._formatText(tText, tFmt)}
\n")
- self._result = "".join(lines)
- self._fullHTML.append(self._result)
+ self._pages.append("".join(lines))
return
@@ -277,9 +261,7 @@ class ToHtml(Tokenizer):
lines.append(f"\n")
lines.append("\n")
- result = "".join(lines)
- self._result += result
- self._fullHTML.append(result)
+ self._pages.append("".join(lines))
return
@@ -296,7 +278,7 @@ class ToHtml(Tokenizer):
},
"text": {
"css": self.getStyleSheet(),
- "html": [t.replace("\t", " ").rstrip().split("\n") for t in self.fullHTML],
+ "html": [t.replace("\t", " ").rstrip().split("\n") for t in self._pages],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
@@ -323,7 +305,7 @@ class ToHtml(Tokenizer):
).format(
title=self._project.data.name,
style="\n".join(self.getStyleSheet()),
- body=("".join(self._fullHTML)).replace("\t", " ").rstrip(),
+ body=("".join(self._pages)).replace("\t", " ").rstrip(),
))
logger.info("Wrote file: %s", path)
@@ -332,12 +314,11 @@ class ToHtml(Tokenizer):
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces in the html."""
- htmlText = []
+ pages = []
tabSpace = spaceChar*nSpaces
- for aLine in self._fullHTML:
- htmlText.append(aLine.replace("\t", tabSpace))
-
- self._fullHTML = htmlText
+ for aLine in self._pages:
+ pages.append(aLine.replace("\t", tabSpace))
+ self._pages = pages
return
def getStyleSheet(self) -> list[str]:
diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py
index 3bd064c5..93caa96e 100644
--- a/novelwriter/formats/tokenizer.py
+++ b/novelwriter/formats/tokenizer.py
@@ -96,7 +96,6 @@ class Tokenizer(ABC):
# Data Variables
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
- self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Blocks and Meta Data (Per Document)
@@ -104,9 +103,10 @@ class Tokenizer(ABC):
self._footnotes: dict[str, T_Note] = {}
# Blocks and Meta Data (Per Instance)
+ self._raw: list[str] = []
+ self._pages: list[str] = []
self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {}
- self._markdown: list[str] = []
# User Settings
self._textFont = QFont("Serif", 11) # Output text font
@@ -211,16 +211,6 @@ class Tokenizer(ABC):
# Properties
##
- @property
- def result(self) -> str:
- """The result of the build process."""
- return self._result
-
- @property
- def allMarkdown(self) -> list[str]:
- """The combined novelWriter Markdown text."""
- return self._markdown
-
@property
def textStats(self) -> dict[str, int]:
"""The collected stats about the text."""
@@ -486,7 +476,7 @@ class Tokenizer(ABC):
BlockTyp.TITLE, f"{self._handle}:T0001", title, [], textAlign
))
if self._keepRaw:
- self._markdown.append(f"#! {title}\n\n")
+ self._raw.append(f"#! {title}\n\n")
return
@@ -841,8 +831,7 @@ class Tokenizer(ABC):
# Make sure the blocks array doesn't start with a page break
# on the very first page, adding a blank first page.
- if tBlocks[1][4] & BlockFmt.PBB:
- cBlock = tBlocks[1]
+ if (cBlock := tBlocks[1])[4] & BlockFmt.PBB:
tBlocks[1] = (
cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB
)
@@ -851,7 +840,7 @@ class Tokenizer(ABC):
tBlocks.append(B_EMPTY)
if keepRaw:
tmpMarkdown.append("\n")
- self._markdown.append("".join(tmpMarkdown))
+ self._raw.append("".join(tmpMarkdown))
# Second Pass
# ===========
@@ -1047,7 +1036,7 @@ class Tokenizer(ABC):
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
- "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
+ "nwd": [page.rstrip("\n").split("\n") for page in self._raw],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
@@ -1055,7 +1044,7 @@ class Tokenizer(ABC):
else:
with open(path, mode="w", encoding="utf-8") as outFile:
- for nwdPage in self._markdown:
+ for nwdPage in self._raw:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
diff --git a/novelwriter/formats/tomarkdown.py b/novelwriter/formats/tomarkdown.py
index 878167c0..198b7e7a 100644
--- a/novelwriter/formats/tomarkdown.py
+++ b/novelwriter/formats/tomarkdown.py
@@ -84,32 +84,20 @@ class ToMarkdown(Tokenizer):
def __init__(self, project: NWProject, extended: bool) -> None:
super().__init__(project)
- self._fullMD: list[str] = []
- self._usedNotes: dict[str, int] = {}
self._extended = extended
+ self._usedNotes: dict[str, int] = {}
return
- ##
- # Properties
- ##
-
- @property
- def fullMD(self) -> list[str]:
- """Return the markdown as a list."""
- return self._fullMD
-
##
# Class Methods
##
def getFullResultSize(self) -> int:
"""Return the size of the full Markdown result."""
- return sum(len(x) for x in self._fullMD)
+ return sum(len(x) for x in self._pages)
def doConvert(self) -> None:
"""Convert the list of text tokens into a Markdown document."""
- self._result = ""
-
if self._extended:
mTags = EXT_MD
cSkip = nwUnicode.U_MMSP
@@ -157,8 +145,7 @@ class ToMarkdown(Tokenizer):
end = " \n" if tStyle & BlockFmt.Z_BTM else "\n\n"
lines.append(f"{self._formatText(tText, tFormat, mTags)}{end}")
- self._result = "".join(lines)
- self._fullMD.append(self._result)
+ self._pages.append("".join(lines))
return
@@ -176,24 +163,21 @@ class ToMarkdown(Tokenizer):
text = self._formatText(content[0], content[1], tags)
lines.append(f"{marker}{text}\n")
lines.append("\n")
-
- result = "".join(lines)
- self._result += result
- self._fullMD.append(result)
+ self._pages.append("".join(lines))
return
def saveDocument(self, path: Path) -> None:
"""Save the data to a plain text file."""
with open(path, mode="w", encoding="utf-8") as outFile:
- outFile.write("".join(self._fullMD))
+ outFile.write("".join(self._pages))
logger.info("Wrote file: %s", path)
return
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
- self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
+ self._pages = [p.replace("\t", spaces) for p in self._pages]
return
##
diff --git a/novelwriter/formats/toraw.py b/novelwriter/formats/toraw.py
index d1c701b1..8e1ed47e 100644
--- a/novelwriter/formats/toraw.py
+++ b/novelwriter/formats/toraw.py
@@ -64,7 +64,7 @@ class ToRaw(Tokenizer):
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
- "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
+ "nwd": [page.rstrip("\n").split("\n") for page in self._raw],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
@@ -72,7 +72,7 @@ class ToRaw(Tokenizer):
else:
with open(path, mode="w", encoding="utf-8") as outFile:
- for nwdPage in self._markdown:
+ for nwdPage in self._raw:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
@@ -82,5 +82,5 @@ class ToRaw(Tokenizer):
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
- self._markdown = [p.replace("\t", spaces) for p in self._markdown]
+ self._raw = [p.replace("\t", spaces) for p in self._raw]
return
diff --git a/tests/test_formats/test_fmt_tohtml.py b/tests/test_formats/test_fmt_tohtml.py
index 3535f982..68db421c 100644
--- a/tests/test_formats/test_fmt_tohtml.py
+++ b/tests/test_formats/test_fmt_tohtml.py
@@ -47,7 +47,7 @@ def testFmtToHtml_ConvertHeaders(mockGUI):
html._text = "# Partition\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Partition
\n"
)
@@ -55,7 +55,7 @@ def testFmtToHtml_ConvertHeaders(mockGUI):
html._text = "## Chapter Title\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Chapter Title
\n"
)
@@ -63,19 +63,19 @@ def testFmtToHtml_ConvertHeaders(mockGUI):
html._text = "### Scene Title\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Scene Title
\n"
+ assert html._pages[-1] == "Scene Title
\n"
# Header 4
html._text = "#### Section Title\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Section Title
\n"
+ assert html._pages[-1] == "Section Title
\n"
# Title
html._text = "#! Title\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Title
\n"
)
@@ -83,7 +83,7 @@ def testFmtToHtml_ConvertHeaders(mockGUI):
html._text = "##! Prologue\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Prologue
\n"
+ assert html._pages[-1] == "Prologue
\n"
# Note Files Headers
# ==================
@@ -97,31 +97,31 @@ def testFmtToHtml_ConvertHeaders(mockGUI):
html._text = "# Heading One\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Heading One
\n"
+ assert html._pages[-1] == "Heading One
\n"
# Header 2
html._text = "## Heading Two\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Heading Two
\n"
+ assert html._pages[-1] == "Heading Two
\n"
# Header 3
html._text = "### Heading Three\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Heading Three
\n"
+ assert html._pages[-1] == "Heading Three
\n"
# Header 4
html._text = "#### Heading Four\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Heading Four
\n"
+ assert html._pages[-1] == "Heading Four
\n"
# Title
html._text = "#! Heading One\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
""
"Heading One
\n"
)
@@ -130,7 +130,7 @@ def testFmtToHtml_ConvertHeaders(mockGUI):
html._text = "##! Heading Two\n"
html.tokenizeText()
html.doConvert()
- assert html.result == "Heading Two
\n"
+ assert html._pages[-1] == "Heading Two
\n"
@pytest.mark.core
@@ -150,7 +150,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Some nested bold and italic and "
"strikethrough text here
\n"
)
@@ -162,7 +162,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
)
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Some bold, italic, strike, "
"underline, mark, "
"superscript, subscript here
\n"
@@ -172,7 +172,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "Line one\nLine two\nLine three\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Line one
Line two
Line three
\n"
)
@@ -180,13 +180,13 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "%synopsis: The synopsis ...\n"
html.tokenizeText()
html.doConvert()
- assert html.result == ""
+ assert html._pages[-1] == ""
html.setSynopsis(True)
html._text = "%synopsis: The synopsis ...\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
""
"Characters: "
"Bod, "
@@ -242,7 +242,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "@tag: Bod\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"
Tag: "
"Bod
\n"
)
@@ -250,7 +250,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "@tag: Bod | Nobody Owens\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Tag: "
"Bod | "
"Nobody Owens
\n"
@@ -262,7 +262,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "## Chapter\n\n@pov: Bod\n@plot: Main\n@location: Europe\n\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Chapter
\n"
""
"Point of View: "
@@ -280,7 +280,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "## Chapter\n\nThis text \u201chas dialogue\u201d in it.\n\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"
Chapter
\n"
"This text “has dialogue” in it.
\n"
)
@@ -292,7 +292,7 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html._text = "## Chapter\n\nThis text ::has alt dialogue:: in it.\n\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Chapter
\n"
"This text ::has alt dialogue:: in it.
\n"
)
@@ -306,15 +306,17 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
)
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Text with one1 "
"or twoERR footnotes.
\n"
)
html.appendFootnotes()
- assert html.result == (
+ assert html._pages[-2] == (
"Text with one1 "
"or twoERR footnotes.
\n"
+ )
+ assert html._pages[-1] == (
"Footnotes
\n"
"\n"
"\n"
@@ -336,7 +338,7 @@ def testFmtToHtml_CloseTags(mockGUI):
html._text = "Text [b][i][s][u][m][sup][sub]text text text.\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Text "
"text text text.
\n"
)
@@ -345,7 +347,7 @@ def testFmtToHtml_CloseTags(mockGUI):
html._text = "Text [b][i][s][u][m][sup][sub]text [b][i][s][u][m][sup][sub]text text.\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Text "
"text text text.
\n"
)
@@ -354,7 +356,7 @@ def testFmtToHtml_CloseTags(mockGUI):
html._text = "Text text [/b][/i][/s][/u][/m][/sup][/sub]text text.\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Text text text text.
\n"
)
@@ -380,7 +382,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.TITLE, tMeta, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
""
"A Title
\n"
)
@@ -390,7 +392,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.HEAD2, tMeta, "Prologue", [], BlockFmt.PBB),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
""
"Prologue
\n"
)
@@ -403,14 +405,14 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.SEP, tMeta, "* * *", [], BlockFmt.CENTRE),
]
html.doConvert()
- assert html.result == "* * *
\n"
+ assert html._pages[-1] == "* * *
\n"
# Skip
html._blocks = [
(BlockTyp.SKIP, tMeta, "", [], BlockFmt.NONE),
]
html.doConvert()
- assert html.result == "
\n"
+ assert html._pages[-1] == "
\n"
# Alignment
# =========
@@ -423,7 +425,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.HEAD1, tMeta, "A Title", [], BlockFmt.LEFT),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"A Title
\n"
)
@@ -434,7 +436,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.HEAD1, tMeta, "A Title", [], BlockFmt.LEFT),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"A Title
\n"
)
@@ -443,7 +445,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.HEAD1, tMeta, "A Title", [], BlockFmt.RIGHT),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"A Title
\n"
)
@@ -452,7 +454,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.HEAD1, tMeta, "A Title", [], BlockFmt.CENTRE),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"A Title
\n"
)
@@ -461,7 +463,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.HEAD1, tMeta, "A Title", [], BlockFmt.JUSTIFY),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"A Title
\n"
)
@@ -473,7 +475,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.HEAD1, tMeta, "A Title", [], BlockFmt.PBB | BlockFmt.PBA),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"A Title
\n"
)
@@ -486,7 +488,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.TEXT, tMeta, "Some text ...", [], BlockFmt.IND_L),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Some text ...
\n"
)
@@ -495,7 +497,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.TEXT, tMeta, "Some text ...", [], BlockFmt.IND_R),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Some text ...
\n"
)
@@ -504,7 +506,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
(BlockTyp.TEXT, tMeta, "Some text ...", [], BlockFmt.IND_T),
]
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Some text ...
\n"
)
@@ -523,28 +525,28 @@ def testFmtToHtml_SpecialCases(mockGUI):
html._text = "Text with > and < with some **bold text** in it.\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Text with > and < with some bold text in it.
\n"
)
html._text = "Text with some <**bold text**> in it.\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Text with some <bold text> in it.
\n"
)
html._text = "Let's > be > _difficult **shall** > we_?\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Let's > be > difficult shall > we?
\n"
)
html._text = "Test > text _<**bold**>_ and more.\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"Test > text <bold> and more.
\n"
)
@@ -556,7 +558,7 @@ def testFmtToHtml_SpecialCases(mockGUI):
html._text = "% Test > text _<**bold**>_ and more.\n"
html.tokenizeText()
html.doConvert()
- assert html.result == (
+ assert html._pages[-1] == (
"