Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Combine the different ways of storing generated text data in formats

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-23 17:12:13 +02:00
parent 0ddc1131dc
commit 3712b2e93f
8 changed files with 252 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/formats/todocx.py
-2
View File
@@ -214,8 +214,6 @@ class ToDocX(Tokenizer):
def doConvert(self) -> None:
"""Convert the list of text tokens into XML elements."""
self._result = "" # Not used, but cleared just in case
bIndent = self._fontSize * self._blockIndent
for tType, _, tText, tFormat, tStyle in self._blocks:
novelwriter/formats/tohtml.py
+10 -29
View File
@@ -79,25 +79,12 @@ class ToHtml(Tokenizer):
def __init__(self, project: NWProject) -> None:
super().__init__(project)
self._cssStyles = True
self._fullHTML: list[str] = []
# Internals
self._trMap = {}
self._cssStyles = True
self._usedNotes: dict[str, int] = {}
self.setReplaceUnicode(False)
return
##
# Properties
##
@property
def fullHTML(self) -> list[str]:
return self._fullHTML
##
# Setters
##
@@ -128,7 +115,7 @@ class ToHtml(Tokenizer):
def getFullResultSize(self) -> int:
"""Return the size of the full HTML result."""
return sum(len(x) for x in self._fullHTML)
return sum(len(x) for x in self._pages)
def doPreProcessing(self) -> None:
"""Extend the auto-replace to also properly encode some unicode
@@ -140,8 +127,6 @@ class ToHtml(Tokenizer):
def doConvert(self) -> None:
"""Convert the list of text tokens into an HTML document."""
self._result = ""
if self._isNovel:
# For story files, we bump the titles one level up
h1Cl = " class='title'"
@@ -258,8 +243,7 @@ class ToHtml(Tokenizer):
tClass = f"meta meta-{tMeta}"
lines.append(f"<p class='{tClass}'{hStyle}>{self._formatText(tText, tFmt)}</p>\n")
self._result = "".join(lines)
self._fullHTML.append(self._result)
self._pages.append("".join(lines))
return
@@ -277,9 +261,7 @@ class ToHtml(Tokenizer):
lines.append(f"<li id='footnote_{index}'><p>{text}</p></li>\n")
lines.append("</ol>\n")
result = "".join(lines)
self._result += result
self._fullHTML.append(result)
self._pages.append("".join(lines))
return
@@ -296,7 +278,7 @@ class ToHtml(Tokenizer):
},
"text": {
"css": self.getStyleSheet(),
"html": [t.replace("\t", "&#09;").rstrip().split("\n") for t in self.fullHTML],
"html": [t.replace("\t", "&#09;").rstrip().split("\n") for t in self._pages],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
@@ -323,7 +305,7 @@ class ToHtml(Tokenizer):
).format(
title=self._project.data.name,
style="\n".join(self.getStyleSheet()),
body=("".join(self._fullHTML)).replace("\t", "&#09;").rstrip(),
body=("".join(self._pages)).replace("\t", "&#09;").rstrip(),
))
logger.info("Wrote file: %s", path)
@@ -332,12 +314,11 @@ class ToHtml(Tokenizer):
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = "&nbsp;") -> None:
"""Replace tabs with spaces in the html."""
htmlText = []
pages = []
tabSpace = spaceChar*nSpaces
for aLine in self._fullHTML:
htmlText.append(aLine.replace("\t", tabSpace))
self._fullHTML = htmlText
for aLine in self._pages:
pages.append(aLine.replace("\t", tabSpace))
self._pages = pages
return
def getStyleSheet(self) -> list[str]:
novelwriter/formats/tokenizer.py
+7 -18
View File
@@ -96,7 +96,6 @@ class Tokenizer(ABC):
# Data Variables
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Blocks and Meta Data (Per Document)
@@ -104,9 +103,10 @@ class Tokenizer(ABC):
self._footnotes: dict[str, T_Note] = {}
# Blocks and Meta Data (Per Instance)
self._raw: list[str] = []
self._pages: list[str] = []
self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {}
self._markdown: list[str] = []
# User Settings
self._textFont = QFont("Serif", 11) # Output text font
@@ -211,16 +211,6 @@ class Tokenizer(ABC):
# Properties
##
@property
def result(self) -> str:
"""The result of the build process."""
return self._result
@property
def allMarkdown(self) -> list[str]:
"""The combined novelWriter Markdown text."""
return self._markdown
@property
def textStats(self) -> dict[str, int]:
"""The collected stats about the text."""
@@ -486,7 +476,7 @@ class Tokenizer(ABC):
BlockTyp.TITLE, f"{self._handle}:T0001", title, [], textAlign
))
if self._keepRaw:
self._markdown.append(f"#! {title}\n\n")
self._raw.append(f"#! {title}\n\n")
return
@@ -841,8 +831,7 @@ class Tokenizer(ABC):
# Make sure the blocks array doesn't start with a page break
# on the very first page, adding a blank first page.
if tBlocks[1][4] & BlockFmt.PBB:
cBlock = tBlocks[1]
if (cBlock := tBlocks[1])[4] & BlockFmt.PBB:
tBlocks[1] = (
cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB
)
@@ -851,7 +840,7 @@ class Tokenizer(ABC):
tBlocks.append(B_EMPTY)
if keepRaw:
tmpMarkdown.append("\n")
self._markdown.append("".join(tmpMarkdown))
self._raw.append("".join(tmpMarkdown))
# Second Pass
# ===========
@@ -1047,7 +1036,7 @@ class Tokenizer(ABC):
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
"nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
"nwd": [page.rstrip("\n").split("\n") for page in self._raw],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
@@ -1055,7 +1044,7 @@ class Tokenizer(ABC):
else:
with open(path, mode="w", encoding="utf-8") as outFile:
for nwdPage in self._markdown:
for nwdPage in self._raw:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
novelwriter/formats/tomarkdown.py
+6 -22
View File
@@ -84,32 +84,20 @@ class ToMarkdown(Tokenizer):
def __init__(self, project: NWProject, extended: bool) -> None:
super().__init__(project)
self._fullMD: list[str] = []
self._usedNotes: dict[str, int] = {}
self._extended = extended
self._usedNotes: dict[str, int] = {}
return
##
# Properties
##
@property
def fullMD(self) -> list[str]:
"""Return the markdown as a list."""
return self._fullMD
##
# Class Methods
##
def getFullResultSize(self) -> int:
"""Return the size of the full Markdown result."""
return sum(len(x) for x in self._fullMD)
return sum(len(x) for x in self._pages)
def doConvert(self) -> None:
"""Convert the list of text tokens into a Markdown document."""
self._result = ""
if self._extended:
mTags = EXT_MD
cSkip = nwUnicode.U_MMSP
@@ -157,8 +145,7 @@ class ToMarkdown(Tokenizer):
end = " \n" if tStyle & BlockFmt.Z_BTM else "\n\n"
lines.append(f"{self._formatText(tText, tFormat, mTags)}{end}")
self._result = "".join(lines)
self._fullMD.append(self._result)
self._pages.append("".join(lines))
return
@@ -176,24 +163,21 @@ class ToMarkdown(Tokenizer):
text = self._formatText(content[0], content[1], tags)
lines.append(f"{marker}{text}\n")
lines.append("\n")
result = "".join(lines)
self._result += result
self._fullMD.append(result)
self._pages.append("".join(lines))
return
def saveDocument(self, path: Path) -> None:
"""Save the data to a plain text file."""
with open(path, mode="w", encoding="utf-8") as outFile:
outFile.write("".join(self._fullMD))
outFile.write("".join(self._pages))
logger.info("Wrote file: %s", path)
return
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
self._pages = [p.replace("\t", spaces) for p in self._pages]
return
##
novelwriter/formats/toraw.py
+3 -3
View File
@@ -64,7 +64,7 @@ class ToRaw(Tokenizer):
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
"nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
"nwd": [page.rstrip("\n").split("\n") for page in self._raw],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
@@ -72,7 +72,7 @@ class ToRaw(Tokenizer):
else:
with open(path, mode="w", encoding="utf-8") as outFile:
for nwdPage in self._markdown:
for nwdPage in self._raw:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
@@ -82,5 +82,5 @@ class ToRaw(Tokenizer):
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._markdown = [p.replace("\t", spaces) for p in self._markdown]
self._raw = [p.replace("\t", spaces) for p in self._raw]
return