Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify JSON and raw text writing

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-15 23:59:25 +02:00
parent 34696ba3a8
commit c633a56bfb
13 changed files with 178 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/docbuild.py
+7 -13
View File
@@ -38,10 +38,11 @@ from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
from novelwriter.error import formatException, logException
from novelwriter.formats.tohtml import ToHtml
from novelwriter.formats.tokenizer import Tokenizer, ToRaw
from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toodt import ToOdt
from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
from novelwriter.formats.toraw import ToRaw
logger = logging.getLogger(__name__)
@@ -146,12 +147,10 @@ class NWBuildDocument:
return
def iterBuild(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
def iterBuildDocument(self, path: Path, bFormat: nwBuildFmt) -> Iterable[tuple[int, bool]]:
"""Wrapper for builders based on format."""
asJson = False
if bFormat in (nwBuildFmt.ODT, nwBuildFmt.FODT):
makeObj = ToOdt(self._project, isFlat=(bFormat == nwBuildFmt.FODT))
makeObj = ToOdt(self._project, bFormat == nwBuildFmt.FODT)
filtered = self._setupBuild(makeObj)
makeObj.initDocument()
yield from self._iterBuild(makeObj, filtered)
@@ -164,7 +163,6 @@ class NWBuildDocument:
makeObj.appendFootnotes()
if not self._build.getBool("html.preserveTabs"):
makeObj.replaceTabs()
asJson = (bFormat == nwBuildFmt.J_HTML)
elif bFormat in (nwBuildFmt.STD_MD, nwBuildFmt.EXT_MD):
makeObj = ToMarkdown(self._project)
@@ -177,20 +175,16 @@ class NWBuildDocument:
elif bFormat in (nwBuildFmt.NWD, nwBuildFmt.J_NWD):
makeObj = ToRaw(self._project)
makeObj.setKeepMarkdown(True)
filtered = self._setupBuild(makeObj)
yield from self._iterBuild(makeObj, filtered)
asJson = (bFormat == nwBuildFmt.J_NWD)
if self._build.getBool("format.replaceTabs"):
makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
self._error = None
self._cache = makeObj
try:
if isinstance(makeObj, ToHtml | ToRaw):
makeObj.saveDocument(path, asJson=asJson)
else:
makeObj.saveDocument(path)
makeObj.saveDocument(path)
except Exception as exc:
logException()
self._error = formatException(exc)
novelwriter/formats/tohtml.py
+2 -2
View File
@@ -290,9 +290,9 @@ class ToHtml(Tokenizer):
return
def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to an HTML file."""
if asJson:
if path.suffix.lower() == ".json":
ts = time()
data = {
"meta": {
novelwriter/formats/tokenizer.py
+18 -61
View File
@@ -134,10 +134,10 @@ class Tokenizer(ABC):
self._project = project
# Data Variables
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._keepMD = False # Whether to keep the markdown text
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Tokens and Meta Data (Per Document)
self._tokens: list[T_Token] = []
@@ -473,11 +473,6 @@ class Tokenizer(ABC):
self._keepBreaks = state
return
def setKeepMarkdown(self, state: bool) -> None:
"""Keep original markdown during build."""
self._keepMD = state
return
##
# Class Methods
##
@@ -487,7 +482,7 @@ class Tokenizer(ABC):
raise NotImplementedError
@abstractmethod
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
raise NotImplementedError
def addRootHeading(self, tHandle: str) -> None:
@@ -509,7 +504,7 @@ class Tokenizer(ABC):
self._tokens.append((
self.T_TITLE, 1, title, [], textAlign
))
if self._keepMD:
if self._keepRaw:
self._markdown.append(f"#! {title}\n\n")
return
@@ -574,7 +569,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append("\n")
continue
@@ -632,26 +627,26 @@ class Tokenizer(ABC):
tokens.append((
self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepMD:
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_SHORT, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepMD:
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.FOOTNOTE:
tLine, tFmt = self._extractFormats(cText, skip=self.FMT_FNOTE)
self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt)
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
self.T_COMMENT, nHead, tLine, tFmt, sAlign
))
if self._doComments and self._keepMD:
if self._doComments and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("@"):
@@ -668,7 +663,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepMD:
if self._doKeywords and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("# ", "#! ")):
@@ -704,7 +699,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("## ", "##! ")):
@@ -739,7 +734,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith(("### ", "###! ")):
@@ -780,7 +775,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif aLine.startswith("#### "):
@@ -810,7 +805,7 @@ class Tokenizer(ABC):
tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
else:
@@ -858,7 +853,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_TEXT, nHead, tLine, tFmt, sAlign
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag
@@ -877,7 +872,7 @@ class Tokenizer(ABC):
tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMD:
if self._keepRaw:
tmpMarkdown.append("\n")
self._markdown.append("".join(tmpMarkdown))
@@ -1245,41 +1240,3 @@ class HeadingFormatter:
hFormat = hFormat.replace(nwHeadFmt.CHAR_FOCUS, fText)
return hFormat
class ToRaw(Tokenizer):
def __init__(self, project: NWProject) -> None:
super().__init__(project)
self._keepMD = True
return
def doConvert(self) -> None:
return
def saveDocument(self, path: str | Path, asJson: bool = False) -> None:
"""Save the raw text to a plain text file."""
if asJson:
ts = time()
data = {
"meta": {
"projectName": self._project.data.name,
"novelAuthor": self._project.data.author,
"buildTime": int(ts),
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
"nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
json.dump(data, fObj, indent=2)
else:
with open(path, mode="w", encoding="utf-8") as outFile:
for nwdPage in self._markdown:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
return
novelwriter/formats/tomarkdown.py
+1 -3
View File
@@ -199,7 +199,7 @@ class ToMarkdown(Tokenizer):
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to a plain text file."""
with open(path, mode="w", encoding="utf-8") as outFile:
outFile.write("".join(self._fullMD))
@@ -210,8 +210,6 @@ class ToMarkdown(Tokenizer):
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._fullMD = [p.replace("\t", spaces) for p in self._fullMD]
if self._keepMD:
self._markdown = [p.replace("\t", spaces) for p in self._markdown]
return
##
novelwriter/formats/toodt.py
+1 -1
View File
@@ -543,7 +543,7 @@ class ToOdt(Tokenizer):
self._xText.insert(0, xFields)
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Save the data to an .fodt or .odt file."""
if self._isFlat:
with open(path, mode="wb") as fObj:
novelwriter/formats/toqdoc.py
+1 -1
View File
@@ -275,7 +275,7 @@ class ToQTextDocument(Tokenizer):
return
def saveDocument(self, path: str | Path) -> None:
def saveDocument(self, path: Path) -> None:
"""Not implemented."""
return
novelwriter/formats/toraw.py
+86
View File
@@ -0,0 +1,86 @@
"""
novelWriter – Raw NW Text Format
================================
File History:
Created: 2024-10-15 [2.6b1] ToRaw
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from time import time
from novelwriter.common import formatTimeStamp
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__)
class ToRaw(Tokenizer):
"""Core: Raw novelWriter Text Writer
A class that will collect the minimally altered original source text
and write it to either a text or JSON file.
"""
def __init__(self, project: NWProject) -> None:
super().__init__(project)
self._keepRaw = True
return
def doConvert(self) -> None:
"""No conversion to perform."""
return
def saveDocument(self, path: Path) -> None:
"""Save the raw text to a plain text file."""
if path.suffix.lower() == ".json":
ts = time()
data = {
"meta": {
"projectName": self._project.data.name,
"novelAuthor": self._project.data.author,
"buildTime": int(ts),
"buildTimeStr": formatTimeStamp(ts),
},
"text": {
"nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
}
}
with open(path, mode="w", encoding="utf-8") as fObj:
json.dump(data, fObj, indent=2)
else:
with open(path, mode="w", encoding="utf-8") as outFile:
for nwdPage in self._markdown:
outFile.write(nwdPage)
logger.info("Wrote file: %s", path)
return
def replaceTabs(self, nSpaces: int = 8, spaceChar: str = " ") -> None:
"""Replace tabs with spaces."""
spaces = spaceChar*nSpaces
self._markdown = [p.replace("\t", spaces) for p in self._markdown]
return
novelwriter/tools/manusbuild.py
+1 -1
View File
@@ -333,7 +333,7 @@ class GuiManuscriptBuild(NDialog):
docBuild.queueAll()
self.buildProgress.setMaximum(len(docBuild))
for i, _ in docBuild.iterBuild(buildPath, bFormat):
for i, _ in docBuild.iterBuildDocument(buildPath, bFormat):
self.buildProgress.setValue(i+1)
self._build.setLastBuildPath(bPath)