Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use format list also for marker keys

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-04-15 01:11:32 +02:00
parent 9823677983
commit 7c4d4dec5c
4 changed files with 56 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tohtml.py
+12 -7
View File
@@ -171,7 +171,7 @@ class ToHtml(Tokenizer):
tHandle = self._handle
for tType, nHead, tText, tFormat, tMarkers, tStyle in self._tokens:
for tType, nHead, tText, tFormat, tStyle in self._tokens:
# Replace < and > with HTML entities
if tFormat:
@@ -181,11 +181,11 @@ class ToHtml(Tokenizer):
for c in tText:
if c == "<":
cText.append("&lt;")
tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat]
tFormat = [[p + 3 if p > i else p, f, k] for p, f, k in tFormat]
i += 4
elif c == ">":
cText.append("&gt;")
tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat]
tFormat = [[p + 3 if p > i else p, f, k] for p, f, k in tFormat]
i += 4
else:
cText.append(c)
@@ -278,10 +278,15 @@ class ToHtml(Tokenizer):
tTemp = tText
if pStyle is None:
pStyle = hStyle
for pos, fmt in reversed(tFormat):
tTemp = f"{tTemp[:pos]}{htmlTags[fmt]}{tTemp[pos:]}"
for pos, fmt, key in reversed(tMarkers):
tTemp = f"{tTemp[:pos]}<sup>[x]</sub>{tTemp[pos:]}"
for pos, fmt, key in reversed(tFormat):
if fmt > self.MRK_BOUNDARY:
if key in self._markers:
index = self._markers[key][0]
if fmt == self.MRK_FOOTNOTE:
ref = f"<sup><a href='#footnote_{index}'>[{index+1}]</a></sup>"
tTemp = f"{tTemp[:pos]}{ref}{tTemp[pos:]}"
else:
tTemp = f"{tTemp[:pos]}{htmlTags[fmt]}{tTemp[pos:]}"
para.append(stripEscape(tTemp.rstrip()))
elif tType == self.T_SYNOPSIS and self._doSynopsis:
novelwriter/core/tokenizer.py
+37 -37
View File
@@ -48,8 +48,7 @@ logger = logging.getLogger(__name__)
ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
T_Formats = list[tuple[int, int]]
T_Markers = list[tuple[int, int, str]]
T_Formats = list[tuple[int, int, str]]
def stripEscape(text: str) -> str:
@@ -132,7 +131,7 @@ class Tokenizer(ABC):
self._allMarkdown = [] # The result novelWriter markdown of all documents
# Processed Tokens and Meta Data
self._tokens: list[tuple[int, int, str, T_Formats, T_Markers, int]] = []
self._tokens: list[tuple[int, int, str, T_Formats, int]] = []
self._markers: dict[str, tuple[int, list[str]]] = {}
self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {}
@@ -426,7 +425,7 @@ class Tokenizer(ABC):
title = f"{trNotes}: {tItem.itemName}"
self._tokens = []
self._tokens.append((
self.T_TITLE, 1, title, [], [], textAlign
self.T_TITLE, 1, title, [], textAlign
))
if self._keepMarkdown:
self._allMarkdown.append(f"#! {title}\n\n")
@@ -490,7 +489,7 @@ class Tokenizer(ABC):
# Check for blank lines
if len(sLine) == 0:
self._tokens.append((
self.T_EMPTY, nHead, "", [], [], self.A_NONE
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMarkdown:
tmpMarkdown.append("\n")
@@ -519,7 +518,7 @@ class Tokenizer(ABC):
elif sLine == "[vspace]":
self._tokens.append(
(self.T_SKIP, nHead, "", [], [], sAlign)
(self.T_SKIP, nHead, "", [], sAlign)
)
continue
@@ -527,11 +526,11 @@ class Tokenizer(ABC):
nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1:
self._tokens.append(
(self.T_SKIP, nHead, "", [], [], sAlign)
(self.T_SKIP, nHead, "", [], sAlign)
)
if nSkip > 1:
self._tokens += (nSkip - 1) * [
(self.T_SKIP, nHead, "", [], [], self.A_NONE)
(self.T_SKIP, nHead, "", [], self.A_NONE)
]
continue
@@ -544,22 +543,26 @@ class Tokenizer(ABC):
if aLine.startswith("%~"):
continue
cStyle, cMod, cText, _, _ = processComment(aLine)
cStyle, cKey, cText, _, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS:
self._tokens.append((
self.T_SYNOPSIS, nHead, cText, [], [], sAlign
self.T_SYNOPSIS, nHead, cText, [], sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
self._tokens.append((
self.T_SHORT, nHead, cText, [], [], sAlign
self.T_SHORT, nHead, cText, [], sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.FOOTNOTE:
if cKey not in self._markers:
self._markers[cKey] = (len(self._markers), [])
self._markers[cKey][1].append(cText)
else:
self._tokens.append((
self.T_COMMENT, nHead, cText, [], [], sAlign
self.T_COMMENT, nHead, cText, [], sAlign
))
if self._doComments and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -573,7 +576,7 @@ class Tokenizer(ABC):
valid, bits, _ = self._project.index.scanThis(aLine)
if valid and bits and bits[0] not in self._skipKeywords:
self._tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), [], [], sAlign
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -609,7 +612,7 @@ class Tokenizer(ABC):
self._noSep = True
self._tokens.append((
tType, nHead, tText, [], [], tStyle
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -644,7 +647,7 @@ class Tokenizer(ABC):
self._noSep = True
self._tokens.append((
tType, nHead, tText, [], [], tStyle
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -685,7 +688,7 @@ class Tokenizer(ABC):
self._noSep = False
self._tokens.append((
tType, nHead, tText, [], [], tStyle
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -715,7 +718,7 @@ class Tokenizer(ABC):
tStyle = self.A_CENTRE
self._tokens.append((
tType, nHead, tText, [], [], tStyle
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -761,9 +764,9 @@ class Tokenizer(ABC):
sAlign |= self.A_IND_R
# Process formats
tLine, fmtPos, insMrk = self._extractFormats(aLine)
tLine, fmtPos = self._extractFormats(aLine)
self._tokens.append((
self.T_TEXT, nHead, tLine, fmtPos, insMrk, sAlign
self.T_TEXT, nHead, tLine, fmtPos, sAlign
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -774,15 +777,15 @@ class Tokenizer(ABC):
# Make sure the token array doesn't start with a page break
# on the very first page, adding a blank first page.
if self._tokens[0][5] & self.A_PBB:
if self._tokens[0][4] & self.A_PBB:
token = self._tokens[0]
self._tokens[0] = (
token[0], token[1], token[2], token[3], token[4], token[5] & ~self.A_PBB
token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
)
# Always add an empty line at the end of the file
self._tokens.append((
self.T_EMPTY, nHead, "", [], [], self.A_NONE
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMarkdown:
tmpMarkdown.append("\n")
@@ -792,8 +795,8 @@ class Tokenizer(ABC):
# ===========
# Some items need a second pass
pToken = (self.T_EMPTY, 0, "", [], [], self.A_NONE)
nToken = (self.T_EMPTY, 0, "", [], [], self.A_NONE)
pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
tCount = len(self._tokens)
for n, token in enumerate(self._tokens):
@@ -803,22 +806,24 @@ class Tokenizer(ABC):
nToken = self._tokens[n+1]
if token[0] == self.T_KEYWORD:
aStyle = token[5]
aStyle = token[4]
if pToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_TOPMRG
if nToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_BTMMRG
self._tokens[n] = (
token[0], token[1], token[2], token[3], token[4], aStyle
token[0], token[1], token[2], token[3], aStyle
)
print(self._markers)
return
def buildOutline(self) -> None:
"""Build an outline of the text up to level 3 headings."""
tHandle = self._handle or ""
isNovel = self._isNovel
for tType, nHead, tText, _, _, _ in self._tokens:
for tType, nHead, tText, _, _ in self._tokens:
if tType == self.T_TITLE:
prefix = "TT"
elif tType == self.T_HEAD1:
@@ -854,7 +859,7 @@ class Tokenizer(ABC):
titleWordChars = self._counts.get("titleWordChars", 0)
para = []
for tType, _, tText, _, _, _ in self._tokens:
for tType, _, tText, _, _ in self._tokens:
tText = tText.replace(nwUnicode.U_ENDASH, " ")
tText = tText.replace(nwUnicode.U_EMDASH, " ")
@@ -974,7 +979,7 @@ class Tokenizer(ABC):
# Internal Functions
##
def _extractFormats(self, text: str) -> tuple[str, T_Formats, T_Markers]:
def _extractFormats(self, text: str) -> tuple[str, T_Formats]:
"""Extract format markers from a text paragraph."""
temp: list[tuple[int, int, int, str]] = []
@@ -1013,18 +1018,13 @@ class Tokenizer(ABC):
# Post-process text and format markers
result = text
formats = []
markers = []
for pos, n, fmt, key in reversed(sorted(temp, key=lambda x: x[0])):
if fmt > 0:
result = result[:pos] + result[pos+n:]
formats = [(p-n, f) for p, f in formats]
markers = [(p-n, f, k) for p, f, k in markers]
if fmt > self.MRK_BOUNDARY:
markers.insert(0, (pos, fmt, key))
else:
formats.insert(0, (pos, fmt))
formats = [(p-n, f, k) for p, f, k in formats]
formats.insert(0, (pos, fmt, key))
return result, formats, markers
return result, formats
# END Class Tokenizer
novelwriter/core/tomd.py
+2 -2
View File
@@ -135,7 +135,7 @@ class ToMarkdown(Tokenizer):
lines = []
lineSep = " \n" if self._preserveBreaks else " "
for tType, _, tText, tFormat, tMarkers, tStyle in self._tokens:
for tType, _, tText, tFormat, tStyle in self._tokens:
if tType == self.T_EMPTY:
if para:
@@ -171,7 +171,7 @@ class ToMarkdown(Tokenizer):
elif tType == self.T_TEXT:
tTemp = tText
for pos, fmt in reversed(tFormat):
for pos, fmt, _ in reversed(tFormat):
tTemp = f"{tTemp[:pos]}{mdTags[fmt]}{tTemp[pos:]}"
para.append(tTemp.rstrip())
novelwriter/core/toodt.py
+5 -5
View File
@@ -39,7 +39,7 @@ from novelwriter import __version__
from novelwriter.common import xmlIndent
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels
from novelwriter.core.project import NWProject
from novelwriter.core.tokenizer import Tokenizer, stripEscape
from novelwriter.core.tokenizer import T_Formats, Tokenizer, stripEscape
logger = logging.getLogger(__name__)
@@ -399,11 +399,11 @@ class ToOdt(Tokenizer):
"""Convert the list of text tokens into XML elements."""
self._result = "" # Not used, but cleared just in case
pFmt = []
pFmt: list[T_Formats] = []
pText = []
pStyle = None
pIndent = True
for tType, _, tText, tFormat, tMarkers, tStyle in self._tokens:
for tType, _, tText, tFormat, tStyle in self._tokens:
# Styles
oStyle = ODTParagraphStyle("New")
@@ -444,11 +444,11 @@ class ToOdt(Tokenizer):
if len(pText) > 0 and pStyle is not None:
tTxt = ""
tFmt = []
tFmt: list[tuple[int, int]] = []
for nText, nFmt in zip(pText, pFmt):
tLen = len(tTxt)
tTxt += f"{nText}\n"
tFmt.extend((p+tLen, fmt) for p, fmt in nFmt)
tFmt.extend((p+tLen, fmt) for p, fmt, _ in nFmt)
# Don't indent a paragraph if it has alignment set
tIndent = self._firstIndent and pIndent and pStyle.isUnaligned()