Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use format list also for marker keys

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-04-15 01:11:32 +02:00
parent 9823677983
commit 7c4d4dec5c
4 changed files with 56 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tohtml.py
+12 -7
View File
@@ -171,7 +171,7 @@ class ToHtml(Tokenizer):
tHandle = self._handle tHandle = self._handle
for tType, nHead, tText, tFormat, tMarkers, tStyle in self._tokens: for tType, nHead, tText, tFormat, tStyle in self._tokens:
# Replace < and > with HTML entities # Replace < and > with HTML entities
if tFormat: if tFormat:
@@ -181,11 +181,11 @@ class ToHtml(Tokenizer):
for c in tText: for c in tText:
if c == "<": if c == "<":
cText.append("&lt;") cText.append("&lt;")
tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat] tFormat = [[p + 3 if p > i else p, f, k] for p, f, k in tFormat]
i += 4 i += 4
elif c == ">": elif c == ">":
cText.append("&gt;") cText.append("&gt;")
tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat] tFormat = [[p + 3 if p > i else p, f, k] for p, f, k in tFormat]
i += 4 i += 4
else: else:
cText.append(c) cText.append(c)
@@ -278,10 +278,15 @@ class ToHtml(Tokenizer):
tTemp = tText tTemp = tText
if pStyle is None: if pStyle is None:
pStyle = hStyle pStyle = hStyle
for pos, fmt in reversed(tFormat): for pos, fmt, key in reversed(tFormat):
tTemp = f"{tTemp[:pos]}{htmlTags[fmt]}{tTemp[pos:]}" if fmt > self.MRK_BOUNDARY:
for pos, fmt, key in reversed(tMarkers): if key in self._markers:
tTemp = f"{tTemp[:pos]}<sup>[x]</sub>{tTemp[pos:]}" index = self._markers[key][0]
if fmt == self.MRK_FOOTNOTE:
ref = f"<sup><a href='#footnote_{index}'>[{index+1}]</a></sup>"
tTemp = f"{tTemp[:pos]}{ref}{tTemp[pos:]}"
else:
tTemp = f"{tTemp[:pos]}{htmlTags[fmt]}{tTemp[pos:]}"
para.append(stripEscape(tTemp.rstrip())) para.append(stripEscape(tTemp.rstrip()))
elif tType == self.T_SYNOPSIS and self._doSynopsis: elif tType == self.T_SYNOPSIS and self._doSynopsis:
novelwriter/core/tokenizer.py
+37 -37
View File
@@ -48,8 +48,7 @@ logger = logging.getLogger(__name__)
ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""} ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL) RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
T_Formats = list[tuple[int, int]] T_Formats = list[tuple[int, int, str]]
T_Markers = list[tuple[int, int, str]]
def stripEscape(text: str) -> str: def stripEscape(text: str) -> str:
@@ -132,7 +131,7 @@ class Tokenizer(ABC):
self._allMarkdown = [] # The result novelWriter markdown of all documents self._allMarkdown = [] # The result novelWriter markdown of all documents
# Processed Tokens and Meta Data # Processed Tokens and Meta Data
self._tokens: list[tuple[int, int, str, T_Formats, T_Markers, int]] = [] self._tokens: list[tuple[int, int, str, T_Formats, int]] = []
self._markers: dict[str, tuple[int, list[str]]] = {} self._markers: dict[str, tuple[int, list[str]]] = {}
self._counts: dict[str, int] = {} self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {} self._outline: dict[str, str] = {}
@@ -426,7 +425,7 @@ class Tokenizer(ABC):
title = f"{trNotes}: {tItem.itemName}" title = f"{trNotes}: {tItem.itemName}"
self._tokens = [] self._tokens = []
self._tokens.append(( self._tokens.append((
self.T_TITLE, 1, title, [], [], textAlign self.T_TITLE, 1, title, [], textAlign
)) ))
if self._keepMarkdown: if self._keepMarkdown:
self._allMarkdown.append(f"#! {title}\n\n") self._allMarkdown.append(f"#! {title}\n\n")
@@ -490,7 +489,7 @@ class Tokenizer(ABC):
# Check for blank lines # Check for blank lines
if len(sLine) == 0: if len(sLine) == 0:
self._tokens.append(( self._tokens.append((
self.T_EMPTY, nHead, "", [], [], self.A_NONE self.T_EMPTY, nHead, "", [], self.A_NONE
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append("\n") tmpMarkdown.append("\n")
@@ -519,7 +518,7 @@ class Tokenizer(ABC):
elif sLine == "[vspace]": elif sLine == "[vspace]":
self._tokens.append( self._tokens.append(
(self.T_SKIP, nHead, "", [], [], sAlign) (self.T_SKIP, nHead, "", [], sAlign)
) )
continue continue
@@ -527,11 +526,11 @@ class Tokenizer(ABC):
nSkip = checkInt(sLine[8:-1], 0) nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1: if nSkip >= 1:
self._tokens.append( self._tokens.append(
(self.T_SKIP, nHead, "", [], [], sAlign) (self.T_SKIP, nHead, "", [], sAlign)
) )
if nSkip > 1: if nSkip > 1:
self._tokens += (nSkip - 1) * [ self._tokens += (nSkip - 1) * [
(self.T_SKIP, nHead, "", [], [], self.A_NONE) (self.T_SKIP, nHead, "", [], self.A_NONE)
] ]
continue continue
@@ -544,22 +543,26 @@ class Tokenizer(ABC):
if aLine.startswith("%~"): if aLine.startswith("%~"):
continue continue
cStyle, cMod, cText, _, _ = processComment(aLine) cStyle, cKey, cText, _, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS: if cStyle == nwComment.SYNOPSIS:
self._tokens.append(( self._tokens.append((
self.T_SYNOPSIS, nHead, cText, [], [], sAlign self.T_SYNOPSIS, nHead, cText, [], sAlign
)) ))
if self._doSynopsis and self._keepMarkdown: if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT: elif cStyle == nwComment.SHORT:
self._tokens.append(( self._tokens.append((
self.T_SHORT, nHead, cText, [], [], sAlign self.T_SHORT, nHead, cText, [], sAlign
)) ))
if self._doSynopsis and self._keepMarkdown: if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.FOOTNOTE:
if cKey not in self._markers:
self._markers[cKey] = (len(self._markers), [])
self._markers[cKey][1].append(cText)
else: else:
self._tokens.append(( self._tokens.append((
self.T_COMMENT, nHead, cText, [], [], sAlign self.T_COMMENT, nHead, cText, [], sAlign
)) ))
if self._doComments and self._keepMarkdown: if self._doComments and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
@@ -573,7 +576,7 @@ class Tokenizer(ABC):
valid, bits, _ = self._project.index.scanThis(aLine) valid, bits, _ = self._project.index.scanThis(aLine)
if valid and bits and bits[0] not in self._skipKeywords: if valid and bits and bits[0] not in self._skipKeywords:
self._tokens.append(( self._tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), [], [], sAlign self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
)) ))
if self._doKeywords and self._keepMarkdown: if self._doKeywords and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
@@ -609,7 +612,7 @@ class Tokenizer(ABC):
self._noSep = True self._noSep = True
self._tokens.append(( self._tokens.append((
tType, nHead, tText, [], [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
@@ -644,7 +647,7 @@ class Tokenizer(ABC):
self._noSep = True self._noSep = True
self._tokens.append(( self._tokens.append((
tType, nHead, tText, [], [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
@@ -685,7 +688,7 @@ class Tokenizer(ABC):
self._noSep = False self._noSep = False
self._tokens.append(( self._tokens.append((
tType, nHead, tText, [], [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
@@ -715,7 +718,7 @@ class Tokenizer(ABC):
tStyle = self.A_CENTRE tStyle = self.A_CENTRE
self._tokens.append(( self._tokens.append((
tType, nHead, tText, [], [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
@@ -761,9 +764,9 @@ class Tokenizer(ABC):
sAlign |= self.A_IND_R sAlign |= self.A_IND_R
# Process formats # Process formats
tLine, fmtPos, insMrk = self._extractFormats(aLine) tLine, fmtPos = self._extractFormats(aLine)
self._tokens.append(( self._tokens.append((
self.T_TEXT, nHead, tLine, fmtPos, insMrk, sAlign self.T_TEXT, nHead, tLine, fmtPos, sAlign
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
@@ -774,15 +777,15 @@ class Tokenizer(ABC):
# Make sure the token array doesn't start with a page break # Make sure the token array doesn't start with a page break
# on the very first page, adding a blank first page. # on the very first page, adding a blank first page.
if self._tokens[0][5] & self.A_PBB: if self._tokens[0][4] & self.A_PBB:
token = self._tokens[0] token = self._tokens[0]
self._tokens[0] = ( self._tokens[0] = (
token[0], token[1], token[2], token[3], token[4], token[5] & ~self.A_PBB token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
) )
# Always add an empty line at the end of the file # Always add an empty line at the end of the file
self._tokens.append(( self._tokens.append((
self.T_EMPTY, nHead, "", [], [], self.A_NONE self.T_EMPTY, nHead, "", [], self.A_NONE
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append("\n") tmpMarkdown.append("\n")
@@ -792,8 +795,8 @@ class Tokenizer(ABC):
# =========== # ===========
# Some items need a second pass # Some items need a second pass
pToken = (self.T_EMPTY, 0, "", [], [], self.A_NONE) pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
nToken = (self.T_EMPTY, 0, "", [], [], self.A_NONE) nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
tCount = len(self._tokens) tCount = len(self._tokens)
for n, token in enumerate(self._tokens): for n, token in enumerate(self._tokens):
@@ -803,22 +806,24 @@ class Tokenizer(ABC):
nToken = self._tokens[n+1] nToken = self._tokens[n+1]
if token[0] == self.T_KEYWORD: if token[0] == self.T_KEYWORD:
aStyle = token[5] aStyle = token[4]
if pToken[0] == self.T_KEYWORD: if pToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_TOPMRG aStyle |= self.A_Z_TOPMRG
if nToken[0] == self.T_KEYWORD: if nToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_BTMMRG aStyle |= self.A_Z_BTMMRG
self._tokens[n] = ( self._tokens[n] = (
token[0], token[1], token[2], token[3], token[4], aStyle token[0], token[1], token[2], token[3], aStyle
) )
print(self._markers)
return return
def buildOutline(self) -> None: def buildOutline(self) -> None:
"""Build an outline of the text up to level 3 headings.""" """Build an outline of the text up to level 3 headings."""
tHandle = self._handle or "" tHandle = self._handle or ""
isNovel = self._isNovel isNovel = self._isNovel
for tType, nHead, tText, _, _, _ in self._tokens: for tType, nHead, tText, _, _ in self._tokens:
if tType == self.T_TITLE: if tType == self.T_TITLE:
prefix = "TT" prefix = "TT"
elif tType == self.T_HEAD1: elif tType == self.T_HEAD1:
@@ -854,7 +859,7 @@ class Tokenizer(ABC):
titleWordChars = self._counts.get("titleWordChars", 0) titleWordChars = self._counts.get("titleWordChars", 0)
para = [] para = []
for tType, _, tText, _, _, _ in self._tokens: for tType, _, tText, _, _ in self._tokens:
tText = tText.replace(nwUnicode.U_ENDASH, " ") tText = tText.replace(nwUnicode.U_ENDASH, " ")
tText = tText.replace(nwUnicode.U_EMDASH, " ") tText = tText.replace(nwUnicode.U_EMDASH, " ")
@@ -974,7 +979,7 @@ class Tokenizer(ABC):
# Internal Functions # Internal Functions
## ##
def _extractFormats(self, text: str) -> tuple[str, T_Formats, T_Markers]: def _extractFormats(self, text: str) -> tuple[str, T_Formats]:
"""Extract format markers from a text paragraph.""" """Extract format markers from a text paragraph."""
temp: list[tuple[int, int, int, str]] = [] temp: list[tuple[int, int, int, str]] = []
@@ -1013,18 +1018,13 @@ class Tokenizer(ABC):
# Post-process text and format markers # Post-process text and format markers
result = text result = text
formats = [] formats = []
markers = []
for pos, n, fmt, key in reversed(sorted(temp, key=lambda x: x[0])): for pos, n, fmt, key in reversed(sorted(temp, key=lambda x: x[0])):
if fmt > 0: if fmt > 0:
result = result[:pos] + result[pos+n:] result = result[:pos] + result[pos+n:]
formats = [(p-n, f) for p, f in formats] formats = [(p-n, f, k) for p, f, k in formats]
markers = [(p-n, f, k) for p, f, k in markers] formats.insert(0, (pos, fmt, key))
if fmt > self.MRK_BOUNDARY:
markers.insert(0, (pos, fmt, key))
else:
formats.insert(0, (pos, fmt))
return result, formats, markers return result, formats
# END Class Tokenizer # END Class Tokenizer
novelwriter/core/tomd.py
+2 -2
View File
@@ -135,7 +135,7 @@ class ToMarkdown(Tokenizer):
lines = [] lines = []
lineSep = " \n" if self._preserveBreaks else " " lineSep = " \n" if self._preserveBreaks else " "
for tType, _, tText, tFormat, tMarkers, tStyle in self._tokens: for tType, _, tText, tFormat, tStyle in self._tokens:
if tType == self.T_EMPTY: if tType == self.T_EMPTY:
if para: if para:
@@ -171,7 +171,7 @@ class ToMarkdown(Tokenizer):
elif tType == self.T_TEXT: elif tType == self.T_TEXT:
tTemp = tText tTemp = tText
for pos, fmt in reversed(tFormat): for pos, fmt, _ in reversed(tFormat):
tTemp = f"{tTemp[:pos]}{mdTags[fmt]}{tTemp[pos:]}" tTemp = f"{tTemp[:pos]}{mdTags[fmt]}{tTemp[pos:]}"
para.append(tTemp.rstrip()) para.append(tTemp.rstrip())
novelwriter/core/toodt.py
+5 -5
View File
@@ -39,7 +39,7 @@ from novelwriter import __version__
from novelwriter.common import xmlIndent from novelwriter.common import xmlIndent
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.core.tokenizer import Tokenizer, stripEscape from novelwriter.core.tokenizer import T_Formats, Tokenizer, stripEscape
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -399,11 +399,11 @@ class ToOdt(Tokenizer):
"""Convert the list of text tokens into XML elements.""" """Convert the list of text tokens into XML elements."""
self._result = "" # Not used, but cleared just in case self._result = "" # Not used, but cleared just in case
pFmt = [] pFmt: list[T_Formats] = []
pText = [] pText = []
pStyle = None pStyle = None
pIndent = True pIndent = True
for tType, _, tText, tFormat, tMarkers, tStyle in self._tokens: for tType, _, tText, tFormat, tStyle in self._tokens:
# Styles # Styles
oStyle = ODTParagraphStyle("New") oStyle = ODTParagraphStyle("New")
@@ -444,11 +444,11 @@ class ToOdt(Tokenizer):
if len(pText) > 0 and pStyle is not None: if len(pText) > 0 and pStyle is not None:
tTxt = "" tTxt = ""
tFmt = [] tFmt: list[tuple[int, int]] = []
for nText, nFmt in zip(pText, pFmt): for nText, nFmt in zip(pText, pFmt):
tLen = len(tTxt) tLen = len(tTxt)
tTxt += f"{nText}\n" tTxt += f"{nText}\n"
tFmt.extend((p+tLen, fmt) for p, fmt in nFmt) tFmt.extend((p+tLen, fmt) for p, fmt, _ in nFmt)
# Don't indent a paragraph if it has alignment set # Don't indent a paragraph if it has alignment set
tIndent = self._firstIndent and pIndent and pStyle.isUnaligned() tIndent = self._firstIndent and pIndent and pStyle.isUnaligned()