diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py
index f027ae67..176a6aed 100644
--- a/novelwriter/core/tohtml.py
+++ b/novelwriter/core/tohtml.py
@@ -171,7 +171,7 @@ class ToHtml(Tokenizer):
tHandle = self._handle
- for tType, nHead, tText, tFormat, tMarkers, tStyle in self._tokens:
+ for tType, nHead, tText, tFormat, tStyle in self._tokens:
# Replace < and > with HTML entities
if tFormat:
@@ -181,11 +181,11 @@ class ToHtml(Tokenizer):
for c in tText:
if c == "<":
cText.append("<")
- tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat]
+ tFormat = [[p + 3 if p > i else p, f, k] for p, f, k in tFormat]
i += 4
elif c == ">":
cText.append(">")
- tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat]
+ tFormat = [[p + 3 if p > i else p, f, k] for p, f, k in tFormat]
i += 4
else:
cText.append(c)
@@ -278,10 +278,15 @@ class ToHtml(Tokenizer):
tTemp = tText
if pStyle is None:
pStyle = hStyle
- for pos, fmt in reversed(tFormat):
- tTemp = f"{tTemp[:pos]}{htmlTags[fmt]}{tTemp[pos:]}"
- for pos, fmt, key in reversed(tMarkers):
- tTemp = f"{tTemp[:pos]}[x]{tTemp[pos:]}"
+ for pos, fmt, key in reversed(tFormat):
+ if fmt > self.MRK_BOUNDARY:
+ if key in self._markers:
+ index = self._markers[key][0]
+ if fmt == self.MRK_FOOTNOTE:
+ ref = f"[{index+1}]"
+ tTemp = f"{tTemp[:pos]}{ref}{tTemp[pos:]}"
+ else:
+ tTemp = f"{tTemp[:pos]}{htmlTags[fmt]}{tTemp[pos:]}"
para.append(stripEscape(tTemp.rstrip()))
elif tType == self.T_SYNOPSIS and self._doSynopsis:
diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py
index ec671322..eb72d70e 100644
--- a/novelwriter/core/tokenizer.py
+++ b/novelwriter/core/tokenizer.py
@@ -48,8 +48,7 @@ logger = logging.getLogger(__name__)
ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
-T_Formats = list[tuple[int, int]]
-T_Markers = list[tuple[int, int, str]]
+T_Formats = list[tuple[int, int, str]]
def stripEscape(text: str) -> str:
@@ -132,7 +131,7 @@ class Tokenizer(ABC):
self._allMarkdown = [] # The result novelWriter markdown of all documents
# Processed Tokens and Meta Data
- self._tokens: list[tuple[int, int, str, T_Formats, T_Markers, int]] = []
+ self._tokens: list[tuple[int, int, str, T_Formats, int]] = []
self._markers: dict[str, tuple[int, list[str]]] = {}
self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {}
@@ -426,7 +425,7 @@ class Tokenizer(ABC):
title = f"{trNotes}: {tItem.itemName}"
self._tokens = []
self._tokens.append((
- self.T_TITLE, 1, title, [], [], textAlign
+ self.T_TITLE, 1, title, [], textAlign
))
if self._keepMarkdown:
self._allMarkdown.append(f"#! {title}\n\n")
@@ -490,7 +489,7 @@ class Tokenizer(ABC):
# Check for blank lines
if len(sLine) == 0:
self._tokens.append((
- self.T_EMPTY, nHead, "", [], [], self.A_NONE
+ self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMarkdown:
tmpMarkdown.append("\n")
@@ -519,7 +518,7 @@ class Tokenizer(ABC):
elif sLine == "[vspace]":
self._tokens.append(
- (self.T_SKIP, nHead, "", [], [], sAlign)
+ (self.T_SKIP, nHead, "", [], sAlign)
)
continue
@@ -527,11 +526,11 @@ class Tokenizer(ABC):
nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1:
self._tokens.append(
- (self.T_SKIP, nHead, "", [], [], sAlign)
+ (self.T_SKIP, nHead, "", [], sAlign)
)
if nSkip > 1:
self._tokens += (nSkip - 1) * [
- (self.T_SKIP, nHead, "", [], [], self.A_NONE)
+ (self.T_SKIP, nHead, "", [], self.A_NONE)
]
continue
@@ -544,22 +543,26 @@ class Tokenizer(ABC):
if aLine.startswith("%~"):
continue
- cStyle, cMod, cText, _, _ = processComment(aLine)
+ cStyle, cKey, cText, _, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS:
self._tokens.append((
- self.T_SYNOPSIS, nHead, cText, [], [], sAlign
+ self.T_SYNOPSIS, nHead, cText, [], sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
self._tokens.append((
- self.T_SHORT, nHead, cText, [], [], sAlign
+ self.T_SHORT, nHead, cText, [], sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
+ elif cStyle == nwComment.FOOTNOTE:
+ if cKey not in self._markers:
+ self._markers[cKey] = (len(self._markers), [])
+ self._markers[cKey][1].append(cText)
else:
self._tokens.append((
- self.T_COMMENT, nHead, cText, [], [], sAlign
+ self.T_COMMENT, nHead, cText, [], sAlign
))
if self._doComments and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -573,7 +576,7 @@ class Tokenizer(ABC):
valid, bits, _ = self._project.index.scanThis(aLine)
if valid and bits and bits[0] not in self._skipKeywords:
self._tokens.append((
- self.T_KEYWORD, nHead, aLine[1:].strip(), [], [], sAlign
+ self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -609,7 +612,7 @@ class Tokenizer(ABC):
self._noSep = True
self._tokens.append((
- tType, nHead, tText, [], [], tStyle
+ tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -644,7 +647,7 @@ class Tokenizer(ABC):
self._noSep = True
self._tokens.append((
- tType, nHead, tText, [], [], tStyle
+ tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -685,7 +688,7 @@ class Tokenizer(ABC):
self._noSep = False
self._tokens.append((
- tType, nHead, tText, [], [], tStyle
+ tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -715,7 +718,7 @@ class Tokenizer(ABC):
tStyle = self.A_CENTRE
self._tokens.append((
- tType, nHead, tText, [], [], tStyle
+ tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -761,9 +764,9 @@ class Tokenizer(ABC):
sAlign |= self.A_IND_R
# Process formats
- tLine, fmtPos, insMrk = self._extractFormats(aLine)
+ tLine, fmtPos = self._extractFormats(aLine)
self._tokens.append((
- self.T_TEXT, nHead, tLine, fmtPos, insMrk, sAlign
+ self.T_TEXT, nHead, tLine, fmtPos, sAlign
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
@@ -774,15 +777,15 @@ class Tokenizer(ABC):
# Make sure the token array doesn't start with a page break
# on the very first page, adding a blank first page.
- if self._tokens[0][5] & self.A_PBB:
+ if self._tokens[0][4] & self.A_PBB:
token = self._tokens[0]
self._tokens[0] = (
- token[0], token[1], token[2], token[3], token[4], token[5] & ~self.A_PBB
+ token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
)
# Always add an empty line at the end of the file
self._tokens.append((
- self.T_EMPTY, nHead, "", [], [], self.A_NONE
+ self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMarkdown:
tmpMarkdown.append("\n")
@@ -792,8 +795,8 @@ class Tokenizer(ABC):
# ===========
# Some items need a second pass
- pToken = (self.T_EMPTY, 0, "", [], [], self.A_NONE)
- nToken = (self.T_EMPTY, 0, "", [], [], self.A_NONE)
+ pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
+ nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
tCount = len(self._tokens)
for n, token in enumerate(self._tokens):
@@ -803,22 +806,24 @@ class Tokenizer(ABC):
nToken = self._tokens[n+1]
if token[0] == self.T_KEYWORD:
- aStyle = token[5]
+ aStyle = token[4]
if pToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_TOPMRG
if nToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_BTMMRG
self._tokens[n] = (
- token[0], token[1], token[2], token[3], token[4], aStyle
+ token[0], token[1], token[2], token[3], aStyle
)
+ print(self._markers)
+
return
def buildOutline(self) -> None:
"""Build an outline of the text up to level 3 headings."""
tHandle = self._handle or ""
isNovel = self._isNovel
- for tType, nHead, tText, _, _, _ in self._tokens:
+ for tType, nHead, tText, _, _ in self._tokens:
if tType == self.T_TITLE:
prefix = "TT"
elif tType == self.T_HEAD1:
@@ -854,7 +859,7 @@ class Tokenizer(ABC):
titleWordChars = self._counts.get("titleWordChars", 0)
para = []
- for tType, _, tText, _, _, _ in self._tokens:
+ for tType, _, tText, _, _ in self._tokens:
tText = tText.replace(nwUnicode.U_ENDASH, " ")
tText = tText.replace(nwUnicode.U_EMDASH, " ")
@@ -974,7 +979,7 @@ class Tokenizer(ABC):
# Internal Functions
##
- def _extractFormats(self, text: str) -> tuple[str, T_Formats, T_Markers]:
+ def _extractFormats(self, text: str) -> tuple[str, T_Formats]:
"""Extract format markers from a text paragraph."""
temp: list[tuple[int, int, int, str]] = []
@@ -1013,18 +1018,13 @@ class Tokenizer(ABC):
# Post-process text and format markers
result = text
formats = []
- markers = []
for pos, n, fmt, key in reversed(sorted(temp, key=lambda x: x[0])):
if fmt > 0:
result = result[:pos] + result[pos+n:]
- formats = [(p-n, f) for p, f in formats]
- markers = [(p-n, f, k) for p, f, k in markers]
- if fmt > self.MRK_BOUNDARY:
- markers.insert(0, (pos, fmt, key))
- else:
- formats.insert(0, (pos, fmt))
+ formats = [(p-n, f, k) for p, f, k in formats]
+ formats.insert(0, (pos, fmt, key))
- return result, formats, markers
+ return result, formats
# END Class Tokenizer
diff --git a/novelwriter/core/tomd.py b/novelwriter/core/tomd.py
index 8503af7b..482751c1 100644
--- a/novelwriter/core/tomd.py
+++ b/novelwriter/core/tomd.py
@@ -135,7 +135,7 @@ class ToMarkdown(Tokenizer):
lines = []
lineSep = " \n" if self._preserveBreaks else " "
- for tType, _, tText, tFormat, tMarkers, tStyle in self._tokens:
+ for tType, _, tText, tFormat, tStyle in self._tokens:
if tType == self.T_EMPTY:
if para:
@@ -171,7 +171,7 @@ class ToMarkdown(Tokenizer):
elif tType == self.T_TEXT:
tTemp = tText
- for pos, fmt in reversed(tFormat):
+ for pos, fmt, _ in reversed(tFormat):
tTemp = f"{tTemp[:pos]}{mdTags[fmt]}{tTemp[pos:]}"
para.append(tTemp.rstrip())
diff --git a/novelwriter/core/toodt.py b/novelwriter/core/toodt.py
index 4d58c367..61d26fee 100644
--- a/novelwriter/core/toodt.py
+++ b/novelwriter/core/toodt.py
@@ -39,7 +39,7 @@ from novelwriter import __version__
from novelwriter.common import xmlIndent
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels
from novelwriter.core.project import NWProject
-from novelwriter.core.tokenizer import Tokenizer, stripEscape
+from novelwriter.core.tokenizer import T_Formats, Tokenizer, stripEscape
logger = logging.getLogger(__name__)
@@ -399,11 +399,11 @@ class ToOdt(Tokenizer):
"""Convert the list of text tokens into XML elements."""
self._result = "" # Not used, but cleared just in case
- pFmt = []
+ pFmt: list[T_Formats] = []
pText = []
pStyle = None
pIndent = True
- for tType, _, tText, tFormat, tMarkers, tStyle in self._tokens:
+ for tType, _, tText, tFormat, tStyle in self._tokens:
# Styles
oStyle = ODTParagraphStyle("New")
@@ -444,11 +444,11 @@ class ToOdt(Tokenizer):
if len(pText) > 0 and pStyle is not None:
tTxt = ""
- tFmt = []
+ tFmt: list[tuple[int, int]] = []
for nText, nFmt in zip(pText, pFmt):
tLen = len(tTxt)
tTxt += f"{nText}\n"
- tFmt.extend((p+tLen, fmt) for p, fmt in nFmt)
+ tFmt.extend((p+tLen, fmt) for p, fmt, _ in nFmt)
# Don't indent a paragraph if it has alignment set
tIndent = self._firstIndent and pIndent and pStyle.isUnaligned()