Use regex end instead of length in tokenizer format processing function

This commit is contained in:
Veronica Berglyd Olsen
2024-09-22 17:57:13 +02:00
parent 9b08ebbbd5
commit 17b4a7be4d
+7 -7
View File
@@ -1111,14 +1111,14 @@ class Tokenizer(ABC):
for regEx, fmts in self._rxMarkdown:
for match in re.finditer(regEx, text):
temp.extend(
(match.start(n), len(match.group(n)), fmt, "")
(match.start(n), match.end(n), fmt, "")
for n, fmt in enumerate(fmts) if fmt > 0
)
# Match Shortcodes
for match in re.finditer(REGEX_PATTERNS.shortcodePlain, text):
temp.append((
match.start(1), len(match.group(1)),
match.start(1), match.end(1),
self._shortCodeFmt.get(match.group(1).lower(), 0),
"",
))
@@ -1128,7 +1128,7 @@ class Tokenizer(ABC):
for match in re.finditer(REGEX_PATTERNS.shortcodeValue, text):
kind = self._shortCodeVals.get(match.group(1).lower(), 0)
temp.append((
match.start(0), len(match.group(0)),
match.start(0), match.end(0),
self.FMT_STRIP if kind == skip else kind,
f"{tHandle}:{match.group(2)}",
))
@@ -1143,11 +1143,11 @@ class Tokenizer(ABC):
# Post-process text and format
result = text
formats = []
for pos, n, fmt, key in reversed(sorted(temp, key=lambda x: x[0])):
for pos, end, fmt, key in reversed(sorted(temp, key=lambda x: x[0])):
if fmt > 0:
if n > 0:
result = result[:pos] + result[pos+n:]
formats = [(p-n if p > pos else p, f, k) for p, f, k in formats]
if end > pos:
result = result[:pos] + result[end:]
formats = [(p+pos-end if p > pos else p, f, k) for p, f, k in formats]
formats.insert(0, (pos, fmt, key))
return result, formats