Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert Qt regex and use UTF-16 to UCS-4 index map for all highlighting

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2025-07-05 15:53:34 +02:00
parent 8067acdc1c
commit c12a864e8a
2 changed files with 120 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/gui/dochighlight.py
+118 -112
View File
@@ -25,10 +25,11 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
from __future__ import annotations from __future__ import annotations
import logging import logging
import re
from time import time from time import time
from PyQt6.QtCore import QRegularExpression, Qt from PyQt6.QtCore import Qt
from PyQt6.QtGui import ( from PyQt6.QtGui import (
QBrush, QColor, QFont, QSyntaxHighlighter, QTextBlockUserData, QBrush, QColor, QFont, QSyntaxHighlighter, QTextBlockUserData,
QTextCharFormat, QTextDocument QTextCharFormat, QTextDocument
@@ -44,14 +45,11 @@ from novelwriter.types import QtTextUserProperty
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
RX_UNICODE = QRegularExpression.PatternOption.UseUnicodePropertiesOption
RX_URL = REGEX_PATTERNS.url RX_URL = REGEX_PATTERNS.url
RX_WORDS = REGEX_PATTERNS.wordSplit
RX_FMT_SC = REGEX_PATTERNS.shortcodePlain RX_FMT_SC = REGEX_PATTERNS.shortcodePlain
RX_FMT_SV = REGEX_PATTERNS.shortcodeValue RX_FMT_SV = REGEX_PATTERNS.shortcodeValue
RX_WORDS = QRegularExpression(REGEX_PATTERNS.wordSplit.pattern, RX_UNICODE)
BLOCK_NONE = 0 BLOCK_NONE = 0
BLOCK_TEXT = 1 BLOCK_TEXT = 1
BLOCK_META = 2 BLOCK_META = 2
@@ -77,9 +75,9 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._spellErr = QTextCharFormat() self._spellErr = QTextCharFormat()
self._hStyles: dict[str, QTextCharFormat] = {} self._hStyles: dict[str, QTextCharFormat] = {}
self._minRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] self._minRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = []
self._txtRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] self._txtRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = []
self._cmnRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] self._cmnRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = []
self._dialogParser = DialogParser() self._dialogParser = DialogParser()
@@ -143,7 +141,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
# Multiple or Trailing Spaces # Multiple or Trailing Spaces
if CONFIG.showMultiSpaces: if CONFIG.showMultiSpaces:
rxRule = QRegularExpression(r"[ ]{2,}|[ ]*$", RX_UNICODE) rxRule = re.compile(r"[ ]{2,}|[ ]*$", re.UNICODE)
hlRule = { hlRule = {
0: self._hStyles["mspaces"], 0: self._hStyles["mspaces"],
} }
@@ -152,7 +150,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# Non-Breaking Spaces # Non-Breaking Spaces
rxRule = QRegularExpression(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+", RX_UNICODE) rxRule = re.compile(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+", re.UNICODE)
hlRule = { hlRule = {
0: self._hStyles["nobreak"], 0: self._hStyles["nobreak"],
} }
@@ -161,15 +159,14 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# Alt Dialogue # Alt Dialogue
if reRx := REGEX_PATTERNS.altDialogStyle: if rxRule := REGEX_PATTERNS.altDialogStyle:
rxRule = QRegularExpression(reRx.pattern, RX_UNICODE)
hlRule = { hlRule = {
0: self._hStyles["altdialog"], 0: self._hStyles["altdialog"],
} }
self._txtRules.append((rxRule, hlRule)) self._txtRules.append((rxRule, hlRule))
# Markdown Italic # Markdown Italic
rxRule = QRegularExpression(REGEX_PATTERNS.markdownItalic.pattern, RX_UNICODE) rxRule = REGEX_PATTERNS.markdownItalic
hlRule = { hlRule = {
1: self._hStyles["markup"], 1: self._hStyles["markup"],
2: self._hStyles["italic"], 2: self._hStyles["italic"],
@@ -180,7 +177,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# Markdown Bold # Markdown Bold
rxRule = QRegularExpression(REGEX_PATTERNS.markdownBold.pattern, RX_UNICODE) rxRule = REGEX_PATTERNS.markdownBold
hlRule = { hlRule = {
1: self._hStyles["markup"], 1: self._hStyles["markup"],
2: self._hStyles["bold"], 2: self._hStyles["bold"],
@@ -191,7 +188,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# Markdown Strikethrough # Markdown Strikethrough
rxRule = QRegularExpression(REGEX_PATTERNS.markdownStrike.pattern, RX_UNICODE) rxRule = REGEX_PATTERNS.markdownStrike
hlRule = { hlRule = {
1: self._hStyles["markup"], 1: self._hStyles["markup"],
2: self._hStyles["strike"], 2: self._hStyles["strike"],
@@ -213,7 +210,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# Shortcodes # Shortcodes
rxRule = QRegularExpression(REGEX_PATTERNS.shortcodePlain.pattern, RX_UNICODE) rxRule = REGEX_PATTERNS.shortcodePlain
hlRule = { hlRule = {
1: self._hStyles["code"], 1: self._hStyles["code"],
} }
@@ -222,7 +219,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# Shortcodes w/Value # Shortcodes w/Value
rxRule = QRegularExpression(REGEX_PATTERNS.shortcodeValue.pattern, RX_UNICODE) rxRule = REGEX_PATTERNS.shortcodeValue
hlRule = { hlRule = {
1: self._hStyles["code"], 1: self._hStyles["code"],
2: self._hStyles["value"], 2: self._hStyles["value"],
@@ -233,7 +230,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# URLs # URLs
rxRule = QRegularExpression(REGEX_PATTERNS.url.pattern, RX_UNICODE) rxRule = REGEX_PATTERNS.url
hlRule = { hlRule = {
0: self._hStyles["link"], 0: self._hStyles["link"],
} }
@@ -242,7 +239,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._cmnRules.append((rxRule, hlRule)) self._cmnRules.append((rxRule, hlRule))
# Alignment Tags # Alignment Tags
rxRule = QRegularExpression(r"(^>{1,2}|<{1,2}$)", RX_UNICODE) rxRule = re.compile(r"(^>{1,2}|<{1,2}$)", re.UNICODE)
hlRule = { hlRule = {
1: self._hStyles["markup"], 1: self._hStyles["markup"],
} }
@@ -250,7 +247,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._txtRules.append((rxRule, hlRule)) self._txtRules.append((rxRule, hlRule))
# Auto-Replace Tags # Auto-Replace Tags
rxRule = QRegularExpression(r"<(\S+?)>", RX_UNICODE) rxRule = re.compile(r"<(\S+?)>", re.UNICODE)
hlRule = { hlRule = {
0: self._hStyles["replace"], 0: self._hStyles["replace"],
} }
@@ -312,43 +309,36 @@ class GuiDocHighlighter(QSyntaxHighlighter):
if self._tHandle is None or not text: if self._tHandle is None or not text:
return return
bLen = self.currentBlock().length() blockLen = self.currentBlock().length()
isWide = bLen > len(text) + 1 utf16Map = None
if blockLen > len(text) + 1:
# If the lengths are different, the line contains 4 byte
# Unicode characters, and we must use a map between Python
# string indices and the UTF-16 indices used by Qt, where a
# 4 byte character occupies two slots. See #2449.
utf16Map = utf16CharMap(text)
xOff = 0 offset = 0
hRules = None rules = None
if text.startswith("@"): # Keywords and commands if text.startswith("@"): # Keywords and commands
self.setCurrentBlockState(BLOCK_META) self.setCurrentBlockState(BLOCK_META)
index = SHARED.project.index index = SHARED.project.index
isValid, bits, pos = index.scanThis(text) isValid, bits, pos = index.scanThis(text)
isGood = index.checkThese(bits, self._tHandle) isGood = index.checkThese(bits, self._tHandle)
if isValid: if isValid:
posMap = []
if isWide:
posMap = utf16CharMap(text)
for n, bit in enumerate(bits): for n, bit in enumerate(bits):
if posMap: xPos = utf16Map[pos[n]] if utf16Map else pos[n]
xPos = posMap[pos[n]] xLen = utf16Map[pos[n] + len(bit)] - xPos if utf16Map else len(bit)
xLen = posMap[pos[n] + len(bit)] - xPos
else:
xPos = pos[n]
xLen = len(bit)
if n == 0 and isGood[n]: if n == 0 and isGood[n]:
self.setFormat(xPos, xLen, self._hStyles["keyword"]) self.setFormat(xPos, xLen, self._hStyles["keyword"])
elif isGood[n] and not self._isInactive: elif isGood[n] and not self._isInactive:
one, two = index.parseValue(bit) a, b = index.parseValue(bit)
if posMap: aLen = utf16Map[pos[n] + len(a)] - xPos if utf16Map else len(a)
oLen = posMap[pos[n] + len(one)] - xPos self.setFormat(xPos, aLen, self._hStyles["tag"])
else: if b:
oLen = len(one) blockLen = utf16Map[pos[n] + len(b)] - xPos if utf16Map else len(b)
self.setFormat(xPos, oLen, self._hStyles["tag"]) bPos = xPos + xLen - blockLen
if two: self.setFormat(bPos, blockLen, self._hStyles["optional"])
if posMap:
yLen = posMap[pos[n] + len(two)] - xPos
else:
yLen = len(two)
yPos = xPos + xLen - yLen
self.setFormat(yPos, yLen, self._hStyles["optional"])
elif not self._isInactive: elif not self._isInactive:
self.setFormat(xPos, xLen, self._hStyles["invalid"]) self.setFormat(xPos, xLen, self._hStyles["invalid"])
@@ -361,63 +351,66 @@ class GuiDocHighlighter(QSyntaxHighlighter):
if text.startswith("# "): # Heading 1 if text.startswith("# "): # Heading 1
self.setFormat(0, 1, self._hStyles["head1h"]) self.setFormat(0, 1, self._hStyles["head1h"])
self.setFormat(1, bLen, self._hStyles["header1"]) self.setFormat(1, blockLen, self._hStyles["header1"])
elif text.startswith("## "): # Heading 2 elif text.startswith("## "): # Heading 2
self.setFormat(0, 2, self._hStyles["head2h"]) self.setFormat(0, 2, self._hStyles["head2h"])
self.setFormat(2, bLen, self._hStyles["header2"]) self.setFormat(2, blockLen, self._hStyles["header2"])
elif text.startswith("### "): # Heading 3 elif text.startswith("### "): # Heading 3
self.setFormat(0, 3, self._hStyles["head3h"]) self.setFormat(0, 3, self._hStyles["head3h"])
self.setFormat(3, bLen, self._hStyles["header3"]) self.setFormat(3, blockLen, self._hStyles["header3"])
elif text.startswith("#### "): # Heading 4 elif text.startswith("#### "): # Heading 4
self.setFormat(0, 4, self._hStyles["head4h"]) self.setFormat(0, 4, self._hStyles["head4h"])
self.setFormat(4, bLen, self._hStyles["header4"]) self.setFormat(4, blockLen, self._hStyles["header4"])
elif text.startswith("#! "): # Title elif text.startswith("#! "): # Title
self.setFormat(0, 2, self._hStyles["head1h"]) self.setFormat(0, 2, self._hStyles["head1h"])
self.setFormat(2, bLen, self._hStyles["header1"]) self.setFormat(2, blockLen, self._hStyles["header1"])
elif text.startswith("##! "): # Unnumbered elif text.startswith("##! "): # Unnumbered
self.setFormat(0, 3, self._hStyles["head2h"]) self.setFormat(0, 3, self._hStyles["head2h"])
self.setFormat(3, bLen, self._hStyles["header2"]) self.setFormat(3, blockLen, self._hStyles["header2"])
elif text.startswith("###! "): # Alternative Scene elif text.startswith("###! "): # Alternative Scene
self.setFormat(0, 4, self._hStyles["head3h"]) self.setFormat(0, 4, self._hStyles["head3h"])
self.setFormat(4, bLen, self._hStyles["header3"]) self.setFormat(4, blockLen, self._hStyles["header3"])
elif text.startswith("%"): # Comments elif text.startswith("%"): # Comments
self.setCurrentBlockState(BLOCK_TEXT) self.setCurrentBlockState(BLOCK_TEXT)
hRules = self._cmnRules rules = self._cmnRules
cStyle, cMod, _, cDot, cPos = processComment(text) style, mod, _, dot, pos = processComment(text)
cLen = bLen - cPos offset = pos
xOff = cPos if utf16Map:
if cStyle == nwComment.PLAIN: dot = utf16Map[dot]
pos = utf16Map[pos]
cLen = blockLen - pos
if style == nwComment.PLAIN:
self.setFormat(0, cLen, self._hStyles["hidden"]) self.setFormat(0, cLen, self._hStyles["hidden"])
elif cStyle == nwComment.IGNORE: elif style == nwComment.IGNORE:
self.setFormat(0, cLen, self._hStyles["strike"]) self.setFormat(0, cLen, self._hStyles["strike"])
return # No more processing for these return # No more processing for these
elif cMod: elif mod:
self.setFormat(0, cDot, self._hStyles["modifier"]) self.setFormat(0, dot, self._hStyles["modifier"])
self.setFormat(cDot, cPos - cDot, self._hStyles["value"]) self.setFormat(dot, pos - dot, self._hStyles["value"])
self.setFormat(cPos, cLen, self._hStyles["note"]) self.setFormat(pos, cLen, self._hStyles["note"])
else: else:
self.setFormat(0, cPos, self._hStyles["modifier"]) self.setFormat(0, pos, self._hStyles["modifier"])
self.setFormat(cPos, cLen, self._hStyles["note"]) self.setFormat(pos, cLen, self._hStyles["note"])
elif text.startswith("["): # Special Command elif text.startswith("["): # Special Command
self.setCurrentBlockState(BLOCK_TEXT) self.setCurrentBlockState(BLOCK_TEXT)
hRules = self._txtRules if self._isNovel else self._minRules rules = self._txtRules if self._isNovel else self._minRules
sText = text.rstrip().lower() check = text.rstrip().lower()
if sText in ("[newpage]", "[new page]", "[vspace]"): if check in ("[newpage]", "[new page]", "[vspace]"):
self.setFormat(0, bLen, self._hStyles["code"]) self.setFormat(0, blockLen, self._hStyles["code"])
return return
elif sText.startswith("[vspace:") and sText.endswith("]"): elif check.startswith("[vspace:") and check.endswith("]"):
tLen = len(sText) tLen = len(check)
tVal = checkInt(sText[8:-1], 0) tVal = checkInt(check[8:-1], 0)
cVal = "value" if tVal > 0 else "invalid" cVal = "value" if tVal > 0 else "invalid"
self.setFormat(0, 8, self._hStyles["code"]) self.setFormat(0, 8, self._hStyles["code"])
self.setFormat(8, tLen-9, self._hStyles[cVal]) self.setFormat(8, tLen-9, self._hStyles[cVal])
@@ -426,36 +419,62 @@ class GuiDocHighlighter(QSyntaxHighlighter):
else: # Text Paragraph else: # Text Paragraph
self.setCurrentBlockState(BLOCK_TEXT) self.setCurrentBlockState(BLOCK_TEXT)
hRules = self._txtRules if self._isNovel else self._minRules rules = self._txtRules if self._isNovel else self._minRules
if self._isNovel and self._dialogParser.enabled: if self._isNovel and self._dialogParser.enabled:
for pos, end in self._dialogParser(text, isWide): if utf16Map:
length = end - pos for pos, end in self._dialogParser(text):
self.setFormat(pos, length, self._hStyles["dialog"]) pos = utf16Map[pos]
end = utf16Map[end]
self.setFormat(pos, end - pos, self._hStyles["dialog"])
else:
for pos, end in self._dialogParser(text):
self.setFormat(pos, end - pos, self._hStyles["dialog"])
if hRules: if rules:
for rX, hRule in hRules: if utf16Map:
rxItt = rX.globalMatch(text, xOff) for rX, hRule in rules:
while rxItt.hasNext(): for res in re.finditer(rX, text[offset:]):
rxMatch = rxItt.next() for x, hFmt in hRule.items():
for xM, hFmt in hRule.items(): pos = res.start(x) + offset
for x in range(rxMatch.capturedStart(xM), rxMatch.capturedEnd(xM)): end = res.end(x) + offset
cFmt = self.format(x) for x in range(pos, end):
if not cFmt.property(QtTextUserProperty): m = utf16Map[x]
cFmt.merge(hFmt) cFmt = self.format(m)
self.setFormat(x, 1, cFmt) if not cFmt.property(QtTextUserProperty):
cFmt.merge(hFmt)
self.setFormat(m, utf16Map[x+1] - m, cFmt)
else:
for rX, hRule in rules:
for res in re.finditer(rX, text[offset:]):
for x, hFmt in hRule.items():
pos = res.start(x) + offset
end = res.end(x) + offset
for x in range(pos, end):
cFmt = self.format(x)
if not cFmt.property(QtTextUserProperty):
cFmt.merge(hFmt)
self.setFormat(x, 1, cFmt)
data = self.currentBlockUserData() data = self.currentBlockUserData()
if not isinstance(data, TextBlockData): if not isinstance(data, TextBlockData):
data = TextBlockData() data = TextBlockData()
self.setCurrentBlockUserData(data) self.setCurrentBlockUserData(data)
data.processText(text, xOff) data.processText(text, offset)
if self._spellCheck: if self._spellCheck:
for xPos, xEnd in data.spellCheck(): if utf16Map:
for x in range(xPos, xEnd): for pos, end in data.spellCheck():
cFmt = self.format(x) for x in range(pos, end):
cFmt.merge(self._spellErr) m = utf16Map[x]
self.setFormat(x, 1, cFmt) cFmt = self.format(m)
cFmt.merge(self._spellErr)
self.setFormat(m, utf16Map[x+1] - m, cFmt)
else:
for pos, end in data.spellCheck():
for x in range(pos, end):
cFmt = self.format(x)
cFmt.merge(self._spellErr)
self.setFormat(x, 1, cFmt)
return return
@@ -550,22 +569,9 @@ class TextBlockData(QTextBlockUserData):
"""Run the spell checker and cache the result, and return the """Run the spell checker and cache the result, and return the
list of spell check errors. list of spell check errors.
""" """
spell = [] # Spell check replace points spell = SHARED.spelling
utf16 = [] # Mapped for UTF-16 for highlighting (See #2449) self._spellErrors = [
(r.start(0), r.end(0)) for r in RX_WORDS.finditer(self._text, self._offset)
checker = SHARED.spelling if (w := r.group(0)) and not (w.isnumeric() or w.isupper() or spell.checkWord(w))
rxSpell = RX_WORDS.globalMatch(self._text, self._offset) ]
while rxSpell.hasNext(): return self._spellErrors
rxMatch = rxSpell.next()
if (
(word := rxMatch.captured(0))
and not (word.isnumeric() or word.isupper() or checker.checkWord(word))
):
xPos = rxMatch.capturedStart(0)
xEnd = rxMatch.capturedEnd(0)
spell.append((xPos, xPos + len(word)))
utf16.append((xPos, xEnd))
self._spellErrors = spell
return utf16
novelwriter/text/patterns.py
+2 -6
View File
@@ -27,7 +27,7 @@ from __future__ import annotations
import re import re
from novelwriter import CONFIG from novelwriter import CONFIG
from novelwriter.common import compact, uniqueCompact, utf16CharMap from novelwriter.common import compact, uniqueCompact
from novelwriter.constants import nwRegEx, nwUnicode from novelwriter.constants import nwRegEx, nwUnicode
@@ -176,7 +176,7 @@ class DialogParser:
return return
def __call__(self, text: str, wideChar: bool = False) -> list[tuple[int, int]]: def __call__(self, text: str) -> list[tuple[int, int]]:
"""Caller wrapper for dialogue processing.""" """Caller wrapper for dialogue processing."""
temp: list[int] = [] temp: list[int] = []
result: list[tuple[int, int]] = [] result: list[tuple[int, int]] = []
@@ -224,8 +224,4 @@ class DialogParser:
result.append((start, pos)) result.append((start, pos))
start = None start = None
if wideChar:
posMap = utf16CharMap(text)
result = [(posMap[s], posMap[p]) for s, p in result]
return result return result