From 9af498d69a6e56aae0daf283287ffb23f79e676d Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Fri, 4 Jul 2025 20:20:48 +0200 Subject: [PATCH 01/11] Drop redundant re.UNICODE flag --- novelwriter/core/coretools.py | 6 ++---- novelwriter/formats/todocx.py | 2 +- novelwriter/gui/dochighlight.py | 8 ++++---- novelwriter/text/patterns.py | 22 +++++++++++----------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/novelwriter/core/coretools.py b/novelwriter/core/coretools.py index a4b2a0b1..85341f0e 100644 --- a/novelwriter/core/coretools.py +++ b/novelwriter/core/coretools.py @@ -289,7 +289,7 @@ class DocSearch: def __init__(self) -> None: self._regEx = re.compile(r"") - self._opts = re.UNICODE | re.IGNORECASE + self._opts = re.IGNORECASE self._words = False self._escape = True return @@ -300,9 +300,7 @@ class DocSearch: def setCaseSensitive(self, state: bool) -> None: """Set the case sensitive search flag.""" - self._opts = re.UNICODE - if not state: - self._opts |= re.IGNORECASE + self._opts = 0 if state else re.IGNORECASE return def setWholeWords(self, state: bool) -> None: diff --git a/novelwriter/formats/todocx.py b/novelwriter/formats/todocx.py index 366191ef..78aaf71d 100644 --- a/novelwriter/formats/todocx.py +++ b/novelwriter/formats/todocx.py @@ -51,7 +51,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) # RegEx -RX_TEXT = re.compile(r"([\n\t])", re.UNICODE) +RX_TEXT = re.compile(r"([\n\t])") # Types and Relationships OOXML_SCM = "http://schemas.openxmlformats.org" diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index 8103fd79..8bdb84de 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -139,7 +139,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): # Multiple or Trailing Spaces if CONFIG.showMultiSpaces: - rxRule = re.compile(r"[ ]{2,}|[ ]*$", re.UNICODE) + rxRule = re.compile(r"[ ]{2,}|[ ]*$") hlRule = { 0: self._hStyles["mspaces"], } @@ -148,7 +148,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Non-Breaking Spaces - rxRule = re.compile(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+", re.UNICODE) + rxRule = re.compile(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+") hlRule = { 0: self._hStyles["nobreak"], } @@ -226,7 +226,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Alignment Tags - rxRule = re.compile(r"(^>{1,2}|<{1,2}$)", re.UNICODE) + rxRule = re.compile(r"(^>{1,2}|<{1,2}$)") hlRule = { 1: self._hStyles["markup"], } @@ -234,7 +234,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._txtRules.append((rxRule, hlRule)) # Auto-Replace Tags - rxRule = re.compile(r"<(\S+?)>", re.UNICODE) + rxRule = re.compile(r"<(\S+?)>") hlRule = { 0: self._hStyles["replace"], } diff --git a/novelwriter/text/patterns.py b/novelwriter/text/patterns.py index a5d2b3e9..e0231d78 100644 --- a/novelwriter/text/patterns.py +++ b/novelwriter/text/patterns.py @@ -37,13 +37,13 @@ class RegExPatterns: # Static RegExes _rxUrl = re.compile(nwRegEx.URL, re.ASCII) - _rxWords = re.compile(nwRegEx.WORDS, re.UNICODE) - _rxBreak = re.compile(nwRegEx.BREAK, re.UNICODE) - _rxItalic = re.compile(nwRegEx.FMT_EI, re.UNICODE) - _rxBold = re.compile(nwRegEx.FMT_EB, re.UNICODE) - _rxStrike = re.compile(nwRegEx.FMT_ST, re.UNICODE) - _rxSCPlain = re.compile(nwRegEx.FMT_SC, re.UNICODE) - _rxSCValue = re.compile(nwRegEx.FMT_SV, re.UNICODE) + _rxWords = re.compile(nwRegEx.WORDS) + _rxBreak = re.compile(nwRegEx.BREAK) + _rxItalic = re.compile(nwRegEx.FMT_EI) + _rxBold = re.compile(nwRegEx.FMT_EB) + _rxStrike = re.compile(nwRegEx.FMT_ST) + _rxSCPlain = re.compile(nwRegEx.FMT_SC) + _rxSCValue = re.compile(nwRegEx.FMT_SV) @property def url(self) -> re.Pattern: @@ -108,7 +108,7 @@ class RegExPatterns: rx.append(f"(?:{qO}[^{qO}]+{qC})") if CONFIG.allowOpenDial: rx.append(f"(?:{qO}.+?$)") - return re.compile("|".join(rx), re.UNICODE) + return re.compile("|".join(rx)) return None @property @@ -118,7 +118,7 @@ class RegExPatterns: qO = re.escape(compact(CONFIG.altDialogOpen)) qC = re.escape(compact(CONFIG.altDialogClose)) qB = r"\B" if (qO == qC or qC in self.AMBIGUOUS) else "" - return re.compile(f"{qO}.*?{qC}{qB}", re.UNICODE) + return re.compile(f"{qO}.*?{qC}{qB}") return None @@ -163,8 +163,8 @@ class DialogParser: # Build narrator break RegExes if narrator := CONFIG.narratorBreak.strip()[:1]: punct = re.escape(".,:;!?") - self._breakD = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?|$)", re.UNICODE) - self._breakQ = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?)", re.UNICODE) + self._breakD = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?|$)") + self._breakQ = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?)") self._narrator = narrator self._mode = f" {narrator}" From 680fc32786198f3083142da5b274cbc54bfc9f2b Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Fri, 4 Jul 2025 23:24:23 +0200 Subject: [PATCH 02/11] Fix highlighting of markup and spell checking for 4 byte Unicode --- novelwriter/gui/dochighlight.py | 91 +++++++++++++++++++-------------- novelwriter/types.py | 2 + 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index 8bdb84de..beb006c1 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -25,11 +25,10 @@ along with this program. If not, see . from __future__ import annotations import logging -import re from time import time -from PyQt6.QtCore import Qt +from PyQt6.QtCore import QRegularExpression, Qt from PyQt6.QtGui import ( QBrush, QColor, QFont, QSyntaxHighlighter, QTextBlockUserData, QTextCharFormat, QTextDocument @@ -41,14 +40,18 @@ from novelwriter.constants import nwStyles, nwUnicode from novelwriter.enum import nwComment from novelwriter.text.comments import processComment from novelwriter.text.patterns import REGEX_PATTERNS, DialogParser +from novelwriter.types import QtTextUserProperty logger = logging.getLogger(__name__) +RX_UNICODE = QRegularExpression.PatternOption.UseUnicodePropertiesOption + RX_URL = REGEX_PATTERNS.url -RX_WORDS = REGEX_PATTERNS.wordSplit RX_FMT_SC = REGEX_PATTERNS.shortcodePlain RX_FMT_SV = REGEX_PATTERNS.shortcodeValue +RX_WORDS = QRegularExpression(REGEX_PATTERNS.wordSplit.pattern, RX_UNICODE) + BLOCK_NONE = 0 BLOCK_TEXT = 1 BLOCK_META = 2 @@ -74,9 +77,9 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._spellErr = QTextCharFormat() self._hStyles: dict[str, QTextCharFormat] = {} - self._minRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = [] - self._txtRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = [] - self._cmnRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = [] + self._minRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] + self._txtRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] + self._cmnRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] self._dialogParser = DialogParser() @@ -139,7 +142,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): # Multiple or Trailing Spaces if CONFIG.showMultiSpaces: - rxRule = re.compile(r"[ ]{2,}|[ ]*$") + rxRule = QRegularExpression(r"[ ]{2,}|[ ]*$", RX_UNICODE) hlRule = { 0: self._hStyles["mspaces"], } @@ -148,7 +151,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Non-Breaking Spaces - rxRule = re.compile(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+") + rxRule = QRegularExpression(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+", RX_UNICODE) hlRule = { 0: self._hStyles["nobreak"], } @@ -157,14 +160,15 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Alt Dialogue - if rxRule := REGEX_PATTERNS.altDialogStyle: + if reRx := REGEX_PATTERNS.altDialogStyle: + rxRule = QRegularExpression(reRx.pattern, RX_UNICODE) hlRule = { 0: self._hStyles["altdialog"], } self._txtRules.append((rxRule, hlRule)) # Markdown Italic - rxRule = REGEX_PATTERNS.markdownItalic + rxRule = QRegularExpression(REGEX_PATTERNS.markdownItalic.pattern, RX_UNICODE) hlRule = { 1: self._hStyles["markup"], 2: self._hStyles["italic"], @@ -175,7 +179,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Markdown Bold - rxRule = REGEX_PATTERNS.markdownBold + rxRule = QRegularExpression(REGEX_PATTERNS.markdownBold.pattern, RX_UNICODE) hlRule = { 1: self._hStyles["markup"], 2: self._hStyles["bold"], @@ -186,7 +190,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Markdown Strikethrough - rxRule = REGEX_PATTERNS.markdownStrike + rxRule = QRegularExpression(REGEX_PATTERNS.markdownStrike.pattern, RX_UNICODE) hlRule = { 1: self._hStyles["markup"], 2: self._hStyles["strike"], @@ -197,7 +201,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Shortcodes - rxRule = REGEX_PATTERNS.shortcodePlain + rxRule = QRegularExpression(REGEX_PATTERNS.shortcodePlain.pattern, RX_UNICODE) hlRule = { 1: self._hStyles["code"], } @@ -206,7 +210,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Shortcodes w/Value - rxRule = REGEX_PATTERNS.shortcodeValue + rxRule = QRegularExpression(REGEX_PATTERNS.shortcodeValue.pattern, RX_UNICODE) hlRule = { 1: self._hStyles["code"], 2: self._hStyles["value"], @@ -217,7 +221,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # URLs - rxRule = REGEX_PATTERNS.url + rxRule = QRegularExpression(REGEX_PATTERNS.url.pattern, RX_UNICODE) hlRule = { 0: self._hStyles["link"], } @@ -226,7 +230,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Alignment Tags - rxRule = re.compile(r"(^>{1,2}|<{1,2}$)") + rxRule = QRegularExpression(r"(^>{1,2}|<{1,2}$)", RX_UNICODE) hlRule = { 1: self._hStyles["markup"], } @@ -234,7 +238,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._txtRules.append((rxRule, hlRule)) # Auto-Replace Tags - rxRule = re.compile(r"<(\S+?)>") + rxRule = QRegularExpression(r"<(\S+?)>", RX_UNICODE) hlRule = { 0: self._hStyles["replace"], } @@ -296,6 +300,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): if self._tHandle is None or not text: return + bLen = self.currentBlock().length() xOff = 0 hRules = None if text.startswith("@"): # Keywords and commands @@ -327,38 +332,38 @@ class GuiDocHighlighter(QSyntaxHighlighter): if text.startswith("# "): # Heading 1 self.setFormat(0, 1, self._hStyles["head1h"]) - self.setFormat(1, len(text), self._hStyles["header1"]) + self.setFormat(1, bLen, self._hStyles["header1"]) elif text.startswith("## "): # Heading 2 self.setFormat(0, 2, self._hStyles["head2h"]) - self.setFormat(2, len(text), self._hStyles["header2"]) + self.setFormat(2, bLen, self._hStyles["header2"]) elif text.startswith("### "): # Heading 3 self.setFormat(0, 3, self._hStyles["head3h"]) - self.setFormat(3, len(text), self._hStyles["header3"]) + self.setFormat(3, bLen, self._hStyles["header3"]) elif text.startswith("#### "): # Heading 4 self.setFormat(0, 4, self._hStyles["head4h"]) - self.setFormat(4, len(text), self._hStyles["header4"]) + self.setFormat(4, bLen, self._hStyles["header4"]) elif text.startswith("#! "): # Title self.setFormat(0, 2, self._hStyles["head1h"]) - self.setFormat(2, len(text), self._hStyles["header1"]) + self.setFormat(2, bLen, self._hStyles["header1"]) elif text.startswith("##! "): # Unnumbered self.setFormat(0, 3, self._hStyles["head2h"]) - self.setFormat(3, len(text), self._hStyles["header2"]) + self.setFormat(3, bLen, self._hStyles["header2"]) elif text.startswith("###! "): # Alternative Scene self.setFormat(0, 4, self._hStyles["head3h"]) - self.setFormat(4, len(text), self._hStyles["header3"]) + self.setFormat(4, bLen, self._hStyles["header3"]) elif text.startswith("%"): # Comments self.setCurrentBlockState(BLOCK_TEXT) hRules = self._cmnRules cStyle, cMod, _, cDot, cPos = processComment(text) - cLen = len(text) - cPos + cLen = bLen - cPos xOff = cPos if cStyle == nwComment.PLAIN: self.setFormat(0, cLen, self._hStyles["hidden"]) @@ -379,7 +384,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): sText = text.rstrip().lower() if sText in ("[newpage]", "[new page]", "[vspace]"): - self.setFormat(0, len(text), self._hStyles["code"]) + self.setFormat(0, bLen, self._hStyles["code"]) return elif sText.startswith("[vspace:") and sText.endswith("]"): tLen = len(sText) @@ -400,13 +405,13 @@ class GuiDocHighlighter(QSyntaxHighlighter): if hRules: for rX, hRule in hRules: - for res in re.finditer(rX, text[xOff:]): + rxItt = rX.globalMatch(text, xOff) + while rxItt.hasNext(): + rxMatch = rxItt.next() for xM, hFmt in hRule.items(): - xPos = res.start(xM) + xOff - xEnd = res.end(xM) + xOff - for x in range(xPos, xEnd): + for x in range(rxMatch.capturedStart(xM), rxMatch.capturedEnd(xM)): cFmt = self.format(x) - if cFmt.fontStyleName() != "markup": + if not cFmt.property(QtTextUserProperty): cFmt.merge(hFmt) self.setFormat(x, 1, cFmt) @@ -435,7 +440,8 @@ class GuiDocHighlighter(QSyntaxHighlighter): ) -> None: """Generate a highlighter character format.""" charFormat = QTextCharFormat() - charFormat.setFontStyleName(name) + blockMerge = name == "markup" + charFormat.setProperty(QtTextUserProperty, blockMerge) if color: charFormat.setForeground(color) @@ -505,7 +511,7 @@ class TextBlockData(QTextBlockUserData): text = f"{text[:s]}{pad}{text[e:]}" self._metaData.append((s, e, res.group(0), "url")) - self._text = text.replace("\u02bc", "'") + self._text = text.replace("\u02bc", "'").replace("_", " ") self._offset = offset return @@ -514,13 +520,22 @@ class TextBlockData(QTextBlockUserData): """Run the spell checker and cache the result, and return the list of spell check errors. """ - self._spellErrors = [] + spell = [] # Spell check replace points + utf16 = [] # Mapped for UTF-16 for highlighting (See #2449) + checker = SHARED.spelling - for res in RX_WORDS.finditer(self._text.replace("_", " "), self._offset): + rxSpell = RX_WORDS.globalMatch(self._text, self._offset) + while rxSpell.hasNext(): + rxMatch = rxSpell.next() if ( - (word := res.group(0)) + (word := rxMatch.captured(0)) and not (word.isnumeric() or word.isupper() or checker.checkWord(word)) ): - self._spellErrors.append((res.start(0), res.end(0))) + xPos = rxMatch.capturedStart(0) + xEnd = rxMatch.capturedEnd(0) + spell.append((xPos, xPos + len(word))) + utf16.append((xPos, xEnd)) - return self._spellErrors + self._spellErrors = spell + + return utf16 diff --git a/novelwriter/types.py b/novelwriter/types.py index fb0faebd..f9db3e7f 100644 --- a/novelwriter/types.py +++ b/novelwriter/types.py @@ -54,6 +54,8 @@ QtPageBreakBefore = QTextFormat.PageBreakFlag.PageBreak_AlwaysBefore QtPageBreakAfter = QTextFormat.PageBreakFlag.PageBreak_AlwaysAfter QtPageBreakAuto = QTextFormat.PageBreakFlag.PageBreak_Auto +QtTextUserProperty = QTextFormat.Property.UserProperty + QtPropLineHeight = 1 # QTextBlockFormat.LineHeightTypes.ProportionalHeight # Qt Painter Types From f8a333c4ea7aff7a8ee62bbbcec26aade071fb6f Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 5 Jul 2025 01:50:03 +0200 Subject: [PATCH 03/11] Make editor auto-replace work with 4 byte Unicode --- novelwriter/gui/doceditor.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index 846b6317..1c900ce7 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -2278,11 +2278,15 @@ class TextAutoReplace: Returns True if anything was changed. """ pos = cursor.positionInBlock() - length = len(text) + apos = cursor.position() + block = cursor.block() + length = block.length() - 1 if length < 1 or pos-1 > length: return False - delete, insert = self._determine(text, pos) + cursor.movePosition(QtMoveLeft, QtKeepAnchor, min(4, pos)) + last = cursor.selectedText() + delete, insert = self._determine(last, pos) if insert == "": return False @@ -2290,8 +2294,8 @@ class TextAutoReplace: if self._doPadBefore and check in self._padBefore: if not (check == ":" and length > 1 and text[0] == "@"): delete = max(delete, 1) - chkPos = pos - delete - 1 - if chkPos >= 0 and text[chkPos].isspace(): + chkPos = len(last) - delete - 1 + if chkPos >= 0 and last[chkPos].isspace(): # Strip existing space before inserting a new (#1061) delete += 1 insert = self._padChar + insert @@ -2302,6 +2306,7 @@ class TextAutoReplace: insert = insert + self._padChar if delete > 0: + cursor.setPosition(apos) cursor.movePosition(QtMoveLeft, QtKeepAnchor, delete) cursor.insertText(insert) return True @@ -2310,10 +2315,10 @@ class TextAutoReplace: def _determine(self, text: str, pos: int) -> tuple[int, str]: """Determine what to replace, if anything.""" - t1 = text[pos-1:pos] - t2 = text[pos-2:pos] - t3 = text[pos-3:pos] - t4 = text[pos-4:pos] + t1 = text[-1:] + t2 = text[-2:] + t3 = text[-3:] + t4 = text[-4:] if t1 == "": # Return early if there is nothing to check return 0, "" From 57bed18ccf259827ed65c8d664aef2229b1deffa Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 5 Jul 2025 01:50:47 +0200 Subject: [PATCH 04/11] Make dialog parser and keyword lines work with 4 byte unicode --- novelwriter/common.py | 15 +++++++++++++++ novelwriter/gui/dochighlight.py | 31 ++++++++++++++++++++++++------- novelwriter/text/patterns.py | 8 ++++++-- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/novelwriter/common.py b/novelwriter/common.py index a4e37f62..83133f69 100644 --- a/novelwriter/common.py +++ b/novelwriter/common.py @@ -493,6 +493,21 @@ def decodeMimeHandles(mimeData: QMimeData) -> list[str]: return mimeData.data(nwConst.MIME_HANDLE).data().decode().split("|") +def utf16CharMap(text: str) -> list[int]: + """Compute mapping from Python string index to QString index. + Python strings are always one character per position in either + ASCII, UCS-2 or UCS-4. QStrings are in UTF-16, so wide characters + use 2 indices, and thus creates an offset. + """ + posMap = list(range(0, len(text) + 1)) + offset = 0 + for i, c in enumerate(text): + if ord(c) > 0xffff: + offset += 1 + posMap[i + 1] = i + 1 + offset + return posMap + + ## # Encoder Functions ## diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index beb006c1..ce0b9c8b 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -35,7 +35,7 @@ from PyQt6.QtGui import ( ) from novelwriter import CONFIG, SHARED -from novelwriter.common import checkInt +from novelwriter.common import checkInt, utf16CharMap from novelwriter.constants import nwStyles, nwUnicode from novelwriter.enum import nwComment from novelwriter.text.comments import processComment @@ -301,6 +301,8 @@ class GuiDocHighlighter(QSyntaxHighlighter): return bLen = self.currentBlock().length() + isWide = bLen > len(text) + 1 + xOff = 0 hRules = None if text.startswith("@"): # Keywords and commands @@ -309,17 +311,32 @@ class GuiDocHighlighter(QSyntaxHighlighter): isValid, bits, pos = index.scanThis(text) isGood = index.checkThese(bits, self._tHandle) if isValid: + posMap = [] + if isWide: + posMap = utf16CharMap(text) for n, bit in enumerate(bits): - xPos = pos[n] - xLen = len(bit) + if posMap: + xPos = posMap[pos[n]] + xLen = posMap[pos[n] + len(bit)] - xPos + else: + xPos = pos[n] + xLen = len(bit) if n == 0 and isGood[n]: self.setFormat(xPos, xLen, self._hStyles["keyword"]) elif isGood[n] and not self._isInactive: one, two = index.parseValue(bit) - self.setFormat(xPos, len(one), self._hStyles["tag"]) + if posMap: + oLen = posMap[pos[n] + len(one)] - xPos + else: + oLen = len(one) + self.setFormat(xPos, oLen, self._hStyles["tag"]) if two: - yPos = xPos + len(bit) - len(two) - self.setFormat(yPos, len(two), self._hStyles["optional"]) + if posMap: + yLen = posMap[pos[n] + len(two)] - xPos + else: + yLen = len(two) + yPos = xPos + xLen - yLen + self.setFormat(yPos, yLen, self._hStyles["optional"]) elif not self._isInactive: self.setFormat(xPos, xLen, self._hStyles["invalid"]) @@ -399,7 +416,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self.setCurrentBlockState(BLOCK_TEXT) hRules = self._txtRules if self._isNovel else self._minRules if self._isNovel and self._dialogParser.enabled: - for pos, end in self._dialogParser(text): + for pos, end in self._dialogParser(text, isWide): length = end - pos self.setFormat(pos, length, self._hStyles["dialog"]) diff --git a/novelwriter/text/patterns.py b/novelwriter/text/patterns.py index e0231d78..2391dcb0 100644 --- a/novelwriter/text/patterns.py +++ b/novelwriter/text/patterns.py @@ -27,7 +27,7 @@ from __future__ import annotations import re from novelwriter import CONFIG -from novelwriter.common import compact, uniqueCompact +from novelwriter.common import compact, uniqueCompact, utf16CharMap from novelwriter.constants import nwRegEx, nwUnicode @@ -170,7 +170,7 @@ class DialogParser: return - def __call__(self, text: str) -> list[tuple[int, int]]: + def __call__(self, text: str, wideChar: bool = False) -> list[tuple[int, int]]: """Caller wrapper for dialogue processing.""" temp: list[int] = [] result: list[tuple[int, int]] = [] @@ -218,4 +218,8 @@ class DialogParser: result.append((start, pos)) start = None + if wideChar: + posMap = utf16CharMap(text) + result = [(posMap[s], posMap[p]) for s, p in result] + return result From c12a864e8ab91336d32baad663b625ddfa96df80 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 5 Jul 2025 15:53:34 +0200 Subject: [PATCH 05/11] Revert Qt regex and use UTF-16 to UCS-4 index map for all highlighting --- novelwriter/gui/dochighlight.py | 230 ++++++++++++++++---------------- novelwriter/text/patterns.py | 8 +- 2 files changed, 120 insertions(+), 118 deletions(-) diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index 3d110c66..e1e172c5 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -25,10 +25,11 @@ along with this program. If not, see . from __future__ import annotations import logging +import re from time import time -from PyQt6.QtCore import QRegularExpression, Qt +from PyQt6.QtCore import Qt from PyQt6.QtGui import ( QBrush, QColor, QFont, QSyntaxHighlighter, QTextBlockUserData, QTextCharFormat, QTextDocument @@ -44,14 +45,11 @@ from novelwriter.types import QtTextUserProperty logger = logging.getLogger(__name__) -RX_UNICODE = QRegularExpression.PatternOption.UseUnicodePropertiesOption - RX_URL = REGEX_PATTERNS.url +RX_WORDS = REGEX_PATTERNS.wordSplit RX_FMT_SC = REGEX_PATTERNS.shortcodePlain RX_FMT_SV = REGEX_PATTERNS.shortcodeValue -RX_WORDS = QRegularExpression(REGEX_PATTERNS.wordSplit.pattern, RX_UNICODE) - BLOCK_NONE = 0 BLOCK_TEXT = 1 BLOCK_META = 2 @@ -77,9 +75,9 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._spellErr = QTextCharFormat() self._hStyles: dict[str, QTextCharFormat] = {} - self._minRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] - self._txtRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] - self._cmnRules: list[tuple[QRegularExpression, dict[int, QTextCharFormat]]] = [] + self._minRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = [] + self._txtRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = [] + self._cmnRules: list[tuple[re.Pattern, dict[int, QTextCharFormat]]] = [] self._dialogParser = DialogParser() @@ -143,7 +141,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): # Multiple or Trailing Spaces if CONFIG.showMultiSpaces: - rxRule = QRegularExpression(r"[ ]{2,}|[ ]*$", RX_UNICODE) + rxRule = re.compile(r"[ ]{2,}|[ ]*$", re.UNICODE) hlRule = { 0: self._hStyles["mspaces"], } @@ -152,7 +150,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Non-Breaking Spaces - rxRule = QRegularExpression(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+", RX_UNICODE) + rxRule = re.compile(f"[{nwUnicode.U_NBSP}{nwUnicode.U_THNBSP}]+", re.UNICODE) hlRule = { 0: self._hStyles["nobreak"], } @@ -161,15 +159,14 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Alt Dialogue - if reRx := REGEX_PATTERNS.altDialogStyle: - rxRule = QRegularExpression(reRx.pattern, RX_UNICODE) + if rxRule := REGEX_PATTERNS.altDialogStyle: hlRule = { 0: self._hStyles["altdialog"], } self._txtRules.append((rxRule, hlRule)) # Markdown Italic - rxRule = QRegularExpression(REGEX_PATTERNS.markdownItalic.pattern, RX_UNICODE) + rxRule = REGEX_PATTERNS.markdownItalic hlRule = { 1: self._hStyles["markup"], 2: self._hStyles["italic"], @@ -180,7 +177,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Markdown Bold - rxRule = QRegularExpression(REGEX_PATTERNS.markdownBold.pattern, RX_UNICODE) + rxRule = REGEX_PATTERNS.markdownBold hlRule = { 1: self._hStyles["markup"], 2: self._hStyles["bold"], @@ -191,7 +188,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Markdown Strikethrough - rxRule = QRegularExpression(REGEX_PATTERNS.markdownStrike.pattern, RX_UNICODE) + rxRule = REGEX_PATTERNS.markdownStrike hlRule = { 1: self._hStyles["markup"], 2: self._hStyles["strike"], @@ -213,7 +210,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Shortcodes - rxRule = QRegularExpression(REGEX_PATTERNS.shortcodePlain.pattern, RX_UNICODE) + rxRule = REGEX_PATTERNS.shortcodePlain hlRule = { 1: self._hStyles["code"], } @@ -222,7 +219,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Shortcodes w/Value - rxRule = QRegularExpression(REGEX_PATTERNS.shortcodeValue.pattern, RX_UNICODE) + rxRule = REGEX_PATTERNS.shortcodeValue hlRule = { 1: self._hStyles["code"], 2: self._hStyles["value"], @@ -233,7 +230,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # URLs - rxRule = QRegularExpression(REGEX_PATTERNS.url.pattern, RX_UNICODE) + rxRule = REGEX_PATTERNS.url hlRule = { 0: self._hStyles["link"], } @@ -242,7 +239,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._cmnRules.append((rxRule, hlRule)) # Alignment Tags - rxRule = QRegularExpression(r"(^>{1,2}|<{1,2}$)", RX_UNICODE) + rxRule = re.compile(r"(^>{1,2}|<{1,2}$)", re.UNICODE) hlRule = { 1: self._hStyles["markup"], } @@ -250,7 +247,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._txtRules.append((rxRule, hlRule)) # Auto-Replace Tags - rxRule = QRegularExpression(r"<(\S+?)>", RX_UNICODE) + rxRule = re.compile(r"<(\S+?)>", re.UNICODE) hlRule = { 0: self._hStyles["replace"], } @@ -312,43 +309,36 @@ class GuiDocHighlighter(QSyntaxHighlighter): if self._tHandle is None or not text: return - bLen = self.currentBlock().length() - isWide = bLen > len(text) + 1 + blockLen = self.currentBlock().length() + utf16Map = None + if blockLen > len(text) + 1: + # If the lengths are different, the line contains 4 byte + # Unicode characters, and we must use a map between Python + # string indices and the UTF-16 indices used by Qt, where a + # 4 byte character occupies two slots. See #2449. + utf16Map = utf16CharMap(text) - xOff = 0 - hRules = None + offset = 0 + rules = None if text.startswith("@"): # Keywords and commands self.setCurrentBlockState(BLOCK_META) index = SHARED.project.index isValid, bits, pos = index.scanThis(text) isGood = index.checkThese(bits, self._tHandle) if isValid: - posMap = [] - if isWide: - posMap = utf16CharMap(text) for n, bit in enumerate(bits): - if posMap: - xPos = posMap[pos[n]] - xLen = posMap[pos[n] + len(bit)] - xPos - else: - xPos = pos[n] - xLen = len(bit) + xPos = utf16Map[pos[n]] if utf16Map else pos[n] + xLen = utf16Map[pos[n] + len(bit)] - xPos if utf16Map else len(bit) if n == 0 and isGood[n]: self.setFormat(xPos, xLen, self._hStyles["keyword"]) elif isGood[n] and not self._isInactive: - one, two = index.parseValue(bit) - if posMap: - oLen = posMap[pos[n] + len(one)] - xPos - else: - oLen = len(one) - self.setFormat(xPos, oLen, self._hStyles["tag"]) - if two: - if posMap: - yLen = posMap[pos[n] + len(two)] - xPos - else: - yLen = len(two) - yPos = xPos + xLen - yLen - self.setFormat(yPos, yLen, self._hStyles["optional"]) + a, b = index.parseValue(bit) + aLen = utf16Map[pos[n] + len(a)] - xPos if utf16Map else len(a) + self.setFormat(xPos, aLen, self._hStyles["tag"]) + if b: + blockLen = utf16Map[pos[n] + len(b)] - xPos if utf16Map else len(b) + bPos = xPos + xLen - blockLen + self.setFormat(bPos, blockLen, self._hStyles["optional"]) elif not self._isInactive: self.setFormat(xPos, xLen, self._hStyles["invalid"]) @@ -361,63 +351,66 @@ class GuiDocHighlighter(QSyntaxHighlighter): if text.startswith("# "): # Heading 1 self.setFormat(0, 1, self._hStyles["head1h"]) - self.setFormat(1, bLen, self._hStyles["header1"]) + self.setFormat(1, blockLen, self._hStyles["header1"]) elif text.startswith("## "): # Heading 2 self.setFormat(0, 2, self._hStyles["head2h"]) - self.setFormat(2, bLen, self._hStyles["header2"]) + self.setFormat(2, blockLen, self._hStyles["header2"]) elif text.startswith("### "): # Heading 3 self.setFormat(0, 3, self._hStyles["head3h"]) - self.setFormat(3, bLen, self._hStyles["header3"]) + self.setFormat(3, blockLen, self._hStyles["header3"]) elif text.startswith("#### "): # Heading 4 self.setFormat(0, 4, self._hStyles["head4h"]) - self.setFormat(4, bLen, self._hStyles["header4"]) + self.setFormat(4, blockLen, self._hStyles["header4"]) elif text.startswith("#! "): # Title self.setFormat(0, 2, self._hStyles["head1h"]) - self.setFormat(2, bLen, self._hStyles["header1"]) + self.setFormat(2, blockLen, self._hStyles["header1"]) elif text.startswith("##! "): # Unnumbered self.setFormat(0, 3, self._hStyles["head2h"]) - self.setFormat(3, bLen, self._hStyles["header2"]) + self.setFormat(3, blockLen, self._hStyles["header2"]) elif text.startswith("###! "): # Alternative Scene self.setFormat(0, 4, self._hStyles["head3h"]) - self.setFormat(4, bLen, self._hStyles["header3"]) + self.setFormat(4, blockLen, self._hStyles["header3"]) elif text.startswith("%"): # Comments self.setCurrentBlockState(BLOCK_TEXT) - hRules = self._cmnRules + rules = self._cmnRules - cStyle, cMod, _, cDot, cPos = processComment(text) - cLen = bLen - cPos - xOff = cPos - if cStyle == nwComment.PLAIN: + style, mod, _, dot, pos = processComment(text) + offset = pos + if utf16Map: + dot = utf16Map[dot] + pos = utf16Map[pos] + cLen = blockLen - pos + if style == nwComment.PLAIN: self.setFormat(0, cLen, self._hStyles["hidden"]) - elif cStyle == nwComment.IGNORE: + elif style == nwComment.IGNORE: self.setFormat(0, cLen, self._hStyles["strike"]) return # No more processing for these - elif cMod: - self.setFormat(0, cDot, self._hStyles["modifier"]) - self.setFormat(cDot, cPos - cDot, self._hStyles["value"]) - self.setFormat(cPos, cLen, self._hStyles["note"]) + elif mod: + self.setFormat(0, dot, self._hStyles["modifier"]) + self.setFormat(dot, pos - dot, self._hStyles["value"]) + self.setFormat(pos, cLen, self._hStyles["note"]) else: - self.setFormat(0, cPos, self._hStyles["modifier"]) - self.setFormat(cPos, cLen, self._hStyles["note"]) + self.setFormat(0, pos, self._hStyles["modifier"]) + self.setFormat(pos, cLen, self._hStyles["note"]) elif text.startswith("["): # Special Command self.setCurrentBlockState(BLOCK_TEXT) - hRules = self._txtRules if self._isNovel else self._minRules + rules = self._txtRules if self._isNovel else self._minRules - sText = text.rstrip().lower() - if sText in ("[newpage]", "[new page]", "[vspace]"): - self.setFormat(0, bLen, self._hStyles["code"]) + check = text.rstrip().lower() + if check in ("[newpage]", "[new page]", "[vspace]"): + self.setFormat(0, blockLen, self._hStyles["code"]) return - elif sText.startswith("[vspace:") and sText.endswith("]"): - tLen = len(sText) - tVal = checkInt(sText[8:-1], 0) + elif check.startswith("[vspace:") and check.endswith("]"): + tLen = len(check) + tVal = checkInt(check[8:-1], 0) cVal = "value" if tVal > 0 else "invalid" self.setFormat(0, 8, self._hStyles["code"]) self.setFormat(8, tLen-9, self._hStyles[cVal]) @@ -426,36 +419,62 @@ class GuiDocHighlighter(QSyntaxHighlighter): else: # Text Paragraph self.setCurrentBlockState(BLOCK_TEXT) - hRules = self._txtRules if self._isNovel else self._minRules + rules = self._txtRules if self._isNovel else self._minRules if self._isNovel and self._dialogParser.enabled: - for pos, end in self._dialogParser(text, isWide): - length = end - pos - self.setFormat(pos, length, self._hStyles["dialog"]) + if utf16Map: + for pos, end in self._dialogParser(text): + pos = utf16Map[pos] + end = utf16Map[end] + self.setFormat(pos, end - pos, self._hStyles["dialog"]) + else: + for pos, end in self._dialogParser(text): + self.setFormat(pos, end - pos, self._hStyles["dialog"]) - if hRules: - for rX, hRule in hRules: - rxItt = rX.globalMatch(text, xOff) - while rxItt.hasNext(): - rxMatch = rxItt.next() - for xM, hFmt in hRule.items(): - for x in range(rxMatch.capturedStart(xM), rxMatch.capturedEnd(xM)): - cFmt = self.format(x) - if not cFmt.property(QtTextUserProperty): - cFmt.merge(hFmt) - self.setFormat(x, 1, cFmt) + if rules: + if utf16Map: + for rX, hRule in rules: + for res in re.finditer(rX, text[offset:]): + for x, hFmt in hRule.items(): + pos = res.start(x) + offset + end = res.end(x) + offset + for x in range(pos, end): + m = utf16Map[x] + cFmt = self.format(m) + if not cFmt.property(QtTextUserProperty): + cFmt.merge(hFmt) + self.setFormat(m, utf16Map[x+1] - m, cFmt) + else: + for rX, hRule in rules: + for res in re.finditer(rX, text[offset:]): + for x, hFmt in hRule.items(): + pos = res.start(x) + offset + end = res.end(x) + offset + for x in range(pos, end): + cFmt = self.format(x) + if not cFmt.property(QtTextUserProperty): + cFmt.merge(hFmt) + self.setFormat(x, 1, cFmt) data = self.currentBlockUserData() if not isinstance(data, TextBlockData): data = TextBlockData() self.setCurrentBlockUserData(data) - data.processText(text, xOff) + data.processText(text, offset) if self._spellCheck: - for xPos, xEnd in data.spellCheck(): - for x in range(xPos, xEnd): - cFmt = self.format(x) - cFmt.merge(self._spellErr) - self.setFormat(x, 1, cFmt) + if utf16Map: + for pos, end in data.spellCheck(): + for x in range(pos, end): + m = utf16Map[x] + cFmt = self.format(m) + cFmt.merge(self._spellErr) + self.setFormat(m, utf16Map[x+1] - m, cFmt) + else: + for pos, end in data.spellCheck(): + for x in range(pos, end): + cFmt = self.format(x) + cFmt.merge(self._spellErr) + self.setFormat(x, 1, cFmt) return @@ -550,22 +569,9 @@ class TextBlockData(QTextBlockUserData): """Run the spell checker and cache the result, and return the list of spell check errors. """ - spell = [] # Spell check replace points - utf16 = [] # Mapped for UTF-16 for highlighting (See #2449) - - checker = SHARED.spelling - rxSpell = RX_WORDS.globalMatch(self._text, self._offset) - while rxSpell.hasNext(): - rxMatch = rxSpell.next() - if ( - (word := rxMatch.captured(0)) - and not (word.isnumeric() or word.isupper() or checker.checkWord(word)) - ): - xPos = rxMatch.capturedStart(0) - xEnd = rxMatch.capturedEnd(0) - spell.append((xPos, xPos + len(word))) - utf16.append((xPos, xEnd)) - - self._spellErrors = spell - - return utf16 + spell = SHARED.spelling + self._spellErrors = [ + (r.start(0), r.end(0)) for r in RX_WORDS.finditer(self._text, self._offset) + if (w := r.group(0)) and not (w.isnumeric() or w.isupper() or spell.checkWord(w)) + ] + return self._spellErrors diff --git a/novelwriter/text/patterns.py b/novelwriter/text/patterns.py index 64e38d1d..8a1ad205 100644 --- a/novelwriter/text/patterns.py +++ b/novelwriter/text/patterns.py @@ -27,7 +27,7 @@ from __future__ import annotations import re from novelwriter import CONFIG -from novelwriter.common import compact, uniqueCompact, utf16CharMap +from novelwriter.common import compact, uniqueCompact from novelwriter.constants import nwRegEx, nwUnicode @@ -176,7 +176,7 @@ class DialogParser: return - def __call__(self, text: str, wideChar: bool = False) -> list[tuple[int, int]]: + def __call__(self, text: str) -> list[tuple[int, int]]: """Caller wrapper for dialogue processing.""" temp: list[int] = [] result: list[tuple[int, int]] = [] @@ -224,8 +224,4 @@ class DialogParser: result.append((start, pos)) start = None - if wideChar: - posMap = utf16CharMap(text) - result = [(posMap[s], posMap[p]) for s, p in result] - return result From 3c8a0ace6096d537b8ac2cb30f1c8b2a8ac45684 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 5 Jul 2025 17:18:04 +0200 Subject: [PATCH 06/11] Add test for utf16CharMap --- novelwriter/common.py | 8 ++++---- tests/test_base/test_base_common.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/novelwriter/common.py b/novelwriter/common.py index 006d651d..e9346a24 100644 --- a/novelwriter/common.py +++ b/novelwriter/common.py @@ -500,13 +500,13 @@ def utf16CharMap(text: str) -> list[int]: ASCII, UCS-2 or UCS-4. QStrings are in UTF-16, so wide characters use 2 indices, and thus creates an offset. """ - posMap = list(range(0, len(text) + 1)) + utf16Map = list(range(0, len(text) + 1)) offset = 0 - for i, c in enumerate(text): + for i, c in enumerate(text, 1): if ord(c) > 0xffff: offset += 1 - posMap[i + 1] = i + 1 + offset - return posMap + utf16Map[i] = i + offset + return utf16Map ## diff --git a/tests/test_base/test_base_common.py b/tests/test_base/test_base_common.py index ef16baaa..76a11bd7 100644 --- a/tests/test_base/test_base_common.py +++ b/tests/test_base/test_base_common.py @@ -38,8 +38,8 @@ from novelwriter.common import ( fuzzyTime, getFileSize, hexToInt, isHandle, isItemClass, isItemLayout, isItemType, isListInstance, isTitleTag, jsonEncode, makeFileNameSafe, minmax, numberToRoman, openExternalPath, processDialogSymbols, - readTextFile, simplified, transferCase, uniqueCompact, xmlElement, - xmlIndent, xmlSubElem, yesNo + readTextFile, simplified, transferCase, uniqueCompact, utf16CharMap, + xmlElement, xmlIndent, xmlSubElem, yesNo ) from novelwriter.enum import nwItemClass @@ -557,6 +557,14 @@ def testBaseCommon_encodeDecodeMimeHandles(monkeypatch): assert decodeMimeHandles(mimeData) == handles +@pytest.mark.base +def testBaseCommon_utf16CharMap(monkeypatch): + """Test the utf16CharMap function.""" + assert utf16CharMap("abc") == [0, 1, 2, 3] + assert utf16CharMap("a\u2014b\u2014c") == [0, 1, 2, 3, 4, 5] + assert utf16CharMap("a\U0001F605b\U0001F605c") == [0, 1, 3, 4, 6, 7] + + @pytest.mark.base def testBaseCommon_jsonEncode(): """Test the jsonEncode function.""" From ad3760762f63bc5b2eeeee48b4ac40b687b88d7a Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 5 Jul 2025 17:19:02 +0200 Subject: [PATCH 07/11] Fix spell checking when 4 byte are in use --- novelwriter/gui/doceditor.py | 8 +-- novelwriter/gui/dochighlight.py | 79 ++++++++++++++-------------- novelwriter/gui/editordocument.py | 16 +++--- tests/test_gui/test_gui_doceditor.py | 7 ++- 4 files changed, 54 insertions(+), 56 deletions(-) diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index 1dfee673..7af87ac9 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -1202,13 +1202,13 @@ class GuiDocEditor(QPlainTextEdit): # Spell Checking if SHARED.project.data.spellCheck: - word, cPos, cLen, suggest = self._qDocument.spellErrorAtPos(pCursor.position()) - if word and cPos >= 0 and cLen > 0: + word, offset, suggest = self._qDocument.spellErrorAtPos(pCursor.position()) + if word and offset >= 0: logger.debug("Word '%s' is misspelled", word) block = pCursor.block() sCursor = self.textCursor() - sCursor.setPosition(block.position() + cPos) - sCursor.movePosition(QtMoveRight, QtKeepAnchor, cLen) + sCursor.setPosition(block.position() + offset) + sCursor.movePosition(QtMoveRight, QtKeepAnchor, len(word)) if suggest: ctxMenu.addSeparator() qtAddAction(ctxMenu, self.tr("Spelling Suggestion(s)")) diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index e1e172c5..24e976fd 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -323,24 +323,23 @@ class GuiDocHighlighter(QSyntaxHighlighter): if text.startswith("@"): # Keywords and commands self.setCurrentBlockState(BLOCK_META) index = SHARED.project.index - isValid, bits, pos = index.scanThis(text) + isValid, bits, loc = index.scanThis(text) isGood = index.checkThese(bits, self._tHandle) if isValid: for n, bit in enumerate(bits): - xPos = utf16Map[pos[n]] if utf16Map else pos[n] - xLen = utf16Map[pos[n] + len(bit)] - xPos if utf16Map else len(bit) + pos = utf16Map[loc[n]] if utf16Map else loc[n] + length = utf16Map[loc[n] + len(bit)] - pos if utf16Map else len(bit) if n == 0 and isGood[n]: - self.setFormat(xPos, xLen, self._hStyles["keyword"]) + self.setFormat(pos, length, self._hStyles["keyword"]) elif isGood[n] and not self._isInactive: a, b = index.parseValue(bit) - aLen = utf16Map[pos[n] + len(a)] - xPos if utf16Map else len(a) - self.setFormat(xPos, aLen, self._hStyles["tag"]) + aLen = utf16Map[loc[n] + len(a)] - pos if utf16Map else len(a) + self.setFormat(pos, aLen, self._hStyles["tag"]) if b: - blockLen = utf16Map[pos[n] + len(b)] - xPos if utf16Map else len(b) - bPos = xPos + xLen - blockLen - self.setFormat(bPos, blockLen, self._hStyles["optional"]) + bLen = utf16Map[loc[n] + len(b)] - pos if utf16Map else len(b) + self.setFormat(pos + length - bLen, bLen, self._hStyles["optional"]) elif not self._isInactive: - self.setFormat(xPos, xLen, self._hStyles["invalid"]) + self.setFormat(pos, length, self._hStyles["invalid"]) # We never want to run the spell checker on keyword/values, # so we force a return here @@ -386,19 +385,19 @@ class GuiDocHighlighter(QSyntaxHighlighter): if utf16Map: dot = utf16Map[dot] pos = utf16Map[pos] - cLen = blockLen - pos + length = blockLen - pos if style == nwComment.PLAIN: - self.setFormat(0, cLen, self._hStyles["hidden"]) + self.setFormat(0, length, self._hStyles["hidden"]) elif style == nwComment.IGNORE: - self.setFormat(0, cLen, self._hStyles["strike"]) + self.setFormat(0, length, self._hStyles["strike"]) return # No more processing for these elif mod: self.setFormat(0, dot, self._hStyles["modifier"]) self.setFormat(dot, pos - dot, self._hStyles["value"]) - self.setFormat(pos, cLen, self._hStyles["note"]) + self.setFormat(pos, length, self._hStyles["note"]) else: self.setFormat(0, pos, self._hStyles["modifier"]) - self.setFormat(pos, cLen, self._hStyles["note"]) + self.setFormat(pos, length, self._hStyles["note"]) elif text.startswith("["): # Special Command self.setCurrentBlockState(BLOCK_TEXT) @@ -409,12 +408,12 @@ class GuiDocHighlighter(QSyntaxHighlighter): self.setFormat(0, blockLen, self._hStyles["code"]) return elif check.startswith("[vspace:") and check.endswith("]"): - tLen = len(check) - tVal = checkInt(check[8:-1], 0) - cVal = "value" if tVal > 0 else "invalid" + length = len(check) + value = checkInt(check[8:-1], 0) + style = "value" if value > 0 else "invalid" self.setFormat(0, 8, self._hStyles["code"]) - self.setFormat(8, tLen-9, self._hStyles[cVal]) - self.setFormat(tLen-1, tLen, self._hStyles["code"]) + self.setFormat(8, length-9, self._hStyles[style]) + self.setFormat(length-1, length, self._hStyles["code"]) return else: # Text Paragraph @@ -462,19 +461,11 @@ class GuiDocHighlighter(QSyntaxHighlighter): data.processText(text, offset) if self._spellCheck: - if utf16Map: - for pos, end in data.spellCheck(): - for x in range(pos, end): - m = utf16Map[x] - cFmt = self.format(m) - cFmt.merge(self._spellErr) - self.setFormat(m, utf16Map[x+1] - m, cFmt) - else: - for pos, end in data.spellCheck(): - for x in range(pos, end): - cFmt = self.format(x) - cFmt.merge(self._spellErr) - self.setFormat(x, 1, cFmt) + for pos, end, _ in data.spellCheck(utf16Map): + for x in range(pos, end): + cFmt = self.format(x) + cFmt.merge(self._spellErr) + self.setFormat(x, 1, cFmt) return @@ -528,7 +519,7 @@ class TextBlockData(QTextBlockUserData): self._text = "" self._offset = 0 self._metaData: list[tuple[int, int, str, str]] = [] - self._spellErrors: list[tuple[int, int]] = [] + self._spellErrors: list[tuple[int, int, str]] = [] return @property @@ -537,7 +528,7 @@ class TextBlockData(QTextBlockUserData): return self._metaData @property - def spellErrors(self) -> list[tuple[int, int]]: + def spellErrors(self) -> list[tuple[int, int, str]]: """Return spell error data from last check.""" return self._spellErrors @@ -565,13 +556,21 @@ class TextBlockData(QTextBlockUserData): return - def spellCheck(self) -> list[tuple[int, int]]: + def spellCheck(self, utf16Map: list[int] | None) -> list[tuple[int, int, str]]: """Run the spell checker and cache the result, and return the list of spell check errors. """ spell = SHARED.spelling - self._spellErrors = [ - (r.start(0), r.end(0)) for r in RX_WORDS.finditer(self._text, self._offset) - if (w := r.group(0)) and not (w.isnumeric() or w.isupper() or spell.checkWord(w)) - ] + if utf16Map: + self._spellErrors = [ + (utf16Map[r.start(0)], utf16Map[r.end(0)], w) + for r in RX_WORDS.finditer(self._text, self._offset) + if (w := r.group(0)) and not (w.isnumeric() or w.isupper() or spell.checkWord(w)) + ] + else: + self._spellErrors = [ + (r.start(0), r.end(0), w) + for r in RX_WORDS.finditer(self._text, self._offset) + if (w := r.group(0)) and not (w.isnumeric() or w.isupper() or spell.checkWord(w)) + ] return self._spellErrors diff --git a/novelwriter/gui/editordocument.py b/novelwriter/gui/editordocument.py index d42ffaf1..04286e9f 100644 --- a/novelwriter/gui/editordocument.py +++ b/novelwriter/gui/editordocument.py @@ -113,7 +113,7 @@ class GuiTextDocument(QTextDocument): return cData, cType return "", "" - def spellErrorAtPos(self, pos: int) -> tuple[str, int, int, list[str]]: + def spellErrorAtPos(self, pos: int) -> tuple[str, int, list[str]]: """Check if there is a misspelled word at a given position in the document, and if so, return it. """ @@ -122,15 +122,11 @@ class GuiTextDocument(QTextDocument): block = cursor.block() data = block.userData() if block.isValid() and isinstance(data, TextBlockData): - text = block.text() - check = pos - block.position() - if check >= 0: - for cPos, cEnd in data.spellErrors: - cLen = cEnd - cPos - if cPos <= check <= cEnd: - word = text[cPos:cEnd] - return word, cPos, cLen, SHARED.spelling.suggestWords(word) - return "", -1, -1, [] + if (check := pos - block.position()) >= 0: + for start, end, word in data.spellErrors: + if start <= check <= end: + return word, start, SHARED.spelling.suggestWords(word) + return "", -1, [] def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Iterable[QTextBlock]: """Iterate over all text blocks of a given type.""" diff --git a/tests/test_gui/test_gui_doceditor.py b/tests/test_gui/test_gui_doceditor.py index 52cd8826..699bbaaa 100644 --- a/tests/test_gui/test_gui_doceditor.py +++ b/tests/test_gui/test_gui_doceditor.py @@ -520,10 +520,10 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText, data = cursor.block().userData() assert cursor.block().text().startswith("Lorem") assert isinstance(data, TextBlockData) - data._spellErrors = [(0, 5)] + data._spellErrors = [(0, 5, "Lorem")] # No known position - assert docEditor._qDocument.spellErrorAtPos(-1) == ("", -1, -1, []) + assert docEditor._qDocument.spellErrorAtPos(-1) == ("", -1, []) # With Suggestion with monkeypatch.context() as mp: @@ -540,6 +540,9 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText, ctxMenu.setObjectName("") ctxMenu.deleteLater() + # Update Entry + data._spellErrors = [(0, 5, "Lorax")] + # Without Suggestion with monkeypatch.context() as mp: mp.setattr(SHARED.spelling, "suggestWords", lambda *a: []) From 58eda950f82f6c8ef56cb8fabe9d785d707da762 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 5 Jul 2025 17:52:19 +0200 Subject: [PATCH 08/11] Add test coverage for spell checker with 4 byte Unicode --- novelwriter/gui/doceditor.py | 5 ----- tests/test_gui/test_gui_doceditor.py | 17 +++++++++-------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index 7af87ac9..99a14ec2 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -2303,8 +2303,6 @@ class TextAutoReplace: cursor.movePosition(QtMoveLeft, QtKeepAnchor, min(4, pos)) last = cursor.selectedText() delete, insert = self._determine(last, pos) - if insert == "": - return False check = insert if self._doPadBefore and check in self._padBefore: @@ -2335,9 +2333,6 @@ class TextAutoReplace: t2 = text[-2:] t3 = text[-3:] t4 = text[-4:] - if t1 == "": - # Return early if there is nothing to check - return 0, "" leading = t2[:1].isspace() if self._replaceDQuote: diff --git a/tests/test_gui/test_gui_doceditor.py b/tests/test_gui/test_gui_doceditor.py index 699bbaaa..6480df93 100644 --- a/tests/test_gui/test_gui_doceditor.py +++ b/tests/test_gui/test_gui_doceditor.py @@ -514,6 +514,7 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText, # Run SpellCheck # ============== SHARED.project.data.setSpellCheck(True) + LORAX = "Lorax\U0001F03A" cursor = docEditor.textCursor() cursor.setPosition(16) @@ -527,21 +528,21 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText, # With Suggestion with monkeypatch.context() as mp: - mp.setattr(SHARED.spelling, "suggestWords", lambda *a: ["Lorax"]) + mp.setattr(SHARED.spelling, "suggestWords", lambda *a: [LORAX]) ctxMenu = getMenuForPos(docEditor, 16) assert ctxMenu is not None actions = [x.text() for x in ctxMenu.actions() if x.text()] assert "Spelling Suggestion(s)" in actions - assert f"{nwUnicode.U_ENDASH} Lorax" in actions + assert f"{nwUnicode.U_ENDASH} {LORAX}" in actions ctxMenu.actions()[7].trigger() QApplication.processEvents() - assert docEditor.getText() == text.replace("Lorem", "Lorax", 1) + assert docEditor.getText() == text.replace("Lorem", LORAX, 1) ctxMenu.setObjectName("") ctxMenu.deleteLater() # Update Entry - data._spellErrors = [(0, 5, "Lorax")] + data._spellErrors = [(0, 7, LORAX)] # Without Suggestion with monkeypatch.context() as mp: @@ -551,7 +552,7 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText, assert ctxMenu is not None actions = [x.text() for x in ctxMenu.actions() if x.text()] assert f"{nwUnicode.U_ENDASH} No Suggestions" in actions - assert docEditor.getText() == text.replace("Lorem", "Lorax", 1) + assert docEditor.getText() == text.replace("Lorem", LORAX, 1) ctxMenu.setObjectName("") ctxMenu.deleteLater() @@ -565,11 +566,11 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText, assert "Ignore Word" in actions assert "Add Word to Dictionary" in actions - assert "Lorax" not in SHARED.spelling._userDict + assert LORAX not in SHARED.spelling._userDict ctxMenu.actions()[7].trigger() # Ignore - assert "Lorax" not in SHARED.spelling._userDict + assert LORAX not in SHARED.spelling._userDict ctxMenu.actions()[8].trigger() # Add - assert "Lorax" in SHARED.spelling._userDict + assert LORAX in SHARED.spelling._userDict ctxMenu.setObjectName("") ctxMenu.deleteLater() From d8d60faa6b82f59592b0c7e97b9f2130a416fde7 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 6 Jul 2025 17:09:21 +0200 Subject: [PATCH 09/11] Add test document to repo for now --- novelwriter/common.py | 2 +- sample/content/f0260d04a8013.nwd | 23 +++++++++++++++++++++++ sample/nwProject.nwx | 18 +++++++++++------- 3 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 sample/content/f0260d04a8013.nwd diff --git a/novelwriter/common.py b/novelwriter/common.py index e9346a24..6bbdda1e 100644 --- a/novelwriter/common.py +++ b/novelwriter/common.py @@ -498,7 +498,7 @@ def utf16CharMap(text: str) -> list[int]: """Compute mapping from Python string index to QString index. Python strings are always one character per position in either ASCII, UCS-2 or UCS-4. QStrings are in UTF-16, so wide characters - use 2 indices, and thus creates an offset. + use 2 indices, and thus create an offset. """ utf16Map = list(range(0, len(text) + 1)) offset = 0 diff --git a/sample/content/f0260d04a8013.nwd b/sample/content/f0260d04a8013.nwd new file mode 100644 index 00000000..8e970ae4 --- /dev/null +++ b/sample/content/f0260d04a8013.nwd @@ -0,0 +1,23 @@ +%%~name: New Scene +%%~path: 6a2d6d5f4f401/f0260d04a8013 +%%~kind: NOVEL/DOCUMENT +%%~hash: d37d33de7c022106f13239a3c310420653871b95 +%%~date: 2025-07-05 14:35:40/2025-07-06 09:55:14 +### Test πŽ€πŽ˜πŽ—πŽš + +@tag: πŸ˜… | Smiley +@char: Jane, πŸ˜…, John +@story: πŸ˜… + +% Test comment πŽ€πŽ˜πŽ—πŽš with stuff and **bold** text. + +%Note.Consistency: πŸ˜… _lol_ … +%Note.πŽ€πŽ˜πŽ—πŽš: Test + +Text **bold πŽ€πŽ˜πŽ—πŽš** and so on and _so forth_. ~~πŸ˜… does it work?~~ + +Yada yada Anata (πŽ€πŽπŽš) – yada yada – yada yada Athirata (πŽ€πŽ˜πŽ—πŽš) – yada yada. β€œTest dialog here.” + +Normal line + +– tttttt \ No newline at end of file diff --git a/sample/nwProject.nwx b/sample/nwProject.nwx index 812795c4..c36e8df6 100644 --- a/sample/nwProject.nwx +++ b/sample/nwProject.nwx @@ -1,6 +1,6 @@ - - + + Sample Project Jane Smith @@ -9,8 +9,8 @@ en_GB None - 636b6aa9b697b - 636b6aa9b697b + f0260d04a8013 + f0260d04a8013 7031beac91f75 7031beac91f75 @@ -20,7 +20,7 @@ D - New + New Notes Started 1st Draft @@ -36,7 +36,7 @@ Main - + Novel @@ -58,13 +58,17 @@ Chapter One - + Making a Scene Another Scene + + + New Scene + Interlude From c40507addbb0db32bf0dd3c79403a6cef48f11f3 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 13 Jul 2025 18:58:52 +0200 Subject: [PATCH 10/11] Fix Unicode bug in syntax highlighter --- novelwriter/gui/dochighlight.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index 24e976fd..114acdb3 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -408,12 +408,11 @@ class GuiDocHighlighter(QSyntaxHighlighter): self.setFormat(0, blockLen, self._hStyles["code"]) return elif check.startswith("[vspace:") and check.endswith("]"): - length = len(check) value = checkInt(check[8:-1], 0) style = "value" if value > 0 else "invalid" self.setFormat(0, 8, self._hStyles["code"]) - self.setFormat(8, length-9, self._hStyles[style]) - self.setFormat(length-1, length, self._hStyles["code"]) + self.setFormat(8, blockLen-10, self._hStyles[style]) + self.setFormat(blockLen-2, blockLen, self._hStyles["code"]) return else: # Text Paragraph From cd474f315226493a1fc9d6172c05890b14421bf7 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 13 Jul 2025 18:59:13 +0200 Subject: [PATCH 11/11] Add full test coverage of syntax highlighter --- sample/content/f0260d04a8013.nwd | 23 - sample/nwProject.nwx | 16 +- tests/test_gui/test_gui_dochighlighter.py | 631 ++++++++++++++++++++++ 3 files changed, 637 insertions(+), 33 deletions(-) delete mode 100644 sample/content/f0260d04a8013.nwd create mode 100644 tests/test_gui/test_gui_dochighlighter.py diff --git a/sample/content/f0260d04a8013.nwd b/sample/content/f0260d04a8013.nwd deleted file mode 100644 index 8e970ae4..00000000 --- a/sample/content/f0260d04a8013.nwd +++ /dev/null @@ -1,23 +0,0 @@ -%%~name: New Scene -%%~path: 6a2d6d5f4f401/f0260d04a8013 -%%~kind: NOVEL/DOCUMENT -%%~hash: d37d33de7c022106f13239a3c310420653871b95 -%%~date: 2025-07-05 14:35:40/2025-07-06 09:55:14 -### Test πŽ€πŽ˜πŽ—πŽš - -@tag: πŸ˜… | Smiley -@char: Jane, πŸ˜…, John -@story: πŸ˜… - -% Test comment πŽ€πŽ˜πŽ—πŽš with stuff and **bold** text. - -%Note.Consistency: πŸ˜… _lol_ … -%Note.πŽ€πŽ˜πŽ—πŽš: Test - -Text **bold πŽ€πŽ˜πŽ—πŽš** and so on and _so forth_. ~~πŸ˜… does it work?~~ - -Yada yada Anata (πŽ€πŽπŽš) – yada yada – yada yada Athirata (πŽ€πŽ˜πŽ—πŽš) – yada yada. β€œTest dialog here.” - -Normal line - -– tttttt \ No newline at end of file diff --git a/sample/nwProject.nwx b/sample/nwProject.nwx index c36e8df6..c70e6a03 100644 --- a/sample/nwProject.nwx +++ b/sample/nwProject.nwx @@ -1,6 +1,6 @@ - - + + Sample Project Jane Smith @@ -9,8 +9,8 @@ en_GB None - f0260d04a8013 - f0260d04a8013 + 636b6aa9b697b + 636b6aa9b697b 7031beac91f75 7031beac91f75 @@ -20,7 +20,7 @@ D - New + New Notes Started 1st Draft @@ -36,7 +36,7 @@ Main - + Novel @@ -65,10 +65,6 @@ Another Scene - - - New Scene - Interlude diff --git a/tests/test_gui/test_gui_dochighlighter.py b/tests/test_gui/test_gui_dochighlighter.py new file mode 100644 index 00000000..87aa6a71 --- /dev/null +++ b/tests/test_gui/test_gui_dochighlighter.py @@ -0,0 +1,631 @@ +""" +novelWriter – GUI Syntax Highlighter Tester +=========================================== + +This file is a part of novelWriter +Copyright (C) 2020 Veronica Berglyd Olsen and novelWriter contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" +from __future__ import annotations + +import pytest + +from PyQt6.QtGui import QTextCharFormat, QTextCursor, QTextDocument + +from novelwriter import CONFIG, SHARED +from novelwriter.core.item import NWItem +from novelwriter.enum import nwItemClass, nwItemLayout, nwItemType, nwTheme +from novelwriter.gui.dochighlight import BLOCK_META, BLOCK_TITLE, GuiDocHighlighter, TextBlockData +from novelwriter.types import QtKeepAnchor + +R_HANDLE = "3456789abcdef" +T_HANDLE = "0123456789abc" + + +@pytest.fixture() +def syntax(nwGUI): + """Create a syntax object for use with testing.""" + CONFIG.lightTheme = "tomorrow" + CONFIG.themeMode = nwTheme.LIGHT + + CONFIG.dialogStyle = 3 + CONFIG.fmtDQuoteOpen = "\u201c" + CONFIG.fmtDQuoteClose = "\u201d" + CONFIG.altDialogOpen = "::" + CONFIG.altDialogClose = "::" + + theme = SHARED.theme + theme.loadTheme(force=True) + assert theme._guiPalette.base().color().getRgb() == (0xff, 0xff, 0xff, 0xff) + assert theme.syntaxTheme.text.getRgb() == (0x4d, 0x4d, 0x4c, 0xff) + + doc = QTextDocument() + syntax = GuiDocHighlighter(doc) + + # Add a Mock Item + tRoot = NWItem(SHARED.project, R_HANDLE) + tRoot.setClass(nwItemClass.NOVEL) + tRoot.setType(nwItemType.ROOT) + + tItem = NWItem(SHARED.project, T_HANDLE) + tItem.setParent(R_HANDLE) + tItem.setLayout(nwItemLayout.NOTE) + tItem.setClass(nwItemClass.NOVEL) + tItem.setType(nwItemType.FILE) + + SHARED.project.tree.add(tRoot) + SHARED.project.tree.add(tItem) + + yield syntax + + +def getFragments( + syntax: GuiDocHighlighter +) -> tuple[list[tuple[int, str]], list[QTextCharFormat]]: + """Extract all syntax highlighter fragments from a document.""" + pieces = [] + formats = [] + doc = syntax.document() + cursor = QTextCursor(doc) + assert doc is not None + for b in range(doc.blockCount()): + block = doc.findBlockByNumber(b) + first = block.position() + syntax.rehighlightBlock(block) + if layout := block.layout(): + for fmt in layout.formats(): + cursor.setPosition(first + fmt.start) + cursor.setPosition(first + fmt.start + fmt.length, QtKeepAnchor) + pieces.append((b, fmt.start, fmt.length, cursor.selectedText())) + formats.append(fmt.format) + return pieces, formats + + +def maxOrd(text: str) -> int: + """Get the max character value of a string.""" + return max(ord(c) for c in text) + + +@pytest.mark.gui +def testGuiDocHighlighter_Basic(syntax): + """Test the basic functionality of the syntax highlighter.""" + # Alternate Spell Check + assert syntax._spellCheck is False + syntax.setSpellCheck(True) + assert syntax._spellCheck is True + + # Check Handle + assert syntax._tHandle is None + syntax.setHandle(T_HANDLE) + assert syntax._tHandle == T_HANDLE + assert syntax._isNovel is False + assert syntax._isInactive is False + + tItem = SHARED.project.tree[T_HANDLE] + assert tItem is not None + tItem.setLayout(nwItemLayout.DOCUMENT) + tItem.setClass(nwItemClass.ARCHIVE) + syntax.setHandle(T_HANDLE) + assert syntax._tHandle == T_HANDLE + assert syntax._isNovel is True + assert syntax._isInactive is True + + +@pytest.mark.gui +def testGuiDocHighlighter_Keywords(syntax): + """Test highlighting of keywords.""" + theme = SHARED.theme + doc = syntax.document() + assert doc is not None + + # Settings + syntax._tHandle = T_HANDLE + + colKey = theme.syntaxTheme.key.getRgb() + colTag = theme.syntaxTheme.tag.getRgb() + colOpt = theme.syntaxTheme.opt.getRgb() + colErr = theme.syntaxTheme.error.getRgb() + + # Ascii + doc.setPlainText( + "@tag: Bob | Robert\n" + "@char: Someone\n" + ) + syntax.rehighlightByType(BLOCK_META) + assert maxOrd(doc.toPlainText()) <= 0x7f + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 4, "@tag"), (0, 6, 3, "Bob"), (0, 12, 6, "Robert"), + (1, 0, 5, "@char"), (1, 7, 7, "Someone"), + ] + assert formats[0].foreground().color().getRgb() == colKey + assert formats[1].foreground().color().getRgb() == colTag + assert formats[2].foreground().color().getRgb() == colOpt + assert formats[3].foreground().color().getRgb() == colKey + assert formats[4].underlineColor().getRgb() == colErr + + # # Unicode <= 0xFFFF + doc.setPlainText( + "@tag: ZoΓ« | ZoΓ« Smith\n" + "@char: ΠžΠ»Π΅ΠΊΡΡ–ΠΉ\n" + ) + syntax.rehighlightByType(BLOCK_META) + assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 4, "@tag"), (0, 6, 3, "ZoΓ«"), (0, 12, 9, "ZoΓ« Smith"), + (1, 0, 5, "@char"), (1, 7, 7, "ΠžΠ»Π΅ΠΊΡΡ–ΠΉ"), + ] + assert formats[0].foreground().color().getRgb() == colKey + assert formats[1].foreground().color().getRgb() == colTag + assert formats[2].foreground().color().getRgb() == colOpt + assert formats[3].foreground().color().getRgb() == colKey + assert formats[4].underlineColor().getRgb() == colErr + + # # Unicode > 0xFFFF + doc.setPlainText( + "@tag: πŸ˜„ | Smiley πŸ˜„\n" + "@char: 😎😎😎\n" + ) + syntax.rehighlightByType(BLOCK_META) + assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 4, "@tag"), (0, 6, 2, "πŸ˜„"), (0, 11, 9, "Smiley πŸ˜„"), + (1, 0, 5, "@char"), (1, 7, 6, "😎😎😎"), + ] + assert formats[0].foreground().color().getRgb() == colKey + assert formats[1].foreground().color().getRgb() == colTag + assert formats[2].foreground().color().getRgb() == colOpt + assert formats[3].foreground().color().getRgb() == colKey + assert formats[4].underlineColor().getRgb() == colErr + + +@pytest.mark.gui +def testGuiDocHighlighter_Titles(syntax): + """Test highlighting of titles.""" + theme = SHARED.theme + doc = syntax.document() + assert doc is not None + + # Settings + syntax._tHandle = T_HANDLE + + colHeadMark = theme.syntaxTheme.headH.getRgb() + colHeadText = theme.syntaxTheme.head.getRgb() + + # Ascii + doc.setPlainText( + "# Heading 1\n\n" + "## Heading 2\n\n" + "### Heading 3\n\n" + "#### Heading 4\n\n" + "#! Heading A1\n\n" + "##! Heading A2\n\n" + "###! Heading A3\n\n" + ) + syntax.rehighlightByType(BLOCK_TITLE) + assert maxOrd(doc.toPlainText()) <= 0x7f + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 1, "#"), (0, 1, 10, " Heading 1"), + (2, 0, 2, "##"), (2, 2, 10, " Heading 2"), + (4, 0, 3, "###"), (4, 3, 10, " Heading 3"), + (6, 0, 4, "####"), (6, 4, 10, " Heading 4"), + (8, 0, 2, "#!"), (8, 2, 11, " Heading A1"), + (10, 0, 3, "##!"), (10, 3, 11, " Heading A2"), + (12, 0, 4, "###!"), (12, 4, 11, " Heading A3"), + ] + for i in range(0, len(formats), 2): + assert formats[i].foreground().color().getRgb() == colHeadMark + assert formats[i+1].foreground().color().getRgb() == colHeadText + + # Unicode <= 0xFFFF + doc.setPlainText( + "# ΘžΗ£Δ‘ 1\n\n" + "## ΘžΗ£Δ‘ 2\n\n" + "### ΘžΗ£Δ‘ 3\n\n" + "#### ΘžΗ£Δ‘ 4\n\n" + "#! ΘžΗ£Δ‘ A1\n\n" + "##! ΘžΗ£Δ‘ A2\n\n" + "###! ΘžΗ£Δ‘ A3\n\n" + ) + syntax.rehighlightByType(BLOCK_TITLE) + assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 1, "#"), (0, 1, 6, " ΘžΗ£Δ‘ 1"), + (2, 0, 2, "##"), (2, 2, 6, " ΘžΗ£Δ‘ 2"), + (4, 0, 3, "###"), (4, 3, 6, " ΘžΗ£Δ‘ 3"), + (6, 0, 4, "####"), (6, 4, 6, " ΘžΗ£Δ‘ 4"), + (8, 0, 2, "#!"), (8, 2, 7, " ΘžΗ£Δ‘ A1"), + (10, 0, 3, "##!"), (10, 3, 7, " ΘžΗ£Δ‘ A2"), + (12, 0, 4, "###!"), (12, 4, 7, " ΘžΗ£Δ‘ A3"), + ] + for i in range(0, len(formats), 2): + assert formats[i].foreground().color().getRgb() == colHeadMark + assert formats[i+1].foreground().color().getRgb() == colHeadText + + # Unicode > 0xFFFF + doc.setPlainText( + "# πŸ˜‡πŸ˜Ž 1\n\n" + "## πŸ˜‡πŸ˜Ž 2\n\n" + "### πŸ˜‡πŸ˜Ž 3\n\n" + "#### πŸ˜‡πŸ˜Ž 4\n\n" + "#! πŸ˜‡πŸ˜Ž A1\n\n" + "##! πŸ˜‡πŸ˜Ž A2\n\n" + "###! πŸ˜‡πŸ˜Ž A3\n\n" + ) + syntax.rehighlightByType(BLOCK_TITLE) + assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 1, "#"), (0, 1, 7, " πŸ˜‡πŸ˜Ž 1"), + (2, 0, 2, "##"), (2, 2, 7, " πŸ˜‡πŸ˜Ž 2"), + (4, 0, 3, "###"), (4, 3, 7, " πŸ˜‡πŸ˜Ž 3"), + (6, 0, 4, "####"), (6, 4, 7, " πŸ˜‡πŸ˜Ž 4"), + (8, 0, 2, "#!"), (8, 2, 8, " πŸ˜‡πŸ˜Ž A1"), + (10, 0, 3, "##!"), (10, 3, 8, " πŸ˜‡πŸ˜Ž A2"), + (12, 0, 4, "###!"), (12, 4, 8, " πŸ˜‡πŸ˜Ž A3"), + ] + for i in range(0, len(formats), 2): + assert formats[i].foreground().color().getRgb() == colHeadMark + assert formats[i+1].foreground().color().getRgb() == colHeadText + + +@pytest.mark.gui +def testGuiDocHighlighter_Comments(syntax): + """Test highlighting of comments.""" + theme = SHARED.theme + doc = syntax.document() + assert doc is not None + + # Settings + syntax._tHandle = T_HANDLE + + colHidden = theme.syntaxTheme.hidden.getRgb() + colMod = theme.syntaxTheme.mod.getRgb() + colValue = theme.syntaxTheme.val.getRgb() + colNote = theme.syntaxTheme.note.getRgb() + + # Ascii + doc.setPlainText( + "% Plain\n" + "%~ Ignored\n" + "%Synopsis: Synopsis\n" + "%Note.Stuff: Note\n" + ) + syntax.rehighlight() + assert maxOrd(doc.toPlainText()) <= 0x7f + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 7, "% Plain"), + (1, 0, 10, "%~ Ignored"), + (2, 0, 10, "%Synopsis:"), (2, 10, 9, " Synopsis"), + (3, 0, 6, "%Note."), (3, 6, 6, "Stuff:"), (3, 12, 5, " Note"), + ] + assert formats[0].foreground().color().getRgb() == colHidden + assert formats[1].foreground().color().getRgb() == colHidden + assert formats[1].fontStrikeOut() is True + assert formats[2].foreground().color().getRgb() == colMod + assert formats[3].foreground().color().getRgb() == colNote + assert formats[4].foreground().color().getRgb() == colMod + assert formats[5].foreground().color().getRgb() == colValue + assert formats[6].foreground().color().getRgb() == colNote + + # Unicode <= 0xFFFF + doc.setPlainText( + "% Π Ρ–Π²Π½ΠΈΠ½Π°\n" + "%~ Π†Π³Π½ΠΎΡ€ΠΎΠ²Π°Π½ΠΎ\n" + "%Synopsis: Бинопсис\n" + "%Note.Stuff: ΠŸΡ€ΠΈΠΌΡ–Ρ‚ΠΊΠ°\n" + ) + syntax.rehighlight() + assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 9, "% Π Ρ–Π²Π½ΠΈΠ½Π°"), + (1, 0, 13, "%~ Π†Π³Π½ΠΎΡ€ΠΎΠ²Π°Π½ΠΎ"), + (2, 0, 10, "%Synopsis:"), (2, 10, 9, " Бинопсис"), + (3, 0, 6, "%Note."), (3, 6, 6, "Stuff:"), (3, 12, 9, " ΠŸΡ€ΠΈΠΌΡ–Ρ‚ΠΊΠ°"), + ] + assert formats[0].foreground().color().getRgb() == colHidden + assert formats[1].foreground().color().getRgb() == colHidden + assert formats[1].fontStrikeOut() is True + assert formats[2].foreground().color().getRgb() == colMod + assert formats[3].foreground().color().getRgb() == colNote + assert formats[4].foreground().color().getRgb() == colMod + assert formats[5].foreground().color().getRgb() == colValue + assert formats[6].foreground().color().getRgb() == colNote + + # Unicode > 0xFFFF + doc.setPlainText( + "% 😎😎\n" + "%~ πŸ™ˆπŸ™ˆπŸ™ˆ\n" + "%Synopsis: 😍😍😍😍\n" + "%Note.Stuff: 😑😑😑😑😑\n" + ) + syntax.rehighlight() + assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 6, "% 😎😎"), + (1, 0, 9, "%~ πŸ™ˆπŸ™ˆπŸ™ˆ"), + (2, 0, 10, "%Synopsis:"), (2, 10, 9, " 😍😍😍😍"), + (3, 0, 6, "%Note."), (3, 6, 6, "Stuff:"), (3, 12, 11, " 😑😑😑😑😑"), + ] + assert formats[0].foreground().color().getRgb() == colHidden + assert formats[1].foreground().color().getRgb() == colHidden + assert formats[1].fontStrikeOut() is True + assert formats[2].foreground().color().getRgb() == colMod + assert formats[3].foreground().color().getRgb() == colNote + assert formats[4].foreground().color().getRgb() == colMod + assert formats[5].foreground().color().getRgb() == colValue + assert formats[6].foreground().color().getRgb() == colNote + + +@pytest.mark.gui +def testGuiDocHighlighter_Special(syntax): + """Test highlighting of special commands.""" + theme = SHARED.theme + doc = syntax.document() + assert doc is not None + + # Settings + syntax._tHandle = T_HANDLE + + colErr = theme.syntaxTheme.error.getRgb() + colCode = theme.syntaxTheme.code.getRgb() + colValue = theme.syntaxTheme.val.getRgb() + + # Ascii + doc.setPlainText( + "[NewPage]\n" + "[New Page]\n" + "[VSpace]\n" + "[VSpace:123]\n" + "[VSpace:Meh]\n" + ) + syntax.rehighlight() + assert maxOrd(doc.toPlainText()) <= 0x7f + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 9, "[NewPage]"), + (1, 0, 10, "[New Page]"), + (2, 0, 8, "[VSpace]"), + (3, 0, 8, "[VSpace:"), (3, 8, 3, "123"), (3, 11, 1, "]"), + (4, 0, 8, "[VSpace:"), (4, 8, 3, "Meh"), (4, 11, 1, "]"), + ] + assert formats[0].foreground().color().getRgb() == colCode + assert formats[1].foreground().color().getRgb() == colCode + assert formats[2].foreground().color().getRgb() == colCode + assert formats[3].foreground().color().getRgb() == colCode + assert formats[4].foreground().color().getRgb() == colValue + assert formats[5].foreground().color().getRgb() == colCode + assert formats[6].foreground().color().getRgb() == colCode + assert formats[7].underlineColor().getRgb() == colErr + assert formats[8].foreground().color().getRgb() == colCode + + # Unicode <= 0xFFFF + doc.setPlainText( + "[NewPage]\n" + "[New Page]\n" + "[VSpace]\n" + "[VSpace:123]\n" + "[VSpace:β…˜]\n" + ) + syntax.rehighlight() + assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 9, "[NewPage]"), + (1, 0, 10, "[New Page]"), + (2, 0, 8, "[VSpace]"), + (3, 0, 8, "[VSpace:"), (3, 8, 3, "123"), (3, 11, 1, "]"), + (4, 0, 8, "[VSpace:"), (4, 8, 1, "β…˜"), (4, 9, 1, "]"), + ] + assert formats[0].foreground().color().getRgb() == colCode + assert formats[1].foreground().color().getRgb() == colCode + assert formats[2].foreground().color().getRgb() == colCode + assert formats[3].foreground().color().getRgb() == colCode + assert formats[4].foreground().color().getRgb() == colValue + assert formats[5].foreground().color().getRgb() == colCode + assert formats[6].foreground().color().getRgb() == colCode + assert formats[7].underlineColor().getRgb() == colErr + assert formats[8].foreground().color().getRgb() == colCode + + # Unicode > 0xFFFF + doc.setPlainText( + "[NewPage]\n" + "[New Page]\n" + "[VSpace]\n" + "[VSpace:123]\n" + "[VSpace:πŸ™ˆπŸ™ˆ]\n" + ) + syntax.rehighlight() + assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 9, "[NewPage]"), + (1, 0, 10, "[New Page]"), + (2, 0, 8, "[VSpace]"), + (3, 0, 8, "[VSpace:"), (3, 8, 3, "123"), (3, 11, 1, "]"), + (4, 0, 8, "[VSpace:"), (4, 8, 4, "πŸ™ˆπŸ™ˆ"), (4, 12, 1, "]"), + ] + assert formats[0].foreground().color().getRgb() == colCode + assert formats[1].foreground().color().getRgb() == colCode + assert formats[2].foreground().color().getRgb() == colCode + assert formats[3].foreground().color().getRgb() == colCode + assert formats[4].foreground().color().getRgb() == colValue + assert formats[5].foreground().color().getRgb() == colCode + assert formats[6].foreground().color().getRgb() == colCode + assert formats[7].underlineColor().getRgb() == colErr + assert formats[8].foreground().color().getRgb() == colCode + + +@pytest.mark.gui +def testGuiDocHighlighter_Text(monkeypatch, syntax): + """Test highlighting of text.""" + theme = SHARED.theme + doc = syntax.document() + assert doc is not None + + # Settings + syntax._tHandle = T_HANDLE + syntax._isNovel = True + syntax.setSpellCheck(True) + monkeypatch.setattr(SHARED.spelling, "checkWord", lambda *a: False) + + colHidden = theme.syntaxTheme.hidden.getRgb() + colEmph = theme.syntaxTheme.emph.getRgb() + colLink = theme.syntaxTheme.link.getRgb() + colSpell = theme.syntaxTheme.spell.getRgb() + colCode = theme.syntaxTheme.code.getRgb() + colDialogue = theme.syntaxTheme.dialN.getRgb() + colAltDialogue = theme.syntaxTheme.dialA.getRgb() + + # Ascii + doc.setPlainText( + "Text **bold** text _italic_ text ~~strike~~ text [b]bold[/b], http://example.com\n\n" + ) + syntax.rehighlight() + assert maxOrd(doc.toPlainText()) <= 0x7f + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 4, "Text"), + (0, 5, 2, "**"), (0, 7, 4, "bold"), (0, 11, 2, "**"), + (0, 14, 4, "text"), + (0, 19, 1, "_"), (0, 20, 6, "italic"), (0, 26, 1, "_"), + (0, 28, 4, "text"), + (0, 33, 2, "~~"), (0, 35, 6, "strike"), (0, 41, 2, "~~"), + (0, 44, 4, "text"), + (0, 49, 3, "[b]"), (0, 52, 4, "bold"), (0, 56, 4, "[/b]"), + (0, 62, 18, "http://example.com"), + ] + assert formats[0].underlineColor().getRgb() == colSpell # Text + assert formats[1].foreground().color().getRgb() == colHidden # ** + assert formats[2].foreground().color().getRgb() == colEmph # bold + assert formats[2].underlineColor().getRgb() == colSpell + assert formats[3].foreground().color().getRgb() == colHidden # ** + assert formats[4].underlineColor().getRgb() == colSpell # text + assert formats[5].foreground().color().getRgb() == colHidden # _ + assert formats[6].foreground().color().getRgb() == colEmph # italic + assert formats[6].underlineColor().getRgb() == colSpell + assert formats[7].foreground().color().getRgb() == colHidden # _ + assert formats[8].underlineColor().getRgb() == colSpell # text + assert formats[9].foreground().color().getRgb() == colHidden # ~~ + assert formats[10].foreground().color().getRgb() == colHidden # strike + assert formats[10].fontStrikeOut() is True + assert formats[10].underlineColor().getRgb() == colSpell + assert formats[11].foreground().color().getRgb() == colHidden # ~~ + assert formats[12].underlineColor().getRgb() == colSpell # text + assert formats[13].foreground().color().getRgb() == colCode # [b] + assert formats[14].underlineColor().getRgb() == colSpell # bold + assert formats[15].foreground().color().getRgb() == colCode # [/b] + assert formats[16].foreground().color().getRgb() == colLink # http://example.com + + # Spell Check + data = doc.findBlockByNumber(0).userData() + assert isinstance(data, TextBlockData) + assert data.metaData == [(62, 80, "http://example.com", "url")] + assert data.spellErrors == [ + (0, 4, "Text"), (7, 11, "bold"), + (14, 18, "text"), (20, 26, "italic"), + (28, 32, "text"), (35, 41, "strike"), + (44, 48, "text"), (52, 56, "bold"), + ] + + # Unicode <= 0xFFFF + doc.setPlainText( + "\u201cDialogue,\u201d and then ::dialogue::, http://example.com\n\n" + ) + syntax.rehighlight() + assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 1, "\u201c"), (0, 1, 8, "Dialogue"), (0, 9, 2, ",\u201d"), + (0, 12, 3, "and"), (0, 16, 4, "then"), + (0, 21, 2, "::"), (0, 23, 8, "dialogue"), (0, 31, 2, "::"), + (0, 35, 18, "http://example.com"), + ] + assert formats[0].foreground().color().getRgb() == colDialogue # Quote + assert formats[1].foreground().color().getRgb() == colDialogue # Dialogue + assert formats[1].underlineColor().getRgb() == colSpell + assert formats[2].foreground().color().getRgb() == colDialogue # Quote + assert formats[3].underlineColor().getRgb() == colSpell # and + assert formats[4].underlineColor().getRgb() == colSpell # then + assert formats[5].foreground().color().getRgb() == colAltDialogue # :: + assert formats[6].foreground().color().getRgb() == colAltDialogue # dialogue + assert formats[6].underlineColor().getRgb() == colSpell + assert formats[7].foreground().color().getRgb() == colAltDialogue # :: + assert formats[8].foreground().color().getRgb() == colLink # http://example.com + + # Spell Check + data = doc.findBlockByNumber(0).userData() + assert isinstance(data, TextBlockData) + assert data.metaData == [(35, 53, "http://example.com", "url")] + assert data.spellErrors == [ + (1, 9, "Dialogue"), (12, 15, "and"), + (16, 20, "then"), (23, 31, "dialogue"), + ] + + # Unicode > 0xFFFF + doc.setPlainText( + "\u201c😁 Grinning 😁,\u201d and then ::πŸ™Š shush πŸ™Š::, http://example.com\n\n" + ) + syntax.rehighlight() + assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff + + pieces, formats = getFragments(syntax) + assert pieces == [ + (0, 0, 4, "\u201c😁 "), (0, 4, 8, "Grinning"), (0, 12, 5, " 😁,\u201d"), + (0, 18, 3, "and"), (0, 22, 4, "then"), + (0, 27, 5, "::πŸ™Š "), (0, 32, 5, "shush"), (0, 37, 5, " πŸ™Š::"), + (0, 44, 18, "http://example.com"), + ] + assert formats[0].foreground().color().getRgb() == colDialogue # Quote😁 + assert formats[1].foreground().color().getRgb() == colDialogue # Grinning + assert formats[1].underlineColor().getRgb() == colSpell + assert formats[2].foreground().color().getRgb() == colDialogue # 😁Quote + assert formats[3].underlineColor().getRgb() == colSpell # and + assert formats[4].underlineColor().getRgb() == colSpell # then + assert formats[5].foreground().color().getRgb() == colAltDialogue # ::πŸ™Š + assert formats[6].foreground().color().getRgb() == colAltDialogue # shush + assert formats[6].underlineColor().getRgb() == colSpell + assert formats[7].foreground().color().getRgb() == colAltDialogue # πŸ™Š:: + assert formats[8].foreground().color().getRgb() == colLink # http://example.com + + # Spell Check + data = doc.findBlockByNumber(0).userData() + assert isinstance(data, TextBlockData) + assert data.metaData == [(40, 58, "http://example.com", "url")] + assert data.spellErrors == [ + (4, 12, "Grinning"), (18, 21, "and"), + (22, 26, "then"), (32, 37, "shush"), + ]