Fix 4 byte Unicode support in editor (#2451)

novelwriter/common.py

@@ -494,6 +494,21 @@ def decodeMimeHandles(mimeData: QMimeData) -> list[str]:
     return mimeData.data(nwConst.MIME_HANDLE).data().decode().split("|")
 
 
+def utf16CharMap(text: str) -> list[int]:
+    """Compute mapping from Python string index to QString index.
+    Python strings are always one character per position in either
+    ASCII, UCS-2 or UCS-4. QStrings are in UTF-16, so wide characters
+    use 2 indices, and thus create an offset.
+    """
+    utf16Map = list(range(0, len(text) + 1))
+    offset = 0
+    for i, c in enumerate(text, 1):
+        if ord(c) > 0xffff:
+            offset += 1
+        utf16Map[i] = i + offset
+    return utf16Map
+
+
 ##
 #  Encoder Functions
 ##

novelwriter/core/coretools.py

@@ -289,7 +289,7 @@ class DocSearch:
 
     def __init__(self) -> None:
         self._regEx = re.compile(r"")
-        self._opts = re.UNICODE | re.IGNORECASE
+        self._opts = re.IGNORECASE
         self._words = False
         self._escape = True
         return
@@ -300,9 +300,7 @@ class DocSearch:
 
     def setCaseSensitive(self, state: bool) -> None:
         """Set the case sensitive search flag."""
-        self._opts = re.UNICODE
-        if not state:
-            self._opts |= re.IGNORECASE
+        self._opts = 0 if state else re.IGNORECASE
         return
 
     def setWholeWords(self, state: bool) -> None:

novelwriter/formats/todocx.py

@@ -51,7 +51,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 # RegEx
-RX_TEXT = re.compile(r"([\n\t])", re.UNICODE)
+RX_TEXT = re.compile(r"([\n\t])")
 
 # Types and Relationships
 OOXML_SCM = "http://schemas.openxmlformats.org"

novelwriter/gui/doceditor.py

@@ -1202,13 +1202,13 @@ class GuiDocEditor(QPlainTextEdit):
 
         # Spell Checking
         if SHARED.project.data.spellCheck:
-            word, cPos, cLen, suggest = self._qDocument.spellErrorAtPos(pCursor.position())
-            if word and cPos >= 0 and cLen > 0:
+            word, offset, suggest = self._qDocument.spellErrorAtPos(pCursor.position())
+            if word and offset >= 0:
                 logger.debug("Word '%s' is misspelled", word)
                 block = pCursor.block()
                 sCursor = self.textCursor()
-                sCursor.setPosition(block.position() + cPos)
-                sCursor.movePosition(QtMoveRight, QtKeepAnchor, cLen)
+                sCursor.setPosition(block.position() + offset)
+                sCursor.movePosition(QtMoveRight, QtKeepAnchor, len(word))
                 if suggest:
                     ctxMenu.addSeparator()
                     qtAddAction(ctxMenu, self.tr("Spelling Suggestion(s)"))
@@ -2294,20 +2294,22 @@ class TextAutoReplace:
         Returns True if anything was changed.
         """
         pos = cursor.positionInBlock()
-        length = len(text)
+        apos = cursor.position()
+        block = cursor.block()
+        length = block.length() - 1
         if length < 1 or pos-1 > length:
             return False
 
-        delete, insert = self._determine(text, pos)
-        if insert == "":
-            return False
+        cursor.movePosition(QtMoveLeft, QtKeepAnchor, min(4, pos))
+        last = cursor.selectedText()
+        delete, insert = self._determine(last, pos)
 
         check = insert
         if self._doPadBefore and check in self._padBefore:
             if not (check == ":" and length > 1 and text[0] == "@"):
                 delete = max(delete, 1)
-                chkPos = pos - delete - 1
-                if chkPos >= 0 and text[chkPos].isspace():
+                chkPos = len(last) - delete - 1
+                if chkPos >= 0 and last[chkPos].isspace():
                     # Strip existing space before inserting a new (#1061)
                     delete += 1
                 insert = self._padChar + insert
@@ -2318,6 +2320,7 @@ class TextAutoReplace:
                 insert = insert + self._padChar
 
         if delete > 0:
+            cursor.setPosition(apos)
             cursor.movePosition(QtMoveLeft, QtKeepAnchor, delete)
             cursor.insertText(insert)
             return True
@@ -2326,13 +2329,10 @@ class TextAutoReplace:
 
     def _determine(self, text: str, pos: int) -> tuple[int, str]:
         """Determine what to replace, if anything."""
-        t1 = text[pos-1:pos]
-        t2 = text[pos-2:pos]
-        t3 = text[pos-3:pos]
-        t4 = text[pos-4:pos]
-        if t1 == "":
-            # Return early if there is nothing to check
-            return 0, ""
+        t1 = text[-1:]
+        t2 = text[-2:]
+        t3 = text[-3:]
+        t4 = text[-4:]
 
         leading = t2[:1].isspace()
         if self._replaceDQuote:

novelwriter/gui/dochighlight.py

@@ -36,11 +36,12 @@ from PyQt6.QtGui import (
 )
 
 from novelwriter import CONFIG, SHARED
-from novelwriter.common import checkInt
+from novelwriter.common import checkInt, utf16CharMap
 from novelwriter.constants import nwStyles, nwUnicode
 from novelwriter.enum import nwComment
 from novelwriter.text.comments import processComment
 from novelwriter.text.patterns import REGEX_PATTERNS, DialogParser
+from novelwriter.types import QtTextUserProperty
 
 logger = logging.getLogger(__name__)
 
@@ -308,27 +309,37 @@ class GuiDocHighlighter(QSyntaxHighlighter):
         if self._tHandle is None or not text:
             return
 
-        xOff = 0
-        hRules = None
+        blockLen = self.currentBlock().length()
+        utf16Map = None
+        if blockLen > len(text) + 1:
+            # If the lengths are different, the line contains 4 byte
+            # Unicode characters, and we must use a map between Python
+            # string indices and the UTF-16 indices used by Qt, where a
+            # 4 byte character occupies two slots. See #2449.
+            utf16Map = utf16CharMap(text)
+
+        offset = 0
+        rules = None
         if text.startswith("@"):  # Keywords and commands
             self.setCurrentBlockState(BLOCK_META)
             index = SHARED.project.index
-            isValid, bits, pos = index.scanThis(text)
+            isValid, bits, loc = index.scanThis(text)
             isGood = index.checkThese(bits, self._tHandle)
             if isValid:
                 for n, bit in enumerate(bits):
-                    xPos = pos[n]
-                    xLen = len(bit)
+                    pos = utf16Map[loc[n]] if utf16Map else loc[n]
+                    length = utf16Map[loc[n] + len(bit)] - pos if utf16Map else len(bit)
                     if n == 0 and isGood[n]:
-                        self.setFormat(xPos, xLen, self._hStyles["keyword"])
+                        self.setFormat(pos, length, self._hStyles["keyword"])
                     elif isGood[n] and not self._isInactive:
-                        one, two = index.parseValue(bit)
-                        self.setFormat(xPos, len(one), self._hStyles["tag"])
-                        if two:
-                            yPos = xPos + len(bit) - len(two)
-                            self.setFormat(yPos, len(two), self._hStyles["optional"])
+                        a, b = index.parseValue(bit)
+                        aLen = utf16Map[loc[n] + len(a)] - pos if utf16Map else len(a)
+                        self.setFormat(pos, aLen, self._hStyles["tag"])
+                        if b:
+                            bLen = utf16Map[loc[n] + len(b)] - pos if utf16Map else len(b)
+                            self.setFormat(pos + length - bLen, bLen, self._hStyles["optional"])
                     elif not self._isInactive:
-                        self.setFormat(xPos, xLen, self._hStyles["invalid"])
+                        self.setFormat(pos, length, self._hStyles["invalid"])
 
             # We never want to run the spell checker on keyword/values,
             # so we force a return here
@@ -339,98 +350,118 @@ class GuiDocHighlighter(QSyntaxHighlighter):
 
             if text.startswith("# "):  # Heading 1
                 self.setFormat(0, 1, self._hStyles["head1h"])
-                self.setFormat(1, len(text), self._hStyles["header1"])
+                self.setFormat(1, blockLen, self._hStyles["header1"])
 
             elif text.startswith("## "):  # Heading 2
                 self.setFormat(0, 2, self._hStyles["head2h"])
-                self.setFormat(2, len(text), self._hStyles["header2"])
+                self.setFormat(2, blockLen, self._hStyles["header2"])
 
             elif text.startswith("### "):  # Heading 3
                 self.setFormat(0, 3, self._hStyles["head3h"])
-                self.setFormat(3, len(text), self._hStyles["header3"])
+                self.setFormat(3, blockLen, self._hStyles["header3"])
 
             elif text.startswith("#### "):  # Heading 4
                 self.setFormat(0, 4, self._hStyles["head4h"])
-                self.setFormat(4, len(text), self._hStyles["header4"])
+                self.setFormat(4, blockLen, self._hStyles["header4"])
 
             elif text.startswith("#! "):  # Title
                 self.setFormat(0, 2, self._hStyles["head1h"])
-                self.setFormat(2, len(text), self._hStyles["header1"])
+                self.setFormat(2, blockLen, self._hStyles["header1"])
 
             elif text.startswith("##! "):  # Unnumbered
                 self.setFormat(0, 3, self._hStyles["head2h"])
-                self.setFormat(3, len(text), self._hStyles["header2"])
+                self.setFormat(3, blockLen, self._hStyles["header2"])
 
             elif text.startswith("###! "):  # Alternative Scene
                 self.setFormat(0, 4, self._hStyles["head3h"])
-                self.setFormat(4, len(text), self._hStyles["header3"])
+                self.setFormat(4, blockLen, self._hStyles["header3"])
 
         elif text.startswith("%"):  # Comments
             self.setCurrentBlockState(BLOCK_TEXT)
-            hRules = self._cmnRules
+            rules = self._cmnRules
 
-            cStyle, cMod, _, cDot, cPos = processComment(text)
-            cLen = len(text) - cPos
-            xOff = cPos
-            if cStyle == nwComment.PLAIN:
-                self.setFormat(0, cLen, self._hStyles["hidden"])
-            elif cStyle == nwComment.IGNORE:
-                self.setFormat(0, cLen, self._hStyles["strike"])
+            style, mod, _, dot, pos = processComment(text)
+            offset = pos
+            if utf16Map:
+                dot = utf16Map[dot]
+                pos = utf16Map[pos]
+            length = blockLen - pos
+            if style == nwComment.PLAIN:
+                self.setFormat(0, length, self._hStyles["hidden"])
+            elif style == nwComment.IGNORE:
+                self.setFormat(0, length, self._hStyles["strike"])
                 return  # No more processing for these
-            elif cMod:
-                self.setFormat(0, cDot, self._hStyles["modifier"])
-                self.setFormat(cDot, cPos - cDot, self._hStyles["value"])
-                self.setFormat(cPos, cLen, self._hStyles["note"])
+            elif mod:
+                self.setFormat(0, dot, self._hStyles["modifier"])
+                self.setFormat(dot, pos - dot, self._hStyles["value"])
+                self.setFormat(pos, length, self._hStyles["note"])
             else:
-                self.setFormat(0, cPos, self._hStyles["modifier"])
-                self.setFormat(cPos, cLen, self._hStyles["note"])
+                self.setFormat(0, pos, self._hStyles["modifier"])
+                self.setFormat(pos, length, self._hStyles["note"])
 
         elif text.startswith("["):  # Special Command
             self.setCurrentBlockState(BLOCK_TEXT)
-            hRules = self._txtRules if self._isNovel else self._minRules
+            rules = self._txtRules if self._isNovel else self._minRules
 
-            sText = text.rstrip().lower()
-            if sText in ("[newpage]", "[new page]", "[vspace]"):
-                self.setFormat(0, len(text), self._hStyles["code"])
+            check = text.rstrip().lower()
+            if check in ("[newpage]", "[new page]", "[vspace]"):
+                self.setFormat(0, blockLen, self._hStyles["code"])
                 return
-            elif sText.startswith("[vspace:") and sText.endswith("]"):
-                tLen = len(sText)
-                tVal = checkInt(sText[8:-1], 0)
-                cVal = "value" if tVal > 0 else "invalid"
+            elif check.startswith("[vspace:") and check.endswith("]"):
+                value = checkInt(check[8:-1], 0)
+                style = "value" if value > 0 else "invalid"
                 self.setFormat(0, 8, self._hStyles["code"])
-                self.setFormat(8, tLen-9, self._hStyles[cVal])
-                self.setFormat(tLen-1, tLen, self._hStyles["code"])
+                self.setFormat(8, blockLen-10, self._hStyles[style])
+                self.setFormat(blockLen-2, blockLen, self._hStyles["code"])
                 return
 
         else:  # Text Paragraph
             self.setCurrentBlockState(BLOCK_TEXT)
-            hRules = self._txtRules if self._isNovel else self._minRules
+            rules = self._txtRules if self._isNovel else self._minRules
             if self._isNovel and self._dialogParser.enabled:
-                for pos, end in self._dialogParser(text):
-                    length = end - pos
-                    self.setFormat(pos, length, self._hStyles["dialog"])
+                if utf16Map:
+                    for pos, end in self._dialogParser(text):
+                        pos = utf16Map[pos]
+                        end = utf16Map[end]
+                        self.setFormat(pos, end - pos, self._hStyles["dialog"])
+                else:
+                    for pos, end in self._dialogParser(text):
+                        self.setFormat(pos, end - pos, self._hStyles["dialog"])
 
-        if hRules:
-            for rX, hRule in hRules:
-                for res in re.finditer(rX, text[xOff:]):
-                    for xM, hFmt in hRule.items():
-                        xPos = res.start(xM) + xOff
-                        xEnd = res.end(xM) + xOff
-                        for x in range(xPos, xEnd):
-                            cFmt = self.format(x)
-                            if cFmt.fontStyleName() != "markup":
-                                cFmt.merge(hFmt)
-                                self.setFormat(x, 1, cFmt)
+        if rules:
+            if utf16Map:
+                for rX, hRule in rules:
+                    for res in re.finditer(rX, text[offset:]):
+                        for x, hFmt in hRule.items():
+                            pos = res.start(x) + offset
+                            end = res.end(x) + offset
+                            for x in range(pos, end):
+                                m = utf16Map[x]
+                                cFmt = self.format(m)
+                                if not cFmt.property(QtTextUserProperty):
+                                    cFmt.merge(hFmt)
+                                    self.setFormat(m, utf16Map[x+1] - m, cFmt)
+            else:
+                for rX, hRule in rules:
+                    for res in re.finditer(rX, text[offset:]):
+                        for x, hFmt in hRule.items():
+                            pos = res.start(x) + offset
+                            end = res.end(x) + offset
+                            for x in range(pos, end):
+                                cFmt = self.format(x)
+                                if not cFmt.property(QtTextUserProperty):
+                                    cFmt.merge(hFmt)
+                                    self.setFormat(x, 1, cFmt)
 
         data = self.currentBlockUserData()
         if not isinstance(data, TextBlockData):
             data = TextBlockData()
             self.setCurrentBlockUserData(data)
 
-        data.processText(text, xOff)
+        data.processText(text, offset)
         if self._spellCheck:
-            for xPos, xEnd in data.spellCheck():
-                for x in range(xPos, xEnd):
+            for pos, end, _ in data.spellCheck(utf16Map):
+                for x in range(pos, end):
                     cFmt = self.format(x)
                     cFmt.merge(self._spellErr)
                     self.setFormat(x, 1, cFmt)
@@ -447,7 +478,8 @@ class GuiDocHighlighter(QSyntaxHighlighter):
     ) -> None:
         """Generate a highlighter character format."""
         charFormat = QTextCharFormat()
-        charFormat.setFontStyleName(name)
+        blockMerge = name == "markup"
+        charFormat.setProperty(QtTextUserProperty, blockMerge)
 
         if style:
             styles = style.split(",")
@@ -486,7 +518,7 @@ class TextBlockData(QTextBlockUserData):
         self._text = ""
         self._offset = 0
         self._metaData: list[tuple[int, int, str, str]] = []
-        self._spellErrors: list[tuple[int, int]] = []
+        self._spellErrors: list[tuple[int, int, str]] = []
         return
 
     @property
@@ -495,7 +527,7 @@ class TextBlockData(QTextBlockUserData):
         return self._metaData
 
     @property
-    def spellErrors(self) -> list[tuple[int, int]]:
+    def spellErrors(self) -> list[tuple[int, int, str]]:
         """Return spell error data from last check."""
         return self._spellErrors
 
@@ -518,22 +550,26 @@ class TextBlockData(QTextBlockUserData):
                     text = f"{text[:s]}{pad}{text[e:]}"
                     self._metaData.append((s, e, res.group(0), "url"))
 
-        self._text = text.replace("\u02bc", "'")
+        self._text = text.replace("\u02bc", "'").replace("_", " ")
         self._offset = offset
 
         return
 
-    def spellCheck(self) -> list[tuple[int, int]]:
+    def spellCheck(self, utf16Map: list[int] | None) -> list[tuple[int, int, str]]:
         """Run the spell checker and cache the result, and return the
         list of spell check errors.
         """
-        self._spellErrors = []
-        checker = SHARED.spelling
-        for res in RX_WORDS.finditer(self._text.replace("_", " "), self._offset):
-            if (
-                (word := res.group(0))
-                and not (word.isnumeric() or word.isupper() or checker.checkWord(word))
-            ):
-                self._spellErrors.append((res.start(0), res.end(0)))
-
+        spell = SHARED.spelling
+        if utf16Map:
+            self._spellErrors = [
+                (utf16Map[r.start(0)], utf16Map[r.end(0)], w)
+                for r in RX_WORDS.finditer(self._text, self._offset)
+                if (w := r.group(0)) and not (w.isnumeric() or w.isupper() or spell.checkWord(w))
+            ]
+        else:
+            self._spellErrors = [
+                (r.start(0), r.end(0), w)
+                for r in RX_WORDS.finditer(self._text, self._offset)
+                if (w := r.group(0)) and not (w.isnumeric() or w.isupper() or spell.checkWord(w))
+            ]
         return self._spellErrors

novelwriter/gui/editordocument.py

@@ -113,7 +113,7 @@ class GuiTextDocument(QTextDocument):
                         return cData, cType
         return "", ""
 
-    def spellErrorAtPos(self, pos: int) -> tuple[str, int, int, list[str]]:
+    def spellErrorAtPos(self, pos: int) -> tuple[str, int, list[str]]:
         """Check if there is a misspelled word at a given position in
         the document, and if so, return it.
         """
@@ -122,15 +122,11 @@ class GuiTextDocument(QTextDocument):
         block = cursor.block()
         data = block.userData()
         if block.isValid() and isinstance(data, TextBlockData):
-            text = block.text()
-            check = pos - block.position()
-            if check >= 0:
-                for cPos, cEnd in data.spellErrors:
-                    cLen = cEnd - cPos
-                    if cPos <= check <= cEnd:
-                        word = text[cPos:cEnd]
-                        return word, cPos, cLen, SHARED.spelling.suggestWords(word)
-        return "", -1, -1, []
+            if (check := pos - block.position()) >= 0:
+                for start, end, word in data.spellErrors:
+                    if start <= check <= end:
+                        return word, start, SHARED.spelling.suggestWords(word)
+        return "", -1, []
 
     def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Iterable[QTextBlock]:
         """Iterate over all text blocks of a given type."""

novelwriter/text/patterns.py

@@ -37,14 +37,14 @@ class RegExPatterns:
 
     # Static RegExes
     _rxUrl     = re.compile(nwRegEx.URL, re.ASCII)
-    _rxWords   = re.compile(nwRegEx.WORDS, re.UNICODE)
-    _rxBreak   = re.compile(nwRegEx.BREAK, re.UNICODE)
-    _rxItalic  = re.compile(nwRegEx.FMT_EI, re.UNICODE)
-    _rxBold    = re.compile(nwRegEx.FMT_EB, re.UNICODE)
-    _rxStrike  = re.compile(nwRegEx.FMT_ST, re.UNICODE)
-    _rxMark    = re.compile(nwRegEx.FMT_HL, re.UNICODE)
-    _rxSCPlain = re.compile(nwRegEx.FMT_SC, re.UNICODE)
-    _rxSCValue = re.compile(nwRegEx.FMT_SV, re.UNICODE)
+    _rxWords   = re.compile(nwRegEx.WORDS)
+    _rxBreak   = re.compile(nwRegEx.BREAK)
+    _rxItalic  = re.compile(nwRegEx.FMT_EI)
+    _rxBold    = re.compile(nwRegEx.FMT_EB)
+    _rxStrike  = re.compile(nwRegEx.FMT_ST)
+    _rxMark    = re.compile(nwRegEx.FMT_HL)
+    _rxSCPlain = re.compile(nwRegEx.FMT_SC)
+    _rxSCValue = re.compile(nwRegEx.FMT_SV)
 
     @property
     def url(self) -> re.Pattern:
@@ -114,7 +114,7 @@ class RegExPatterns:
                     rx.append(f"(?:{qO}[^{qO}]+{qC})")
                 if CONFIG.allowOpenDial:
                     rx.append(f"(?:{qO}.+?$)")
-            return re.compile("|".join(rx), re.UNICODE)
+            return re.compile("|".join(rx))
         return None
 
     @property
@@ -124,7 +124,7 @@ class RegExPatterns:
             qO = re.escape(compact(CONFIG.altDialogOpen))
             qC = re.escape(compact(CONFIG.altDialogClose))
             qB = r"\B" if (qO == qC or qC in self.AMBIGUOUS) else ""
-            return re.compile(f"{qO}.*?{qC}{qB}", re.UNICODE)
+            return re.compile(f"{qO}.*?{qC}{qB}")
         return None
 
 
@@ -169,8 +169,8 @@ class DialogParser:
         # Build narrator break RegExes
         if narrator := CONFIG.narratorBreak.strip()[:1]:
             punct = re.escape(".,:;!?")
-            self._breakD = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?|$)", re.UNICODE)
-            self._breakQ = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?)", re.UNICODE)
+            self._breakD = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?|$)")
+            self._breakQ = re.compile(f"{narrator}.*?(?:{narrator}[{punct}]?)")
             self._narrator = narrator
             self._mode = f" {narrator}"
 

novelwriter/types.py

@@ -54,6 +54,8 @@ QtPageBreakBefore = QTextFormat.PageBreakFlag.PageBreak_AlwaysBefore
 QtPageBreakAfter = QTextFormat.PageBreakFlag.PageBreak_AlwaysAfter
 QtPageBreakAuto = QTextFormat.PageBreakFlag.PageBreak_Auto
 
+QtTextUserProperty = QTextFormat.Property.UserProperty
+
 QtPropLineHeight = 1  # QTextBlockFormat.LineHeightTypes.ProportionalHeight
 
 # Qt Painter Types

sample/nwProject.nwx

@@ -1,6 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<novelWriterXML appVersion="2.8a1" hexVersion="0x020800a1" fileVersion="1.5" fileRevision="6" timeStamp="2025-06-24 17:12:20">
-  <project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="2236" autoCount="290" editTime="98026">
+<novelWriterXML appVersion="2.8a1" hexVersion="0x020800a1" fileVersion="1.5" fileRevision="6" timeStamp="2025-07-13 18:58:02">
+  <project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="2279" autoCount="301" editTime="99977">
     <name>Sample Project</name>
     <author>Jane Smith</author>
   </project>
@@ -20,7 +20,7 @@
       <entry key="C">D</entry>
     </autoReplace>
     <status>
-      <entry key="sf12341" count="8" color="faded" shape="STAR">New</entry>
+      <entry key="sf12341" count="8" color="#969896" shape="STAR">New</entry>
       <entry key="sf24ce6" count="2" color="red" shape="STAR">Notes</entry>
       <entry key="sc24b8f" count="3" color="yellow" shape="BARS_1">Started</entry>
       <entry key="s90e6c9" count="5" color="yellow" shape="BARS_2">1st Draft</entry>
@@ -58,7 +58,7 @@
       <name status="sf24ce6" import="ia857f0" active="yes">Chapter One</name>
     </item>
     <item handle="636b6aa9b697b" parent="6a2d6d5f4f401" root="7031beac91f75" order="0" type="FILE" class="NOVEL" layout="DOCUMENT">
-      <meta expanded="no" heading="H3" charCount="3003" wordCount="530" paraCount="16" cursorPos="586" />
+      <meta expanded="no" heading="H3" charCount="3003" wordCount="530" paraCount="16" cursorPos="19" />
       <name status="s90e6c9" import="ia857f0" active="yes">Making a Scene</name>
     </item>
     <item handle="bc0cbd2a407f3" parent="6a2d6d5f4f401" root="7031beac91f75" order="1" type="FILE" class="NOVEL" layout="DOCUMENT">

tests/test_base/test_base_common.py

@@ -38,8 +38,8 @@ from novelwriter.common import (
     fuzzyTime, getFileSize, hexToInt, isHandle, isItemClass, isItemLayout,
     isItemType, isListInstance, isTitleTag, jsonEncode, makeFileNameSafe,
     minmax, numberToRoman, openExternalPath, processDialogSymbols,
-    readTextFile, simplified, transferCase, uniqueCompact, xmlElement,
-    xmlIndent, xmlSubElem, yesNo
+    readTextFile, simplified, transferCase, uniqueCompact, utf16CharMap,
+    xmlElement, xmlIndent, xmlSubElem, yesNo
 )
 from novelwriter.enum import nwItemClass
 
@@ -557,6 +557,14 @@ def testBaseCommon_encodeDecodeMimeHandles(monkeypatch):
     assert decodeMimeHandles(mimeData) == handles
 
 
+@pytest.mark.base
+def testBaseCommon_utf16CharMap(monkeypatch):
+    """Test the utf16CharMap function."""
+    assert utf16CharMap("abc") == [0, 1, 2, 3]
+    assert utf16CharMap("a\u2014b\u2014c") == [0, 1, 2, 3, 4, 5]
+    assert utf16CharMap("a\U0001F605b\U0001F605c") == [0, 1, 3, 4, 6, 7]
+
+
 @pytest.mark.base
 def testBaseCommon_jsonEncode():
     """Test the jsonEncode function."""

tests/test_gui/test_gui_doceditor.py

@@ -514,32 +514,36 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText,
     # Run SpellCheck
     # ==============
     SHARED.project.data.setSpellCheck(True)
+    LORAX = "Lorax\U0001F03A"
 
     cursor = docEditor.textCursor()
     cursor.setPosition(16)
     data = cursor.block().userData()
     assert cursor.block().text().startswith("Lorem")
     assert isinstance(data, TextBlockData)
-    data._spellErrors = [(0, 5)]
+    data._spellErrors = [(0, 5, "Lorem")]
 
     # No known position
-    assert docEditor._qDocument.spellErrorAtPos(-1) == ("", -1, -1, [])
+    assert docEditor._qDocument.spellErrorAtPos(-1) == ("", -1, [])
 
     # With Suggestion
     with monkeypatch.context() as mp:
-        mp.setattr(SHARED.spelling, "suggestWords", lambda *a: ["Lorax"])
+        mp.setattr(SHARED.spelling, "suggestWords", lambda *a: [LORAX])
 
         ctxMenu = getMenuForPos(docEditor, 16)
         assert ctxMenu is not None
         actions = [x.text() for x in ctxMenu.actions() if x.text()]
         assert "Spelling Suggestion(s)" in actions
-        assert f"{nwUnicode.U_ENDASH} Lorax" in actions
+        assert f"{nwUnicode.U_ENDASH} {LORAX}" in actions
         ctxMenu.actions()[7].trigger()
         QApplication.processEvents()
-        assert docEditor.getText() == text.replace("Lorem", "Lorax", 1)
+        assert docEditor.getText() == text.replace("Lorem", LORAX, 1)
         ctxMenu.setObjectName("")
         ctxMenu.deleteLater()
 
+    # Update Entry
+    data._spellErrors = [(0, 7, LORAX)]
+
     # Without Suggestion
     with monkeypatch.context() as mp:
         mp.setattr(SHARED.spelling, "suggestWords", lambda *a: [])
@@ -548,7 +552,7 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText,
         assert ctxMenu is not None
         actions = [x.text() for x in ctxMenu.actions() if x.text()]
         assert f"{nwUnicode.U_ENDASH} No Suggestions" in actions
-        assert docEditor.getText() == text.replace("Lorem", "Lorax", 1)
+        assert docEditor.getText() == text.replace("Lorem", LORAX, 1)
         ctxMenu.setObjectName("")
         ctxMenu.deleteLater()
 
@@ -562,11 +566,11 @@ def testGuiEditor_SpellChecking(qtbot, monkeypatch, nwGUI, projPath, ipsumText,
         assert "Ignore Word" in actions
         assert "Add Word to Dictionary" in actions
 
-        assert "Lorax" not in SHARED.spelling._userDict
+        assert LORAX not in SHARED.spelling._userDict
         ctxMenu.actions()[7].trigger()  # Ignore
-        assert "Lorax" not in SHARED.spelling._userDict
+        assert LORAX not in SHARED.spelling._userDict
         ctxMenu.actions()[8].trigger()  # Add
-        assert "Lorax" in SHARED.spelling._userDict
+        assert LORAX in SHARED.spelling._userDict
         ctxMenu.setObjectName("")
         ctxMenu.deleteLater()
 

tests/test_gui/test_gui_dochighlighter.py

@@ -0,0 +1,631 @@
+"""
+novelWriter – GUI Syntax Highlighter Tester
+===========================================
+
+This file is a part of novelWriter
+Copyright (C) 2020 Veronica Berglyd Olsen and novelWriter contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+from __future__ import annotations
+
+import pytest
+
+from PyQt6.QtGui import QTextCharFormat, QTextCursor, QTextDocument
+
+from novelwriter import CONFIG, SHARED
+from novelwriter.core.item import NWItem
+from novelwriter.enum import nwItemClass, nwItemLayout, nwItemType, nwTheme
+from novelwriter.gui.dochighlight import BLOCK_META, BLOCK_TITLE, GuiDocHighlighter, TextBlockData
+from novelwriter.types import QtKeepAnchor
+
+R_HANDLE = "3456789abcdef"
+T_HANDLE = "0123456789abc"
+
+
+@pytest.fixture()
+def syntax(nwGUI):
+    """Create a syntax object for use with testing."""
+    CONFIG.lightTheme = "tomorrow"
+    CONFIG.themeMode = nwTheme.LIGHT
+
+    CONFIG.dialogStyle = 3
+    CONFIG.fmtDQuoteOpen = "\u201c"
+    CONFIG.fmtDQuoteClose = "\u201d"
+    CONFIG.altDialogOpen = "::"
+    CONFIG.altDialogClose = "::"
+
+    theme = SHARED.theme
+    theme.loadTheme(force=True)
+    assert theme._guiPalette.base().color().getRgb() == (0xff, 0xff, 0xff, 0xff)
+    assert theme.syntaxTheme.text.getRgb() == (0x4d, 0x4d, 0x4c, 0xff)
+
+    doc = QTextDocument()
+    syntax = GuiDocHighlighter(doc)
+
+    # Add a Mock Item
+    tRoot = NWItem(SHARED.project, R_HANDLE)
+    tRoot.setClass(nwItemClass.NOVEL)
+    tRoot.setType(nwItemType.ROOT)
+
+    tItem = NWItem(SHARED.project, T_HANDLE)
+    tItem.setParent(R_HANDLE)
+    tItem.setLayout(nwItemLayout.NOTE)
+    tItem.setClass(nwItemClass.NOVEL)
+    tItem.setType(nwItemType.FILE)
+
+    SHARED.project.tree.add(tRoot)
+    SHARED.project.tree.add(tItem)
+
+    yield syntax
+
+
+def getFragments(
+    syntax: GuiDocHighlighter
+) -> tuple[list[tuple[int, str]], list[QTextCharFormat]]:
+    """Extract all syntax highlighter fragments from a document."""
+    pieces = []
+    formats = []
+    doc = syntax.document()
+    cursor = QTextCursor(doc)
+    assert doc is not None
+    for b in range(doc.blockCount()):
+        block = doc.findBlockByNumber(b)
+        first = block.position()
+        syntax.rehighlightBlock(block)
+        if layout := block.layout():
+            for fmt in layout.formats():
+                cursor.setPosition(first + fmt.start)
+                cursor.setPosition(first + fmt.start + fmt.length, QtKeepAnchor)
+                pieces.append((b, fmt.start, fmt.length, cursor.selectedText()))
+                formats.append(fmt.format)
+    return pieces, formats
+
+
+def maxOrd(text: str) -> int:
+    """Get the max character value of a string."""
+    return max(ord(c) for c in text)
+
+
+@pytest.mark.gui
+def testGuiDocHighlighter_Basic(syntax):
+    """Test the basic functionality of the syntax highlighter."""
+    # Alternate Spell Check
+    assert syntax._spellCheck is False
+    syntax.setSpellCheck(True)
+    assert syntax._spellCheck is True
+
+    # Check Handle
+    assert syntax._tHandle is None
+    syntax.setHandle(T_HANDLE)
+    assert syntax._tHandle == T_HANDLE
+    assert syntax._isNovel is False
+    assert syntax._isInactive is False
+
+    tItem = SHARED.project.tree[T_HANDLE]
+    assert tItem is not None
+    tItem.setLayout(nwItemLayout.DOCUMENT)
+    tItem.setClass(nwItemClass.ARCHIVE)
+    syntax.setHandle(T_HANDLE)
+    assert syntax._tHandle == T_HANDLE
+    assert syntax._isNovel is True
+    assert syntax._isInactive is True
+
+
+@pytest.mark.gui
+def testGuiDocHighlighter_Keywords(syntax):
+    """Test highlighting of keywords."""
+    theme = SHARED.theme
+    doc = syntax.document()
+    assert doc is not None
+
+    # Settings
+    syntax._tHandle = T_HANDLE
+
+    colKey = theme.syntaxTheme.key.getRgb()
+    colTag = theme.syntaxTheme.tag.getRgb()
+    colOpt = theme.syntaxTheme.opt.getRgb()
+    colErr = theme.syntaxTheme.error.getRgb()
+
+    # Ascii
+    doc.setPlainText(
+        "@tag: Bob | Robert\n"
+        "@char: Someone\n"
+    )
+    syntax.rehighlightByType(BLOCK_META)
+    assert maxOrd(doc.toPlainText()) <= 0x7f
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 4, "@tag"), (0, 6, 3, "Bob"), (0, 12, 6, "Robert"),
+        (1, 0, 5, "@char"), (1, 7, 7, "Someone"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colKey
+    assert formats[1].foreground().color().getRgb() == colTag
+    assert formats[2].foreground().color().getRgb() == colOpt
+    assert formats[3].foreground().color().getRgb() == colKey
+    assert formats[4].underlineColor().getRgb() == colErr
+
+    # # Unicode <= 0xFFFF
+    doc.setPlainText(
+        "@tag: Zoë | Zoë Smith\n"
+        "@char: Олексій\n"
+    )
+    syntax.rehighlightByType(BLOCK_META)
+    assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 4, "@tag"), (0, 6, 3, "Zoë"), (0, 12, 9, "Zoë Smith"),
+        (1, 0, 5, "@char"), (1, 7, 7, "Олексій"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colKey
+    assert formats[1].foreground().color().getRgb() == colTag
+    assert formats[2].foreground().color().getRgb() == colOpt
+    assert formats[3].foreground().color().getRgb() == colKey
+    assert formats[4].underlineColor().getRgb() == colErr
+
+    # # Unicode > 0xFFFF
+    doc.setPlainText(
+        "@tag: 😄 | Smiley 😄\n"
+        "@char: 😎😎😎\n"
+    )
+    syntax.rehighlightByType(BLOCK_META)
+    assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 4, "@tag"), (0, 6, 2, "😄"), (0, 11, 9, "Smiley 😄"),
+        (1, 0, 5, "@char"), (1, 7, 6, "😎😎😎"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colKey
+    assert formats[1].foreground().color().getRgb() == colTag
+    assert formats[2].foreground().color().getRgb() == colOpt
+    assert formats[3].foreground().color().getRgb() == colKey
+    assert formats[4].underlineColor().getRgb() == colErr
+
+
+@pytest.mark.gui
+def testGuiDocHighlighter_Titles(syntax):
+    """Test highlighting of titles."""
+    theme = SHARED.theme
+    doc = syntax.document()
+    assert doc is not None
+
+    # Settings
+    syntax._tHandle = T_HANDLE
+
+    colHeadMark = theme.syntaxTheme.headH.getRgb()
+    colHeadText = theme.syntaxTheme.head.getRgb()
+
+    # Ascii
+    doc.setPlainText(
+        "# Heading 1\n\n"
+        "## Heading 2\n\n"
+        "### Heading 3\n\n"
+        "#### Heading 4\n\n"
+        "#! Heading A1\n\n"
+        "##! Heading A2\n\n"
+        "###! Heading A3\n\n"
+    )
+    syntax.rehighlightByType(BLOCK_TITLE)
+    assert maxOrd(doc.toPlainText()) <= 0x7f
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 1, "#"), (0, 1, 10, " Heading 1"),
+        (2, 0, 2, "##"), (2, 2, 10, " Heading 2"),
+        (4, 0, 3, "###"), (4, 3, 10, " Heading 3"),
+        (6, 0, 4, "####"), (6, 4, 10, " Heading 4"),
+        (8, 0, 2, "#!"), (8, 2, 11, " Heading A1"),
+        (10, 0, 3, "##!"), (10, 3, 11, " Heading A2"),
+        (12, 0, 4, "###!"), (12, 4, 11, " Heading A3"),
+    ]
+    for i in range(0, len(formats), 2):
+        assert formats[i].foreground().color().getRgb() == colHeadMark
+        assert formats[i+1].foreground().color().getRgb() == colHeadText
+
+    # Unicode <= 0xFFFF
+    doc.setPlainText(
+        "# Ȟǣđ 1\n\n"
+        "## Ȟǣđ 2\n\n"
+        "### Ȟǣđ 3\n\n"
+        "#### Ȟǣđ 4\n\n"
+        "#! Ȟǣđ A1\n\n"
+        "##! Ȟǣđ A2\n\n"
+        "###! Ȟǣđ A3\n\n"
+    )
+    syntax.rehighlightByType(BLOCK_TITLE)
+    assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 1, "#"), (0, 1, 6, " Ȟǣđ 1"),
+        (2, 0, 2, "##"), (2, 2, 6, " Ȟǣđ 2"),
+        (4, 0, 3, "###"), (4, 3, 6, " Ȟǣđ 3"),
+        (6, 0, 4, "####"), (6, 4, 6, " Ȟǣđ 4"),
+        (8, 0, 2, "#!"), (8, 2, 7, " Ȟǣđ A1"),
+        (10, 0, 3, "##!"), (10, 3, 7, " Ȟǣđ A2"),
+        (12, 0, 4, "###!"), (12, 4, 7, " Ȟǣđ A3"),
+    ]
+    for i in range(0, len(formats), 2):
+        assert formats[i].foreground().color().getRgb() == colHeadMark
+        assert formats[i+1].foreground().color().getRgb() == colHeadText
+
+    # Unicode > 0xFFFF
+    doc.setPlainText(
+        "# 😇😎 1\n\n"
+        "## 😇😎 2\n\n"
+        "### 😇😎 3\n\n"
+        "#### 😇😎 4\n\n"
+        "#! 😇😎 A1\n\n"
+        "##! 😇😎 A2\n\n"
+        "###! 😇😎 A3\n\n"
+    )
+    syntax.rehighlightByType(BLOCK_TITLE)
+    assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 1, "#"), (0, 1, 7, " 😇😎 1"),
+        (2, 0, 2, "##"), (2, 2, 7, " 😇😎 2"),
+        (4, 0, 3, "###"), (4, 3, 7, " 😇😎 3"),
+        (6, 0, 4, "####"), (6, 4, 7, " 😇😎 4"),
+        (8, 0, 2, "#!"), (8, 2, 8, " 😇😎 A1"),
+        (10, 0, 3, "##!"), (10, 3, 8, " 😇😎 A2"),
+        (12, 0, 4, "###!"), (12, 4, 8, " 😇😎 A3"),
+    ]
+    for i in range(0, len(formats), 2):
+        assert formats[i].foreground().color().getRgb() == colHeadMark
+        assert formats[i+1].foreground().color().getRgb() == colHeadText
+
+
+@pytest.mark.gui
+def testGuiDocHighlighter_Comments(syntax):
+    """Test highlighting of comments."""
+    theme = SHARED.theme
+    doc = syntax.document()
+    assert doc is not None
+
+    # Settings
+    syntax._tHandle = T_HANDLE
+
+    colHidden = theme.syntaxTheme.hidden.getRgb()
+    colMod = theme.syntaxTheme.mod.getRgb()
+    colValue = theme.syntaxTheme.val.getRgb()
+    colNote = theme.syntaxTheme.note.getRgb()
+
+    # Ascii
+    doc.setPlainText(
+        "% Plain\n"
+        "%~ Ignored\n"
+        "%Synopsis: Synopsis\n"
+        "%Note.Stuff: Note\n"
+    )
+    syntax.rehighlight()
+    assert maxOrd(doc.toPlainText()) <= 0x7f
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 7, "% Plain"),
+        (1, 0, 10, "%~ Ignored"),
+        (2, 0, 10, "%Synopsis:"), (2, 10, 9, " Synopsis"),
+        (3, 0, 6, "%Note."), (3, 6, 6, "Stuff:"), (3, 12, 5, " Note"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colHidden
+    assert formats[1].foreground().color().getRgb() == colHidden
+    assert formats[1].fontStrikeOut() is True
+    assert formats[2].foreground().color().getRgb() == colMod
+    assert formats[3].foreground().color().getRgb() == colNote
+    assert formats[4].foreground().color().getRgb() == colMod
+    assert formats[5].foreground().color().getRgb() == colValue
+    assert formats[6].foreground().color().getRgb() == colNote
+
+    # Unicode <= 0xFFFF
+    doc.setPlainText(
+        "% Рівнина\n"
+        "%~ Ігноровано\n"
+        "%Synopsis: Синопсис\n"
+        "%Note.Stuff: Примітка\n"
+    )
+    syntax.rehighlight()
+    assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 9, "% Рівнина"),
+        (1, 0, 13, "%~ Ігноровано"),
+        (2, 0, 10, "%Synopsis:"), (2, 10, 9, " Синопсис"),
+        (3, 0, 6, "%Note."), (3, 6, 6, "Stuff:"), (3, 12, 9, " Примітка"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colHidden
+    assert formats[1].foreground().color().getRgb() == colHidden
+    assert formats[1].fontStrikeOut() is True
+    assert formats[2].foreground().color().getRgb() == colMod
+    assert formats[3].foreground().color().getRgb() == colNote
+    assert formats[4].foreground().color().getRgb() == colMod
+    assert formats[5].foreground().color().getRgb() == colValue
+    assert formats[6].foreground().color().getRgb() == colNote
+
+    # Unicode > 0xFFFF
+    doc.setPlainText(
+        "% 😎😎\n"
+        "%~ 🙈🙈🙈\n"
+        "%Synopsis: 😍😍😍😍\n"
+        "%Note.Stuff: 😡😡😡😡😡\n"
+    )
+    syntax.rehighlight()
+    assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 6, "% 😎😎"),
+        (1, 0, 9, "%~ 🙈🙈🙈"),
+        (2, 0, 10, "%Synopsis:"), (2, 10, 9, " 😍😍😍😍"),
+        (3, 0, 6, "%Note."), (3, 6, 6, "Stuff:"), (3, 12, 11, " 😡😡😡😡😡"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colHidden
+    assert formats[1].foreground().color().getRgb() == colHidden
+    assert formats[1].fontStrikeOut() is True
+    assert formats[2].foreground().color().getRgb() == colMod
+    assert formats[3].foreground().color().getRgb() == colNote
+    assert formats[4].foreground().color().getRgb() == colMod
+    assert formats[5].foreground().color().getRgb() == colValue
+    assert formats[6].foreground().color().getRgb() == colNote
+
+
+@pytest.mark.gui
+def testGuiDocHighlighter_Special(syntax):
+    """Test highlighting of special commands."""
+    theme = SHARED.theme
+    doc = syntax.document()
+    assert doc is not None
+
+    # Settings
+    syntax._tHandle = T_HANDLE
+
+    colErr = theme.syntaxTheme.error.getRgb()
+    colCode = theme.syntaxTheme.code.getRgb()
+    colValue = theme.syntaxTheme.val.getRgb()
+
+    # Ascii
+    doc.setPlainText(
+        "[NewPage]\n"
+        "[New Page]\n"
+        "[VSpace]\n"
+        "[VSpace:123]\n"
+        "[VSpace:Meh]\n"
+    )
+    syntax.rehighlight()
+    assert maxOrd(doc.toPlainText()) <= 0x7f
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 9, "[NewPage]"),
+        (1, 0, 10, "[New Page]"),
+        (2, 0, 8, "[VSpace]"),
+        (3, 0, 8, "[VSpace:"), (3, 8, 3, "123"), (3, 11, 1, "]"),
+        (4, 0, 8, "[VSpace:"), (4, 8, 3, "Meh"), (4, 11, 1, "]"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colCode
+    assert formats[1].foreground().color().getRgb() == colCode
+    assert formats[2].foreground().color().getRgb() == colCode
+    assert formats[3].foreground().color().getRgb() == colCode
+    assert formats[4].foreground().color().getRgb() == colValue
+    assert formats[5].foreground().color().getRgb() == colCode
+    assert formats[6].foreground().color().getRgb() == colCode
+    assert formats[7].underlineColor().getRgb() == colErr
+    assert formats[8].foreground().color().getRgb() == colCode
+
+    # Unicode <= 0xFFFF
+    doc.setPlainText(
+        "[NewPage]\n"
+        "[New Page]\n"
+        "[VSpace]\n"
+        "[VSpace:123]\n"
+        "[VSpace:⅘]\n"
+    )
+    syntax.rehighlight()
+    assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 9, "[NewPage]"),
+        (1, 0, 10, "[New Page]"),
+        (2, 0, 8, "[VSpace]"),
+        (3, 0, 8, "[VSpace:"), (3, 8, 3, "123"), (3, 11, 1, "]"),
+        (4, 0, 8, "[VSpace:"), (4, 8, 1, "⅘"), (4, 9, 1, "]"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colCode
+    assert formats[1].foreground().color().getRgb() == colCode
+    assert formats[2].foreground().color().getRgb() == colCode
+    assert formats[3].foreground().color().getRgb() == colCode
+    assert formats[4].foreground().color().getRgb() == colValue
+    assert formats[5].foreground().color().getRgb() == colCode
+    assert formats[6].foreground().color().getRgb() == colCode
+    assert formats[7].underlineColor().getRgb() == colErr
+    assert formats[8].foreground().color().getRgb() == colCode
+
+    # Unicode > 0xFFFF
+    doc.setPlainText(
+        "[NewPage]\n"
+        "[New Page]\n"
+        "[VSpace]\n"
+        "[VSpace:123]\n"
+        "[VSpace:🙈🙈]\n"
+    )
+    syntax.rehighlight()
+    assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 9, "[NewPage]"),
+        (1, 0, 10, "[New Page]"),
+        (2, 0, 8, "[VSpace]"),
+        (3, 0, 8, "[VSpace:"), (3, 8, 3, "123"), (3, 11, 1, "]"),
+        (4, 0, 8, "[VSpace:"), (4, 8, 4, "🙈🙈"), (4, 12, 1, "]"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colCode
+    assert formats[1].foreground().color().getRgb() == colCode
+    assert formats[2].foreground().color().getRgb() == colCode
+    assert formats[3].foreground().color().getRgb() == colCode
+    assert formats[4].foreground().color().getRgb() == colValue
+    assert formats[5].foreground().color().getRgb() == colCode
+    assert formats[6].foreground().color().getRgb() == colCode
+    assert formats[7].underlineColor().getRgb() == colErr
+    assert formats[8].foreground().color().getRgb() == colCode
+
+
+@pytest.mark.gui
+def testGuiDocHighlighter_Text(monkeypatch, syntax):
+    """Test highlighting of text."""
+    theme = SHARED.theme
+    doc = syntax.document()
+    assert doc is not None
+
+    # Settings
+    syntax._tHandle = T_HANDLE
+    syntax._isNovel = True
+    syntax.setSpellCheck(True)
+    monkeypatch.setattr(SHARED.spelling, "checkWord", lambda *a: False)
+
+    colHidden = theme.syntaxTheme.hidden.getRgb()
+    colEmph = theme.syntaxTheme.emph.getRgb()
+    colLink = theme.syntaxTheme.link.getRgb()
+    colSpell = theme.syntaxTheme.spell.getRgb()
+    colCode = theme.syntaxTheme.code.getRgb()
+    colDialogue = theme.syntaxTheme.dialN.getRgb()
+    colAltDialogue = theme.syntaxTheme.dialA.getRgb()
+
+    # Ascii
+    doc.setPlainText(
+        "Text **bold** text _italic_ text ~~strike~~ text [b]bold[/b], http://example.com\n\n"
+    )
+    syntax.rehighlight()
+    assert maxOrd(doc.toPlainText()) <= 0x7f
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 4, "Text"),
+        (0, 5, 2, "**"), (0, 7, 4, "bold"), (0, 11, 2, "**"),
+        (0, 14, 4, "text"),
+        (0, 19, 1, "_"), (0, 20, 6, "italic"), (0, 26, 1, "_"),
+        (0, 28, 4, "text"),
+        (0, 33, 2, "~~"), (0, 35, 6, "strike"), (0, 41, 2, "~~"),
+        (0, 44, 4, "text"),
+        (0, 49, 3, "[b]"), (0, 52, 4, "bold"), (0, 56, 4, "[/b]"),
+        (0, 62, 18, "http://example.com"),
+    ]
+    assert formats[0].underlineColor().getRgb() == colSpell  # Text
+    assert formats[1].foreground().color().getRgb() == colHidden  # **
+    assert formats[2].foreground().color().getRgb() == colEmph  # bold
+    assert formats[2].underlineColor().getRgb() == colSpell
+    assert formats[3].foreground().color().getRgb() == colHidden  # **
+    assert formats[4].underlineColor().getRgb() == colSpell  # text
+    assert formats[5].foreground().color().getRgb() == colHidden  # _
+    assert formats[6].foreground().color().getRgb() == colEmph  # italic
+    assert formats[6].underlineColor().getRgb() == colSpell
+    assert formats[7].foreground().color().getRgb() == colHidden  # _
+    assert formats[8].underlineColor().getRgb() == colSpell  # text
+    assert formats[9].foreground().color().getRgb() == colHidden  # ~~
+    assert formats[10].foreground().color().getRgb() == colHidden  # strike
+    assert formats[10].fontStrikeOut() is True
+    assert formats[10].underlineColor().getRgb() == colSpell
+    assert formats[11].foreground().color().getRgb() == colHidden  # ~~
+    assert formats[12].underlineColor().getRgb() == colSpell  # text
+    assert formats[13].foreground().color().getRgb() == colCode  # [b]
+    assert formats[14].underlineColor().getRgb() == colSpell  # bold
+    assert formats[15].foreground().color().getRgb() == colCode  # [/b]
+    assert formats[16].foreground().color().getRgb() == colLink  # http://example.com
+
+    # Spell Check
+    data = doc.findBlockByNumber(0).userData()
+    assert isinstance(data, TextBlockData)
+    assert data.metaData == [(62, 80, "http://example.com", "url")]
+    assert data.spellErrors == [
+        (0, 4, "Text"), (7, 11, "bold"),
+        (14, 18, "text"), (20, 26, "italic"),
+        (28, 32, "text"), (35, 41, "strike"),
+        (44, 48, "text"), (52, 56, "bold"),
+    ]
+
+    # Unicode <= 0xFFFF
+    doc.setPlainText(
+        "\u201cDialogue,\u201d and then ::dialogue::, http://example.com\n\n"
+    )
+    syntax.rehighlight()
+    assert 0x7f < maxOrd(doc.toPlainText()) <= 0xffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 1, "\u201c"), (0, 1, 8, "Dialogue"), (0, 9, 2, ",\u201d"),
+        (0, 12, 3, "and"), (0, 16, 4, "then"),
+        (0, 21, 2, "::"), (0, 23, 8, "dialogue"), (0, 31, 2, "::"),
+        (0, 35, 18, "http://example.com"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colDialogue  # Quote
+    assert formats[1].foreground().color().getRgb() == colDialogue  # Dialogue
+    assert formats[1].underlineColor().getRgb() == colSpell
+    assert formats[2].foreground().color().getRgb() == colDialogue  # Quote
+    assert formats[3].underlineColor().getRgb() == colSpell  # and
+    assert formats[4].underlineColor().getRgb() == colSpell  # then
+    assert formats[5].foreground().color().getRgb() == colAltDialogue  # ::
+    assert formats[6].foreground().color().getRgb() == colAltDialogue  # dialogue
+    assert formats[6].underlineColor().getRgb() == colSpell
+    assert formats[7].foreground().color().getRgb() == colAltDialogue  # ::
+    assert formats[8].foreground().color().getRgb() == colLink  # http://example.com
+
+    # Spell Check
+    data = doc.findBlockByNumber(0).userData()
+    assert isinstance(data, TextBlockData)
+    assert data.metaData == [(35, 53, "http://example.com", "url")]
+    assert data.spellErrors == [
+        (1, 9, "Dialogue"), (12, 15, "and"),
+        (16, 20, "then"), (23, 31, "dialogue"),
+    ]
+
+    # Unicode > 0xFFFF
+    doc.setPlainText(
+        "\u201c😁 Grinning 😁,\u201d and then ::🙊 shush 🙊::, http://example.com\n\n"
+    )
+    syntax.rehighlight()
+    assert 0xffff < maxOrd(doc.toPlainText()) <= 0xffffffff
+
+    pieces, formats = getFragments(syntax)
+    assert pieces == [
+        (0, 0, 4, "\u201c😁 "), (0, 4, 8, "Grinning"), (0, 12, 5, " 😁,\u201d"),
+        (0, 18, 3, "and"), (0, 22, 4, "then"),
+        (0, 27, 5, "::🙊 "), (0, 32, 5, "shush"), (0, 37, 5, " 🙊::"),
+        (0, 44, 18, "http://example.com"),
+    ]
+    assert formats[0].foreground().color().getRgb() == colDialogue  # Quote😁
+    assert formats[1].foreground().color().getRgb() == colDialogue  # Grinning
+    assert formats[1].underlineColor().getRgb() == colSpell
+    assert formats[2].foreground().color().getRgb() == colDialogue  # 😁Quote
+    assert formats[3].underlineColor().getRgb() == colSpell  # and
+    assert formats[4].underlineColor().getRgb() == colSpell  # then
+    assert formats[5].foreground().color().getRgb() == colAltDialogue  # ::🙊
+    assert formats[6].foreground().color().getRgb() == colAltDialogue  # shush
+    assert formats[6].underlineColor().getRgb() == colSpell
+    assert formats[7].foreground().color().getRgb() == colAltDialogue  # 🙊::
+    assert formats[8].foreground().color().getRgb() == colLink  # http://example.com
+
+    # Spell Check
+    data = doc.findBlockByNumber(0).userData()
+    assert isinstance(data, TextBlockData)
+    assert data.metaData == [(40, 58, "http://example.com", "url")]
+    assert data.spellErrors == [
+        (4, 12, "Grinning"), (18, 21, "and"),
+        (22, 26, "then"), (32, 37, "shush"),
+    ]