Make dialog parser and keyword lines work with 4 byte unicode

novelwriter/common.py

@@ -493,6 +493,21 @@ def decodeMimeHandles(mimeData: QMimeData) -> list[str]:
     return mimeData.data(nwConst.MIME_HANDLE).data().decode().split("|")
 
 
+def utf16CharMap(text: str) -> list[int]:
+    """Compute mapping from Python string index to QString index.
+    Python strings are always one character per position in either
+    ASCII, UCS-2 or UCS-4. QStrings are in UTF-16, so wide characters
+    use 2 indices, and thus creates an offset.
+    """
+    posMap = list(range(0, len(text) + 1))
+    offset = 0
+    for i, c in enumerate(text):
+        if ord(c) > 0xffff:
+            offset += 1
+        posMap[i + 1] = i + 1 + offset
+    return posMap
+
+
 ##
 #  Encoder Functions
 ##

novelwriter/gui/dochighlight.py

@@ -35,7 +35,7 @@ from PyQt6.QtGui import (
 )
 
 from novelwriter import CONFIG, SHARED
-from novelwriter.common import checkInt
+from novelwriter.common import checkInt, utf16CharMap
 from novelwriter.constants import nwStyles, nwUnicode
 from novelwriter.enum import nwComment
 from novelwriter.text.comments import processComment
@@ -301,6 +301,8 @@ class GuiDocHighlighter(QSyntaxHighlighter):
             return
 
         bLen = self.currentBlock().length()
+        isWide = bLen > len(text) + 1
+
         xOff = 0
         hRules = None
         if text.startswith("@"):  # Keywords and commands
@@ -309,17 +311,32 @@ class GuiDocHighlighter(QSyntaxHighlighter):
             isValid, bits, pos = index.scanThis(text)
             isGood = index.checkThese(bits, self._tHandle)
             if isValid:
+                posMap = []
+                if isWide:
+                    posMap = utf16CharMap(text)
                 for n, bit in enumerate(bits):
-                    xPos = pos[n]
+                    if posMap:
-                    xLen = len(bit)
+                        xPos = posMap[pos[n]]
+                        xLen = posMap[pos[n] + len(bit)] - xPos
+                    else:
+                        xPos = pos[n]
+                        xLen = len(bit)
                     if n == 0 and isGood[n]:
                         self.setFormat(xPos, xLen, self._hStyles["keyword"])
                     elif isGood[n] and not self._isInactive:
                         one, two = index.parseValue(bit)
-                        self.setFormat(xPos, len(one), self._hStyles["tag"])
+                        if posMap:
+                            oLen = posMap[pos[n] + len(one)] - xPos
+                        else:
+                            oLen = len(one)
+                        self.setFormat(xPos, oLen, self._hStyles["tag"])
                         if two:
-                            yPos = xPos + len(bit) - len(two)
+                            if posMap:
-                            self.setFormat(yPos, len(two), self._hStyles["optional"])
+                                yLen = posMap[pos[n] + len(two)] - xPos
+                            else:
+                                yLen = len(two)
+                            yPos = xPos + xLen - yLen
+                            self.setFormat(yPos, yLen, self._hStyles["optional"])
                     elif not self._isInactive:
                         self.setFormat(xPos, xLen, self._hStyles["invalid"])
 
@@ -399,7 +416,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
             self.setCurrentBlockState(BLOCK_TEXT)
             hRules = self._txtRules if self._isNovel else self._minRules
             if self._isNovel and self._dialogParser.enabled:
-                for pos, end in self._dialogParser(text):
+                for pos, end in self._dialogParser(text, isWide):
                     length = end - pos
                     self.setFormat(pos, length, self._hStyles["dialog"])
 

novelwriter/text/patterns.py

@@ -27,7 +27,7 @@ from __future__ import annotations
 import re
 
 from novelwriter import CONFIG
-from novelwriter.common import compact, uniqueCompact
+from novelwriter.common import compact, uniqueCompact, utf16CharMap
 from novelwriter.constants import nwRegEx, nwUnicode
 
 
@@ -170,7 +170,7 @@ class DialogParser:
 
         return
 
-    def __call__(self, text: str) -> list[tuple[int, int]]:
+    def __call__(self, text: str, wideChar: bool = False) -> list[tuple[int, int]]:
         """Caller wrapper for dialogue processing."""
         temp: list[int] = []
         result: list[tuple[int, int]] = []
@@ -218,4 +218,8 @@ class DialogParser:
                         result.append((start, pos))
                         start = None
 
+                if wideChar:
+                    posMap = utf16CharMap(text)
+                    result = [(posMap[s], posMap[p]) for s, p in result]
+
         return result