From 57bed18ccf259827ed65c8d664aef2229b1deffa Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Sat, 5 Jul 2025 01:50:47 +0200
Subject: [PATCH] Make dialog parser and keyword lines work with 4 byte unicode

---
 novelwriter/common.py           | 15 +++++++++++++++
 novelwriter/gui/dochighlight.py | 31 ++++++++++++++++++++++++-------
 novelwriter/text/patterns.py    |  8 ++++++--
 3 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/novelwriter/common.py b/novelwriter/common.py
index a4e37f62..83133f69 100644
--- a/novelwriter/common.py
+++ b/novelwriter/common.py
@@ -493,6 +493,21 @@ def decodeMimeHandles(mimeData: QMimeData) -> list[str]:
     return mimeData.data(nwConst.MIME_HANDLE).data().decode().split("|")
 
 
+def utf16CharMap(text: str) -> list[int]:
+    """Compute mapping from Python string index to QString index.
+    Python strings are always one character per position in either
+    ASCII, UCS-2 or UCS-4. QStrings are in UTF-16, so wide characters
+    use 2 indices, and thus creates an offset.
+    """
+    posMap = list(range(0, len(text) + 1))
+    offset = 0
+    for i, c in enumerate(text):
+        if ord(c) > 0xffff:
+            offset += 1
+        posMap[i + 1] = i + 1 + offset
+    return posMap
+
+
 ##
 #  Encoder Functions
 ##
diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py
index beb006c1..ce0b9c8b 100644
--- a/novelwriter/gui/dochighlight.py
+++ b/novelwriter/gui/dochighlight.py
@@ -35,7 +35,7 @@ from PyQt6.QtGui import (
 )
 
 from novelwriter import CONFIG, SHARED
-from novelwriter.common import checkInt
+from novelwriter.common import checkInt, utf16CharMap
 from novelwriter.constants import nwStyles, nwUnicode
 from novelwriter.enum import nwComment
 from novelwriter.text.comments import processComment
@@ -301,6 +301,8 @@ class GuiDocHighlighter(QSyntaxHighlighter):
             return
 
         bLen = self.currentBlock().length()
+        isWide = bLen > len(text) + 1
+
         xOff = 0
         hRules = None
         if text.startswith("@"):  # Keywords and commands
@@ -309,17 +311,32 @@ class GuiDocHighlighter(QSyntaxHighlighter):
             isValid, bits, pos = index.scanThis(text)
             isGood = index.checkThese(bits, self._tHandle)
             if isValid:
+                posMap = []
+                if isWide:
+                    posMap = utf16CharMap(text)
                 for n, bit in enumerate(bits):
-                    xPos = pos[n]
-                    xLen = len(bit)
+                    if posMap:
+                        xPos = posMap[pos[n]]
+                        xLen = posMap[pos[n] + len(bit)] - xPos
+                    else:
+                        xPos = pos[n]
+                        xLen = len(bit)
                     if n == 0 and isGood[n]:
                         self.setFormat(xPos, xLen, self._hStyles["keyword"])
                     elif isGood[n] and not self._isInactive:
                         one, two = index.parseValue(bit)
-                        self.setFormat(xPos, len(one), self._hStyles["tag"])
+                        if posMap:
+                            oLen = posMap[pos[n] + len(one)] - xPos
+                        else:
+                            oLen = len(one)
+                        self.setFormat(xPos, oLen, self._hStyles["tag"])
                         if two:
-                            yPos = xPos + len(bit) - len(two)
-                            self.setFormat(yPos, len(two), self._hStyles["optional"])
+                            if posMap:
+                                yLen = posMap[pos[n] + len(two)] - xPos
+                            else:
+                                yLen = len(two)
+                            yPos = xPos + xLen - yLen
+                            self.setFormat(yPos, yLen, self._hStyles["optional"])
                     elif not self._isInactive:
                         self.setFormat(xPos, xLen, self._hStyles["invalid"])
 
@@ -399,7 +416,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
             self.setCurrentBlockState(BLOCK_TEXT)
             hRules = self._txtRules if self._isNovel else self._minRules
             if self._isNovel and self._dialogParser.enabled:
-                for pos, end in self._dialogParser(text):
+                for pos, end in self._dialogParser(text, isWide):
                     length = end - pos
                     self.setFormat(pos, length, self._hStyles["dialog"])
 
diff --git a/novelwriter/text/patterns.py b/novelwriter/text/patterns.py
index e0231d78..2391dcb0 100644
--- a/novelwriter/text/patterns.py
+++ b/novelwriter/text/patterns.py
@@ -27,7 +27,7 @@ from __future__ import annotations
 import re
 
 from novelwriter import CONFIG
-from novelwriter.common import compact, uniqueCompact
+from novelwriter.common import compact, uniqueCompact, utf16CharMap
 from novelwriter.constants import nwRegEx, nwUnicode
 
 
@@ -170,7 +170,7 @@ class DialogParser:
 
         return
 
-    def __call__(self, text: str) -> list[tuple[int, int]]:
+    def __call__(self, text: str, wideChar: bool = False) -> list[tuple[int, int]]:
         """Caller wrapper for dialogue processing."""
         temp: list[int] = []
         result: list[tuple[int, int]] = []
@@ -218,4 +218,8 @@ class DialogParser:
                         result.append((start, pos))
                         start = None
 
+                if wideChar:
+                    posMap = utf16CharMap(text)
+                    result = [(posMap[s], posMap[p]) for s, p in result]
+
         return result