Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix issue of alternative unicode symbols being replaced too early

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-11-13 00:03:38 +01:00
parent 1bd8370864
commit c2b3eb4ac2
2 changed files with 23 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/constants.py
+5
View File
@@ -542,6 +542,11 @@ class nwUnicode:
U_LTRI = "\u25c0" # Left-pointing triangle U_LTRI = "\u25c0" # Left-pointing triangle
U_LTRIS = "\u25c2" # Left-pointing triangle, small U_LTRIS = "\u25c2" # Left-pointing triangle, small
# Special
U_UNKN = "\ufffd" # Unknown character
U_NAC1 = "\ufffe" # Not a character
U_NAC2 = "\uffff" # Not a character
# HTML Equivalents # HTML Equivalents
# ================ # ================
novelwriter/formats/tokenizer.py
+18 -14
View File
@@ -490,22 +490,14 @@ class Tokenizer(ABC):
return return
def doPreProcessing(self) -> None: def doPreProcessing(self) -> None:
"""Run trough the various replace dictionaries.""" """Run pre-processing jobs before the text is tokenized."""
# Process the user's auto-replace dictionary # Process the user's auto-replace dictionary
autoReplace = self._project.data.autoReplace if autoReplace := self._project.data.autoReplace:
if len(autoReplace) > 0:
repDict = {} repDict = {}
for aKey, aVal in autoReplace.items(): for aKey, aVal in autoReplace.items():
repDict[f"<{aKey}>"] = aVal repDict[f"<{aKey}>"] = aVal
xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL) xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
self._text = xRep.sub(lambda x: repDict[x.group(0)], self._text) self._text = xRep.sub(lambda x: repDict[x.group(0)], self._text)
# Process the translation map for placeholder characters
self._text = self._text.translate(str.maketrans({
nwUnicode.U_MAPOS: nwUnicode.U_RSQUO,
nwUnicode.U_HBAR: nwUnicode.U_EMDASH,
}))
return return
def tokenizeText(self) -> None: def tokenizeText(self) -> None:
@@ -538,13 +530,25 @@ class Tokenizer(ABC):
firstIndent = self._firstIndent firstIndent = self._firstIndent
# Replace all instances of [br] with a placeholder character # Replace all instances of [br] with a placeholder character
text = REGEX_PATTERNS.lineBreak.sub("\uffff", self._text) text = REGEX_PATTERNS.lineBreak.sub(nwUnicode.U_NAC2, self._text)
# Translation Maps
transMapA = str.maketrans({
nwUnicode.U_NAC2: "", # Used when [br] is ignored
nwUnicode.U_MAPOS: nwUnicode.U_RSQUO,
nwUnicode.U_HBAR: nwUnicode.U_EMDASH,
})
transMapB = str.maketrans({
nwUnicode.U_NAC2: "\n", # Used when [br] is not ignored
nwUnicode.U_MAPOS: nwUnicode.U_RSQUO,
nwUnicode.U_HBAR: nwUnicode.U_EMDASH,
})
nHead = 0 nHead = 0
tHandle = self._handle or "" tHandle = self._handle or ""
tBlocks: list[T_Block] = [B_EMPTY] tBlocks: list[T_Block] = [B_EMPTY]
for bLine in text.splitlines(): for bLine in text.splitlines():
aLine = bLine.replace("\uffff", "") # Remove placeholder characters aLine = bLine.translate(transMapA)
sLine = aLine.strip().lower() sLine = aLine.strip().lower()
# Check for blank lines # Check for blank lines
@@ -884,7 +888,7 @@ class Tokenizer(ABC):
if doJustify and not cStyle & BlockFmt.ALIGNED: if doJustify and not cStyle & BlockFmt.ALIGNED:
cStyle |= BlockFmt.JUSTIFY cStyle |= BlockFmt.JUSTIFY
pTxt = pLines[0][2].replace("\uffff", "\n") pTxt = pLines[0][2].translate(transMapB)
sBlocks.append(( sBlocks.append((
BlockTyp.TEXT, pLines[0][1], pTxt, pLines[0][3], cStyle BlockTyp.TEXT, pLines[0][1], pTxt, pLines[0][3], cStyle
)) ))
@@ -901,7 +905,7 @@ class Tokenizer(ABC):
tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3]) tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3])
cStyle |= aBlock[4] cStyle |= aBlock[4]
pTxt = tTxt[:-1].replace("\uffff", "\n") pTxt = tTxt[:-1].translate(transMapB)
sBlocks.append(( sBlocks.append((
BlockTyp.TEXT, pLines[0][1], pTxt, tFmt, cStyle BlockTyp.TEXT, pLines[0][1], pTxt, tFmt, cStyle
)) ))