Add horizontal bar to em dash conversion in tokenizer

2024-11-04 17:32:22 +01:00
parent 89f452f3fc
commit e4a2ca2d19
2 changed files with 7 additions and 4 deletions
@@ -493,9 +493,11 @@ class Tokenizer(ABC):
            xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
            self._text = xRep.sub(lambda x: repDict[x.group(0)], self._text)

-        # Process the character translation map
-        trDict = {nwUnicode.U_MAPOS: nwUnicode.U_RSQUO}
-        self._text = self._text.translate(str.maketrans(trDict))
+        # Process the translation map for placeholder characters
+        self._text = self._text.translate(str.maketrans({
+            nwUnicode.U_MAPOS: nwUnicode.U_RSQUO,
+            nwUnicode.U_HBAR: nwUnicode.U_EMDASH,
+        }))

        return