Change the way Unicode is handled in HTML, and clean up the class a bit

nw/constants/__init__.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 from nw.constants.iso import isoLanguage, isoCountry
 from nw.constants.constants import (
-    nwConst, nwLists, nwRegEx, nwFiles, nwKeyWords, nwLabels, nwQuotes, nwUnicode
+    nwConst, nwLists, nwRegEx, nwFiles, nwKeyWords, nwLabels, nwQuotes,
+    nwUnicode, nwHtmlUnicode
 )
 from nw.constants.enum import (
     nwAlert, nwDocAction, nwItemClass, nwItemLayout, nwItemType, nwOutline,
@@ -19,6 +20,7 @@ __all__ = [
     "nwLabels",
     "nwQuotes",
     "nwUnicode",
+    "nwHtmlUnicode",
     "nwAlert",
     "nwDocAction",
     "nwItemClass",

nw/constants/constants.py

@@ -265,7 +265,7 @@ class nwUnicode:
     U_LCQUO  = "\u300c" # Left corner bracket
     U_RCQUO  = "\u300d" # Right corner bracket
     U_LWCQUO = "\u300e" # Left white corner bracket
-    U_RECQUO = "\u300f" # Right white corner bracket
+    U_RWCQUO = "\u300f" # Right white corner bracket
 
     ## Punctuation
     U_FGDASH = "\u2012" # Figure dash
@@ -331,7 +331,7 @@ class nwUnicode:
     H_LCQUO  = "「"
     H_RCQUO  = "」"
     H_LWCQUO = "『"
-    H_LWCQUO = "『"
+    H_RWCQUO = "』"
 
     ## Punctuation
     H_FGDASH = "‒"
@@ -374,3 +374,60 @@ class nwUnicode:
     H_LTRIS  = "◂"
 
 # END Class nwUnicode
+
+class nwHtmlUnicode():
+
+    U_TO_H = {
+        ## Quotes
+        nwUnicode.U_QUOT   : nwUnicode.H_QUOT,
+        nwUnicode.U_APOS   : nwUnicode.H_APOS,
+        nwUnicode.U_LAQUO  : nwUnicode.H_LAQUO,
+        nwUnicode.U_RAQUO  : nwUnicode.H_RAQUO,
+        nwUnicode.U_LSQUO  : nwUnicode.H_LSQUO,
+        nwUnicode.U_RSQUO  : nwUnicode.H_RSQUO,
+        nwUnicode.U_SBQUO  : nwUnicode.H_SBQUO,
+        nwUnicode.U_SUQUO  : nwUnicode.H_SUQUO,
+        nwUnicode.U_LDQUO  : nwUnicode.H_LDQUO,
+        nwUnicode.U_RDQUO  : nwUnicode.H_RDQUO,
+        nwUnicode.U_BDQUO  : nwUnicode.H_BDQUO,
+        nwUnicode.U_UDQUO  : nwUnicode.H_UDQUO,
+        nwUnicode.U_LSAQUO : nwUnicode.H_LSAQUO,
+        nwUnicode.U_RSAQUO : nwUnicode.H_RSAQUO,
+        nwUnicode.U_BDRQUO : nwUnicode.H_BDRQUO,
+        nwUnicode.U_LCQUO  : nwUnicode.H_LCQUO,
+        nwUnicode.U_RCQUO  : nwUnicode.H_RCQUO,
+        nwUnicode.U_LWCQUO : nwUnicode.H_LWCQUO,
+        nwUnicode.U_RWCQUO : nwUnicode.H_RWCQUO,
+
+        ## Punctuation
+        nwUnicode.U_FGDASH : nwUnicode.H_FGDASH,
+        nwUnicode.U_ENDASH : nwUnicode.H_ENDASH,
+        nwUnicode.U_EMDASH : nwUnicode.H_EMDASH,
+        nwUnicode.U_HBAR   : nwUnicode.H_HBAR,
+        nwUnicode.U_HELLIP : nwUnicode.H_HELLIP,
+        nwUnicode.U_MAPOSS : nwUnicode.H_MAPOSS,
+        nwUnicode.U_PRIME  : nwUnicode.H_PRIME,
+        nwUnicode.U_DPRIME : nwUnicode.H_DPRIME,
+
+        ## Spaces
+        nwUnicode.U_NBSP   : nwUnicode.H_NBSP,
+        nwUnicode.U_THSP   : nwUnicode.H_THSP,
+        nwUnicode.U_THNBSP : nwUnicode.H_THNBSP,
+        nwUnicode.U_ENSP   : nwUnicode.H_ENSP,
+        nwUnicode.U_EMSP   : nwUnicode.H_EMSP,
+
+        ## Symbols
+        nwUnicode.U_CHECK  : nwUnicode.H_CHECK,
+        nwUnicode.U_CROSS  : nwUnicode.H_CROSS,
+        nwUnicode.U_BULL   : nwUnicode.H_BULL,
+        nwUnicode.U_TRBULL : nwUnicode.H_TRBULL,
+        nwUnicode.U_HYBULL : nwUnicode.H_HYBULL,
+        nwUnicode.U_FLOWER : nwUnicode.H_FLOWER,
+        nwUnicode.U_PERMIL : nwUnicode.H_PERMIL,
+        nwUnicode.U_DEGREE : nwUnicode.H_DEGREE,
+        nwUnicode.U_MINUS  : nwUnicode.H_MINUS,
+        nwUnicode.U_TIMES  : nwUnicode.H_TIMES,
+        nwUnicode.U_DIVIDE : nwUnicode.H_DIVIDE,
+    }
+
+# END Class nwHtmlUnicode

nw/core/tohtml.py

@@ -25,10 +25,9 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
 
 import logging
-import re
 
 from nw.core.tokenizer import Tokenizer
-from nw.constants import nwUnicode, nwLabels, nwKeyWords
+from nw.constants import nwLabels, nwKeyWords, nwHtmlUnicode
 
 logger = logging.getLogger(__name__)
 
@@ -41,27 +40,13 @@ class ToHtml(Tokenizer):
     def __init__(self, theProject, theParent):
         Tokenizer.__init__(self, theProject, theParent)
 
-        self.genMode = self.M_EXPORT
+        self.genMode   = self.M_EXPORT
         self.cssStyles = True
+        self.fullHTML  = []
 
-        self.repDict = {
+        # Internals
-            "<" : "&lt;",
+        self._trMap = {}
-            ">" : "&gt;",
+        self.setReplaceUnicode(False)
-            "&" : "&amp;",
-            nwUnicode.U_ENDASH : nwUnicode.H_ENDASH,
-            nwUnicode.U_EMDASH : nwUnicode.H_EMDASH,
-            nwUnicode.U_HELLIP : nwUnicode.H_HELLIP,
-            nwUnicode.U_NBSP   : nwUnicode.H_NBSP,
-            nwUnicode.U_THSP   : nwUnicode.H_THSP,
-            nwUnicode.U_THNBSP : nwUnicode.H_THNBSP,
-            nwUnicode.U_MAPOSS : nwUnicode.H_RSQUO,
-        }
-        self.revDict = {}
-        self.reReplace = []
-        self.reReverse = []
-        self._buildRegEx()
-
-        self.fullHTML = []
 
         return
 
@@ -88,6 +73,23 @@ class ToHtml(Tokenizer):
         self.cssStyles = cssStyles
         return
 
+    def setReplaceUnicode(self, doReplace):
+        """Set the translation map to either minimal or full unicode to
+        html entities replacement.
+        """
+        # Control characters must always be replaced
+        self._trMap = str.maketrans({
+            "<" : "&lt;",
+            ">" : "&gt;",
+            "&" : "&amp;",
+        })
+
+        if doReplace:
+            # Extend to all relevant Unicode characters
+            self._trMap.update(str.maketrans(nwHtmlUnicode.U_TO_H))
+
+        return
+
     ##
     #  Class Methods
     ##
@@ -97,30 +99,12 @@ class ToHtml(Tokenizer):
         """
         return sum([len(x) for x in self.fullHTML])
 
-    def doAutoReplace(self):
+    def doPreProcessing(self):
         """Extend the auto-replace to also properly encode some unicode
         characters into their respective HTML entities.
         """
-        Tokenizer.doAutoReplace(self)
+        Tokenizer.doPreProcessing(self)
-        self.theText = self.reReplace.sub(
+        self.theText = self.theText.translate(self._trMap)
-            lambda x: self.repDict[x.group(0)], self.theText
-        )
-        return
-
-    def doPostProcessing(self):
-        """Reverse the html entities replacement on the markdown text.
-        Otherwise, all the &something; bits will also be in there.
-        """
-        Tokenizer.doPostProcessing(self)
-        if self.genMode == self.M_PREVIEW:
-            # Doesn't matter for preview as we don't use the markdown
-            return
-
-        if self.keepMarkdown:
-            self.theMarkdown[-1] = self.reReverse.sub(
-                lambda x: self.revDict[x.group(0)], self.theMarkdown[-1]
-            )
-
         return
 
     def doConvert(self):
@@ -466,16 +450,4 @@ class ToHtml(Tokenizer):
 
         return retText
 
-    def _buildRegEx(self):
-        """Build the regular expressions
-        """
-        self.revDict = dict(map(reversed, self.repDict.items()))
-        self.reReplace = re.compile(
-            "|".join([re.escape(k) for k in self.repDict.keys()]), flags=re.DOTALL
-        )
-        self.reReverse = re.compile(
-            "|".join([re.escape(k) for k in self.revDict.keys()]), flags=re.DOTALL
-        )
-        return
-
 # END Class ToHtml

nw/core/tokenizer.py

@@ -32,7 +32,7 @@ from PyQt5.QtCore import QRegularExpression
 
 from nw.core.document import NWDoc
 from nw.core.tools import numberToWord, numberToRoman
-from nw.constants import nwConst, nwItemLayout, nwItemType, nwRegEx
+from nw.constants import nwConst, nwUnicode, nwItemLayout, nwItemType, nwRegEx
 
 logger = logging.getLogger(__name__)
 
@@ -86,7 +86,7 @@ class Tokenizer():
         self.theResult   = ""   # The result of the last document
 
         self.keepMarkdown = False # Whether to keep the markdown text
-        self.theMarkdown  = []   # The result novelWriter markdown of all documents
+        self.theMarkdown  = []    # The result novelWriter markdown of all documents
 
         # User Settings
         self.textFont    = "Serif" # Output text font
@@ -297,9 +297,10 @@ class Tokenizer():
 
         return True
 
-    def doAutoReplace(self):
+    def doPreProcessing(self):
-        """Run through the user's auto-replace dictionary.
+        """Reun trough the various replace doctionaries.
         """
+        # Process the user's auto-replace dictionary
         if len(self.theProject.autoReplace) > 0:
             repDict = {}
             for aKey, aVal in self.theProject.autoReplace.items():
@@ -307,6 +308,10 @@ class Tokenizer():
             xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
             self.theText = xRep.sub(lambda x: repDict[x.group(0)], self.theText)
 
+        # Process the character translation map
+        trDict = {nwUnicode.U_MAPOSS: nwUnicode.U_RSQUO}
+        self.theText = self.theText.translate(str.maketrans(trDict))
+
         return
 
     def doPostProcessing(self):

nw/gui/docviewer.py

@@ -176,7 +176,7 @@ class GuiDocViewer(QTextBrowser):
         # See issue #298
         try:
             aDoc.setText(tHandle)
-            aDoc.doAutoReplace()
+            aDoc.doPreProcessing()
             aDoc.tokenizeText()
             aDoc.doConvert()
             aDoc.doPostProcessing()