diff --git a/nw/constants/__init__.py b/nw/constants/__init__.py index b58b56c3..1a6748a1 100644 --- a/nw/constants/__init__.py +++ b/nw/constants/__init__.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- from nw.constants.iso import isoLanguage, isoCountry from nw.constants.constants import ( - nwConst, nwLists, nwRegEx, nwFiles, nwKeyWords, nwLabels, nwQuotes, nwUnicode + nwConst, nwLists, nwRegEx, nwFiles, nwKeyWords, nwLabels, nwQuotes, + nwUnicode, nwHtmlUnicode ) from nw.constants.enum import ( nwAlert, nwDocAction, nwItemClass, nwItemLayout, nwItemType, nwOutline, @@ -19,6 +20,7 @@ __all__ = [ "nwLabels", "nwQuotes", "nwUnicode", + "nwHtmlUnicode", "nwAlert", "nwDocAction", "nwItemClass", diff --git a/nw/constants/constants.py b/nw/constants/constants.py index 5e4581b9..0909b1f2 100644 --- a/nw/constants/constants.py +++ b/nw/constants/constants.py @@ -265,7 +265,7 @@ class nwUnicode: U_LCQUO = "\u300c" # Left corner bracket U_RCQUO = "\u300d" # Right corner bracket U_LWCQUO = "\u300e" # Left white corner bracket - U_RECQUO = "\u300f" # Right white corner bracket + U_RWCQUO = "\u300f" # Right white corner bracket ## Punctuation U_FGDASH = "\u2012" # Figure dash @@ -331,7 +331,7 @@ class nwUnicode: H_LCQUO = "「" H_RCQUO = "」" H_LWCQUO = "『" - H_LWCQUO = "『" + H_RWCQUO = "』" ## Punctuation H_FGDASH = "‒" @@ -374,3 +374,60 @@ class nwUnicode: H_LTRIS = "◂" # END Class nwUnicode + +class nwHtmlUnicode(): + + U_TO_H = { + ## Quotes + nwUnicode.U_QUOT : nwUnicode.H_QUOT, + nwUnicode.U_APOS : nwUnicode.H_APOS, + nwUnicode.U_LAQUO : nwUnicode.H_LAQUO, + nwUnicode.U_RAQUO : nwUnicode.H_RAQUO, + nwUnicode.U_LSQUO : nwUnicode.H_LSQUO, + nwUnicode.U_RSQUO : nwUnicode.H_RSQUO, + nwUnicode.U_SBQUO : nwUnicode.H_SBQUO, + nwUnicode.U_SUQUO : nwUnicode.H_SUQUO, + nwUnicode.U_LDQUO : nwUnicode.H_LDQUO, + nwUnicode.U_RDQUO : nwUnicode.H_RDQUO, + nwUnicode.U_BDQUO : nwUnicode.H_BDQUO, + nwUnicode.U_UDQUO : nwUnicode.H_UDQUO, + nwUnicode.U_LSAQUO : nwUnicode.H_LSAQUO, + nwUnicode.U_RSAQUO : nwUnicode.H_RSAQUO, + nwUnicode.U_BDRQUO : nwUnicode.H_BDRQUO, + nwUnicode.U_LCQUO : nwUnicode.H_LCQUO, + nwUnicode.U_RCQUO : nwUnicode.H_RCQUO, + nwUnicode.U_LWCQUO : nwUnicode.H_LWCQUO, + nwUnicode.U_RWCQUO : nwUnicode.H_RWCQUO, + + ## Punctuation + nwUnicode.U_FGDASH : nwUnicode.H_FGDASH, + nwUnicode.U_ENDASH : nwUnicode.H_ENDASH, + nwUnicode.U_EMDASH : nwUnicode.H_EMDASH, + nwUnicode.U_HBAR : nwUnicode.H_HBAR, + nwUnicode.U_HELLIP : nwUnicode.H_HELLIP, + nwUnicode.U_MAPOSS : nwUnicode.H_MAPOSS, + nwUnicode.U_PRIME : nwUnicode.H_PRIME, + nwUnicode.U_DPRIME : nwUnicode.H_DPRIME, + + ## Spaces + nwUnicode.U_NBSP : nwUnicode.H_NBSP, + nwUnicode.U_THSP : nwUnicode.H_THSP, + nwUnicode.U_THNBSP : nwUnicode.H_THNBSP, + nwUnicode.U_ENSP : nwUnicode.H_ENSP, + nwUnicode.U_EMSP : nwUnicode.H_EMSP, + + ## Symbols + nwUnicode.U_CHECK : nwUnicode.H_CHECK, + nwUnicode.U_CROSS : nwUnicode.H_CROSS, + nwUnicode.U_BULL : nwUnicode.H_BULL, + nwUnicode.U_TRBULL : nwUnicode.H_TRBULL, + nwUnicode.U_HYBULL : nwUnicode.H_HYBULL, + nwUnicode.U_FLOWER : nwUnicode.H_FLOWER, + nwUnicode.U_PERMIL : nwUnicode.H_PERMIL, + nwUnicode.U_DEGREE : nwUnicode.H_DEGREE, + nwUnicode.U_MINUS : nwUnicode.H_MINUS, + nwUnicode.U_TIMES : nwUnicode.H_TIMES, + nwUnicode.U_DIVIDE : nwUnicode.H_DIVIDE, + } + +# END Class nwHtmlUnicode diff --git a/nw/core/tohtml.py b/nw/core/tohtml.py index 1198ea1b..67ebcc0f 100644 --- a/nw/core/tohtml.py +++ b/nw/core/tohtml.py @@ -25,10 +25,9 @@ along with this program. If not, see . """ import logging -import re from nw.core.tokenizer import Tokenizer -from nw.constants import nwUnicode, nwLabels, nwKeyWords +from nw.constants import nwLabels, nwKeyWords, nwHtmlUnicode logger = logging.getLogger(__name__) @@ -41,27 +40,13 @@ class ToHtml(Tokenizer): def __init__(self, theProject, theParent): Tokenizer.__init__(self, theProject, theParent) - self.genMode = self.M_EXPORT + self.genMode = self.M_EXPORT self.cssStyles = True + self.fullHTML = [] - self.repDict = { - "<" : "<", - ">" : ">", - "&" : "&", - nwUnicode.U_ENDASH : nwUnicode.H_ENDASH, - nwUnicode.U_EMDASH : nwUnicode.H_EMDASH, - nwUnicode.U_HELLIP : nwUnicode.H_HELLIP, - nwUnicode.U_NBSP : nwUnicode.H_NBSP, - nwUnicode.U_THSP : nwUnicode.H_THSP, - nwUnicode.U_THNBSP : nwUnicode.H_THNBSP, - nwUnicode.U_MAPOSS : nwUnicode.H_RSQUO, - } - self.revDict = {} - self.reReplace = [] - self.reReverse = [] - self._buildRegEx() - - self.fullHTML = [] + # Internals + self._trMap = {} + self.setReplaceUnicode(False) return @@ -88,6 +73,23 @@ class ToHtml(Tokenizer): self.cssStyles = cssStyles return + def setReplaceUnicode(self, doReplace): + """Set the translation map to either minimal or full unicode to + html entities replacement. + """ + # Control characters must always be replaced + self._trMap = str.maketrans({ + "<" : "<", + ">" : ">", + "&" : "&", + }) + + if doReplace: + # Extend to all relevant Unicode characters + self._trMap.update(str.maketrans(nwHtmlUnicode.U_TO_H)) + + return + ## # Class Methods ## @@ -97,30 +99,12 @@ class ToHtml(Tokenizer): """ return sum([len(x) for x in self.fullHTML]) - def doAutoReplace(self): + def doPreProcessing(self): """Extend the auto-replace to also properly encode some unicode characters into their respective HTML entities. """ - Tokenizer.doAutoReplace(self) - self.theText = self.reReplace.sub( - lambda x: self.repDict[x.group(0)], self.theText - ) - return - - def doPostProcessing(self): - """Reverse the html entities replacement on the markdown text. - Otherwise, all the &something; bits will also be in there. - """ - Tokenizer.doPostProcessing(self) - if self.genMode == self.M_PREVIEW: - # Doesn't matter for preview as we don't use the markdown - return - - if self.keepMarkdown: - self.theMarkdown[-1] = self.reReverse.sub( - lambda x: self.revDict[x.group(0)], self.theMarkdown[-1] - ) - + Tokenizer.doPreProcessing(self) + self.theText = self.theText.translate(self._trMap) return def doConvert(self): @@ -466,16 +450,4 @@ class ToHtml(Tokenizer): return retText - def _buildRegEx(self): - """Build the regular expressions - """ - self.revDict = dict(map(reversed, self.repDict.items())) - self.reReplace = re.compile( - "|".join([re.escape(k) for k in self.repDict.keys()]), flags=re.DOTALL - ) - self.reReverse = re.compile( - "|".join([re.escape(k) for k in self.revDict.keys()]), flags=re.DOTALL - ) - return - # END Class ToHtml diff --git a/nw/core/tokenizer.py b/nw/core/tokenizer.py index 96b4be2d..3e0474ae 100644 --- a/nw/core/tokenizer.py +++ b/nw/core/tokenizer.py @@ -32,7 +32,7 @@ from PyQt5.QtCore import QRegularExpression from nw.core.document import NWDoc from nw.core.tools import numberToWord, numberToRoman -from nw.constants import nwConst, nwItemLayout, nwItemType, nwRegEx +from nw.constants import nwConst, nwUnicode, nwItemLayout, nwItemType, nwRegEx logger = logging.getLogger(__name__) @@ -86,7 +86,7 @@ class Tokenizer(): self.theResult = "" # The result of the last document self.keepMarkdown = False # Whether to keep the markdown text - self.theMarkdown = [] # The result novelWriter markdown of all documents + self.theMarkdown = [] # The result novelWriter markdown of all documents # User Settings self.textFont = "Serif" # Output text font @@ -297,9 +297,10 @@ class Tokenizer(): return True - def doAutoReplace(self): - """Run through the user's auto-replace dictionary. + def doPreProcessing(self): + """Reun trough the various replace doctionaries. """ + # Process the user's auto-replace dictionary if len(self.theProject.autoReplace) > 0: repDict = {} for aKey, aVal in self.theProject.autoReplace.items(): @@ -307,6 +308,10 @@ class Tokenizer(): xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL) self.theText = xRep.sub(lambda x: repDict[x.group(0)], self.theText) + # Process the character translation map + trDict = {nwUnicode.U_MAPOSS: nwUnicode.U_RSQUO} + self.theText = self.theText.translate(str.maketrans(trDict)) + return def doPostProcessing(self): diff --git a/nw/gui/docviewer.py b/nw/gui/docviewer.py index 24d53019..ddc0d876 100644 --- a/nw/gui/docviewer.py +++ b/nw/gui/docviewer.py @@ -176,7 +176,7 @@ class GuiDocViewer(QTextBrowser): # See issue #298 try: aDoc.setText(tHandle) - aDoc.doAutoReplace() + aDoc.doPreProcessing() aDoc.tokenizeText() aDoc.doConvert() aDoc.doPostProcessing()