Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change the way Unicode is handled in HTML, and clean up the class a bit

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2021-02-10 16:56:52 +01:00
parent a0fba34ff6
commit f67a50c064
5 changed files with 98 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/constants/__init__.py
+3 -1
View File
@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
from nw.constants.iso import isoLanguage, isoCountry
from nw.constants.constants import (
nwConst, nwLists, nwRegEx, nwFiles, nwKeyWords, nwLabels, nwQuotes, nwUnicode
nwConst, nwLists, nwRegEx, nwFiles, nwKeyWords, nwLabels, nwQuotes,
nwUnicode, nwHtmlUnicode
)
from nw.constants.enum import (
nwAlert, nwDocAction, nwItemClass, nwItemLayout, nwItemType, nwOutline,
@@ -19,6 +20,7 @@ __all__ = [
"nwLabels",
"nwQuotes",
"nwUnicode",
"nwHtmlUnicode",
"nwAlert",
"nwDocAction",
"nwItemClass",
nw/constants/constants.py
+59 -2
View File
@@ -265,7 +265,7 @@ class nwUnicode:
U_LCQUO = "\u300c" # Left corner bracket
U_RCQUO = "\u300d" # Right corner bracket
U_LWCQUO = "\u300e" # Left white corner bracket
U_RECQUO = "\u300f" # Right white corner bracket
U_RWCQUO = "\u300f" # Right white corner bracket
## Punctuation
U_FGDASH = "\u2012" # Figure dash
@@ -331,7 +331,7 @@ class nwUnicode:
H_LCQUO = "「"
H_RCQUO = "」"
H_LWCQUO = "『"
H_LWCQUO = "『"
H_RWCQUO = "』"
## Punctuation
H_FGDASH = "‒"
@@ -374,3 +374,60 @@ class nwUnicode:
H_LTRIS = "◂"
# END Class nwUnicode
class nwHtmlUnicode():
U_TO_H = {
## Quotes
nwUnicode.U_QUOT : nwUnicode.H_QUOT,
nwUnicode.U_APOS : nwUnicode.H_APOS,
nwUnicode.U_LAQUO : nwUnicode.H_LAQUO,
nwUnicode.U_RAQUO : nwUnicode.H_RAQUO,
nwUnicode.U_LSQUO : nwUnicode.H_LSQUO,
nwUnicode.U_RSQUO : nwUnicode.H_RSQUO,
nwUnicode.U_SBQUO : nwUnicode.H_SBQUO,
nwUnicode.U_SUQUO : nwUnicode.H_SUQUO,
nwUnicode.U_LDQUO : nwUnicode.H_LDQUO,
nwUnicode.U_RDQUO : nwUnicode.H_RDQUO,
nwUnicode.U_BDQUO : nwUnicode.H_BDQUO,
nwUnicode.U_UDQUO : nwUnicode.H_UDQUO,
nwUnicode.U_LSAQUO : nwUnicode.H_LSAQUO,
nwUnicode.U_RSAQUO : nwUnicode.H_RSAQUO,
nwUnicode.U_BDRQUO : nwUnicode.H_BDRQUO,
nwUnicode.U_LCQUO : nwUnicode.H_LCQUO,
nwUnicode.U_RCQUO : nwUnicode.H_RCQUO,
nwUnicode.U_LWCQUO : nwUnicode.H_LWCQUO,
nwUnicode.U_RWCQUO : nwUnicode.H_RWCQUO,
## Punctuation
nwUnicode.U_FGDASH : nwUnicode.H_FGDASH,
nwUnicode.U_ENDASH : nwUnicode.H_ENDASH,
nwUnicode.U_EMDASH : nwUnicode.H_EMDASH,
nwUnicode.U_HBAR : nwUnicode.H_HBAR,
nwUnicode.U_HELLIP : nwUnicode.H_HELLIP,
nwUnicode.U_MAPOSS : nwUnicode.H_MAPOSS,
nwUnicode.U_PRIME : nwUnicode.H_PRIME,
nwUnicode.U_DPRIME : nwUnicode.H_DPRIME,
## Spaces
nwUnicode.U_NBSP : nwUnicode.H_NBSP,
nwUnicode.U_THSP : nwUnicode.H_THSP,
nwUnicode.U_THNBSP : nwUnicode.H_THNBSP,
nwUnicode.U_ENSP : nwUnicode.H_ENSP,
nwUnicode.U_EMSP : nwUnicode.H_EMSP,
## Symbols
nwUnicode.U_CHECK : nwUnicode.H_CHECK,
nwUnicode.U_CROSS : nwUnicode.H_CROSS,
nwUnicode.U_BULL : nwUnicode.H_BULL,
nwUnicode.U_TRBULL : nwUnicode.H_TRBULL,
nwUnicode.U_HYBULL : nwUnicode.H_HYBULL,
nwUnicode.U_FLOWER : nwUnicode.H_FLOWER,
nwUnicode.U_PERMIL : nwUnicode.H_PERMIL,
nwUnicode.U_DEGREE : nwUnicode.H_DEGREE,
nwUnicode.U_MINUS : nwUnicode.H_MINUS,
nwUnicode.U_TIMES : nwUnicode.H_TIMES,
nwUnicode.U_DIVIDE : nwUnicode.H_DIVIDE,
}
# END Class nwHtmlUnicode
nw/core/tohtml.py
+26 -54
View File
@@ -25,10 +25,9 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
import logging
import re
from nw.core.tokenizer import Tokenizer
from nw.constants import nwUnicode, nwLabels, nwKeyWords
from nw.constants import nwLabels, nwKeyWords, nwHtmlUnicode
logger = logging.getLogger(__name__)
@@ -41,27 +40,13 @@ class ToHtml(Tokenizer):
def __init__(self, theProject, theParent):
Tokenizer.__init__(self, theProject, theParent)
self.genMode = self.M_EXPORT
self.genMode = self.M_EXPORT
self.cssStyles = True
self.fullHTML = []
self.repDict = {
"<" : "&lt;",
">" : "&gt;",
"&" : "&amp;",
nwUnicode.U_ENDASH : nwUnicode.H_ENDASH,
nwUnicode.U_EMDASH : nwUnicode.H_EMDASH,
nwUnicode.U_HELLIP : nwUnicode.H_HELLIP,
nwUnicode.U_NBSP : nwUnicode.H_NBSP,
nwUnicode.U_THSP : nwUnicode.H_THSP,
nwUnicode.U_THNBSP : nwUnicode.H_THNBSP,
nwUnicode.U_MAPOSS : nwUnicode.H_RSQUO,
}
self.revDict = {}
self.reReplace = []
self.reReverse = []
self._buildRegEx()
self.fullHTML = []
# Internals
self._trMap = {}
self.setReplaceUnicode(False)
return
@@ -88,6 +73,23 @@ class ToHtml(Tokenizer):
self.cssStyles = cssStyles
return
def setReplaceUnicode(self, doReplace):
"""Set the translation map to either minimal or full unicode to
html entities replacement.
"""
# Control characters must always be replaced
self._trMap = str.maketrans({
"<" : "&lt;",
">" : "&gt;",
"&" : "&amp;",
})
if doReplace:
# Extend to all relevant Unicode characters
self._trMap.update(str.maketrans(nwHtmlUnicode.U_TO_H))
return
##
# Class Methods
##
@@ -97,30 +99,12 @@ class ToHtml(Tokenizer):
"""
return sum([len(x) for x in self.fullHTML])
def doAutoReplace(self):
def doPreProcessing(self):
"""Extend the auto-replace to also properly encode some unicode
characters into their respective HTML entities.
"""
Tokenizer.doAutoReplace(self)
self.theText = self.reReplace.sub(
lambda x: self.repDict[x.group(0)], self.theText
)
return
def doPostProcessing(self):
"""Reverse the html entities replacement on the markdown text.
Otherwise, all the &something; bits will also be in there.
"""
Tokenizer.doPostProcessing(self)
if self.genMode == self.M_PREVIEW:
# Doesn't matter for preview as we don't use the markdown
return
if self.keepMarkdown:
self.theMarkdown[-1] = self.reReverse.sub(
lambda x: self.revDict[x.group(0)], self.theMarkdown[-1]
)
Tokenizer.doPreProcessing(self)
self.theText = self.theText.translate(self._trMap)
return
def doConvert(self):
@@ -466,16 +450,4 @@ class ToHtml(Tokenizer):
return retText
def _buildRegEx(self):
"""Build the regular expressions
"""
self.revDict = dict(map(reversed, self.repDict.items()))
self.reReplace = re.compile(
"|".join([re.escape(k) for k in self.repDict.keys()]), flags=re.DOTALL
)
self.reReverse = re.compile(
"|".join([re.escape(k) for k in self.revDict.keys()]), flags=re.DOTALL
)
return
# END Class ToHtml
nw/core/tokenizer.py
+9 -4
View File
@@ -32,7 +32,7 @@ from PyQt5.QtCore import QRegularExpression
from nw.core.document import NWDoc
from nw.core.tools import numberToWord, numberToRoman
from nw.constants import nwConst, nwItemLayout, nwItemType, nwRegEx
from nw.constants import nwConst, nwUnicode, nwItemLayout, nwItemType, nwRegEx
logger = logging.getLogger(__name__)
@@ -86,7 +86,7 @@ class Tokenizer():
self.theResult = "" # The result of the last document
self.keepMarkdown = False # Whether to keep the markdown text
self.theMarkdown = [] # The result novelWriter markdown of all documents
self.theMarkdown = [] # The result novelWriter markdown of all documents
# User Settings
self.textFont = "Serif" # Output text font
@@ -297,9 +297,10 @@ class Tokenizer():
return True
def doAutoReplace(self):
"""Run through the user's auto-replace dictionary.
def doPreProcessing(self):
"""Reun trough the various replace doctionaries.
"""
# Process the user's auto-replace dictionary
if len(self.theProject.autoReplace) > 0:
repDict = {}
for aKey, aVal in self.theProject.autoReplace.items():
@@ -307,6 +308,10 @@ class Tokenizer():
xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
self.theText = xRep.sub(lambda x: repDict[x.group(0)], self.theText)
# Process the character translation map
trDict = {nwUnicode.U_MAPOSS: nwUnicode.U_RSQUO}
self.theText = self.theText.translate(str.maketrans(trDict))
return
def doPostProcessing(self):
nw/gui/docviewer.py
+1 -1
View File
@@ -176,7 +176,7 @@ class GuiDocViewer(QTextBrowser):
# See issue #298
try:
aDoc.setText(tHandle)
aDoc.doAutoReplace()
aDoc.doPreProcessing()
aDoc.tokenizeText()
aDoc.doConvert()
aDoc.doPostProcessing()