Move the coding of angle brackets in HTML to the innermost function in the chain (#821)

nw/core/tohtml.py

@@ -74,19 +74,13 @@ class ToHtml(Tokenizer):
         return
 
     def setReplaceUnicode(self, doReplace):
-        """Set the translation map to either minimal or full unicode to
+        """Set the translation map to either minimal or full unicode for
         html entities replacement.
         """
         # Control characters must always be replaced
-        # This affects alignment and indenting code, so the Tokenizer
-        # must take this into account when parsing for markup using
-        # angle brackets.
-        self._trMap = str.maketrans({
-            "<": "&lt;",
-            ">": "&gt;",
-            "&": "&amp;",
-        })
-
+        # Angle brackets are replaced later as they are also used in
+        # formatting codes
+        self._trMap = str.maketrans({"&": "&amp;"})
         if doReplace:
             # Extend to all relevant Unicode characters
             self._trMap.update(str.maketrans(nwHtmlUnicode.U_TO_H))
@@ -157,6 +151,9 @@ class ToHtml(Tokenizer):
 
         for tType, tLine, tText, tFormat, tStyle in self.theTokens:
 
+            # Replace < and > before adding html tags
+            tText = tText.replace("<", "&lt;").replace(">", "&gt;")
+
             # Styles
             aStyle = []
             if tStyle is not None and self.cssStyles:

nw/core/tokenizer.py

@@ -459,28 +459,16 @@ class Tokenizer():
                 if aLine.startswith(">>"):
                     tagRight = True
                     aLine = aLine[2:].lstrip(" ")
-                elif aLine.startswith(">>"):
-                    tagRight = True
-                    aLine = aLine[8:].lstrip(" ")
                 elif aLine.startswith(">"):
                     indLeft = True
                     aLine = aLine[1:].lstrip(" ")
-                elif aLine.startswith(">"):
-                    indLeft = True
-                    aLine = aLine[4:].lstrip(" ")
 
                 if aLine.endswith("<<"):
                     tagLeft = True
                     aLine = aLine[:-2].rstrip(" ")
-                elif aLine.endswith("&lt;&lt;"):
-                    tagLeft = True
-                    aLine = aLine[:-8].rstrip(" ")
                 elif aLine.endswith("<"):
                     indRight = True
                     aLine = aLine[:-1].rstrip(" ")
-                elif aLine.endswith("&lt;"):
-                    indRight = True
-                    aLine = aLine[:-4].rstrip(" ")
 
                 textAlign = defAlign
                 if tagLeft and tagRight:

tests/test_core/test_core_tohtml.py

@@ -457,11 +457,11 @@ def testCoreToHtml_Methods(mockGUI):
     theHtml.tokenizeText()
     theHtml.doConvert()
     assert theHtml.theMarkdown[-1] == (
-        "Text with <brackets> & short–dash, long—dash …\n\n"
+        "Text with <brackets> &amp; short&ndash;dash, long&mdash;dash &hellip;\n\n"
     )
     theHtml.doPostProcessing()
     assert theHtml.theMarkdown[-1] == (
-        "Text with &lt;brackets&gt; &amp; short&ndash;dash, long&mdash;dash &hellip;\n\n"
+        "Text with <brackets> &amp; short&ndash;dash, long&mdash;dash &hellip;\n\n"
     )
 
     # Result Size

tests/test_core/test_core_tokenizer.py

@@ -452,48 +452,6 @@ def testCoreToken_Tokenize(mockGUI):
         "Right-indent, right-aligned\n\n\n"
     )
 
-    # Alignment w/HTML Codes
-    theToken.theText = (
-        "Some regular text\n\n"
-        "Some left-aligned text <<\n\n"
-        ">> Some right-aligned text\n\n"
-        ">> Some centered text <<\n\n"
-        "> Left-indented block\n\n"
-        "Right-indented block <\n\n"
-        "> Double-indented block <\n\n"
-        ">> Right-indent, right-aligned <\n\n"
-    )
-    theToken.tokenizeText()
-    assert theToken.theTokens == [
-        (Tokenizer.T_TEXT,  1, "Some regular text", [], Tokenizer.A_NONE),
-        (Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT,  3, "Some left-aligned text", [], Tokenizer.A_LEFT),
-        (Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT,  5, "Some right-aligned text", [], Tokenizer.A_RIGHT),
-        (Tokenizer.T_EMPTY, 6, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT,  7, "Some centered text", [], Tokenizer.A_CENTRE),
-        (Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT,  9, "Left-indented block", [], Tokenizer.A_IND_L),
-        (Tokenizer.T_EMPTY, 10, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT,  11, "Right-indented block", [], Tokenizer.A_IND_R),
-        (Tokenizer.T_EMPTY, 12, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT,  13, "Double-indented block", [], dblIndent),
-        (Tokenizer.T_EMPTY, 14, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_TEXT,  15, "Right-indent, right-aligned", [], rIndAlign),
-        (Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
-        (Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
-    ]
-    assert theToken.theMarkdown[-1] == (
-        "Some regular text\n\n"
-        "Some left-aligned text\n\n"
-        "Some right-aligned text\n\n"
-        "Some centered text\n\n"
-        "Left-indented block\n\n"
-        "Right-indented block\n\n"
-        "Double-indented block\n\n"
-        "Right-indent, right-aligned\n\n\n"
-    )
-
 # END Test testCoreToken_Tokenize