From 38f31d6a64a4e9f3192068ee28a259adb4c27921 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 15 Apr 2023 18:30:37 +0200 Subject: [PATCH 1/2] Remove the tokenizer post processing stage and instead run it where needed (Issue #1412) --- novelwriter/core/tohtml.py | 4 ++-- novelwriter/core/tokenizer.py | 28 ++++++++++---------------- novelwriter/core/toodt.py | 9 +++++---- novelwriter/gui/docviewer.py | 1 - novelwriter/tools/build.py | 1 - tests/test_core/test_core_tohtml.py | 4 ---- tests/test_core/test_core_tokenizer.py | 17 ++++++++++------ 7 files changed, 29 insertions(+), 35 deletions(-) diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py index d95d217a..30fb63dc 100644 --- a/novelwriter/core/tohtml.py +++ b/novelwriter/core/tohtml.py @@ -26,7 +26,7 @@ along with this program. If not, see . import logging from novelwriter.constants import nwKeyWords, nwLabels, nwHtmlUnicode -from novelwriter.core.tokenizer import Tokenizer +from novelwriter.core.tokenizer import Tokenizer, stripEscape logger = logging.getLogger(__name__) @@ -269,7 +269,7 @@ class ToHtml(Tokenizer): parStyle = hStyle for xPos, xLen, xFmt in reversed(tFormat): tTemp = tTemp[:xPos] + htmlTags[xFmt] + tTemp[xPos+xLen:] - thisPar.append(tTemp.rstrip()) + thisPar.append(stripEscape(tTemp.rstrip())) elif tType == self.T_SYNOPSIS and self._doSynopsis: tmpResult.append(self._formatSynopsis(tText)) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index f6434fcf..56886657 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -40,6 +40,17 @@ from novelwriter.constants import nwConst, nwRegEx, nwUnicode logger = logging.getLogger(__name__) +def stripEscape(text): + """Helper function to strip escaped markdown characters from + paragraph text. + """ + if "\\" in text: + # Checking first is slightly slower when there are escaped + # characters in the text, but significantly faster when not + return text.replace(r"\*", "*").replace(r"\~", "~").replace(r"\_", "_") + return text + + class Tokenizer(ABC): # In-Text Format @@ -340,23 +351,6 @@ class Tokenizer(ABC): return - def doPostProcessing(self): - """Do some postprocessing. Overloaded by subclasses. This just - does the standard escaped characters. - """ - escapeDict = { - r"\*": "*", - r"\~": "~", - r"\_": "_", - } - escReplace = re.compile( - "|".join([re.escape(k) for k in escapeDict.keys()]), flags=re.DOTALL - ) - self._theResult = escReplace.sub( - lambda x: escapeDict[x.group(0)], self._theResult - ) - return - def tokenizeText(self): """Scan the text for either lines starting with specific characters that indicate headers, comments, commands etc, or diff --git a/novelwriter/core/toodt.py b/novelwriter/core/toodt.py index 2f6e1f0b..6170c7a2 100644 --- a/novelwriter/core/toodt.py +++ b/novelwriter/core/toodt.py @@ -32,7 +32,7 @@ from zipfile import ZipFile from datetime import datetime from novelwriter.constants import nwKeyWords, nwLabels -from novelwriter.core.tokenizer import Tokenizer +from novelwriter.core.tokenizer import Tokenizer, stripEscape logger = logging.getLogger(__name__) @@ -1356,16 +1356,17 @@ class XMLParagraph: return - def appendText(self, tText): + def appendText(self, text): """Append text to the XML element. We do this one character at the time in order to be able to process line breaks, tabs and spaces separately. Multiple spaces above one are concatenated into a single tag, and must therefore be processed separately. """ + text = stripEscape(text) nSpaces = 0 - self._rawTxt += tText + self._rawTxt += text - for c in tText: + for c in text: if c == " ": nSpaces += 1 continue diff --git a/novelwriter/gui/docviewer.py b/novelwriter/gui/docviewer.py index 2449688d..517b88f4 100644 --- a/novelwriter/gui/docviewer.py +++ b/novelwriter/gui/docviewer.py @@ -188,7 +188,6 @@ class GuiDocViewer(QTextBrowser): aDoc.doPreProcessing() aDoc.tokenizeText() aDoc.doConvert() - aDoc.doPostProcessing() except Exception: logger.error("Failed to generate preview for document with handle '%s'", tHandle) logException() diff --git a/novelwriter/tools/build.py b/novelwriter/tools/build.py index f6f9ccd8..f222ebdd 100644 --- a/novelwriter/tools/build.py +++ b/novelwriter/tools/build.py @@ -771,7 +771,6 @@ class GuiBuildNovel(QDialog): bldObj.doHeaders() if doConvert: bldObj.doConvert() - bldObj.doPostProcessing() except Exception: logger.error("Failed to build document '%s'", tItem.itemHandle) diff --git a/tests/test_core/test_core_tohtml.py b/tests/test_core/test_core_tohtml.py index d459bff1..50af2307 100644 --- a/tests/test_core/test_core_tohtml.py +++ b/tests/test_core/test_core_tohtml.py @@ -574,10 +574,6 @@ def testCoreToHtml_Methods(mockGUI): assert theHtml.theMarkdown[-1] == ( "Text with & short–dash, long—dash …\n\n" ) - theHtml.doPostProcessing() - assert theHtml.theMarkdown[-1] == ( - "Text with & short–dash, long—dash …\n\n" - ) # Result Size assert theHtml.getFullResultSize() == 147 diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 4b535730..6b44c93d 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -24,7 +24,7 @@ import pytest from tools import C, buildTestProject, readFile from novelwriter.core.project import NWProject -from novelwriter.core.tokenizer import Tokenizer +from novelwriter.core.tokenizer import Tokenizer, stripEscape class BareTokenizer(Tokenizer): @@ -203,11 +203,6 @@ def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath): theToken.doPreProcessing() assert theToken._theText == docTextR - # Post Processing - theToken._theResult = r"This is text with escapes: \** \~~ \__" - theToken.doPostProcessing() - assert theToken.theResult == "This is text with escapes: ** ~~ __" - # Save File savePath = fncPath / "dump.nwd" theToken.saveRawMarkdown(savePath) @@ -223,6 +218,16 @@ def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath): # END Test testCoreToken_TextOps +@pytest.mark.core +def testCoreToken_StripEscape(): + """Test the stripEscape helper function. + """ + text = r"This is text with escapes: \** \~~ \__" + assert stripEscape(text) == "This is text with escapes: ** ~~ __" + + return + + @pytest.mark.core def testCoreToken_HeaderFormat(mockGUI): """Test the tokenization of header formats in the Tokenizer class. From 574ed1b2f0e451462fd0544ed8893b3b4fb7ff7b Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 15 Apr 2023 18:46:32 +0200 Subject: [PATCH 2/2] Improve test coverage --- tests/test_core/test_core_tohtml.py | 17 ++++++++++++++--- tests/test_core/test_core_tokenizer.py | 8 +++++--- tests/test_core/test_core_toodt.py | 16 +++++++++++++++- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/test_core/test_core_tohtml.py b/tests/test_core/test_core_tohtml.py index 50af2307..9f70852c 100644 --- a/tests/test_core/test_core_tohtml.py +++ b/tests/test_core/test_core_tohtml.py @@ -378,7 +378,7 @@ def testCoreToHtml_ConvertDirect(mockGUI): @pytest.mark.core def testCoreToHtml_SpecialCases(mockGUI): - """Test some special cases that has caused errors in the past. + """Test some special cases that have caused errors in the past. """ theProject = NWProject(mockGUI) theHtml = ToHtml(theProject) @@ -415,8 +415,8 @@ def testCoreToHtml_SpecialCases(mockGUI): "

Test > text <bold> and more.

\n" ) - # Test for bug #950 - # ================= + # Test for issue #950 + # =================== # See: https://github.com/vkbo/novelWriter/issues/950 theHtml.setComments(True) @@ -436,6 +436,17 @@ def testCoreToHtml_SpecialCases(mockGUI): "

Heading <1>

\n" ) + # Test for issue #1412 + # ==================== + # See: https://github.com/vkbo/novelWriter/issues/1412 + + theHtml._theText = "Test text \\**_bold_** and more.\n" + theHtml.tokenizeText() + theHtml.doConvert() + assert theHtml.theResult == ( + "

Test text **bold** and more.

\n" + ) + # END Test testCoreToHtml_SpecialCases diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 6b44c93d..8a2f9631 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -222,10 +222,12 @@ def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath): def testCoreToken_StripEscape(): """Test the stripEscape helper function. """ - text = r"This is text with escapes: \** \~~ \__" - assert stripEscape(text) == "This is text with escapes: ** ~~ __" + text1 = "This is text with escapes: \\** \\~~ \\__" + text2 = "This is text with escapes: ** ~~ __" + assert stripEscape(text1) == "This is text with escapes: ** ~~ __" + assert stripEscape(text2) == "This is text with escapes: ** ~~ __" - return +# END Test testCoreToken_StripEscape @pytest.mark.core diff --git a/tests/test_core/test_core_toodt.py b/tests/test_core/test_core_toodt.py index fedfbcf3..6e3ea446 100644 --- a/tests/test_core/test_core_toodt.py +++ b/tests/test_core/test_core_toodt.py @@ -221,7 +221,21 @@ def testCoreToOdt_TextFormatting(mockGUI): "" ) - # Tabs and Breaks + # Test for issue #1412 + # ==================== + # See: https://github.com/vkbo/novelWriter/issues/1412 + + theDoc.initDocument() + theTxt = "Test text \\**_bold_** and more." + theFmt = " I i " + theDoc._addTextPar("Standard", oStyle, theTxt, theFmt=theFmt) + assert theDoc.getErrors() == [] + assert xmlToText(theDoc._xText) == ( + "" + "Test text **" + "bold** and more." + "" + ) # END Test testCoreToOdt_TextFormatting