From 8ef5eea825e0a3167d9f387e6487a385a73e9507 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Thu, 14 Mar 2024 22:01:44 +0100 Subject: [PATCH] Improve paragraph skips in Markdown output --- novelwriter/constants.py | 1 + novelwriter/core/tomd.py | 8 +- tests/test_core/test_core_tokenizer.py | 110 +++++++++++++++++++++++-- tests/test_core/test_core_tomd.py | 2 +- 4 files changed, 110 insertions(+), 11 deletions(-) diff --git a/novelwriter/constants.py b/novelwriter/constants.py index 130378bd..96e3db99 100644 --- a/novelwriter/constants.py +++ b/novelwriter/constants.py @@ -405,6 +405,7 @@ class nwUnicode: U_THNBSP = "\u202f" # Thin non-breaking space U_ENSP = "\u2002" # Short (en) space U_EMSP = "\u2003" # Long (em) space + U_MMSP = "\u205f" # Medium mathematical space U_LSEP = "\u2028" # Line separator U_PSEP = "\u2029" # Paragraph separator diff --git a/novelwriter/core/tomd.py b/novelwriter/core/tomd.py index 22a72b5f..82fdb78a 100644 --- a/novelwriter/core/tomd.py +++ b/novelwriter/core/tomd.py @@ -27,7 +27,7 @@ import logging from pathlib import Path -from novelwriter.constants import nwHeadFmt, nwLabels +from novelwriter.constants import nwHeadFmt, nwLabels, nwUnicode from novelwriter.core.project import NWProject from novelwriter.core.tokenizer import Tokenizer @@ -108,6 +108,7 @@ class ToMarkdown(Tokenizer): self.FMT_SUB_B: "", self.FMT_SUB_E: "", } + cSkip = "" else: # Extended Markdown mdTags = { @@ -126,6 +127,7 @@ class ToMarkdown(Tokenizer): self.FMT_SUB_B: "~", self.FMT_SUB_E: "~", } + cSkip = nwUnicode.U_MMSP self._result = "" @@ -136,7 +138,7 @@ class ToMarkdown(Tokenizer): for tType, _, tText, tFormat, tStyle in self._tokens: if tType == self.T_EMPTY: - if len(para) > 0: + if para: tTemp = (lineSep.join(para)).rstrip(" ") lines.append(f"{tTemp}\n\n") para = [] @@ -165,7 +167,7 @@ class ToMarkdown(Tokenizer): lines.append(f"{tText}\n\n") elif tType == self.T_SKIP: - lines.append("\n\n\n") + lines.append(f"{cSkip}\n\n") elif tType == self.T_TEXT: tTemp = tText diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 0d27231d..b3b69774 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -1734,10 +1734,8 @@ def testCoreToken_CountStats(mockGUI, ipsumText): @pytest.mark.core -def testCoreToken_HeaderCounterAndVisibility(mockGUI): - """Test the header counter and visibility of the Tokenizer class. - This is a special test to cover issue #1704. - """ +def testCoreToken_SceneSeparators(mockGUI): + """Test the section and scene separators of the Tokenizer class.""" project = NWProject() project.data.setLanguage("en") project._loadProjectLocalisation() @@ -1855,8 +1853,50 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): "Text\n\n" ) - # Show/Hide Headings - # ================== + # Separators with Scenes Only + # =========================== + # Requires a fresh builder class + md = ToMarkdown(project) + md.setExtendedMarkdown() + md._isNone = False + md._isNote = False + md._isNovel = True + + md._text = ( + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "Text\n\n" + "### Scene Three\n\n" + "Text\n\n" + "###! Scene Four\n\n" + "Text\n\n" + ) + + md.setSceneFormat("", False) + md.setHardSceneFormat("* * *", False) + md.tokenizeText() + md.doConvert() + assert md.result == ( + "Text\n\n" + "\u205f\n\n" + "Text\n\n" + "\u205f\n\n" + "Text\n\n" + "* * *\n\n" + "Text\n\n" + ) + +# END Test testCoreToken_SceneSeparators + + +@pytest.mark.core +def testCoreToken_HeaderVisibility(mockGUI): + """Test the heading visibility settings of the Tokenizer class.""" + project = NWProject() + project.data.setLanguage("en") + project._loadProjectLocalisation() + md = ToMarkdown(project) md._text = ( "#! Novel\n\n" @@ -1876,6 +1916,13 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): "Text\n\n" ) + # Novel Files + # =========== + + md._isNone = False + md._isNote = False + md._isNovel = True + # Show All md.setTitleFormat(nwHeadFmt.TITLE, False) md.setChapterFormat(nwHeadFmt.TITLE, False) @@ -1923,6 +1970,55 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): "Text\n\n" ) + # Note Files + # ========== + + md._isNone = False + md._isNote = True + md._isNovel = False + + # Hide All + md.setTitleFormat(nwHeadFmt.TITLE, True) + md.setChapterFormat(nwHeadFmt.TITLE, True) + md.setUnNumberedFormat(nwHeadFmt.TITLE, True) + md.setSceneFormat(nwHeadFmt.TITLE, True) + md.setHardSceneFormat(nwHeadFmt.TITLE, True) + md.setSectionFormat(nwHeadFmt.TITLE, True) + + md.tokenizeText() + md.doConvert() + assert md.result == ( + "# Novel\n\n" + "# Title One\n\n" + "## Prologue\n\n" + "Text\n\n" + "## Chapter One\n\n" + "### Scene One\n\n" + "Text\n\n" + "### Scene Two\n\n" + "#### Section Two\n\n" + "Text\n\n" + "## Chapter Two\n\n" + "### Scene Three\n\n" + "Text\n\n" + "### Scene Four\n\n" + "Text\n\n" + ) + +# END Test testCoreToken_HeaderVisibility + + +@pytest.mark.core +def testCoreToken_CounterHandling(mockGUI): + """Test the heading counter of the Tokenizer class.""" + project = NWProject() + project.data.setLanguage("en") + project._loadProjectLocalisation() + md = ToMarkdown(project) + md._isNone = False + md._isNote = False + md._isNovel = True + # Counter Handling, Novel Titles # ============================== # This also checks that only numbered chapters bump the counter @@ -1998,7 +2094,7 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): "Text\n\n" ) -# END Test testCoreToken_HeaderCounterAndVisibility +# END Test testCoreToken_CounterHandling @pytest.mark.core diff --git a/tests/test_core/test_core_tomd.py b/tests/test_core/test_core_tomd.py index 4d973ead..c67caf63 100644 --- a/tests/test_core/test_core_tomd.py +++ b/tests/test_core/test_core_tomd.py @@ -230,7 +230,7 @@ def testCoreToMarkdown_ConvertDirect(mockGUI): (toMD.T_EMPTY, 1, "", None, toMD.A_NONE), ] toMD.doConvert() - assert toMD.result == "\n\n\n" + assert toMD.result == "\n\n" # END Test testCoreToMarkdown_ConvertDirect