From 37f56a34f421654315d591e6eb752af9920475c2 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Mar 2024 18:21:55 +0100 Subject: [PATCH] Remove the header processing function and add comments to the tokenizer --- novelwriter/core/docbuild.py | 1 - novelwriter/core/tokenizer.py | 97 +++++++++++++++++++------- novelwriter/text/counting.py | 1 - tests/test_core/test_core_tohtml.py | 15 ---- tests/test_core/test_core_tokenizer.py | 86 +++-------------------- tests/test_core/test_core_toodt.py | 11 --- 6 files changed, 80 insertions(+), 131 deletions(-) diff --git a/novelwriter/core/docbuild.py b/novelwriter/core/docbuild.py index e43816f8..f36d7345 100644 --- a/novelwriter/core/docbuild.py +++ b/novelwriter/core/docbuild.py @@ -352,7 +352,6 @@ class NWBuildDocument: bldObj.setText(tHandle) bldObj.doPreProcessing() bldObj.tokenizeText() - bldObj.doHeaders() if self._count: bldObj.countStats() if convert: diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index f34e54b3..d2281734 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -169,7 +169,7 @@ class Tokenizer(ABC): # Instance Variables self._hFormatter = HeadingFormatter(self._project) - self._skipSep = False # Flag to indicate that we skip the scene separator + self._noSep = False # Flag to indicate that we don't want a scene separator # This File self._isNone = False # Document has unknown layout @@ -486,10 +486,11 @@ class Tokenizer(ABC): # ================= if aLine[0] == "[": - # Parse special formatting line - # This must be a separate if statement, as it may not - # reach a continue statement and must therefore proceed - # to check other formats. + # Special Formats + # =============== + # Parse special formatting line. This must be a separate if + # statement, as it may not reach a continue statement and must + # therefore proceed to check other formats. if sLine in ("[newpage]", "[new page]"): breakNext = True @@ -514,8 +515,12 @@ class Tokenizer(ABC): continue if aLine[0] == "%": + # Comments + # ======== + # All style comments are processed and the exact type exact + # style extracted. Ignored comments on the '%~' format are + # skipped completely. if aLine[1] == "~": - # Completely ignore the paragraph continue cStyle, cText, _ = processComment(aLine) @@ -539,6 +544,11 @@ class Tokenizer(ABC): tmpMarkdown.append("%s\n" % aLine) elif aLine[0] == "@": + # Keywords + # ======== + # Only valid keyword lines are parsed, and any ignored keywords + # are automatically skipped. + valid, bits, _ = self._project.index.scanThis(aLine) if valid and bits and bits[0] not in self._skipKeywords: self._tokens.append(( @@ -548,13 +558,19 @@ class Tokenizer(ABC): tmpMarkdown.append("%s\n" % aLine) elif aLine[:2] == "# ": + # Partition Headers + # ================= + # Partition headers are only formatted in novel documents, and + # otherwise unchanged. Scene separators are disabled + # immediately after partitions, and scene numbers are reset. + nHead += 1 tText = aLine[2:].strip() tStyle = self.A_NONE if self._isNovel: tText = self._hFormatter.apply(self._fmtTitle, tText, nHead) tStyle = self._titleStyle - self._skipSep = True + self._noSep = True self._hFormatter.resetScene() self._tokens.append(( @@ -564,6 +580,14 @@ class Tokenizer(ABC): tmpMarkdown.append("%s\n" % aLine) elif aLine[:3] == "## ": + # Chapter Headers + # =============== + # Chapter headers are only formatted in novel documents, and + # otherwise unchanged. Chapter numbers are bumped before the + # heading is formatted. Scene separators are disabled + # immediately after chapter headers, and scene numbers are + # reset. + nHead += 1 tText = aLine[3:].strip() tStyle = self.A_NONE @@ -571,7 +595,7 @@ class Tokenizer(ABC): self._hFormatter.incChapter() tText = self._hFormatter.apply(self._fmtChapter, tText, nHead) tStyle = self._chapterStyle - self._skipSep = True + self._noSep = True self._hFormatter.resetScene() self._tokens.append(( @@ -581,6 +605,17 @@ class Tokenizer(ABC): tmpMarkdown.append("%s\n" % aLine) elif aLine[:4] == "### ": + # Scene Headers + # ============= + # Scene headers in novel documents are treated as centred + # separators if the formatting does not change the text. If the + # format is empty, the scene can be hidden or a blank paragraph + # (skip). When the scene title has static text or no text, it + # is always ignored if the noSep flag is set. This prevents + # separators immediately after other titles. Scene numbers are + # always incremented before formatting. For notes, the header + # is unchanged. + nHead += 1 tText = aLine[4:].strip() tType = self.T_HEAD3 @@ -590,13 +625,13 @@ class Tokenizer(ABC): tText = self._hFormatter.apply(self._fmtScene, tText, nHead) tStyle = self._sceneStyle if tText == "": - tType = self.T_EMPTY if self._skipSep or self._hideScene else self.T_SKIP + tType = self.T_EMPTY if self._noSep or self._hideScene else self.T_SKIP tStyle = self.A_NONE elif tText == self._fmtScene: - tText = "" if self._skipSep else tText - tType = self.T_EMPTY if self._skipSep else self.T_SEP - tStyle = self.A_NONE if self._skipSep else self.A_CENTRE - self._skipSep = False + tText = "" if self._noSep else tText + tType = self.T_EMPTY if self._noSep else self.T_SEP + tStyle = self.A_NONE if self._noSep else self.A_CENTRE + self._noSep = False self._tokens.append(( tType, nHead, tText, [], tStyle @@ -605,6 +640,13 @@ class Tokenizer(ABC): tmpMarkdown.append("%s\n" % aLine) elif aLine[:5] == "#### ": + # Section Headers + # =============== + # Section headers in novel docs are treated as centred + # separators if the formatting does not change the text. If the + # format is empty, the section can be hidden or a blank + # paragraph (skip). For notes, the header is unchanged. + nHead += 1 tText = aLine[5:].strip() tType = self.T_HEAD4 @@ -624,6 +666,12 @@ class Tokenizer(ABC): tmpMarkdown.append("%s\n" % aLine) elif aLine[:3] == "#! ": + # Main Title + # ========== + # Main titles are allowed in any document, and they are always + # centred and start on a new page. For novel documents, we also + # reset all counters when such a title is encountered. + nHead += 1 self._tokens.append(( self.T_TITLE, nHead, aLine[3:].strip(), [], self.A_PBB | self.A_CENTRE @@ -631,11 +679,15 @@ class Tokenizer(ABC): if self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) if self._isNovel: - # For new titles, we reset all counters - self._skipSep = True + self._noSep = True self._hFormatter.resetAll() elif aLine[:4] == "##! ": + # Unnumbered Chapter Header + # ========================= + # Unnumbered chapters are only meaningful in Novel docs, so if + # we're in a note, we convert them to a plain level 2 header. + nHead += 1 tText = aLine[4:].strip() tType = self.T_HEAD2 @@ -644,10 +696,9 @@ class Tokenizer(ABC): tText = self._hFormatter.apply(self._fmtUnNum, tText, nHead) tType = self.T_UNNUM tStyle = self._chapterStyle - self._skipSep = True + self._noSep = True self._hFormatter.resetScene() - # If we're not in a novel section, we just treat this as a regular H2 self._tokens.append(( tType, nHead, tText, [], tStyle )) @@ -655,6 +706,10 @@ class Tokenizer(ABC): tmpMarkdown.append("%s\n" % aLine) else: + # Text Lines + # ========== + # Anything remaining at this point is body text. If body text + # is not disabled, we proceed to process text formatting. if not self._doBodyText: # Skip all body text continue @@ -745,14 +800,6 @@ class Tokenizer(ABC): return - def doHeaders(self) -> bool: - """Apply formatting to the text headers for novel files. This - also applies chapter and scene numbering. - """ - if not self._isNovel: - return False - return True - def countStats(self) -> dict[str, int]: """Count stats on the tokenized text.""" titleCount = self._counts.get("titleCount", 0) diff --git a/novelwriter/text/counting.py b/novelwriter/text/counting.py index b381adea..2e3b0ed5 100644 --- a/novelwriter/text/counting.py +++ b/novelwriter/text/counting.py @@ -31,7 +31,6 @@ from novelwriter.constants import nwRegEx, nwUnicode RX_SC = re.compile(nwRegEx.FMT_SC) RX_LO = re.compile(r"(?i)(?{1,2}\s*|\s*<{1,2}$") def preProcessText(text: str, keepHeaders: bool = True) -> list[str]: diff --git a/tests/test_core/test_core_tohtml.py b/tests/test_core/test_core_tohtml.py index 3ccfecf3..b9be9f23 100644 --- a/tests/test_core/test_core_tohtml.py +++ b/tests/test_core/test_core_tohtml.py @@ -44,7 +44,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI): # Header 1 html._text = "# Partition\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == ( "

Partition

\n" @@ -53,7 +52,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI): # Header 2 html._text = "## Chapter Title\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == ( "

Chapter Title

\n" @@ -62,21 +60,18 @@ def testCoreToHtml_ConvertHeaders(mockGUI): # Header 3 html._text = "### Scene Title\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Scene Title

\n" # Header 4 html._text = "#### Section Title\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Section Title

\n" # Title html._text = "#! Title\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == ( "

Title

\n" @@ -85,7 +80,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI): # Unnumbered html._text = "##! Prologue\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Prologue

\n" @@ -100,35 +94,30 @@ def testCoreToHtml_ConvertHeaders(mockGUI): # Header 1 html._text = "# Heading One\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Heading One

\n" # Header 2 html._text = "## Heading Two\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Heading Two

\n" # Header 3 html._text = "### Heading Three\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Heading Three

\n" # Header 4 html._text = "#### Heading Four\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Heading Four

\n" # Title html._text = "#! Heading One\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == ( "

" @@ -138,7 +127,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI): # Unnumbered html._text = "##! Heading Two\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == "

Heading Two

\n" @@ -245,7 +233,6 @@ def testCoreToHtml_ConvertParagraphs(mockGUI): html.setKeywords(True) html._text = "## Chapter\n\n@pov: Bod\n@plot: Main\n@location: Europe\n\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == ( "

Chapter

\n" @@ -477,7 +464,6 @@ def testCoreToHtml_SpecialCases(mockGUI): html._noBreak = False html._text = "## Heading <1>\n" html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == ( "

Heading <1>

\n" @@ -551,7 +537,6 @@ def testCoreToHtml_Complex(mockGUI, fncPath): html._text = docText[i] html.doPreProcessing() html.tokenizeText() - html.doHeaders() html.doConvert() assert html.result == resText[i] diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index f2a8d827..af220f0b 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -276,7 +276,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "# Novel Title\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Novel Title", [], Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -290,7 +289,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "# Note Title\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -307,7 +305,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "## Chapter One\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -320,7 +317,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "## Heading 2\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Heading 2", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -336,7 +332,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "### Scene One\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Scene One", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -349,7 +344,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "### Heading 3\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Heading 3", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -365,7 +359,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "#### A Section\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD4, 1, "A Section", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -378,7 +371,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "#### Heading 4\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD4, 1, "Heading 4", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -395,7 +387,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "#! Title\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -409,7 +400,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "#! Title\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -425,7 +415,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "##! Prologue\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_UNNUM, 1, "Prologue", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -438,7 +427,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens._text = "##! Prologue\n" tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -459,7 +447,6 @@ def testCoreToken_HeaderStyle(mockGUI): tokens._isFirst = first tokens._noBreak = first tokens.tokenizeText() - tokens.doHeaders() return tokens._tokens[0][4] # No Styles @@ -1120,7 +1107,6 @@ def testCoreToken_SpecialFormat(mockGUI): "# Title Two\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == correctResp # Command w/Space @@ -1131,7 +1117,6 @@ def testCoreToken_SpecialFormat(mockGUI): "# Title Two\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == correctResp # Trailing Spaces @@ -1142,7 +1127,6 @@ def testCoreToken_SpecialFormat(mockGUI): "# Title Two\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == correctResp # Single Empty Paragraph @@ -1154,7 +1138,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1175,7 +1158,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1193,7 +1175,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1213,7 +1194,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1230,7 +1210,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1247,7 +1226,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1268,7 +1246,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1288,7 +1265,6 @@ def testCoreToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1313,17 +1289,6 @@ def testCoreToken_ProcessHeaders(mockGUI): project._loadProjectLocalisation() tokens = BareTokenizer(project) - # Nothing - tokens._text = "Some text ...\n" - assert tokens.doHeaders() is False - tokens._isNone = True - assert tokens.doHeaders() is False - tokens._isNone = False - assert tokens.doHeaders() is False - tokens._isNote = True - assert tokens.doHeaders() is False - tokens._isNote = False - ## # Story Files ## @@ -1340,7 +1305,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "# Part One\n" tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}") tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1351,7 +1315,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "# Part One\n" tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}") tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1364,7 +1327,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "## Chapter One\n" tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}") tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "C: Chapter One", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1374,7 +1336,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "##! Prologue\n" tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}") tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_UNNUM, 1, "U: Prologue", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1385,7 +1346,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}") tokens._hFormatter._chCount = 0 tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1395,7 +1355,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "## Chapter\n" tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}") tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter II", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1405,7 +1364,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "## Chapter\n" tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}") tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter iii", [], Tokenizer.A_PBB), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1418,7 +1376,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "### Scene One\n" tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "S: Scene One", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1428,7 +1385,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "### Scene One\n" tokens.setSceneFormat("", True) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1437,9 +1393,8 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._skipSep = True + tokens._noSep = True tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1448,9 +1403,8 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene wo/Format, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) - tokens._skipSep = False + tokens._noSep = False tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1459,9 +1413,8 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._skipSep = True + tokens._noSep = True tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1470,9 +1423,8 @@ def testCoreToken_ProcessHeaders(mockGUI): # H3: Scene Separator, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) - tokens._skipSep = False + tokens._noSep = False tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1484,7 +1436,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._hFormatter._scAbsCount = 0 tokens._hFormatter._scChCount = 0 tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Scene 1", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1496,7 +1447,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._hFormatter._scAbsCount = 0 tokens._hFormatter._scChCount = 1 tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Scene 3.2", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1509,7 +1459,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat("", True) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1519,7 +1468,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat("", False) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1529,7 +1477,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_HEAD4, 1, "X: A Section", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), @@ -1539,25 +1486,22 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat("* * *", False) tokens.tokenizeText() - tokens.doHeaders() assert tokens._tokens == [ (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Check the first scene detector, plain text - tokens._skipSep = True + tokens._noSep = True tokens._text = "Some text ...\n" tokens.tokenizeText() - tokens.doHeaders() - assert tokens._skipSep is True + assert tokens._noSep is True # Check the first scene detector, text plus scene - tokens._skipSep = True + tokens._noSep = True tokens._text = "Some text ...\n\n### Scene\n\nText" tokens.tokenizeText() - tokens.doHeaders() - assert tokens._skipSep is False + assert tokens._noSep is False # END Test testCoreToken_ProcessHeaders @@ -1580,7 +1524,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens._text = "## A Chapter Title\n\n" tokens._counts = {} tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert tokens._tokens[0][2] == "A Chapter Title" assert tokens.textStats == { @@ -1596,7 +1539,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}") tokens._hFormatter.resetAll() tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert tokens._tokens[0][2] == "C 1: A Chapter Title" assert tokens.textStats == { @@ -1611,7 +1553,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens._text = "Some text\non two lines.\n\nWith a second paragraph.\n\n" tokens._counts = {} tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert tokens.textStats == { "titleCount": 0, "paragraphCount": 2, @@ -1626,7 +1567,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setChapterFormat(nwHeadFmt.TITLE) tokens.setSceneFormat("* * *", False) tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ "Chapter", "", "", "", "Text", "", "* * *", "", "Text", "" @@ -1646,7 +1586,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setSceneFormat("* * *", False) tokens.setSynopsis(True) tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ "Chapter", "", "", "", "Stuff", "", "Text", "" @@ -1666,7 +1605,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setSceneFormat("* * *", False) tokens.setSynopsis(True) tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ "Chapter", "", "", "", "Stuff", "", "Text", "" @@ -1686,7 +1624,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setSceneFormat("* * *", False) tokens.setComments(True) tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ "Chapter", "", "", "", "Stuff", "", "Text", "" @@ -1706,7 +1643,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setSceneFormat("* * *", False) tokens.setKeywords(True) tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert [t[2] for t in tokens._tokens] == [ "Chapter", "", "", "", "pov: Jane", "", "Text", "" @@ -1769,7 +1705,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setKeywords(True) tokens.tokenizeText() - tokens.doHeaders() tokens.countStats() assert tokens.textStats == { "titleCount": 4, "paragraphCount": 5, @@ -1816,7 +1751,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): # Static Separator md.setSceneFormat("* * *", False) md.tokenizeText() - md.doHeaders() md.doConvert() assert md.result == ( "# T: Title One\n\n" @@ -1832,7 +1766,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): # Scene Title Formatted md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) md.tokenizeText() - md.doHeaders() md.doConvert() assert md.result == ( "# T: Title One\n\n" @@ -1870,7 +1803,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): # Static Separator md.setSceneFormat("* * *", False) md.tokenizeText() - md.doHeaders() md.doConvert() assert md.result == ( "# T: Title One\n\n" @@ -1887,7 +1819,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): # Scene Title Formatted md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) md.tokenizeText() - md.doHeaders() md.doConvert() assert md.result == ( "# T: Title One\n\n" @@ -1938,7 +1869,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI): # Two Novel Format md.tokenizeText() - md.doHeaders() md.doConvert() assert md.result == ( "# Novel One\n\n" diff --git a/tests/test_core/test_core_toodt.py b/tests/test_core/test_core_toodt.py index 8c4863e7..eaa808f9 100644 --- a/tests/test_core/test_core_toodt.py +++ b/tests/test_core/test_core_toodt.py @@ -246,7 +246,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI): # Header 1 odt._text = "# Title\n" odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -260,7 +259,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI): # Header 2 odt._text = "## Chapter\n" odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -274,7 +272,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI): # Header 3 odt._text = "### Scene\n" odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -288,7 +285,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI): # Header 4 odt._text = "#### Section\n" odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -303,7 +299,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI): odt._isFirst = True odt._text = "#! Title\n" odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -317,7 +312,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI): # Unnumbered chapter odt._text = "##! Prologue\n" odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -507,7 +501,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI): odt._text = "### Scene One\n\nText\n\n### Scene Two\n\nText" odt.setSceneFormat("* * *", False) odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -525,7 +518,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI): odt._text = "### Scene One\n\nText\n\n### Scene Two\n\nText" odt.setSceneFormat("", False) odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -613,7 +605,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI): "Text\n\n" ) odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -738,7 +729,6 @@ def testCoreToOdt_SaveFlat(mockGUI, fncPath, tstPaths): "Text\n\n" ) odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument() @@ -778,7 +768,6 @@ def testCoreToOdt_SaveFull(mockGUI, fncPath, tstPaths): "Text\n\n" ) odt.tokenizeText() - odt.doHeaders() odt.initDocument() odt.doConvert() odt.closeDocument()