diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py index b0b6557a..85a91f59 100644 --- a/novelwriter/core/tohtml.py +++ b/novelwriter/core/tohtml.py @@ -238,8 +238,8 @@ class ToHtml(Tokenizer): aNm = "" # Process Text Type - if tType == self.T_EMPTY: - pass + if tType == self.T_TEXT: + lines.append(f"{self._formatText(tText, tFormat, hTags)}

\n") elif tType == self.T_TITLE: tHead = tText.replace(nwHeadFmt.BR, "
") @@ -267,9 +267,6 @@ class ToHtml(Tokenizer): elif tType == self.T_SKIP: lines.append(f"

 

\n") - elif tType == self.T_TEXT: - lines.append(f"{self._formatText(tText, tFormat, hTags)}

\n") - elif tType == self.T_SYNOPSIS and self._doSynopsis: lines.append(self._formatSynopsis(self._formatText(tText, tFormat, hTags), True)) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 44417c67..df85b8a2 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -840,9 +840,8 @@ class Tokenizer(ABC): nToken = tokens[n+1] # Look ahead if cToken[0] == self.T_EMPTY: - # Strip multiple empty - if pToken[0] != self.T_EMPTY: - self._tokens.append(cToken) + # We don't need to keep the empty lines after this pass + pass elif cToken[0] == self.T_KEYWORD: # Adjust margins for lines in a list of keyword lines @@ -918,7 +917,6 @@ class Tokenizer(ABC): textWordChars = self._counts.get("textWordChars", 0) titleWordChars = self._counts.get("titleWordChars", 0) - para = [] for tType, _, tText, _, _ in self._tokens: tText = tText.replace(nwUnicode.U_ENDASH, " ") tText = tText.replace(nwUnicode.U_EMDASH, " ") @@ -928,24 +926,7 @@ class Tokenizer(ABC): nChars = len(tText) nWChars = len("".join(tWords)) - if tType == self.T_EMPTY: - if len(para) > 0: - tTemp = "\n".join(para) - tPWords = tTemp.split() - nPWords = len(tPWords) - nPChars = len(tTemp) - nPWChars = len("".join(tPWords)) - - paragraphCount += 1 - allWords += nPWords - textWords += nPWords - allChars += nPChars - textChars += nPChars - allWordChars += nPWChars - textWordChars += nPWChars - para = [] - - elif tType in self.L_HEADINGS: + if tType in self.L_HEADINGS: titleCount += 1 allWords += nWords titleWords += nWords @@ -960,7 +941,18 @@ class Tokenizer(ABC): allWordChars += nWChars elif tType == self.T_TEXT: - para.append(tText.rstrip()) + tPWords = tText.split() + nPWords = len(tPWords) + nPChars = len(tText) + nPWChars = len("".join(tPWords)) + + paragraphCount += 1 + allWords += nPWords + textWords += nPWords + allChars += nPChars + textChars += nPChars + allWordChars += nPWChars + textWordChars += nPWChars elif tType == self.T_SYNOPSIS and self._doSynopsis: text = "{0}: {1}".format(self._localLookup("Synopsis"), tText) diff --git a/novelwriter/core/tomarkdown.py b/novelwriter/core/tomarkdown.py index 371846ad..440da0e0 100644 --- a/novelwriter/core/tomarkdown.py +++ b/novelwriter/core/tomarkdown.py @@ -142,8 +142,9 @@ class ToMarkdown(Tokenizer): lines = [] for tType, _, tText, tFormat, tStyle in self._tokens: - if tType == self.T_EMPTY: - pass + if tType == self.T_TEXT: + tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n") + lines.append(f"{tTemp}\n\n") elif tType == self.T_TITLE: tHead = tText.replace(nwHeadFmt.BR, "\n") @@ -171,10 +172,6 @@ class ToMarkdown(Tokenizer): elif tType == self.T_SKIP: lines.append(f"{cSkip}\n\n") - elif tType == self.T_TEXT: - tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n") - lines.append(f"{tTemp}\n\n") - elif tType == self.T_SYNOPSIS and self._doSynopsis: label = self._localLookup("Synopsis") lines.append(f"**{label}:** {self._formatText(tText, tFormat, mTags)}\n\n") diff --git a/novelwriter/core/toodt.py b/novelwriter/core/toodt.py index 30873704..cd81d100 100644 --- a/novelwriter/core/toodt.py +++ b/novelwriter/core/toodt.py @@ -454,8 +454,12 @@ class ToOdt(Tokenizer): pIndent = False # Process Text Types - if tType == self.T_EMPTY: - pass + if tType == self.T_TEXT: + if self._firstIndent and pIndent and oStyle.isUnaligned(): + self._addTextPar(xText, S_FIND, oStyle, tText, tFmt=tFormat) + else: + self._addTextPar(xText, S_TEXT, oStyle, tText, tFmt=tFormat) + pIndent = True elif tType == self.T_TITLE: # Title must be text:p @@ -484,13 +488,6 @@ class ToOdt(Tokenizer): elif tType == self.T_SKIP: self._addTextPar(xText, S_SEP, oStyle, "") - elif tType == self.T_TEXT: - if self._firstIndent and pIndent and oStyle.isUnaligned(): - self._addTextPar(xText, S_FIND, oStyle, tText, tFmt=tFormat) - else: - self._addTextPar(xText, S_TEXT, oStyle, tText, tFmt=tFormat) - pIndent = True - elif tType == self.T_SYNOPSIS and self._doSynopsis: tTemp, tFmt = self._formatSynopsis(tText, tFormat, True) self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt) diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 3122f182..388b8843 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -262,7 +262,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TITLE, 1, "Novel Title", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Novel Title\n\n" @@ -274,7 +273,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TITLE, 1, "Note Title", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Note Title\n\n" @@ -289,7 +287,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Novel Title", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "# Novel Title\n\n" @@ -301,7 +298,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "# Note Title\n\n" @@ -315,7 +311,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "## Chapter One\n\n" @@ -326,7 +321,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Heading 2", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "## Heading 2\n\n" @@ -340,7 +334,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Scene One", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "### Scene One\n\n" @@ -351,7 +344,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Heading 3", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "### Heading 3\n\n" @@ -365,7 +357,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD4, 1, "A Section", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#### A Section\n\n" @@ -376,7 +367,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD4, 1, "Heading 4", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#### Heading 4\n\n" @@ -391,7 +381,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Title\n\n" @@ -403,7 +392,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Title\n\n" @@ -417,7 +405,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "##! Prologue\n\n" @@ -428,7 +415,6 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "##! Prologue\n\n" @@ -706,7 +692,6 @@ def testCoreToken_MetaFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_COMMENT, 0, "A comment", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -725,13 +710,11 @@ def testCoreToken_MetaFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] tokens._text = "% synopsis: The synopsis\n" tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -745,7 +728,6 @@ def testCoreToken_MetaFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_SHORT, 0, "A short description", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -758,7 +740,6 @@ def testCoreToken_MetaFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_KEYWORD, 0, "char: Bod", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -775,7 +756,6 @@ def testCoreToken_MetaFormat(mockGUI): (Tokenizer.T_KEYWORD, 0, "pov: Bod", [], styTop), (Tokenizer.T_KEYWORD, 0, "plot: Main", [], styMid), (Tokenizer.T_KEYWORD, 0, "location: Europe", [], styBtm), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n" @@ -785,7 +765,6 @@ def testCoreToken_MetaFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_KEYWORD, 0, "pov: Bod", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] @@ -812,21 +791,13 @@ def testCoreToken_MarginFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TEXT, 0, "Some regular text", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Some left-aligned text", [], Tokenizer.A_LEFT), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Some right-aligned text", [], Tokenizer.A_RIGHT), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Some centered text", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Left-indented block", [], Tokenizer.A_IND_L), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Right-indented block", [], Tokenizer.A_IND_R), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Double-indented block", [], dblIndent), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Right-indent, right-aligned", [], rIndAlign), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == ( "Some regular text\n\n" @@ -959,9 +930,7 @@ def testCoreToken_Paragraphs(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TEXT, 0, "First paragraph", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Second paragraph", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] # Combine multi-line paragraphs, keep breaks @@ -970,7 +939,6 @@ def testCoreToken_Paragraphs(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TEXT, 0, "This is text\nspanning multiple\nlines", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] # Combine multi-line paragraphs, remove breaks @@ -979,7 +947,6 @@ def testCoreToken_Paragraphs(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TEXT, 0, "This is text spanning multiple lines", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] # Combine multi-line paragraphs, remove breaks, with formatting @@ -998,7 +965,6 @@ def testCoreToken_Paragraphs(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] # Make sure titles break a paragraph @@ -1022,7 +988,6 @@ def testCoreToken_Paragraphs(mockGUI): (14, Tokenizer.FMT_I_E, ""), ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), ] @@ -1038,7 +1003,6 @@ def testCoreToken_TextFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TEXT, 0, "Some plain text\non two lines", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n" @@ -1061,7 +1025,6 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some **bolded text** on this lines\n\n" @@ -1077,7 +1040,6 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some _italic text_ on this lines\n\n" @@ -1095,7 +1057,6 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n" @@ -1111,7 +1072,6 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n" @@ -1131,7 +1091,6 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == ( "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n" @@ -1151,9 +1110,7 @@ def testCoreToken_SpecialFormat(mockGUI): correctResp = [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_HEAD1, 2, "Title Two", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), ] # Command wo/Space @@ -1197,11 +1154,8 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Multiple Empty Paragraphs @@ -1216,11 +1170,8 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Three Skips @@ -1232,13 +1183,10 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 1 @@ -1250,9 +1198,7 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 2 @@ -1264,9 +1210,7 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 3 @@ -1278,9 +1222,7 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Empty Paragraph and Page Break @@ -1296,11 +1238,8 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Multiple Skip @@ -1313,13 +1252,10 @@ def testCoreToken_SpecialFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] @@ -1347,7 +1283,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H1: Title, Not First Page @@ -1357,7 +1292,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Chapters @@ -1369,7 +1303,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "C: Chapter One", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Unnumbered Chapter @@ -1378,7 +1311,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "U: Prologue", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Chapter Word Number @@ -1388,7 +1320,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Chapter Roman Number Upper Case @@ -1397,7 +1328,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter II", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Chapter Roman Number Lower Case @@ -1406,7 +1336,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD2, 1, "Chapter iii", [], Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Scenes @@ -1418,7 +1347,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "S: Scene One", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene Hidden wo/Format @@ -1441,7 +1369,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene Separator, first @@ -1458,7 +1385,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene w/Absolute Number @@ -1469,7 +1395,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Scene 1", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene w/Chapter Number @@ -1480,7 +1405,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD3, 1, "Scene 3.2", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Sections @@ -1498,7 +1422,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H4: Section w/Format @@ -1507,7 +1430,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_HEAD4, 1, "X: A Section", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H4: Section Separator @@ -1516,7 +1438,6 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Check the first scene detector, plain text @@ -1656,9 +1577,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setSceneFormat("* * *", False) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "Text", "", "* * *", "", "Text", "" - ] + assert [t[2] for t in tokens._tokens] == ["Chapter", "Text", "* * *", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 2, "allWords": 6, "textWords": 2, "titleWords": 1, @@ -1675,9 +1594,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setSynopsis(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "Stuff", "", "Text", "" - ] + assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 4, "textWords": 1, "titleWords": 1, @@ -1694,9 +1611,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setSynopsis(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "Stuff", "", "Text", "" - ] + assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 5, "textWords": 1, "titleWords": 1, @@ -1713,9 +1628,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setComments(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "Stuff", "", "Text", "" - ] + assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 4, "textWords": 1, "titleWords": 1, @@ -1732,9 +1645,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText): tokens.setKeywords(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == [ - "Chapter", "", "pov: Jane", "", "Text", "" - ] + assert [t[2] for t in tokens._tokens] == ["Chapter", "pov: Jane", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 6, "textWords": 1, "titleWords": 1,