diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 77d4a23d..7f0bb7d2 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -121,7 +121,7 @@ class Tokenizer(ABC): self._allMarkdown = [] # The result novelWriter markdown of all documents # Processed Tokens - self._tokens: list[tuple[int, int, str, list[tuple[int, int]] | None, int]] = [] + self._tokens: list[tuple[int, int, str, list[tuple[int, int]], int]] = [] # User Settings self._textFont = "Serif" # Output text font @@ -354,7 +354,7 @@ class Tokenizer(ABC): title = f"{trNotes}: {tItem.itemName}" self._tokens = [] self._tokens.append(( - self.T_TITLE, 0, title, None, textAlign + self.T_TITLE, 0, title, [], textAlign )) if self._keepMarkdown: self._allMarkdown.append(f"# {title}\n\n") @@ -422,7 +422,7 @@ class Tokenizer(ABC): # Check for blank lines if len(sLine) == 0: self._tokens.append(( - self.T_EMPTY, nHead, "", None, self.A_NONE + self.T_EMPTY, nHead, "", [], self.A_NONE )) if self._keepMarkdown: tmpMarkdown.append("\n") @@ -450,7 +450,7 @@ class Tokenizer(ABC): elif sLine == "[vspace]": self._tokens.append( - (self.T_SKIP, nHead, "", None, sAlign) + (self.T_SKIP, nHead, "", [], sAlign) ) continue @@ -458,11 +458,11 @@ class Tokenizer(ABC): nSkip = checkInt(sLine[8:-1], 0) if nSkip >= 1: self._tokens.append( - (self.T_SKIP, nHead, "", None, sAlign) + (self.T_SKIP, nHead, "", [], sAlign) ) if nSkip > 1: self._tokens += (nSkip - 1) * [ - (self.T_SKIP, nHead, "", None, self.A_NONE) + (self.T_SKIP, nHead, "", [], self.A_NONE) ] continue @@ -470,26 +470,26 @@ class Tokenizer(ABC): cStyle, cText, _ = processComment(aLine) if cStyle == nwComment.SYNOPSIS: self._tokens.append(( - self.T_SYNOPSIS, nHead, cText, None, sAlign + self.T_SYNOPSIS, nHead, cText, [], sAlign )) if self._doSynopsis and self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) elif cStyle == nwComment.SHORT: self._tokens.append(( - self.T_SHORT, nHead, cText, None, sAlign + self.T_SHORT, nHead, cText, [], sAlign )) if self._doSynopsis and self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) else: self._tokens.append(( - self.T_COMMENT, nHead, cText, None, sAlign + self.T_COMMENT, nHead, cText, [], sAlign )) if self._doComments and self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) elif aLine[0] == "@": self._tokens.append(( - self.T_KEYWORD, nHead, aLine[1:].strip(), None, sAlign + self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign )) if self._doKeywords and self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) @@ -501,7 +501,7 @@ class Tokenizer(ABC): nHead += 1 self._tokens.append(( - self.T_HEAD1, nHead, aLine[2:].strip(), None, sAlign + self.T_HEAD1, nHead, aLine[2:].strip(), [], sAlign )) if self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) @@ -512,7 +512,7 @@ class Tokenizer(ABC): nHead += 1 self._tokens.append(( - self.T_HEAD2, nHead, aLine[3:].strip(), None, sAlign + self.T_HEAD2, nHead, aLine[3:].strip(), [], sAlign )) if self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) @@ -520,7 +520,7 @@ class Tokenizer(ABC): elif aLine[:4] == "### ": nHead += 1 self._tokens.append(( - self.T_HEAD3, nHead, aLine[4:].strip(), None, sAlign + self.T_HEAD3, nHead, aLine[4:].strip(), [], sAlign )) if self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) @@ -528,7 +528,7 @@ class Tokenizer(ABC): elif aLine[:5] == "#### ": nHead += 1 self._tokens.append(( - self.T_HEAD4, nHead, aLine[5:].strip(), None, sAlign + self.T_HEAD4, nHead, aLine[5:].strip(), [], sAlign )) if self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) @@ -541,7 +541,7 @@ class Tokenizer(ABC): tStyle = self.T_HEAD1 self._tokens.append(( - tStyle, nHead, aLine[3:].strip(), None, sAlign | self.A_CENTRE + tStyle, nHead, aLine[3:].strip(), [], sAlign | self.A_CENTRE )) if self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) @@ -555,7 +555,7 @@ class Tokenizer(ABC): tStyle = self.T_HEAD2 self._tokens.append(( - tStyle, nHead, aLine[4:].strip(), None, sAlign + tStyle, nHead, aLine[4:].strip(), [], sAlign )) if self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) @@ -618,7 +618,7 @@ class Tokenizer(ABC): # Always add an empty line at the end of the file self._tokens.append(( - self.T_EMPTY, nHead, "", None, self.A_NONE + self.T_EMPTY, nHead, "", [], self.A_NONE )) if self._keepMarkdown: tmpMarkdown.append("\n") @@ -630,8 +630,8 @@ class Tokenizer(ABC): # =========== # Some items need a second pass - pToken = (self.T_EMPTY, 0, "", None, self.A_NONE) - nToken = (self.T_EMPTY, 0, "", None, self.A_NONE) + pToken = (self.T_EMPTY, 0, "", [], self.A_NONE) + nToken = (self.T_EMPTY, 0, "", [], self.A_NONE) tCount = len(self._tokens) for n, tToken in enumerate(self._tokens): @@ -672,7 +672,7 @@ class Tokenizer(ABC): tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1]) self._tokens[n] = ( - tToken[0], tToken[1], tTemp, None, tToken[4] + tToken[0], tToken[1], tTemp, [], tToken[4] ) elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): @@ -687,7 +687,7 @@ class Tokenizer(ABC): # Format the chapter header self._tokens[n] = ( - tToken[0], tToken[1], tTemp, None, tToken[4] + tToken[0], tToken[1], tTemp, [], tToken[4] ) # Set scene variables @@ -702,29 +702,29 @@ class Tokenizer(ABC): tTemp = self._hFormatter.apply(self._fmtScene, tToken[2], tToken[1]) if tTemp == "" and self._hideScene: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", None, self.A_NONE + self.T_EMPTY, tToken[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideScene: if self._firstScene: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", None, self.A_NONE + self.T_EMPTY, tToken[1], "", [], self.A_NONE ) else: self._tokens[n] = ( - self.T_SKIP, tToken[1], "", None, tToken[4] + self.T_SKIP, tToken[1], "", [], tToken[4] ) elif tTemp == self._fmtScene: if self._firstScene: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", None, self.A_NONE + self.T_EMPTY, tToken[1], "", [], self.A_NONE ) else: self._tokens[n] = ( - self.T_SEP, tToken[1], tTemp, None, tToken[4] | self.A_CENTRE + self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE ) else: self._tokens[n] = ( - tToken[0], tToken[1], tTemp, None, tToken[4] + tToken[0], tToken[1], tTemp, [], tToken[4] ) self._firstScene = False @@ -735,19 +735,19 @@ class Tokenizer(ABC): tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1]) if tTemp == "" and self._hideSection: self._tokens[n] = ( - self.T_EMPTY, tToken[1], "", None, self.A_NONE + self.T_EMPTY, tToken[1], "", [], self.A_NONE ) elif tTemp == "" and not self._hideSection: self._tokens[n] = ( - self.T_SKIP, tToken[1], "", None, tToken[4] + self.T_SKIP, tToken[1], "", [], tToken[4] ) elif tTemp == self._fmtSection: self._tokens[n] = ( - self.T_SEP, tToken[1], tTemp, None, tToken[4] | self.A_CENTRE + self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE ) else: self._tokens[n] = ( - tToken[0], tToken[1], tTemp, None, tToken[4] + tToken[0], tToken[1], tTemp, [], tToken[4] ) return True diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 910d9916..56099589 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -170,14 +170,14 @@ def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath): assert tokens.addRootHeading(C.hPlotRoot) is True assert tokens.allMarkdown[-1] == "# Notes: Plot\n\n" assert tokens._tokens[-1] == ( - Tokenizer.T_TITLE, 0, "Notes: Plot", None, Tokenizer.A_CENTRE + Tokenizer.T_TITLE, 0, "Notes: Plot", [], Tokenizer.A_CENTRE ) # Not First Page assert tokens.addRootHeading(C.hPlotRoot) is True assert tokens.allMarkdown[-1] == "# Notes: Plot\n\n" assert tokens._tokens[-1] == ( - Tokenizer.T_TITLE, 0, "Notes: Plot", None, Tokenizer.A_CENTRE | Tokenizer.A_PBB + Tokenizer.T_TITLE, 0, "Notes: Plot", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB ) # Set Text @@ -247,8 +247,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_TITLE, 1, "Novel Title", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_TITLE, 1, "Novel Title", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Novel Title\n\n" @@ -260,8 +260,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Note Title", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Note Title\n\n" @@ -276,8 +276,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Novel Title", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Novel Title", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "# Novel Title\n\n" @@ -289,8 +289,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Note Title", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "# Note Title\n\n" @@ -304,8 +304,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "## Chapter One\n\n" @@ -316,8 +316,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD2, 1, "Heading 2", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD2, 1, "Heading 2", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "## Heading 2\n\n" @@ -331,8 +331,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD3, 1, "Scene One", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD3, 1, "Scene One", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "### Scene One\n\n" @@ -343,8 +343,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD3, 1, "Heading 3", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD3, 1, "Heading 3", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "### Heading 3\n\n" @@ -358,8 +358,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD4, 1, "A Section", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD4, 1, "A Section", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#### A Section\n\n" @@ -370,8 +370,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD4, 1, "Heading 4", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD4, 1, "Heading 4", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#### Heading 4\n\n" @@ -385,8 +385,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_TITLE, 1, "Title", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Title\n\n" @@ -397,8 +397,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "#! Title\n\n" @@ -412,8 +412,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_UNNUM, 1, "Prologue", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_UNNUM, 1, "Prologue", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "##! Prologue\n\n" @@ -424,8 +424,8 @@ def testCoreToken_HeaderFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD2, 1, "Prologue", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "##! Prologue\n\n" @@ -443,8 +443,8 @@ def testCoreToken_MetaFormat(mockGUI): tokens._text = "% A comment\n" tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_COMMENT, 0, "A comment", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_COMMENT, 0, "A comment", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -456,14 +456,14 @@ def testCoreToken_MetaFormat(mockGUI): tokens._text = "%synopsis: The synopsis\n" tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_SYNOPSIS, 0, "The synopsis", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] tokens._text = "% synopsis: The synopsis\n" tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_SYNOPSIS, 0, "The synopsis", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -476,8 +476,8 @@ def testCoreToken_MetaFormat(mockGUI): tokens._text = "% short: A short description\n" tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_SHORT, 0, "A short description", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_SHORT, 0, "A short description", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -489,8 +489,8 @@ def testCoreToken_MetaFormat(mockGUI): tokens._text = "@char: Bod\n" tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_KEYWORD, 0, "char: Bod", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_KEYWORD, 0, "char: Bod", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -504,10 +504,10 @@ def testCoreToken_MetaFormat(mockGUI): styMid = Tokenizer.A_NONE | Tokenizer.A_Z_BTMMRG | Tokenizer.A_Z_TOPMRG styBtm = Tokenizer.A_NONE | Tokenizer.A_Z_TOPMRG assert tokens._tokens == [ - (Tokenizer.T_KEYWORD, 0, "pov: Bod", None, styTop), - (Tokenizer.T_KEYWORD, 0, "plot: Main", None, styMid), - (Tokenizer.T_KEYWORD, 0, "location: Europe", None, styBtm), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_KEYWORD, 0, "pov: Bod", [], styTop), + (Tokenizer.T_KEYWORD, 0, "plot: Main", [], styMid), + (Tokenizer.T_KEYWORD, 0, "location: Europe", [], styBtm), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n" @@ -537,22 +537,22 @@ def testCoreToken_MarginFormat(mockGUI): tokens.tokenizeText() assert tokens._tokens == [ (Tokenizer.T_TEXT, 0, "Some regular text", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Some left-aligned text", [], Tokenizer.A_LEFT), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Some right-aligned text", [], Tokenizer.A_RIGHT), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Some centered text", [], Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Left-indented block", [], Tokenizer.A_IND_L), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Right-indented block", [], Tokenizer.A_IND_R), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Double-indented block", [], dblIndent), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Right-indent, right-aligned", [], rIndAlign), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == ( "Some regular text\n\n" @@ -672,18 +672,18 @@ def testCoreToken_TextFormat(mockGUI): assert tokens._tokens == [ (Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n" tokens.setBodyText(False) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "\n\n\n" tokens.setBodyText(True) @@ -701,7 +701,7 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some **bolded text** on this lines\n\n" @@ -717,7 +717,7 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some _italic text_ on this lines\n\n" @@ -735,7 +735,7 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n" @@ -751,7 +751,7 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n" @@ -771,7 +771,7 @@ def testCoreToken_TextFormat(mockGUI): ], Tokenizer.A_NONE ), - (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), ] assert tokens.allMarkdown[-1] == ( "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n" @@ -792,12 +792,12 @@ def testCoreToken_SpecialFormat(mockGUI): # ======== correctResp = [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_HEAD1, 2, "Title Two", None, Tokenizer.A_CENTRE | Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 2, "Title Two", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), ] # Command wo/Space @@ -840,13 +840,13 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Multiple Empty Paragraphs @@ -860,13 +860,13 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Three Skips @@ -877,15 +877,15 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 1 @@ -896,12 +896,12 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 2 @@ -912,12 +912,12 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Malformed Command, Case 3 @@ -928,12 +928,12 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Empty Paragraph and Page Break @@ -948,14 +948,14 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Multiple Skip @@ -967,16 +967,16 @@ def testCoreToken_SpecialFormat(mockGUI): ) tokens.tokenizeText() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_PBB), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # END Test testCoreToken_SpecialFormat @@ -1019,8 +1019,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "T: Part One", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H1: Title, Not First Page @@ -1030,8 +1030,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD1, 1, "T: Part One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Chapters @@ -1043,8 +1043,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD2, 1, "C: Chapter One", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD2, 1, "C: Chapter One", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Unnumbered Chapter @@ -1053,8 +1053,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_UNNUM, 1, "U: Prologue", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_UNNUM, 1, "U: Prologue", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Chapter Word Number @@ -1064,8 +1064,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Chapter Roman Number Upper Case @@ -1074,8 +1074,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD2, 1, "Chapter II", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD2, 1, "Chapter II", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H2: Chapter Roman Number Lower Case @@ -1084,8 +1084,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD2, 1, "Chapter iii", None, Tokenizer.A_PBB), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD2, 1, "Chapter iii", [], Tokenizer.A_PBB), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Scenes @@ -1097,8 +1097,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD3, 1, "S: Scene One", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD3, 1, "S: Scene One", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene Hidden wo/Format @@ -1107,8 +1107,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene wo/Format, first @@ -1118,8 +1118,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene wo/Format, not first @@ -1129,8 +1129,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene Separator, first @@ -1140,8 +1140,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene Separator, not first @@ -1151,8 +1151,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene w/Absolute Number @@ -1163,8 +1163,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD3, 1, "Scene 1", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD3, 1, "Scene 1", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H3: Scene w/Chapter Number @@ -1175,8 +1175,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD3, 1, "Scene 3.2", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD3, 1, "Scene 3.2", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Sections @@ -1188,8 +1188,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H4: Section Visible wo/Format @@ -1198,8 +1198,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H4: Section w/Format @@ -1208,8 +1208,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_HEAD4, 1, "X: A Section", None, Tokenizer.A_NONE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_HEAD4, 1, "X: A Section", [], Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # H4: Section Separator @@ -1218,8 +1218,8 @@ def testCoreToken_ProcessHeaders(mockGUI): tokens.tokenizeText() tokens.doHeaders() assert tokens._tokens == [ - (Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE), - (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE), + (Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE), + (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), ] # Check the first scene detector