Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change tokenizer parsed data to always use a list for format, even when empty

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-01-30 23:25:11 +01:00
parent e6193ed858
commit 76ab9c3908
2 changed files with 190 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tokenizer.py
+31 -31
View File
@@ -121,7 +121,7 @@ class Tokenizer(ABC):
self._allMarkdown = [] # The result novelWriter markdown of all documents
# Processed Tokens
self._tokens: list[tuple[int, int, str, list[tuple[int, int]] | None, int]] = []
self._tokens: list[tuple[int, int, str, list[tuple[int, int]], int]] = []
# User Settings
self._textFont = "Serif" # Output text font
@@ -354,7 +354,7 @@ class Tokenizer(ABC):
title = f"{trNotes}: {tItem.itemName}"
self._tokens = []
self._tokens.append((
self.T_TITLE, 0, title, None, textAlign
self.T_TITLE, 0, title, [], textAlign
))
if self._keepMarkdown:
self._allMarkdown.append(f"# {title}\n\n")
@@ -422,7 +422,7 @@ class Tokenizer(ABC):
# Check for blank lines
if len(sLine) == 0:
self._tokens.append((
self.T_EMPTY, nHead, "", None, self.A_NONE
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMarkdown:
tmpMarkdown.append("\n")
@@ -450,7 +450,7 @@ class Tokenizer(ABC):
elif sLine == "[vspace]":
self._tokens.append(
(self.T_SKIP, nHead, "", None, sAlign)
(self.T_SKIP, nHead, "", [], sAlign)
)
continue
@@ -458,11 +458,11 @@ class Tokenizer(ABC):
nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1:
self._tokens.append(
(self.T_SKIP, nHead, "", None, sAlign)
(self.T_SKIP, nHead, "", [], sAlign)
)
if nSkip > 1:
self._tokens += (nSkip - 1) * [
(self.T_SKIP, nHead, "", None, self.A_NONE)
(self.T_SKIP, nHead, "", [], self.A_NONE)
]
continue
@@ -470,26 +470,26 @@ class Tokenizer(ABC):
cStyle, cText, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS:
self._tokens.append((
self.T_SYNOPSIS, nHead, cText, None, sAlign
self.T_SYNOPSIS, nHead, cText, [], sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
elif cStyle == nwComment.SHORT:
self._tokens.append((
self.T_SHORT, nHead, cText, None, sAlign
self.T_SHORT, nHead, cText, [], sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
else:
self._tokens.append((
self.T_COMMENT, nHead, cText, None, sAlign
self.T_COMMENT, nHead, cText, [], sAlign
))
if self._doComments and self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
elif aLine[0] == "@":
self._tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), None, sAlign
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
@@ -501,7 +501,7 @@ class Tokenizer(ABC):
nHead += 1
self._tokens.append((
self.T_HEAD1, nHead, aLine[2:].strip(), None, sAlign
self.T_HEAD1, nHead, aLine[2:].strip(), [], sAlign
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
@@ -512,7 +512,7 @@ class Tokenizer(ABC):
nHead += 1
self._tokens.append((
self.T_HEAD2, nHead, aLine[3:].strip(), None, sAlign
self.T_HEAD2, nHead, aLine[3:].strip(), [], sAlign
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
@@ -520,7 +520,7 @@ class Tokenizer(ABC):
elif aLine[:4] == "### ":
nHead += 1
self._tokens.append((
self.T_HEAD3, nHead, aLine[4:].strip(), None, sAlign
self.T_HEAD3, nHead, aLine[4:].strip(), [], sAlign
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
@@ -528,7 +528,7 @@ class Tokenizer(ABC):
elif aLine[:5] == "#### ":
nHead += 1
self._tokens.append((
self.T_HEAD4, nHead, aLine[5:].strip(), None, sAlign
self.T_HEAD4, nHead, aLine[5:].strip(), [], sAlign
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
@@ -541,7 +541,7 @@ class Tokenizer(ABC):
tStyle = self.T_HEAD1
self._tokens.append((
tStyle, nHead, aLine[3:].strip(), None, sAlign | self.A_CENTRE
tStyle, nHead, aLine[3:].strip(), [], sAlign | self.A_CENTRE
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
@@ -555,7 +555,7 @@ class Tokenizer(ABC):
tStyle = self.T_HEAD2
self._tokens.append((
tStyle, nHead, aLine[4:].strip(), None, sAlign
tStyle, nHead, aLine[4:].strip(), [], sAlign
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
@@ -618,7 +618,7 @@ class Tokenizer(ABC):
# Always add an empty line at the end of the file
self._tokens.append((
self.T_EMPTY, nHead, "", None, self.A_NONE
self.T_EMPTY, nHead, "", [], self.A_NONE
))
if self._keepMarkdown:
tmpMarkdown.append("\n")
@@ -630,8 +630,8 @@ class Tokenizer(ABC):
# ===========
# Some items need a second pass
pToken = (self.T_EMPTY, 0, "", None, self.A_NONE)
nToken = (self.T_EMPTY, 0, "", None, self.A_NONE)
pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
tCount = len(self._tokens)
for n, tToken in enumerate(self._tokens):
@@ -672,7 +672,7 @@ class Tokenizer(ABC):
tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1])
self._tokens[n] = (
tToken[0], tToken[1], tTemp, None, tToken[4]
tToken[0], tToken[1], tTemp, [], tToken[4]
)
elif tToken[0] in (self.T_HEAD2, self.T_UNNUM):
@@ -687,7 +687,7 @@ class Tokenizer(ABC):
# Format the chapter header
self._tokens[n] = (
tToken[0], tToken[1], tTemp, None, tToken[4]
tToken[0], tToken[1], tTemp, [], tToken[4]
)
# Set scene variables
@@ -702,29 +702,29 @@ class Tokenizer(ABC):
tTemp = self._hFormatter.apply(self._fmtScene, tToken[2], tToken[1])
if tTemp == "" and self._hideScene:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", None, self.A_NONE
self.T_EMPTY, tToken[1], "", [], self.A_NONE
)
elif tTemp == "" and not self._hideScene:
if self._firstScene:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", None, self.A_NONE
self.T_EMPTY, tToken[1], "", [], self.A_NONE
)
else:
self._tokens[n] = (
self.T_SKIP, tToken[1], "", None, tToken[4]
self.T_SKIP, tToken[1], "", [], tToken[4]
)
elif tTemp == self._fmtScene:
if self._firstScene:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", None, self.A_NONE
self.T_EMPTY, tToken[1], "", [], self.A_NONE
)
else:
self._tokens[n] = (
self.T_SEP, tToken[1], tTemp, None, tToken[4] | self.A_CENTRE
self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE
)
else:
self._tokens[n] = (
tToken[0], tToken[1], tTemp, None, tToken[4]
tToken[0], tToken[1], tTemp, [], tToken[4]
)
self._firstScene = False
@@ -735,19 +735,19 @@ class Tokenizer(ABC):
tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1])
if tTemp == "" and self._hideSection:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", None, self.A_NONE
self.T_EMPTY, tToken[1], "", [], self.A_NONE
)
elif tTemp == "" and not self._hideSection:
self._tokens[n] = (
self.T_SKIP, tToken[1], "", None, tToken[4]
self.T_SKIP, tToken[1], "", [], tToken[4]
)
elif tTemp == self._fmtSection:
self._tokens[n] = (
self.T_SEP, tToken[1], tTemp, None, tToken[4] | self.A_CENTRE
self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE
)
else:
self._tokens[n] = (
tToken[0], tToken[1], tTemp, None, tToken[4]
tToken[0], tToken[1], tTemp, [], tToken[4]
)
return True
tests/test_core/test_core_tokenizer.py
+159 -159
View File
@@ -170,14 +170,14 @@ def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
assert tokens.addRootHeading(C.hPlotRoot) is True
assert tokens.allMarkdown[-1] == "# Notes: Plot\n\n"
assert tokens._tokens[-1] == (
Tokenizer.T_TITLE, 0, "Notes: Plot", None, Tokenizer.A_CENTRE
Tokenizer.T_TITLE, 0, "Notes: Plot", [], Tokenizer.A_CENTRE
)
# Not First Page
assert tokens.addRootHeading(C.hPlotRoot) is True
assert tokens.allMarkdown[-1] == "# Notes: Plot\n\n"
assert tokens._tokens[-1] == (
Tokenizer.T_TITLE, 0, "Notes: Plot", None, Tokenizer.A_CENTRE | Tokenizer.A_PBB
Tokenizer.T_TITLE, 0, "Notes: Plot", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB
)
# Set Text
@@ -247,8 +247,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_TITLE, 1, "Novel Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TITLE, 1, "Novel Title", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "#! Novel Title\n\n"
@@ -260,8 +260,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Note Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "#! Note Title\n\n"
@@ -276,8 +276,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Novel Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Novel Title", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "# Novel Title\n\n"
@@ -289,8 +289,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Note Title", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "# Note Title\n\n"
@@ -304,8 +304,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "## Chapter One\n\n"
@@ -316,8 +316,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Heading 2", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD2, 1, "Heading 2", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "## Heading 2\n\n"
@@ -331,8 +331,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene One", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD3, 1, "Scene One", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "### Scene One\n\n"
@@ -343,8 +343,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Heading 3", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD3, 1, "Heading 3", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "### Heading 3\n\n"
@@ -358,8 +358,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "A Section", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD4, 1, "A Section", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "#### A Section\n\n"
@@ -370,8 +370,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "Heading 4", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD4, 1, "Heading 4", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "#### Heading 4\n\n"
@@ -385,8 +385,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_TITLE, 1, "Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "#! Title\n\n"
@@ -397,8 +397,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "#! Title\n\n"
@@ -412,8 +412,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_UNNUM, 1, "Prologue", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_UNNUM, 1, "Prologue", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
@@ -424,8 +424,8 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Prologue", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
@@ -443,8 +443,8 @@ def testCoreToken_MetaFormat(mockGUI):
tokens._text = "% A comment\n"
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_COMMENT, 0, "A comment", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_COMMENT, 0, "A comment", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -456,14 +456,14 @@ def testCoreToken_MetaFormat(mockGUI):
tokens._text = "%synopsis: The synopsis\n"
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_SYNOPSIS, 0, "The synopsis", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
tokens._text = "% synopsis: The synopsis\n"
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_SYNOPSIS, 0, "The synopsis", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -476,8 +476,8 @@ def testCoreToken_MetaFormat(mockGUI):
tokens._text = "% short: A short description\n"
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_SHORT, 0, "A short description", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SHORT, 0, "A short description", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -489,8 +489,8 @@ def testCoreToken_MetaFormat(mockGUI):
tokens._text = "@char: Bod\n"
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_KEYWORD, 0, "char: Bod", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_KEYWORD, 0, "char: Bod", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -504,10 +504,10 @@ def testCoreToken_MetaFormat(mockGUI):
styMid = Tokenizer.A_NONE | Tokenizer.A_Z_BTMMRG | Tokenizer.A_Z_TOPMRG
styBtm = Tokenizer.A_NONE | Tokenizer.A_Z_TOPMRG
assert tokens._tokens == [
(Tokenizer.T_KEYWORD, 0, "pov: Bod", None, styTop),
(Tokenizer.T_KEYWORD, 0, "plot: Main", None, styMid),
(Tokenizer.T_KEYWORD, 0, "location: Europe", None, styBtm),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_KEYWORD, 0, "pov: Bod", [], styTop),
(Tokenizer.T_KEYWORD, 0, "plot: Main", [], styMid),
(Tokenizer.T_KEYWORD, 0, "location: Europe", [], styBtm),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n"
@@ -537,22 +537,22 @@ def testCoreToken_MarginFormat(mockGUI):
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_TEXT, 0, "Some regular text", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Some left-aligned text", [], Tokenizer.A_LEFT),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Some right-aligned text", [], Tokenizer.A_RIGHT),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Some centered text", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Left-indented block", [], Tokenizer.A_IND_L),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Right-indented block", [], Tokenizer.A_IND_R),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Double-indented block", [], dblIndent),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Right-indent, right-aligned", [], rIndAlign),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == (
"Some regular text\n\n"
@@ -672,18 +672,18 @@ def testCoreToken_TextFormat(mockGUI):
assert tokens._tokens == [
(Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"
tokens.setBodyText(False)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "\n\n\n"
tokens.setBodyText(True)
@@ -701,7 +701,7 @@ def testCoreToken_TextFormat(mockGUI):
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "Some **bolded text** on this lines\n\n"
@@ -717,7 +717,7 @@ def testCoreToken_TextFormat(mockGUI):
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "Some _italic text_ on this lines\n\n"
@@ -735,7 +735,7 @@ def testCoreToken_TextFormat(mockGUI):
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n"
@@ -751,7 +751,7 @@ def testCoreToken_TextFormat(mockGUI):
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n"
@@ -771,7 +771,7 @@ def testCoreToken_TextFormat(mockGUI):
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == (
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
@@ -792,12 +792,12 @@ def testCoreToken_SpecialFormat(mockGUI):
# ========
correctResp = [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 2, "Title Two", None, Tokenizer.A_CENTRE | Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 2, "Title Two", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE),
]
# Command wo/Space
@@ -840,13 +840,13 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Multiple Empty Paragraphs
@@ -860,13 +860,13 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Three Skips
@@ -877,15 +877,15 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Malformed Command, Case 1
@@ -896,12 +896,12 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Malformed Command, Case 2
@@ -912,12 +912,12 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Malformed Command, Case 3
@@ -928,12 +928,12 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Empty Paragraph and Page Break
@@ -948,14 +948,14 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Multiple Skip
@@ -967,16 +967,16 @@ def testCoreToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_PBB),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# END Test testCoreToken_SpecialFormat
@@ -1019,8 +1019,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "T: Part One", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H1: Title, Not First Page
@@ -1030,8 +1030,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "T: Part One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Chapters
@@ -1043,8 +1043,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "C: Chapter One", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD2, 1, "C: Chapter One", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H2: Unnumbered Chapter
@@ -1053,8 +1053,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_UNNUM, 1, "U: Prologue", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_UNNUM, 1, "U: Prologue", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H2: Chapter Word Number
@@ -1064,8 +1064,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H2: Chapter Roman Number Upper Case
@@ -1074,8 +1074,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter II", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD2, 1, "Chapter II", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H2: Chapter Roman Number Lower Case
@@ -1084,8 +1084,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter iii", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD2, 1, "Chapter iii", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Scenes
@@ -1097,8 +1097,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "S: Scene One", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD3, 1, "S: Scene One", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene Hidden wo/Format
@@ -1107,8 +1107,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene wo/Format, first
@@ -1118,8 +1118,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene wo/Format, not first
@@ -1129,8 +1129,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene Separator, first
@@ -1140,8 +1140,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene Separator, not first
@@ -1151,8 +1151,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene w/Absolute Number
@@ -1163,8 +1163,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene 1", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD3, 1, "Scene 1", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene w/Chapter Number
@@ -1175,8 +1175,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene 3.2", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD3, 1, "Scene 3.2", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Sections
@@ -1188,8 +1188,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H4: Section Visible wo/Format
@@ -1198,8 +1198,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H4: Section w/Format
@@ -1208,8 +1208,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "X: A Section", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_HEAD4, 1, "X: A Section", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H4: Section Separator
@@ -1218,8 +1218,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Check the first scene detector