Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Strip consecutive empty paragraphs when tokenizing

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-05-21 20:53:45 +02:00
parent aadad6aa8c
commit afe4a29fd1
2 changed files with 72 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tokenizer.py
+43 -26
View File
@@ -152,6 +152,7 @@ class Tokenizer(ABC):
self._doComments = False # Also process comments self._doComments = False # Also process comments
self._doKeywords = False # Also process keywords like tags and references self._doKeywords = False # Also process keywords like tags and references
self._skipKeywords = set() # Keywords to ignore self._skipKeywords = set() # Keywords to ignore
self._keepBreaks = True # Keep line breaks in paragraphs
# Margins # Margins
self._marginTitle = (1.000, 0.500) self._marginTitle = (1.000, 0.500)
@@ -409,6 +410,11 @@ class Tokenizer(ABC):
self._skipKeywords = set(x.lower().strip() for x in keywords.split(",")) self._skipKeywords = set(x.lower().strip() for x in keywords.split(","))
return return
def setKeepLineBreaks(self, state: bool) -> None:
"""Keep line breaks in paragraphs."""
self._keepBreaks = state
return
def setKeepMarkdown(self, state: bool) -> None: def setKeepMarkdown(self, state: bool) -> None:
"""Keep original markdown during build.""" """Keep original markdown during build."""
self._keepMD = state self._keepMD = state
@@ -490,7 +496,6 @@ class Tokenizer(ABC):
4: The internal formatting map of the text, self.FMT_* 4: The internal formatting map of the text, self.FMT_*
5: The style of the block, self.A_* 5: The style of the block, self.A_*
""" """
self._tokens = []
if self._isNovel: if self._isNovel:
self._hFormatter.setHandle(self._handle) self._hFormatter.setHandle(self._handle)
@@ -498,12 +503,13 @@ class Tokenizer(ABC):
breakNext = False breakNext = False
tmpMarkdown = [] tmpMarkdown = []
tHandle = self._handle or "" tHandle = self._handle or ""
tokens = []
for aLine in self._text.splitlines(): for aLine in self._text.splitlines():
sLine = aLine.strip().lower() sLine = aLine.strip().lower()
# Check for blank lines # Check for blank lines
if len(sLine) == 0: if len(sLine) == 0:
self._tokens.append(( tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE self.T_EMPTY, nHead, "", [], self.A_NONE
)) ))
if self._keepMD: if self._keepMD:
@@ -532,7 +538,7 @@ class Tokenizer(ABC):
continue continue
elif sLine == "[vspace]": elif sLine == "[vspace]":
self._tokens.append( tokens.append(
(self.T_SKIP, nHead, "", [], sAlign) (self.T_SKIP, nHead, "", [], sAlign)
) )
continue continue
@@ -540,11 +546,11 @@ class Tokenizer(ABC):
elif sLine.startswith("[vspace:") and sLine.endswith("]"): elif sLine.startswith("[vspace:") and sLine.endswith("]"):
nSkip = checkInt(sLine[8:-1], 0) nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1: if nSkip >= 1:
self._tokens.append( tokens.append(
(self.T_SKIP, nHead, "", [], sAlign) (self.T_SKIP, nHead, "", [], sAlign)
) )
if nSkip > 1: if nSkip > 1:
self._tokens += (nSkip - 1) * [ tokens += (nSkip - 1) * [
(self.T_SKIP, nHead, "", [], self.A_NONE) (self.T_SKIP, nHead, "", [], self.A_NONE)
] ]
continue continue
@@ -561,14 +567,14 @@ class Tokenizer(ABC):
cStyle, cKey, cText, _, _ = processComment(aLine) cStyle, cKey, cText, _, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS: if cStyle == nwComment.SYNOPSIS:
tLine, tFmt = self._extractFormats(cText) tLine, tFmt = self._extractFormats(cText)
self._tokens.append(( tokens.append((
self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
)) ))
if self._doSynopsis and self._keepMD: if self._doSynopsis and self._keepMD:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT: elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText) tLine, tFmt = self._extractFormats(cText)
self._tokens.append(( tokens.append((
self.T_SHORT, nHead, tLine, tFmt, sAlign self.T_SHORT, nHead, tLine, tFmt, sAlign
)) ))
if self._doSynopsis and self._keepMD: if self._doSynopsis and self._keepMD:
@@ -580,7 +586,7 @@ class Tokenizer(ABC):
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
else: else:
tLine, tFmt = self._extractFormats(cText) tLine, tFmt = self._extractFormats(cText)
self._tokens.append(( tokens.append((
self.T_COMMENT, nHead, tLine, tFmt, sAlign self.T_COMMENT, nHead, tLine, tFmt, sAlign
)) ))
if self._doComments and self._keepMD: if self._doComments and self._keepMD:
@@ -594,7 +600,7 @@ class Tokenizer(ABC):
valid, bits, _ = self._project.index.scanThis(aLine) valid, bits, _ = self._project.index.scanThis(aLine)
if valid and bits and bits[0] not in self._skipKeywords: if valid and bits and bits[0] not in self._skipKeywords:
self._tokens.append(( tokens.append((
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
)) ))
if self._doKeywords and self._keepMD: if self._doKeywords and self._keepMD:
@@ -630,7 +636,7 @@ class Tokenizer(ABC):
self._hFormatter.resetAll() self._hFormatter.resetAll()
self._noSep = True self._noSep = True
self._tokens.append(( tokens.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMD: if self._keepMD:
@@ -665,7 +671,7 @@ class Tokenizer(ABC):
self._hFormatter.resetScene() self._hFormatter.resetScene()
self._noSep = True self._noSep = True
self._tokens.append(( tokens.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMD: if self._keepMD:
@@ -706,7 +712,7 @@ class Tokenizer(ABC):
tStyle = self.A_NONE if self._noSep else self.A_CENTRE tStyle = self.A_NONE if self._noSep else self.A_CENTRE
self._noSep = False self._noSep = False
self._tokens.append(( tokens.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMD: if self._keepMD:
@@ -736,7 +742,7 @@ class Tokenizer(ABC):
tType = self.T_SEP tType = self.T_SEP
tStyle = self.A_CENTRE tStyle = self.A_CENTRE
self._tokens.append(( tokens.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMD: if self._keepMD:
@@ -784,26 +790,26 @@ class Tokenizer(ABC):
# Process formats # Process formats
tLine, tFmt = self._extractFormats(aLine) tLine, tFmt = self._extractFormats(aLine)
self._tokens.append(( tokens.append((
self.T_TEXT, nHead, tLine, tFmt, sAlign self.T_TEXT, nHead, tLine, tFmt, sAlign
)) ))
if self._keepMD: if self._keepMD:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag # If we have content, turn off the first page flag
if self._isFirst and self._tokens: if self._isFirst and tokens:
self._isFirst = False # First document has been processed self._isFirst = False # First document has been processed
# Make sure the token array doesn't start with a page break # Make sure the token array doesn't start with a page break
# on the very first page, adding a blank first page. # on the very first page, adding a blank first page.
if self._tokens[0][4] & self.A_PBB: if tokens[0][4] & self.A_PBB:
token = self._tokens[0] token = tokens[0]
self._tokens[0] = ( tokens[0] = (
token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
) )
# Always add an empty line at the end of the file # Always add an empty line at the end of the file
self._tokens.append(( tokens.append((
self.T_EMPTY, nHead, "", [], self.A_NONE self.T_EMPTY, nHead, "", [], self.A_NONE
)) ))
if self._keepMD: if self._keepMD:
@@ -814,25 +820,36 @@ class Tokenizer(ABC):
# =========== # ===========
# Some items need a second pass # Some items need a second pass
self._tokens = []
pToken = (self.T_EMPTY, 0, "", [], self.A_NONE) pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
nToken = (self.T_EMPTY, 0, "", [], self.A_NONE) nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
tCount = len(self._tokens)
for n, token in enumerate(self._tokens): tCount = len(tokens)
for n, token in enumerate(tokens):
if n > 0: if n > 0:
pToken = self._tokens[n-1] pToken = tokens[n-1] # Look behind
if n < tCount - 1: if n < tCount - 1:
nToken = self._tokens[n+1] nToken = tokens[n+1] # Look ahead
if token[0] == self.T_KEYWORD: if token[0] == self.T_EMPTY:
# Strip multiple empty
if pToken[0] != self.T_EMPTY:
self._tokens.append(token)
elif token[0] == self.T_KEYWORD:
# Adjust margins for lines in a list of keyword lines
aStyle = token[4] aStyle = token[4]
if pToken[0] == self.T_KEYWORD: if pToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_TOPMRG aStyle |= self.A_Z_TOPMRG
if nToken[0] == self.T_KEYWORD: if nToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_BTMMRG aStyle |= self.A_Z_BTMMRG
self._tokens[n] = ( self._tokens.append((
token[0], token[1], token[2], token[3], aStyle token[0], token[1], token[2], token[3], aStyle
) ))
else:
self._tokens.append(token)
return return
tests/test_core/test_core_tokenizer.py
+29 -47
View File
@@ -717,9 +717,7 @@ def testCoreToken_MetaFormat(mockGUI):
# Ignore Text # Ignore Text
tokens._text = "%~ Some text\n" tokens._text = "%~ Some text\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._tokens == []
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "\n" assert tokens.allMarkdown[-1] == "\n"
# Synopsis # Synopsis
@@ -829,7 +827,6 @@ def testCoreToken_MarginFormat(mockGUI):
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Right-indent, right-aligned", [], rIndAlign), (Tokenizer.T_TEXT, 0, "Right-indent, right-aligned", [], rIndAlign),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
] ]
assert tokens.allMarkdown[-1] == ( assert tokens.allMarkdown[-1] == (
"Some regular text\n\n" "Some regular text\n\n"
@@ -950,6 +947,24 @@ def testCoreToken_ExtractFormats(mockGUI):
] ]
@pytest.mark.core
def testCoreToken_Paragraphs(mockGUI):
"""Test the splitting of paragraphs."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setKeepMarkdown(True)
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_TEXT, 0, "First paragraph", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "Second paragraph", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
@pytest.mark.core @pytest.mark.core
def testCoreToken_TextFormat(mockGUI): def testCoreToken_TextFormat(mockGUI):
"""Test the tokenization of text formats in the Tokenizer class.""" """Test the tokenization of text formats in the Tokenizer class."""
@@ -964,18 +979,12 @@ def testCoreToken_TextFormat(mockGUI):
(Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
] ]
assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n" assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"
tokens.setBodyText(False) tokens.setBodyText(False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._tokens == []
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
]
assert tokens.allMarkdown[-1] == "\n\n\n" assert tokens.allMarkdown[-1] == "\n\n\n"
tokens.setBodyText(True) tokens.setBodyText(True)
@@ -1083,10 +1092,8 @@ def testCoreToken_SpecialFormat(mockGUI):
correctResp = [ correctResp = [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_CENTRE), (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_HEAD1, 2, "Title Two", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB), (Tokenizer.T_HEAD1, 2, "Title Two", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE),
] ]
# Command wo/Space # Command wo/Space
@@ -1135,7 +1142,6 @@ def testCoreToken_SpecialFormat(mockGUI):
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
# Multiple Empty Paragraphs # Multiple Empty Paragraphs
@@ -1155,7 +1161,6 @@ def testCoreToken_SpecialFormat(mockGUI):
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
# Three Skips # Three Skips
@@ -1174,7 +1179,6 @@ def testCoreToken_SpecialFormat(mockGUI):
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
# Malformed Command, Case 1 # Malformed Command, Case 1
@@ -1187,10 +1191,8 @@ def testCoreToken_SpecialFormat(mockGUI):
assert tokens._tokens == [ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
# Malformed Command, Case 2 # Malformed Command, Case 2
@@ -1203,10 +1205,8 @@ def testCoreToken_SpecialFormat(mockGUI):
assert tokens._tokens == [ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
# Malformed Command, Case 3 # Malformed Command, Case 3
@@ -1219,10 +1219,8 @@ def testCoreToken_SpecialFormat(mockGUI):
assert tokens._tokens == [ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
# Empty Paragraph and Page Break # Empty Paragraph and Page Break
@@ -1239,12 +1237,10 @@ def testCoreToken_SpecialFormat(mockGUI):
assert tokens._tokens == [ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
# Multiple Skip # Multiple Skip
@@ -1258,14 +1254,12 @@ def testCoreToken_SpecialFormat(mockGUI):
assert tokens._tokens == [ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE), (Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0), (Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
] ]
@@ -1371,20 +1365,14 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.setSceneFormat("", True) tokens.setSceneFormat("", True)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._tokens == []
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene wo/Format, first # H3: Scene wo/Format, first
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False) tokens.setSceneFormat("", False)
tokens._noSep = True tokens._noSep = True
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._tokens == []
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene wo/Format, not first # H3: Scene wo/Format, not first
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
@@ -1401,10 +1389,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.setSceneFormat("* * *", False) tokens.setSceneFormat("* * *", False)
tokens._noSep = True tokens._noSep = True
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._tokens == []
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H3: Scene Separator, not first # H3: Scene Separator, not first
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
@@ -1445,10 +1430,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n" tokens._text = "#### A Section\n"
tokens.setSectionFormat("", True) tokens.setSectionFormat("", True)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._tokens == []
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# H4: Section Visible wo/Format # H4: Section Visible wo/Format
tokens._text = "#### A Section\n" tokens._text = "#### A Section\n"
@@ -1615,7 +1597,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == [ assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Text", "", "* * *", "", "Text", "" "Chapter", "", "Text", "", "* * *", "", "Text", ""
] ]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 2, "titleCount": 1, "paragraphCount": 2,
@@ -1634,7 +1616,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == [ assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Stuff", "", "Text", "" "Chapter", "", "Stuff", "", "Text", ""
] ]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
@@ -1653,7 +1635,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == [ assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Stuff", "", "Text", "" "Chapter", "", "Stuff", "", "Text", ""
] ]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
@@ -1672,7 +1654,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == [ assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Stuff", "", "Text", "" "Chapter", "", "Stuff", "", "Text", ""
] ]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
@@ -1691,7 +1673,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == [ assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "pov: Jane", "", "Text", "" "Chapter", "", "pov: Jane", "", "Text", ""
] ]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,