Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Tokenizer class (#1724)

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-03-02 18:27:40 +01:00
committed by GitHub
parent d932beb37d 20c996e87e
commit 210c1079cd
7 changed files with 151 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/docbuild.py
-1
View File
@@ -352,7 +352,6 @@ class NWBuildDocument:
bldObj.setText(tHandle)
bldObj.doPreProcessing()
bldObj.tokenizeText()
bldObj.doHeaders()
if self._count:
bldObj.countStats()
if convert:
novelwriter/core/tokenizer.py
+129 -117
View File
@@ -169,7 +169,7 @@ class Tokenizer(ABC):
# Instance Variables
self._hFormatter = HeadingFormatter(self._project)
self._skipSeparator = False # Flag to indicate that we skip the scene separator
self._noSep = False # Flag to indicate that we don't want a scene separator
# This File
self._isNone = False # Document has unknown layout
@@ -457,9 +457,12 @@ class Tokenizer(ABC):
5: The style of the block, self.A_*
"""
self._tokens = []
tmpMarkdown = []
if self._isNovel:
self._hFormatter.setHandle(self._nwItem.itemHandle if self._nwItem else None)
nHead = 0
breakNext = False
tmpMarkdown = []
for aLine in self._text.splitlines():
sLine = aLine.strip().lower()
@@ -483,10 +486,11 @@ class Tokenizer(ABC):
# =================
if aLine[0] == "[":
# Parse special formatting line
# This must be a separate if statement, as it may not
# reach a continue statement and must therefore proceed
# to check other formats.
# Special Formats
# ===============
# Parse special formatting line. This must be a separate if
# statement, as it may not reach a continue statement and must
# therefore proceed to check other formats.
if sLine in ("[newpage]", "[new page]"):
breakNext = True
@@ -511,8 +515,12 @@ class Tokenizer(ABC):
continue
if aLine[0] == "%":
# Comments
# ========
# All style comments are processed and the exact type exact
# style extracted. Ignored comments on the '%~' format are
# skipped completely.
if aLine[1] == "~":
# Completely ignore the paragraph
continue
cStyle, cText, _ = processComment(aLine)
@@ -536,6 +544,11 @@ class Tokenizer(ABC):
tmpMarkdown.append("%s\n" % aLine)
elif aLine[0] == "@":
# Keywords
# ========
# Only valid keyword lines are parsed, and any ignored keywords
# are automatically skipped.
valid, bits, _ = self._project.index.scanThis(aLine)
if valid and bits and bits[0] not in self._skipKeywords:
self._tokens.append((
@@ -545,56 +558,158 @@ class Tokenizer(ABC):
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:2] == "# ":
# Partition Headers
# =================
# Partition headers are only formatted in novel documents, and
# otherwise unchanged. Scene separators are disabled
# immediately after partitions, and scene numbers are reset.
nHead += 1
tText = aLine[2:].strip()
tStyle = self.A_NONE
if self._isNovel:
tText = self._hFormatter.apply(self._fmtTitle, tText, nHead)
tStyle = self._titleStyle
self._noSep = True
self._hFormatter.resetScene()
self._tokens.append((
self.T_HEAD1, nHead, aLine[2:].strip(), [], self.A_NONE
self.T_HEAD1, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:3] == "## ":
# Chapter Headers
# ===============
# Chapter headers are only formatted in novel documents, and
# otherwise unchanged. Chapter numbers are bumped before the
# heading is formatted. Scene separators are disabled
# immediately after chapter headers, and scene numbers are
# reset.
nHead += 1
tText = aLine[3:].strip()
tStyle = self.A_NONE
if self._isNovel:
self._hFormatter.incChapter()
tText = self._hFormatter.apply(self._fmtChapter, tText, nHead)
tStyle = self._chapterStyle
self._noSep = True
self._hFormatter.resetScene()
self._tokens.append((
self.T_HEAD2, nHead, aLine[3:].strip(), [], self.A_NONE
self.T_HEAD2, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:4] == "### ":
# Scene Headers
# =============
# Scene headers in novel documents are treated as centred
# separators if the formatting does not change the text. If the
# format is empty, the scene can be hidden or a blank paragraph
# (skip). When the scene title has static text or no text, it
# is always ignored if the noSep flag is set. This prevents
# separators immediately after other titles. Scene numbers are
# always incremented before formatting. For notes, the header
# is unchanged.
nHead += 1
tText = aLine[4:].strip()
tType = self.T_HEAD3
tStyle = self.A_NONE
if self._isNovel:
self._hFormatter.incScene()
tText = self._hFormatter.apply(self._fmtScene, tText, nHead)
tStyle = self._sceneStyle
if tText == "":
tType = self.T_EMPTY if self._noSep or self._hideScene else self.T_SKIP
tStyle = self.A_NONE
elif tText == self._fmtScene:
tText = "" if self._noSep else tText
tType = self.T_EMPTY if self._noSep else self.T_SEP
tStyle = self.A_NONE if self._noSep else self.A_CENTRE
self._noSep = False
self._tokens.append((
self.T_HEAD3, nHead, aLine[4:].strip(), [], self.A_NONE
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:5] == "#### ":
# Section Headers
# ===============
# Section headers in novel docs are treated as centred
# separators if the formatting does not change the text. If the
# format is empty, the section can be hidden or a blank
# paragraph (skip). For notes, the header is unchanged.
nHead += 1
tText = aLine[5:].strip()
tType = self.T_HEAD4
tStyle = self.A_NONE
if self._isNovel:
tText = self._hFormatter.apply(self._fmtSection, tText, nHead)
if tText == "":
tType = self.T_EMPTY if self._hideSection else self.T_SKIP
elif tText == self._fmtSection:
tType = self.T_SEP
tStyle = self.A_CENTRE
self._tokens.append((
self.T_HEAD4, nHead, aLine[5:].strip(), [], self.A_NONE
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:3] == "#! ":
# Main Title
# ==========
# Main titles are allowed in any document, and they are always
# centred and start on a new page. For novel documents, we also
# reset all counters when such a title is encountered.
nHead += 1
self._tokens.append((
self.T_TITLE, nHead, aLine[3:].strip(), [], self.A_PBB | self.A_CENTRE
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
if self._isNovel:
self._noSep = True
self._hFormatter.resetAll()
elif aLine[:4] == "##! ":
# Unnumbered Chapter Header
# =========================
# Unnumbered chapters are only meaningful in Novel docs, so if
# we're in a note, we convert them to a plain level 2 header.
nHead += 1
# If we're not in a novel section, we just treat this as a regular H2
tStyle = self.T_UNNUM if self._isNovel else self.T_HEAD2
tText = aLine[4:].strip()
tType = self.T_HEAD2
tStyle = self.A_NONE
if self._isNovel:
tText = self._hFormatter.apply(self._fmtUnNum, tText, nHead)
tType = self.T_UNNUM
tStyle = self._chapterStyle
self._noSep = True
self._hFormatter.resetScene()
self._tokens.append((
tStyle, nHead, aLine[4:].strip(), [], self.A_NONE
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
else:
# Text Lines
# ==========
# Anything remaining at this point is body text. If body text
# is not disabled, we proceed to process text formatting.
if not self._doBodyText:
# Skip all body text
continue
@@ -685,109 +800,6 @@ class Tokenizer(ABC):
return
def doHeaders(self) -> bool:
"""Apply formatting to the text headers for novel files. This
also applies chapter and scene numbering.
"""
if not self._isNovel:
return False
self._hFormatter.setHandle(self._nwItem.itemHandle if self._nwItem else None)
for n, token in enumerate(self._tokens):
if token[0] == self.T_TITLE: # Title
# For new titles, we reset all counters
self._skipSeparator = True
self._hFormatter.resetAll()
elif token[0] == self.T_HEAD1: # Partition
tTemp = self._hFormatter.apply(self._fmtTitle, token[2], token[1])
self._tokens[n] = (
token[0], token[1], tTemp, [], self._titleStyle
)
# Set scene variables
self._skipSeparator = True
self._hFormatter.resetScene()
elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter, Unnumbered
# Numbered or Unnumbered
if token[0] == self.T_UNNUM:
tTemp = self._hFormatter.apply(self._fmtUnNum, token[2], token[1])
else:
self._hFormatter.incChapter()
tTemp = self._hFormatter.apply(self._fmtChapter, token[2], token[1])
# Format the chapter header
self._tokens[n] = (
token[0], token[1], tTemp, [], self._chapterStyle
)
# Set scene variables
self._skipSeparator = True
self._hFormatter.resetScene()
elif token[0] == self.T_HEAD3: # Scene
self._hFormatter.incScene()
tTemp = self._hFormatter.apply(self._fmtScene, token[2], token[1])
if tTemp == "" and self._hideScene:
self._tokens[n] = (
self.T_EMPTY, token[1], "", [], self.A_NONE
)
elif tTemp == "" and not self._hideScene:
self._tokens[n] = (
self.T_EMPTY if self._skipSeparator else self.T_SKIP, token[1],
"", [], self.A_NONE if self._skipSeparator else token[4]
)
elif tTemp == self._fmtScene:
self._tokens[n] = (
self.T_EMPTY if self._skipSeparator else self.T_SEP, token[1],
"" if self._skipSeparator else tTemp, [],
self.A_NONE if self._skipSeparator else self.A_CENTRE
)
else:
self._tokens[n] = (
token[0], token[1], tTemp, [], self._sceneStyle
)
self._skipSeparator = False
elif token[0] == self.T_HEAD4: # Section
tTemp = self._hFormatter.apply(self._fmtSection, token[2], token[1])
if tTemp == "" and self._hideSection:
self._tokens[n] = (
self.T_EMPTY, token[1], "", [], self.A_NONE
)
elif tTemp == "" and not self._hideSection:
self._tokens[n] = (
self.T_SKIP, token[1], "", [], token[4]
)
elif tTemp == self._fmtSection:
self._tokens[n] = (
self.T_SEP, token[1], tTemp, [], token[4] | self.A_CENTRE
)
else:
self._tokens[n] = (
token[0], token[1], tTemp, [], token[4]
)
if n == 0 and self._noBreak:
# Make sure again that the token array doesn't start with a page break
self._noBreak = False
if self._tokens[0][4] & self.A_PBB:
token = self._tokens[0]
self._tokens[0] = (
token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
)
return True
def countStats(self) -> dict[str, int]:
"""Count stats on the tokenized text."""
titleCount = self._counts.get("titleCount", 0)
novelwriter/text/counting.py
-1
View File
@@ -31,7 +31,6 @@ from novelwriter.constants import nwRegEx, nwUnicode
RX_SC = re.compile(nwRegEx.FMT_SC)
RX_LO = re.compile(r"(?i)(?<!\\)(\[(?:vspace|newpage|new page)(:\d+)?)(?<!\\)(\])")
RX_IN = re.compile(r"^>{1,2}\s*|\s*<{1,2}$")
def preProcessText(text: str, keepHeaders: bool = True) -> list[str]:
novelwriter/tools/manuscript.py
+13 -13
View File
@@ -968,10 +968,10 @@ class _StatsWidget(QWidget):
"""Build the minimal stats page."""
mPx = CONFIG.pxInt(8)
self.lblWordCount = QLabel(self.tr("Words:"))
self.lblWordCount = QLabel(self.tr("Words"))
self.minWordCount = QLabel(self)
self.lblCharCount = QLabel(self.tr("Characters:"))
self.lblCharCount = QLabel(self.tr("Characters"))
self.minCharCount = QLabel(self)
# Assemble
@@ -1010,12 +1010,12 @@ class _StatsWidget(QWidget):
self.maxParCount.setAlignment(alignRight)
self.leftForm = QFormLayout()
self.leftForm.addRow(self.tr("Words:"), self.maxTotalWords)
self.leftForm.addRow(self.tr("Header Words:"), self.maxHeaderWords)
self.leftForm.addRow(self.tr("Body Text Words:"), self.maxTextWords)
self.leftForm.addRow(self.tr("Words"), self.maxTotalWords)
self.leftForm.addRow(self.tr("Header Words"), self.maxHeaderWords)
self.leftForm.addRow(self.tr("Body Text Words"), self.maxTextWords)
self.leftForm.addRow("", QLabel(self))
self.leftForm.addRow(self.tr("Headers:"), self.maxTitleCount)
self.leftForm.addRow(self.tr("Paragraphs:"), self.maxParCount)
self.leftForm.addRow(self.tr("Headers"), self.maxTitleCount)
self.leftForm.addRow(self.tr("Paragraphs"), self.maxParCount)
self.leftForm.setHorizontalSpacing(hPx)
self.leftForm.setVerticalSpacing(vPx)
@@ -1037,12 +1037,12 @@ class _StatsWidget(QWidget):
self.maxTextWordChars.setAlignment(alignRight)
self.rightForm = QFormLayout()
self.rightForm.addRow(self.tr("Characters:"), self.maxTotalChars)
self.rightForm.addRow(self.tr("Header Characters:"), self.maxHeaderChars)
self.rightForm.addRow(self.tr("Body Text Characters:"), self.maxTextChars)
self.rightForm.addRow(self.tr("Characters, No Spaces:"), self.maxTotalWordChars)
self.rightForm.addRow(self.tr("Header Characters, No Spaces:"), self.maxHeaderWordChars)
self.rightForm.addRow(self.tr("Body Text Characters, No Spaces:"), self.maxTextWordChars)
self.rightForm.addRow(self.tr("Characters"), self.maxTotalChars)
self.rightForm.addRow(self.tr("Header Characters"), self.maxHeaderChars)
self.rightForm.addRow(self.tr("Body Text Characters"), self.maxTextChars)
self.rightForm.addRow(self.tr("Characters, No Spaces"), self.maxTotalWordChars)
self.rightForm.addRow(self.tr("Header Characters, No Spaces"), self.maxHeaderWordChars)
self.rightForm.addRow(self.tr("Body Text Characters, No Spaces"), self.maxTextWordChars)
self.rightForm.setHorizontalSpacing(hPx)
self.rightForm.setVerticalSpacing(vPx)
tests/test_core/test_core_tohtml.py
-15
View File
@@ -44,7 +44,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
# Header 1
html._text = "# Partition\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == (
"<h1 class='title' style='text-align: center;'>Partition</h1>\n"
@@ -53,7 +52,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
# Header 2
html._text = "## Chapter Title\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == (
"<h1 style='page-break-before: always;'>Chapter Title</h1>\n"
@@ -62,21 +60,18 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
# Header 3
html._text = "### Scene Title\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h2>Scene Title</h2>\n"
# Header 4
html._text = "#### Section Title\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h3>Section Title</h3>\n"
# Title
html._text = "#! Title\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == (
"<h1 class='title' style='text-align: center; page-break-before: always;'>Title</h1>\n"
@@ -85,7 +80,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
# Unnumbered
html._text = "##! Prologue\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h1 style='page-break-before: always;'>Prologue</h1>\n"
@@ -100,35 +94,30 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
# Header 1
html._text = "# Heading One\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h1><a name='T0001'></a>Heading One</h1>\n"
# Header 2
html._text = "## Heading Two\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h2><a name='T0001'></a>Heading Two</h2>\n"
# Header 3
html._text = "### Heading Three\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h3><a name='T0001'></a>Heading Three</h3>\n"
# Header 4
html._text = "#### Heading Four\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h4><a name='T0001'></a>Heading Four</h4>\n"
# Title
html._text = "#! Heading One\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == (
"<h1 class='title' style='text-align: center; page-break-before: always;'>"
@@ -138,7 +127,6 @@ def testCoreToHtml_ConvertHeaders(mockGUI):
# Unnumbered
html._text = "##! Heading Two\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == "<h2><a name='T0001'></a>Heading Two</h2>\n"
@@ -245,7 +233,6 @@ def testCoreToHtml_ConvertParagraphs(mockGUI):
html.setKeywords(True)
html._text = "## Chapter\n\n@pov: Bod\n@plot: Main\n@location: Europe\n\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == (
"<h1 style='page-break-before: always;'>Chapter</h1>\n"
@@ -477,7 +464,6 @@ def testCoreToHtml_SpecialCases(mockGUI):
html._noBreak = False
html._text = "## Heading <1>\n"
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == (
"<h1 style='page-break-before: always;'>Heading &lt;1&gt;</h1>\n"
@@ -551,7 +537,6 @@ def testCoreToHtml_Complex(mockGUI, fncPath):
html._text = docText[i]
html.doPreProcessing()
html.tokenizeText()
html.doHeaders()
html.doConvert()
assert html.result == resText[i]
tests/test_core/test_core_tokenizer.py
+8 -78
View File
@@ -276,7 +276,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "# Novel Title\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Novel Title", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -290,7 +289,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "# Note Title\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -307,7 +305,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "## Chapter One\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -320,7 +317,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "## Heading 2\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Heading 2", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -336,7 +332,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "### Scene One\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene One", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -349,7 +344,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "### Heading 3\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Heading 3", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -365,7 +359,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "#### A Section\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "A Section", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -378,7 +371,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "#### Heading 4\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "Heading 4", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -395,7 +387,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "#! Title\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -409,7 +400,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "#! Title\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -425,7 +415,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "##! Prologue\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_UNNUM, 1, "Prologue", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -438,7 +427,6 @@ def testCoreToken_HeaderFormat(mockGUI):
tokens._text = "##! Prologue\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -459,7 +447,6 @@ def testCoreToken_HeaderStyle(mockGUI):
tokens._isFirst = first
tokens._noBreak = first
tokens.tokenizeText()
tokens.doHeaders()
return tokens._tokens[0][4]
# No Styles
@@ -1120,7 +1107,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"# Title Two\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == correctResp
# Command w/Space
@@ -1131,7 +1117,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"# Title Two\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == correctResp
# Trailing Spaces
@@ -1142,7 +1127,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"# Title Two\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == correctResp
# Single Empty Paragraph
@@ -1154,7 +1138,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1175,7 +1158,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1193,7 +1175,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1213,7 +1194,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1230,7 +1210,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1247,7 +1226,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1268,7 +1246,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1288,7 +1265,6 @@ def testCoreToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1313,17 +1289,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
# Nothing
tokens._text = "Some text ...\n"
assert tokens.doHeaders() is False
tokens._isNone = True
assert tokens.doHeaders() is False
tokens._isNone = False
assert tokens.doHeaders() is False
tokens._isNote = True
assert tokens.doHeaders() is False
tokens._isNote = False
##
# Story Files
##
@@ -1340,7 +1305,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "# Part One\n"
tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1351,7 +1315,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "# Part One\n"
tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1364,7 +1327,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter One\n"
tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "C: Chapter One", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1374,7 +1336,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "##! Prologue\n"
tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_UNNUM, 1, "U: Prologue", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1385,7 +1346,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}")
tokens._hFormatter._chCount = 0
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1395,7 +1355,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter II", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1405,7 +1364,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter iii", [], Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1418,7 +1376,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "### Scene One\n"
tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "S: Scene One", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1428,7 +1385,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", True)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1437,9 +1393,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene wo/Format, first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._skipSeparator = True
tokens._noSep = True
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1448,9 +1403,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene wo/Format, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._skipSeparator = False
tokens._noSep = False
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1459,9 +1413,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene Separator, first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False)
tokens._skipSeparator = True
tokens._noSep = True
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1470,9 +1423,8 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene Separator, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False)
tokens._skipSeparator = False
tokens._noSep = False
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1484,7 +1436,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 0
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene 1", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1496,7 +1447,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 1
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene 3.2", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1509,7 +1459,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n"
tokens.setSectionFormat("", True)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1519,7 +1468,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n"
tokens.setSectionFormat("", False)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1529,7 +1477,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n"
tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "X: A Section", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
@@ -1539,25 +1486,22 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n"
tokens.setSectionFormat("* * *", False)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
(Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
]
# Check the first scene detector, plain text
tokens._skipSeparator = True
tokens._noSep = True
tokens._text = "Some text ...\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._skipSeparator is True
assert tokens._noSep is True
# Check the first scene detector, text plus scene
tokens._skipSeparator = True
tokens._noSep = True
tokens._text = "Some text ...\n\n### Scene\n\nText"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._skipSeparator is False
assert tokens._noSep is False
# END Test testCoreToken_ProcessHeaders
@@ -1580,7 +1524,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens._text = "## A Chapter Title\n\n"
tokens._counts = {}
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert tokens._tokens[0][2] == "A Chapter Title"
assert tokens.textStats == {
@@ -1596,7 +1539,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}")
tokens._hFormatter.resetAll()
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert tokens._tokens[0][2] == "C 1: A Chapter Title"
assert tokens.textStats == {
@@ -1611,7 +1553,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens._text = "Some text\non two lines.\n\nWith a second paragraph.\n\n"
tokens._counts = {}
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert tokens.textStats == {
"titleCount": 0, "paragraphCount": 2,
@@ -1626,7 +1567,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.setChapterFormat(nwHeadFmt.TITLE)
tokens.setSceneFormat("* * *", False)
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Text", "", "* * *", "", "Text", ""
@@ -1646,7 +1586,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.setSceneFormat("* * *", False)
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Stuff", "", "Text", ""
@@ -1666,7 +1605,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.setSceneFormat("* * *", False)
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Stuff", "", "Text", ""
@@ -1686,7 +1624,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.setSceneFormat("* * *", False)
tokens.setComments(True)
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "Stuff", "", "Text", ""
@@ -1706,7 +1643,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.setSceneFormat("* * *", False)
tokens.setKeywords(True)
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert [t[2] for t in tokens._tokens] == [
"Chapter", "", "", "", "pov: Jane", "", "Text", ""
@@ -1769,7 +1705,6 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
tokens.setKeywords(True)
tokens.tokenizeText()
tokens.doHeaders()
tokens.countStats()
assert tokens.textStats == {
"titleCount": 4, "paragraphCount": 5,
@@ -1816,7 +1751,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
# Static Separator
md.setSceneFormat("* * *", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
@@ -1832,7 +1766,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
# Scene Title Formatted
md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
@@ -1870,7 +1803,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
# Static Separator
md.setSceneFormat("* * *", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
@@ -1887,7 +1819,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
# Scene Title Formatted
md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
@@ -1938,7 +1869,6 @@ def testCoreToken_HeaderCounterAndVisibility(mockGUI):
# Two Novel Format
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# Novel One\n\n"
tests/test_core/test_core_toodt.py
+1 -11
View File
@@ -246,7 +246,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
# Header 1
odt._text = "# Title\n"
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -260,7 +259,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
# Header 2
odt._text = "## Chapter\n"
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -274,7 +272,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
# Header 3
odt._text = "### Scene\n"
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -288,7 +285,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
# Header 4
odt._text = "#### Section\n"
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -303,7 +299,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
odt._isFirst = True
odt._text = "#! Title\n"
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -317,7 +312,6 @@ def testCoreToOdt_ConvertHeaders(mockGUI):
# Unnumbered chapter
odt._text = "##! Prologue\n"
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -507,7 +501,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
odt._text = "### Scene One\n\nText\n\n### Scene Two\n\nText"
odt.setSceneFormat("* * *", False)
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -525,7 +518,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
odt._text = "### Scene One\n\nText\n\n### Scene Two\n\nText"
odt.setSceneFormat("", False)
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -551,6 +543,7 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
"> Left indent\n\n"
"Right indent <\n\n"
)
odt.setSceneFormat(nwHeadFmt.TITLE, False)
odt.setKeywords(True)
odt.tokenizeText()
odt.initDocument()
@@ -612,7 +605,6 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
"Text\n\n"
)
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -737,7 +729,6 @@ def testCoreToOdt_SaveFlat(mockGUI, fncPath, tstPaths):
"Text\n\n"
)
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()
@@ -777,7 +768,6 @@ def testCoreToOdt_SaveFull(mockGUI, fncPath, tstPaths):
"Text\n\n"
)
odt.tokenizeText()
odt.doHeaders()
odt.initDocument()
odt.doConvert()
odt.closeDocument()