diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py
index c27fd9de..2db60f85 100644
--- a/tests/test_core/test_core_tokenizer.py
+++ b/tests/test_core/test_core_tokenizer.py
@@ -19,6 +19,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see .
"""
+import json
import pytest
from tools import C, buildTestProject, readFile
@@ -36,96 +37,96 @@ class BareTokenizer(Tokenizer):
@pytest.mark.core
def testCoreToken_Setters(mockGUI):
"""Test all the setters for the Tokenizer class."""
- theProject = NWProject()
- theToken = BareTokenizer(theProject)
+ project = NWProject()
+ tokens = BareTokenizer(project)
# Verify defaults
- assert theToken._fmtTitle == nwHeadFmt.TITLE
- assert theToken._fmtChapter == nwHeadFmt.TITLE
- assert theToken._fmtUnNum == nwHeadFmt.TITLE
- assert theToken._fmtScene == nwHeadFmt.TITLE
- assert theToken._fmtSection == nwHeadFmt.TITLE
- assert theToken._textFont == "Serif"
- assert theToken._textSize == 11
- assert theToken._textFixed is False
- assert theToken._lineHeight == 1.15
- assert theToken._blockIndent == 4.0
- assert theToken._doJustify is False
- assert theToken._marginTitle == (1.000, 0.500)
- assert theToken._marginHead1 == (1.000, 0.500)
- assert theToken._marginHead2 == (0.834, 0.500)
- assert theToken._marginHead3 == (0.584, 0.500)
- assert theToken._marginHead4 == (0.584, 0.500)
- assert theToken._marginText == (0.000, 0.584)
- assert theToken._marginMeta == (0.000, 0.584)
- assert theToken._hideScene is False
- assert theToken._hideSection is False
- assert theToken._linkHeaders is False
- assert theToken._doBodyText is True
- assert theToken._doSynopsis is False
- assert theToken._doComments is False
- assert theToken._doKeywords is False
+ assert tokens._fmtTitle == nwHeadFmt.TITLE
+ assert tokens._fmtChapter == nwHeadFmt.TITLE
+ assert tokens._fmtUnNum == nwHeadFmt.TITLE
+ assert tokens._fmtScene == nwHeadFmt.TITLE
+ assert tokens._fmtSection == nwHeadFmt.TITLE
+ assert tokens._textFont == "Serif"
+ assert tokens._textSize == 11
+ assert tokens._textFixed is False
+ assert tokens._lineHeight == 1.15
+ assert tokens._blockIndent == 4.0
+ assert tokens._doJustify is False
+ assert tokens._marginTitle == (1.000, 0.500)
+ assert tokens._marginHead1 == (1.000, 0.500)
+ assert tokens._marginHead2 == (0.834, 0.500)
+ assert tokens._marginHead3 == (0.584, 0.500)
+ assert tokens._marginHead4 == (0.584, 0.500)
+ assert tokens._marginText == (0.000, 0.584)
+ assert tokens._marginMeta == (0.000, 0.584)
+ assert tokens._hideScene is False
+ assert tokens._hideSection is False
+ assert tokens._linkHeaders is False
+ assert tokens._doBodyText is True
+ assert tokens._doSynopsis is False
+ assert tokens._doComments is False
+ assert tokens._doKeywords is False
# Set new values
- theToken.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
- theToken.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
- theToken.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
- theToken.setSceneFormat(f"S: {nwHeadFmt.TITLE}", True)
- theToken.setSectionFormat(f"X: {nwHeadFmt.TITLE}", True)
- theToken.setFont("Monospace", 10, True)
- theToken.setLineHeight(2.0)
- theToken.setBlockIndent(6.0)
- theToken.setJustify(True)
- theToken.setTitleMargins(2.0, 2.0)
- theToken.setHead1Margins(2.0, 2.0)
- theToken.setHead2Margins(2.0, 2.0)
- theToken.setHead3Margins(2.0, 2.0)
- theToken.setHead4Margins(2.0, 2.0)
- theToken.setTextMargins(2.0, 2.0)
- theToken.setMetaMargins(2.0, 2.0)
- theToken.setLinkHeaders(True)
- theToken.setBodyText(False)
- theToken.setSynopsis(True)
- theToken.setComments(True)
- theToken.setKeywords(True)
+ tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
+ tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
+ tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
+ tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", True)
+ tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", True)
+ tokens.setFont("Monospace", 10, True)
+ tokens.setLineHeight(2.0)
+ tokens.setBlockIndent(6.0)
+ tokens.setJustify(True)
+ tokens.setTitleMargins(2.0, 2.0)
+ tokens.setHead1Margins(2.0, 2.0)
+ tokens.setHead2Margins(2.0, 2.0)
+ tokens.setHead3Margins(2.0, 2.0)
+ tokens.setHead4Margins(2.0, 2.0)
+ tokens.setTextMargins(2.0, 2.0)
+ tokens.setMetaMargins(2.0, 2.0)
+ tokens.setLinkHeaders(True)
+ tokens.setBodyText(False)
+ tokens.setSynopsis(True)
+ tokens.setComments(True)
+ tokens.setKeywords(True)
# Check new values
- assert theToken._fmtTitle == f"T: {nwHeadFmt.TITLE}"
- assert theToken._fmtChapter == f"C: {nwHeadFmt.TITLE}"
- assert theToken._fmtUnNum == f"U: {nwHeadFmt.TITLE}"
- assert theToken._fmtScene == f"S: {nwHeadFmt.TITLE}"
- assert theToken._fmtSection == f"X: {nwHeadFmt.TITLE}"
- assert theToken._textFont == "Monospace"
- assert theToken._textSize == 10
- assert theToken._textFixed is True
- assert theToken._lineHeight == 2.0
- assert theToken._blockIndent == 6.0
- assert theToken._doJustify is True
- assert theToken._marginTitle == (2.0, 2.0)
- assert theToken._marginHead1 == (2.0, 2.0)
- assert theToken._marginHead2 == (2.0, 2.0)
- assert theToken._marginHead3 == (2.0, 2.0)
- assert theToken._marginHead4 == (2.0, 2.0)
- assert theToken._marginText == (2.0, 2.0)
- assert theToken._marginMeta == (2.0, 2.0)
- assert theToken._hideScene is True
- assert theToken._hideSection is True
- assert theToken._linkHeaders is True
- assert theToken._doBodyText is False
- assert theToken._doSynopsis is True
- assert theToken._doComments is True
- assert theToken._doKeywords is True
+ assert tokens._fmtTitle == f"T: {nwHeadFmt.TITLE}"
+ assert tokens._fmtChapter == f"C: {nwHeadFmt.TITLE}"
+ assert tokens._fmtUnNum == f"U: {nwHeadFmt.TITLE}"
+ assert tokens._fmtScene == f"S: {nwHeadFmt.TITLE}"
+ assert tokens._fmtSection == f"X: {nwHeadFmt.TITLE}"
+ assert tokens._textFont == "Monospace"
+ assert tokens._textSize == 10
+ assert tokens._textFixed is True
+ assert tokens._lineHeight == 2.0
+ assert tokens._blockIndent == 6.0
+ assert tokens._doJustify is True
+ assert tokens._marginTitle == (2.0, 2.0)
+ assert tokens._marginHead1 == (2.0, 2.0)
+ assert tokens._marginHead2 == (2.0, 2.0)
+ assert tokens._marginHead3 == (2.0, 2.0)
+ assert tokens._marginHead4 == (2.0, 2.0)
+ assert tokens._marginText == (2.0, 2.0)
+ assert tokens._marginMeta == (2.0, 2.0)
+ assert tokens._hideScene is True
+ assert tokens._hideSection is True
+ assert tokens._linkHeaders is True
+ assert tokens._doBodyText is False
+ assert tokens._doSynopsis is True
+ assert tokens._doComments is True
+ assert tokens._doKeywords is True
# Check Limits
- theToken.setLineHeight(0.0)
- assert theToken._lineHeight == 0.5
- theToken.setLineHeight(10.0)
- assert theToken._lineHeight == 5.0
+ tokens.setLineHeight(0.0)
+ assert tokens._lineHeight == 0.5
+ tokens.setLineHeight(10.0)
+ assert tokens._lineHeight == 5.0
- theToken.setBlockIndent(-6.0)
- assert theToken._blockIndent == 0.0
- theToken.setBlockIndent(60.0)
- assert theToken._blockIndent == 10.0
+ tokens.setBlockIndent(-6.0)
+ assert tokens._blockIndent == 0.0
+ tokens.setBlockIndent(60.0)
+ assert tokens._blockIndent == 10.0
# END Test testCoreToken_Setters
@@ -133,15 +134,15 @@ def testCoreToken_Setters(mockGUI):
@pytest.mark.core
def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
"""Test handling files and text in the Tokenizer class."""
- theProject = NWProject()
+ project = NWProject()
mockRnd.reset()
- buildTestProject(theProject, fncPath)
+ buildTestProject(project, fncPath)
- theProject.data.setLanguage("en")
- theProject._loadProjectLocalisation()
+ project.data.setLanguage("en")
+ project._loadProjectLocalisation()
- theToken = BareTokenizer(theProject)
- theToken.setKeepMarkdown(True)
+ tokens = BareTokenizer(project)
+ tokens.setKeepMarkdown(True)
# Set some content to work with
docText = (
@@ -153,58 +154,65 @@ def testCoreToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
)
docTextR = docText.replace("", "this").replace("", "that")
- nDoc = theProject.storage.getDocument(C.hSceneDoc)
+ nDoc = project.storage.getDocument(C.hSceneDoc)
assert nDoc.writeDocument(docText)
- theProject.data.setAutoReplace({"A": "this", "B": "that"})
+ project.data.setAutoReplace({"A": "this", "B": "that"})
- assert theProject.saveProject()
+ assert project.saveProject()
# Root Heading
- assert theToken.addRootHeading("stuff") is False
- assert theToken.addRootHeading(C.hSceneDoc) is False
+ assert tokens.addRootHeading("stuff") is False
+ assert tokens.addRootHeading(C.hSceneDoc) is False
# First Page
- assert theToken.addRootHeading(C.hPlotRoot) is True
- assert theToken.theMarkdown[-1] == "# Notes: Plot\n\n"
- assert theToken._tokens[-1] == (
+ assert tokens.addRootHeading(C.hPlotRoot) is True
+ assert tokens.theMarkdown[-1] == "# Notes: Plot\n\n"
+ assert tokens._tokens[-1] == (
Tokenizer.T_TITLE, 0, "Notes: Plot", None, Tokenizer.A_CENTRE
)
# Not First Page
- assert theToken.addRootHeading(C.hPlotRoot) is True
- assert theToken.theMarkdown[-1] == "# Notes: Plot\n\n"
- assert theToken._tokens[-1] == (
+ assert tokens.addRootHeading(C.hPlotRoot) is True
+ assert tokens.theMarkdown[-1] == "# Notes: Plot\n\n"
+ assert tokens._tokens[-1] == (
Tokenizer.T_TITLE, 0, "Notes: Plot", None, Tokenizer.A_CENTRE | Tokenizer.A_PBB
)
# Set Text
- assert theToken.setText("stuff") is False
- assert theToken.setText(C.hSceneDoc) is True
- assert theToken._text == docText
+ assert tokens.setText("stuff") is False
+ assert tokens.setText(C.hSceneDoc) is True
+ assert tokens._text == docText
- assert theToken.setText(C.hSceneDoc, docText) is True
- assert theToken._text == docText
+ assert tokens.setText(C.hSceneDoc, docText) is True
+ assert tokens._text == docText
- assert theToken._isNone is False
- assert theToken._isNovel is True
- assert theToken._isNote is False
+ assert tokens._isNone is False
+ assert tokens._isNovel is True
+ assert tokens._isNote is False
# Pre Processing
- theToken.doPreProcessing()
- assert theToken._text == docTextR
+ tokens.doPreProcessing()
+ assert tokens._text == docTextR
# Save File
savePath = fncPath / "dump.nwd"
- theToken.saveRawMarkdown(savePath)
+ tokens.saveRawMarkdown(savePath)
assert readFile(savePath) == (
"# Notes: Plot\n\n"
"# Notes: Plot\n\n"
)
+ tokens.saveRawMarkdownJSON(savePath)
+ assert json.loads(readFile(savePath))["text"] == {
+ "nwd": [
+ ["# Notes: Plot"],
+ ["# Notes: Plot"]
+ ]
+ }
# Check abstract method
with pytest.raises(NotImplementedError):
- theToken.doConvert()
+ tokens.doConvert()
# END Test testCoreToken_TextOps
@@ -223,202 +231,202 @@ def testCoreToken_StripEscape():
@pytest.mark.core
def testCoreToken_HeaderFormat(mockGUI):
"""Test the tokenization of header formats in the Tokenizer class."""
- theProject = NWProject()
- theToken = BareTokenizer(theProject)
- theToken.setKeepMarkdown(True)
+ project = NWProject()
+ tokens = BareTokenizer(project)
+ tokens.setKeepMarkdown(True)
# Title
# =====
# Story File
- theToken._isNovel = True
- theToken._isNote = False
- theToken._isFirst = True
- theToken._text = "#! Novel Title\n"
+ tokens._isNovel = True
+ tokens._isNote = False
+ tokens._isFirst = True
+ tokens._text = "#! Novel Title\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_TITLE, 1, "Novel Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "#! Novel Title\n\n"
+ assert tokens.theMarkdown[-1] == "#! Novel Title\n\n"
# Note File
- theToken._isNovel = False
- theToken._isNote = True
- theToken._isFirst = True
- theToken._text = "#! Note Title\n"
+ tokens._isNovel = False
+ tokens._isNote = True
+ tokens._isFirst = True
+ tokens._text = "#! Note Title\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Note Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "#! Note Title\n\n"
+ assert tokens.theMarkdown[-1] == "#! Note Title\n\n"
# Header 1
# ========
# Story File
- theToken._isNovel = True
- theToken._isNote = False
- theToken._isFirst = True
- theToken._text = "# Novel Title\n"
+ tokens._isNovel = True
+ tokens._isNote = False
+ tokens._isFirst = True
+ tokens._text = "# Novel Title\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Novel Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "# Novel Title\n\n"
+ assert tokens.theMarkdown[-1] == "# Novel Title\n\n"
# Note File
- theToken._isNovel = False
- theToken._isNote = True
- theToken._isFirst = True
- theToken._text = "# Note Title\n"
+ tokens._isNovel = False
+ tokens._isNote = True
+ tokens._isFirst = True
+ tokens._text = "# Note Title\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Note Title", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "# Note Title\n\n"
+ assert tokens.theMarkdown[-1] == "# Note Title\n\n"
# Header 2
# ========
# Story File
- theToken._isNovel = True
- theToken._isNote = False
- theToken._text = "## Chapter One\n"
+ tokens._isNovel = True
+ tokens._isNote = False
+ tokens._text = "## Chapter One\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "## Chapter One\n\n"
+ assert tokens.theMarkdown[-1] == "## Chapter One\n\n"
# Note File
- theToken._isNovel = False
- theToken._isNote = True
- theToken._text = "## Heading 2\n"
+ tokens._isNovel = False
+ tokens._isNote = True
+ tokens._text = "## Heading 2\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Heading 2", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "## Heading 2\n\n"
+ assert tokens.theMarkdown[-1] == "## Heading 2\n\n"
# Header 3
# ========
# Story File
- theToken._isNovel = True
- theToken._isNote = False
- theToken._text = "### Scene One\n"
+ tokens._isNovel = True
+ tokens._isNote = False
+ tokens._text = "### Scene One\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene One", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "### Scene One\n\n"
+ assert tokens.theMarkdown[-1] == "### Scene One\n\n"
# Note File
- theToken._isNovel = False
- theToken._isNote = True
- theToken._text = "### Heading 3\n"
+ tokens._isNovel = False
+ tokens._isNote = True
+ tokens._text = "### Heading 3\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Heading 3", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "### Heading 3\n\n"
+ assert tokens.theMarkdown[-1] == "### Heading 3\n\n"
# Header 4
# ========
# Story File
- theToken._isNovel = True
- theToken._isNote = False
- theToken._text = "#### A Section\n"
+ tokens._isNovel = True
+ tokens._isNote = False
+ tokens._text = "#### A Section\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "A Section", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "#### A Section\n\n"
+ assert tokens.theMarkdown[-1] == "#### A Section\n\n"
# Note File
- theToken._isNovel = False
- theToken._isNote = True
- theToken._text = "#### Heading 4\n"
+ tokens._isNovel = False
+ tokens._isNote = True
+ tokens._text = "#### Heading 4\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "Heading 4", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "#### Heading 4\n\n"
+ assert tokens.theMarkdown[-1] == "#### Heading 4\n\n"
# Title
# =====
# Story File
- theToken._isNovel = True
- theToken._isNote = False
- theToken._text = "#! Title\n"
+ tokens._isNovel = True
+ tokens._isNote = False
+ tokens._text = "#! Title\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_TITLE, 1, "Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "#! Title\n\n"
+ assert tokens.theMarkdown[-1] == "#! Title\n\n"
# Note File
- theToken._isNovel = False
- theToken._isNote = True
- theToken._text = "#! Title\n"
+ tokens._isNovel = False
+ tokens._isNote = True
+ tokens._text = "#! Title\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "#! Title\n\n"
+ assert tokens.theMarkdown[-1] == "#! Title\n\n"
# Unnumbered
# ==========
# Story File
- theToken._isNovel = True
- theToken._isNote = False
- theToken._text = "##! Prologue\n"
+ tokens._isNovel = True
+ tokens._isNote = False
+ tokens._text = "##! Prologue\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_UNNUM, 1, "Prologue", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "##! Prologue\n\n"
+ assert tokens.theMarkdown[-1] == "##! Prologue\n\n"
# Note File
- theToken._isNovel = False
- theToken._isNote = True
- theToken._text = "##! Prologue\n"
+ tokens._isNovel = False
+ tokens._isNote = True
+ tokens._text = "##! Prologue\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Prologue", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "##! Prologue\n\n"
+ assert tokens.theMarkdown[-1] == "##! Prologue\n\n"
# END Test testCoreToken_HeaderFormat
@@ -426,67 +434,67 @@ def testCoreToken_HeaderFormat(mockGUI):
@pytest.mark.core
def testCoreToken_MetaFormat(mockGUI):
"""Test the tokenization of meta formats in the Tokenizer class."""
- theProject = NWProject()
- theToken = BareTokenizer(theProject)
- theToken.setKeepMarkdown(True)
+ project = NWProject()
+ tokens = BareTokenizer(project)
+ tokens.setKeepMarkdown(True)
# Comment
- theToken._text = "% A comment\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "% A comment\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_COMMENT, 1, "A comment", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "\n"
+ assert tokens.theMarkdown[-1] == "\n"
- theToken.setComments(True)
- theToken.tokenizeText()
- assert theToken.theMarkdown[-1] == "% A comment\n\n"
+ tokens.setComments(True)
+ tokens.tokenizeText()
+ assert tokens.theMarkdown[-1] == "% A comment\n\n"
# Synopsis
- theToken._text = "%synopsis: The synopsis\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "%synopsis: The synopsis\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_SYNOPSIS, 1, "The synopsis", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- theToken._text = "% synopsis: The synopsis\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "% synopsis: The synopsis\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_SYNOPSIS, 1, "The synopsis", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "\n"
+ assert tokens.theMarkdown[-1] == "\n"
- theToken.setSynopsis(True)
- theToken.tokenizeText()
- assert theToken.theMarkdown[-1] == "% synopsis: The synopsis\n\n"
+ tokens.setSynopsis(True)
+ tokens.tokenizeText()
+ assert tokens.theMarkdown[-1] == "% synopsis: The synopsis\n\n"
# Keyword
- theToken._text = "@char: Bod\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "@char: Bod\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_KEYWORD, 1, "char: Bod", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "\n"
+ assert tokens.theMarkdown[-1] == "\n"
- theToken.setKeywords(True)
- theToken.tokenizeText()
- assert theToken.theMarkdown[-1] == "@char: Bod\n\n"
+ tokens.setKeywords(True)
+ tokens.tokenizeText()
+ assert tokens.theMarkdown[-1] == "@char: Bod\n\n"
- theToken._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
- theToken.tokenizeText()
+ tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
+ tokens.tokenizeText()
styTop = Tokenizer.A_NONE | Tokenizer.A_Z_BTMMRG
styMid = Tokenizer.A_NONE | Tokenizer.A_Z_BTMMRG | Tokenizer.A_Z_TOPMRG
styBtm = Tokenizer.A_NONE | Tokenizer.A_Z_TOPMRG
- assert theToken._tokens == [
+ assert tokens._tokens == [
(Tokenizer.T_KEYWORD, 1, "pov: Bod", None, styTop),
(Tokenizer.T_KEYWORD, 2, "plot: Main", None, styMid),
(Tokenizer.T_KEYWORD, 3, "location: Europe", None, styBtm),
(Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n"
+ assert tokens.theMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n"
# END Test testCoreToken_MetaFormat
@@ -494,14 +502,14 @@ def testCoreToken_MetaFormat(mockGUI):
@pytest.mark.core
def testCoreToken_MarginFormat(mockGUI):
"""Test the tokenization of margin formats in the Tokenizer class."""
- theProject = NWProject()
- theToken = BareTokenizer(theProject)
- theToken.setKeepMarkdown(True)
+ project = NWProject()
+ tokens = BareTokenizer(project)
+ tokens.setKeepMarkdown(True)
# Alignment and Indentation
dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
rIndAlign = Tokenizer.A_RIGHT | Tokenizer.A_IND_R
- theToken._text = (
+ tokens._text = (
"Some regular text\n\n"
"Some left-aligned text <<\n\n"
">> Some right-aligned text\n\n"
@@ -511,8 +519,8 @@ def testCoreToken_MarginFormat(mockGUI):
"> Double-indented block <\n\n"
">> Right-indent, right-aligned <\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_TEXT, 1, "Some regular text", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 3, "Some left-aligned text", [], Tokenizer.A_LEFT),
@@ -531,7 +539,7 @@ def testCoreToken_MarginFormat(mockGUI):
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == (
+ assert tokens.theMarkdown[-1] == (
"Some regular text\n\n"
"Some left-aligned text\n\n"
"Some right-aligned text\n\n"
@@ -545,121 +553,212 @@ def testCoreToken_MarginFormat(mockGUI):
# END Test testCoreToken_MarginFormat
+@pytest.mark.core
+def testCoreToken_ExtractFormats(mockGUI):
+ """Test the extraction of formats in the Tokenizer class."""
+ project = NWProject()
+ tokens = BareTokenizer(project)
+ tokens.setKeepMarkdown(True)
+
+ # Markdown
+ # ========
+
+ # Plain bold
+ text, fmt = tokens._extractFormats("Text with **bold** in it.")
+ assert text == "Text with bold in it."
+ assert fmt == [(10, tokens.FMT_B_B), (14, tokens.FMT_B_E)]
+
+ # Plain italics
+ text, fmt = tokens._extractFormats("Text with _italics_ in it.")
+ assert text == "Text with italics in it."
+ assert fmt == [(10, tokens.FMT_I_B), (17, tokens.FMT_I_E)]
+
+ # Plain strikethrough
+ text, fmt = tokens._extractFormats("Text with ~~strikethrough~~ in it.")
+ assert text == "Text with strikethrough in it."
+ assert fmt == [(10, tokens.FMT_D_B), (23, tokens.FMT_D_E)]
+
+ # Nested bold/italics
+ text, fmt = tokens._extractFormats("Text with **bold and _italics_** in it.")
+ assert text == "Text with bold and italics in it."
+ assert fmt == [
+ (10, tokens.FMT_B_B), (19, tokens.FMT_I_B), (26, tokens.FMT_I_E), (26, tokens.FMT_B_E)
+ ]
+
+ # Bold with overlapping italics
+ # Here, bold is ignored because it is not on word boundary
+ text, fmt = tokens._extractFormats("Text with **bold and overlapping _italics**_ in it.")
+ assert text == "Text with **bold and overlapping italics** in it."
+ assert fmt == [(33, tokens.FMT_I_B), (42, tokens.FMT_I_E)]
+
+ # Shortcodes
+ # ==========
+
+ # Plain bold
+ text, fmt = tokens._extractFormats("Text with [b]bold[/b] in it.")
+ assert text == "Text with bold in it."
+ assert fmt == [(10, tokens.FMT_B_B), (14, tokens.FMT_B_E)]
+
+ # Plain italics
+ text, fmt = tokens._extractFormats("Text with [i]italics[/i] in it.")
+ assert text == "Text with italics in it."
+ assert fmt == [(10, tokens.FMT_I_B), (17, tokens.FMT_I_E)]
+
+ # Plain strikethrough
+ text, fmt = tokens._extractFormats("Text with [s]strikethrough[/s] in it.")
+ assert text == "Text with strikethrough in it."
+ assert fmt == [(10, tokens.FMT_D_B), (23, tokens.FMT_D_E)]
+
+ # Plain underline
+ text, fmt = tokens._extractFormats("Text with [u]underline[/u] in it.")
+ assert text == "Text with underline in it."
+ assert fmt == [(10, tokens.FMT_U_B), (19, tokens.FMT_U_E)]
+
+ # Plain superscript
+ text, fmt = tokens._extractFormats("Text with super[sup]script[/sup] in it.")
+ assert text == "Text with superscript in it."
+ assert fmt == [(15, tokens.FMT_SUP_B), (21, tokens.FMT_SUP_E)]
+
+ # Plain subscript
+ text, fmt = tokens._extractFormats("Text with sub[sub]script[/sub] in it.")
+ assert text == "Text with subscript in it."
+ assert fmt == [(13, tokens.FMT_SUB_B), (19, tokens.FMT_SUB_E)]
+
+ # Nested bold/italics
+ text, fmt = tokens._extractFormats("Text with [b]bold and [i]italics[/i][/b] in it.")
+ assert text == "Text with bold and italics in it."
+ assert fmt == [
+ (10, tokens.FMT_B_B), (19, tokens.FMT_I_B), (26, tokens.FMT_I_E), (26, tokens.FMT_B_E)
+ ]
+
+ # Bold with overlapping italics
+ # With shortcodes, this works
+ text, fmt = tokens._extractFormats(
+ "Text with [b]bold and overlapping [i]italics[/b][/i] in it."
+ )
+ assert text == "Text with bold and overlapping italics in it."
+ assert fmt == [
+ (10, tokens.FMT_B_B), (31, tokens.FMT_I_B), (38, tokens.FMT_B_E), (38, tokens.FMT_I_E)
+ ]
+
+# END Test testCoreToken_ExtractFormats
+
+
@pytest.mark.core
def testCoreToken_TextFormat(mockGUI):
"""Test the tokenization of text formats in the Tokenizer class."""
- theProject = NWProject()
- theToken = BareTokenizer(theProject)
- theToken.setKeepMarkdown(True)
+ project = NWProject()
+ tokens = BareTokenizer(project)
+ tokens.setKeepMarkdown(True)
# Text
- theToken._text = "Some plain text\non two lines\n\n\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "Some plain text\non two lines\n\n\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_TEXT, 1, "Some plain text", [], Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 2, "on two lines", [], Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"
+ assert tokens.theMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"
- theToken.setBodyText(False)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.setBodyText(False)
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "\n\n\n"
- theToken.setBodyText(True)
+ assert tokens.theMarkdown[-1] == "\n\n\n"
+ tokens.setBodyText(True)
# Text Emphasis
- theToken._text = "Some **bolded text** on this lines\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "Some **bolded text** on this lines\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(
Tokenizer.T_TEXT, 1,
- "Some **bolded text** on this lines",
+ "Some bolded text on this lines",
[
- [5, 2, Tokenizer.FMT_B_B],
- [18, 2, Tokenizer.FMT_B_E],
+ (5, Tokenizer.FMT_B_B),
+ (16, Tokenizer.FMT_B_E),
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "Some **bolded text** on this lines\n\n"
+ assert tokens.theMarkdown[-1] == "Some **bolded text** on this lines\n\n"
- theToken._text = "Some _italic text_ on this lines\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "Some _italic text_ on this lines\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(
Tokenizer.T_TEXT, 1,
- "Some _italic text_ on this lines",
+ "Some italic text on this lines",
[
- [5, 1, Tokenizer.FMT_I_B],
- [17, 1, Tokenizer.FMT_I_E],
+ (5, Tokenizer.FMT_I_B),
+ (16, Tokenizer.FMT_I_E),
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "Some _italic text_ on this lines\n\n"
+ assert tokens.theMarkdown[-1] == "Some _italic text_ on this lines\n\n"
- theToken._text = "Some **_bold italic text_** on this lines\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "Some **_bold italic text_** on this lines\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(
Tokenizer.T_TEXT, 1,
- "Some **_bold italic text_** on this lines",
+ "Some bold italic text on this lines",
[
- [5, 2, Tokenizer.FMT_B_B],
- [7, 1, Tokenizer.FMT_I_B],
- [24, 1, Tokenizer.FMT_I_E],
- [25, 2, Tokenizer.FMT_B_E],
+ (5, Tokenizer.FMT_B_B),
+ (5, Tokenizer.FMT_I_B),
+ (21, Tokenizer.FMT_I_E),
+ (21, Tokenizer.FMT_B_E),
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n"
+ assert tokens.theMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n"
- theToken._text = "Some ~~strikethrough text~~ on this lines\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "Some ~~strikethrough text~~ on this lines\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(
Tokenizer.T_TEXT, 1,
- "Some ~~strikethrough text~~ on this lines",
+ "Some strikethrough text on this lines",
[
- [5, 2, Tokenizer.FMT_D_B],
- [25, 2, Tokenizer.FMT_D_E],
+ (5, Tokenizer.FMT_D_B),
+ (23, Tokenizer.FMT_D_E),
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n"
+ assert tokens.theMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n"
- theToken._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(
Tokenizer.T_TEXT, 1,
- "Some **nested bold and _italic_ and ~~strikethrough~~ text** here",
+ "Some nested bold and italic and strikethrough text here",
[
- [5, 2, Tokenizer.FMT_B_B],
- [23, 1, Tokenizer.FMT_I_B],
- [30, 1, Tokenizer.FMT_I_E],
- [36, 2, Tokenizer.FMT_D_B],
- [51, 2, Tokenizer.FMT_D_E],
- [58, 2, Tokenizer.FMT_B_E],
+ (5, Tokenizer.FMT_B_B),
+ (21, Tokenizer.FMT_I_B),
+ (27, Tokenizer.FMT_I_E),
+ (32, Tokenizer.FMT_D_B),
+ (45, Tokenizer.FMT_D_E),
+ (50, Tokenizer.FMT_B_E),
],
Tokenizer.A_NONE
),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
- assert theToken.theMarkdown[-1] == (
+ assert tokens.theMarkdown[-1] == (
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
)
@@ -669,10 +768,10 @@ def testCoreToken_TextFormat(mockGUI):
@pytest.mark.core
def testCoreToken_SpecialFormat(mockGUI):
"""Test the tokenization of special formats in the Tokenizer class."""
- theProject = NWProject()
- theToken = BareTokenizer(theProject)
+ project = NWProject()
+ tokens = BareTokenizer(project)
- theToken._isNovel = True
+ tokens._isNovel = True
# New Page
# ========
@@ -687,45 +786,45 @@ def testCoreToken_SpecialFormat(mockGUI):
]
# Command wo/Space
- theToken._isFirst = True
- theToken._text = (
+ tokens._isFirst = True
+ tokens._text = (
"# Title One\n\n"
"[NEWPAGE]\n\n"
"# Title Two\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == correctResp
+ tokens.tokenizeText()
+ assert tokens._tokens == correctResp
# Command w/Space
- theToken._isFirst = True
- theToken._text = (
+ tokens._isFirst = True
+ tokens._text = (
"# Title One\n\n"
"[NEW PAGE]\n\n"
"# Title Two\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == correctResp
+ tokens.tokenizeText()
+ assert tokens._tokens == correctResp
# Trailing Spaces
- theToken._isFirst = True
- theToken._text = (
+ tokens._isFirst = True
+ tokens._text = (
"# Title One\n\n"
"[NEW PAGE] \t\n\n"
"# Title Two\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == correctResp
+ tokens.tokenizeText()
+ assert tokens._tokens == correctResp
# Single Empty Paragraph
# ======================
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[VSPACE] \n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 3, "", None, Tokenizer.A_NONE),
@@ -739,13 +838,13 @@ def testCoreToken_SpecialFormat(mockGUI):
# =========================
# One Skip
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[VSPACE:1] \n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 3, "", None, Tokenizer.A_NONE),
@@ -756,13 +855,13 @@ def testCoreToken_SpecialFormat(mockGUI):
]
# Three Skips
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[VSPACE:3] \n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_SKIP, 3, "", None, Tokenizer.A_NONE),
@@ -775,13 +874,13 @@ def testCoreToken_SpecialFormat(mockGUI):
]
# Malformed Command, Case 1
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[VSPACE:3xa] \n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
@@ -791,13 +890,13 @@ def testCoreToken_SpecialFormat(mockGUI):
]
# Malformed Command, Case 2
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[VSPACE:3.5]\n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
@@ -807,13 +906,13 @@ def testCoreToken_SpecialFormat(mockGUI):
]
# Malformed Command, Case 3
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[VSPACE:-1]\n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
@@ -826,14 +925,14 @@ def testCoreToken_SpecialFormat(mockGUI):
# ==============================
# Single Skip
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[NEW PAGE]\n\n"
"[VSPACE]\n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
@@ -845,14 +944,14 @@ def testCoreToken_SpecialFormat(mockGUI):
]
# Multiple Skip
- theToken._text = (
+ tokens._text = (
"# Title One\n\n"
"[NEW PAGE]\n\n"
"[VSPACE:3]\n\n"
"Some text to go here ...\n\n"
)
- theToken.tokenizeText()
- assert theToken._tokens == [
+ tokens.tokenizeText()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "Title One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
@@ -871,51 +970,51 @@ def testCoreToken_SpecialFormat(mockGUI):
@pytest.mark.core
def testCoreToken_ProcessHeaders(mockGUI):
"""Test the header and page parser of the Tokenizer class."""
- theProject = NWProject()
- theProject.data.setLanguage("en")
- theProject._loadProjectLocalisation()
- theToken = BareTokenizer(theProject)
+ project = NWProject()
+ project.data.setLanguage("en")
+ project._loadProjectLocalisation()
+ tokens = BareTokenizer(project)
# Nothing
- theToken._text = "Some text ...\n"
- assert theToken.doHeaders() is False
- theToken._isNone = True
- assert theToken.doHeaders() is False
- theToken._isNone = False
- assert theToken.doHeaders() is False
- theToken._isNote = True
- assert theToken.doHeaders() is False
- theToken._isNote = False
+ tokens._text = "Some text ...\n"
+ assert tokens.doHeaders() is False
+ tokens._isNone = True
+ assert tokens.doHeaders() is False
+ tokens._isNone = False
+ assert tokens.doHeaders() is False
+ tokens._isNote = True
+ assert tokens.doHeaders() is False
+ tokens._isNote = False
##
# Story FIles
##
- theToken._isNone = False
- theToken._isNote = False
- theToken._isNovel = True
+ tokens._isNone = False
+ tokens._isNote = False
+ tokens._isNovel = True
# Titles
# ======
# H1: Title, First Page
- assert theToken._isFirst is True
- theToken._text = "# Part One\n"
- theToken.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ assert tokens._isFirst is True
+ tokens._text = "# Part One\n"
+ tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "T: Part One", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H1: Title, Not First Page
- assert theToken._isFirst is False
- theToken._text = "# Part One\n"
- theToken.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ assert tokens._isFirst is False
+ tokens._text = "# Part One\n"
+ tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD1, 1, "T: Part One", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
@@ -924,52 +1023,52 @@ def testCoreToken_ProcessHeaders(mockGUI):
# ========
# H2: Chapter
- theToken._text = "## Chapter One\n"
- theToken.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "## Chapter One\n"
+ tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "C: Chapter One", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H2: Unnumbered Chapter
- theToken._text = "##! Prologue\n"
- theToken.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "##! Prologue\n"
+ tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_UNNUM, 1, "U: Prologue", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H2: Chapter Word Number
- theToken._text = "## Chapter\n"
- theToken.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}")
- theToken._hFormatter._chCount = 0
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "## Chapter\n"
+ tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}")
+ tokens._hFormatter._chCount = 0
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H2: Chapter Roman Number Upper Case
- theToken._text = "## Chapter\n"
- theToken.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "## Chapter\n"
+ tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter II", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H2: Chapter Roman Number Lower Case
- theToken._text = "## Chapter\n"
- theToken.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "## Chapter\n"
+ tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD2, 1, "Chapter iii", None, Tokenizer.A_PBB),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
@@ -978,89 +1077,89 @@ def testCoreToken_ProcessHeaders(mockGUI):
# ======
# H3: Scene w/Title
- theToken._text = "### Scene One\n"
- theToken.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### Scene One\n"
+ tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "S: Scene One", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H3: Scene Hidden wo/Format
- theToken._text = "### Scene One\n"
- theToken.setSceneFormat("", True)
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### Scene One\n"
+ tokens.setSceneFormat("", True)
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H3: Scene wo/Format, first
- theToken._text = "### Scene One\n"
- theToken.setSceneFormat("", False)
- theToken._firstScene = True
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### Scene One\n"
+ tokens.setSceneFormat("", False)
+ tokens._firstScene = True
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H3: Scene wo/Format, not first
- theToken._text = "### Scene One\n"
- theToken.setSceneFormat("", False)
- theToken._firstScene = False
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### Scene One\n"
+ tokens.setSceneFormat("", False)
+ tokens._firstScene = False
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H3: Scene Separator, first
- theToken._text = "### Scene One\n"
- theToken.setSceneFormat("* * *", False)
- theToken._firstScene = True
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### Scene One\n"
+ tokens.setSceneFormat("* * *", False)
+ tokens._firstScene = True
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H3: Scene Separator, not first
- theToken._text = "### Scene One\n"
- theToken.setSceneFormat("* * *", False)
- theToken._firstScene = False
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### Scene One\n"
+ tokens.setSceneFormat("* * *", False)
+ tokens._firstScene = False
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H3: Scene w/Absolute Number
- theToken._text = "### A Scene\n"
- theToken.setSceneFormat(f"Scene {nwHeadFmt.SC_ABS}", False)
- theToken._hFormatter._scAbsCount = 0
- theToken._hFormatter._scChCount = 0
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### A Scene\n"
+ tokens.setSceneFormat(f"Scene {nwHeadFmt.SC_ABS}", False)
+ tokens._hFormatter._scAbsCount = 0
+ tokens._hFormatter._scChCount = 0
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene 1", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H3: Scene w/Chapter Number
- theToken._text = "### A Scene\n"
- theToken.setSceneFormat(f"Scene {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM}", False)
- theToken._hFormatter._scAbsCount = 0
- theToken._hFormatter._scChCount = 1
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "### A Scene\n"
+ tokens.setSceneFormat(f"Scene {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM}", False)
+ tokens._hFormatter._scAbsCount = 0
+ tokens._hFormatter._scChCount = 1
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD3, 1, "Scene 3.2", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
@@ -1069,51 +1168,51 @@ def testCoreToken_ProcessHeaders(mockGUI):
# ========
# H4: Section Hidden wo/Format
- theToken._text = "#### A Section\n"
- theToken.setSectionFormat(r"", True)
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "#### A Section\n"
+ tokens.setSectionFormat(r"", True)
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H4: Section Visible wo/Format
- theToken._text = "#### A Section\n"
- theToken.setSectionFormat("", False)
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "#### A Section\n"
+ tokens.setSectionFormat("", False)
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H4: Section w/Format
- theToken._text = "#### A Section\n"
- theToken.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "#### A Section\n"
+ tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_HEAD4, 1, "X: A Section", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# H4: Section Separator
- theToken._text = "#### A Section\n"
- theToken.setSectionFormat("* * *", False)
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._tokens == [
+ tokens._text = "#### A Section\n"
+ tokens.setSectionFormat("* * *", False)
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._tokens == [
(Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
]
# Check the first scene detector
- assert theToken._firstScene is False
- theToken._firstScene = True
- theToken._text = "Some text ...\n"
- theToken.tokenizeText()
- theToken.doHeaders()
- assert theToken._firstScene is False
+ assert tokens._firstScene is False
+ tokens._firstScene = True
+ tokens._text = "Some text ...\n"
+ tokens.tokenizeText()
+ tokens.doHeaders()
+ assert tokens._firstScene is False
# END Test testCoreToken_ProcessHeaders