Veronica K. B. Olsen
645 lines
20 KiB
Python
645 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""novelWriter Tokenizer Class Tester
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from nw.core import NWProject, NWDoc
|
|
from nw.core.tokenizer import Tokenizer
|
|
|
|
@pytest.mark.core
|
|
def testCoreToken_Setters(dummyGUI):
|
|
"""Test all the setters for the Tokenizer class.
|
|
"""
|
|
theProject = NWProject(dummyGUI)
|
|
theToken = Tokenizer(theProject, dummyGUI)
|
|
|
|
# Verify defaults
|
|
assert theToken.fmtTitle == "%title%"
|
|
assert theToken.fmtChapter == "%title%"
|
|
assert theToken.fmtUnNum == "%title%"
|
|
assert theToken.fmtScene == "%title%"
|
|
assert theToken.fmtSection == "%title%"
|
|
assert theToken.hideScene is False
|
|
assert theToken.hideSection is False
|
|
assert theToken.linkHeaders is False
|
|
assert theToken.doBodyText is True
|
|
assert theToken.doSynopsis is False
|
|
assert theToken.doComments is False
|
|
assert theToken.doKeywords is False
|
|
assert theToken.doJustify is False
|
|
|
|
# Set new values
|
|
theToken.setTitleFormat("T: %title%")
|
|
theToken.setChapterFormat("C: %title%")
|
|
theToken.setUnNumberedFormat("U: %title%")
|
|
theToken.setSceneFormat("S: %title%", True)
|
|
theToken.setSectionFormat("X: %title%", True)
|
|
theToken.setLinkHeaders(True)
|
|
theToken.setBodyText(False)
|
|
theToken.setSynopsis(True)
|
|
theToken.setComments(True)
|
|
theToken.setKeywords(True)
|
|
theToken.setJustify(True)
|
|
|
|
# Check new values
|
|
assert theToken.fmtTitle == "T: %title%"
|
|
assert theToken.fmtChapter == "C: %title%"
|
|
assert theToken.fmtUnNum == "U: %title%"
|
|
assert theToken.fmtScene == "S: %title%"
|
|
assert theToken.fmtSection == "X: %title%"
|
|
assert theToken.hideScene is True
|
|
assert theToken.hideSection is True
|
|
assert theToken.linkHeaders is True
|
|
assert theToken.doBodyText is False
|
|
assert theToken.doSynopsis is True
|
|
assert theToken.doComments is True
|
|
assert theToken.doKeywords is True
|
|
assert theToken.doJustify is True
|
|
|
|
# END Test testCoreToken_Setters
|
|
|
|
@pytest.mark.core
|
|
def testCoreToken_TextOps(monkeypatch, nwMinimal, dummyGUI):
|
|
"""Test handling files and text in the Tokenizer class.
|
|
"""
|
|
theProject = NWProject(dummyGUI)
|
|
theProject.projTree.setSeed(42)
|
|
theToken = Tokenizer(theProject, dummyGUI)
|
|
|
|
assert theProject.openProject(nwMinimal)
|
|
sHandle = "8c659a11cd429"
|
|
|
|
# Set some content to work with
|
|
|
|
docText = (
|
|
"### Scene Six\n\n"
|
|
"This is text with _italic text_, some **bold text**, some ~~deleted text~~, "
|
|
"and some **_mixed text_** and **some _nested_ text**.\n\n"
|
|
"#### Replace\n\n"
|
|
"Also, replace <A> and <B>.\n\n"
|
|
)
|
|
docTextR = docText.replace("<A>", "this").replace("<B>", "that")
|
|
|
|
nDoc = NWDoc(theProject, dummyGUI)
|
|
nDoc.openDocument(sHandle)
|
|
nDoc.saveDocument(docText)
|
|
nDoc.clearDocument()
|
|
|
|
theProject.setAutoReplace({"A": "this", "B": "that"})
|
|
|
|
assert theProject.saveProject()
|
|
|
|
# Root heading
|
|
assert theToken.addRootHeading("dummy") is False
|
|
assert theToken.addRootHeading(sHandle) is False
|
|
assert theToken.addRootHeading("7695ce551d265") is True
|
|
assert theToken.theMarkdown == "# Notes: Plot\n\n"
|
|
|
|
# Set text
|
|
assert theToken.setText("dummy") is False
|
|
assert theToken.setText(sHandle) is True
|
|
assert theToken.theText == docText
|
|
|
|
monkeypatch.setattr("nw.constants.nwConst.MAX_DOCSIZE", 100)
|
|
assert theToken.setText(sHandle, docText) is True
|
|
assert theToken.theText == (
|
|
"# ERROR\n\n"
|
|
"Document 'New Scene' is too big (0.00 MB). Skipping.\n\n"
|
|
)
|
|
monkeypatch.undo()
|
|
|
|
assert theToken.setText(sHandle, docText) is True
|
|
assert theToken.theText == docText
|
|
|
|
assert theToken.isNone is False
|
|
assert theToken.isTitle is False
|
|
assert theToken.isBook is False
|
|
assert theToken.isPage is False
|
|
assert theToken.isPart is False
|
|
assert theToken.isUnNum is False
|
|
assert theToken.isChap is False
|
|
assert theToken.isScene is True
|
|
assert theToken.isNote is False
|
|
assert theToken.isNovel is True
|
|
|
|
# Auto replace
|
|
theToken.doAutoReplace()
|
|
assert theToken.theText == docTextR
|
|
|
|
# Access
|
|
assert theToken.getResult() is None
|
|
assert theToken.getResultSize() == 0
|
|
theToken.theResult = ""
|
|
assert theToken.getResultSize() == 0
|
|
|
|
# Post Processing
|
|
theToken.theResult = r"This is text with escapes: \** \~~ \__"
|
|
theToken.doPostProcessing()
|
|
assert theToken.theResult == "This is text with escapes: ** ~~ __"
|
|
|
|
# END Test testCoreToken_TextOps
|
|
|
|
@pytest.mark.core
|
|
def testCoreToken_Tokenize(dummyGUI):
|
|
"""Test the tokenization of the Tokenizer class.
|
|
"""
|
|
theProject = NWProject(dummyGUI)
|
|
theToken = Tokenizer(theProject, dummyGUI)
|
|
|
|
# Header 1
|
|
theToken.theText = "# Novel Title\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD1, 1, "Novel Title", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "# Novel Title\n\n"
|
|
|
|
# Header 2
|
|
theToken.theText = "## Chapter One\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "## Chapter One\n\n"
|
|
|
|
# Header 3
|
|
theToken.theText = "### Scene One\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD3, 1, "Scene One", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "### Scene One\n\n"
|
|
|
|
# Header 4
|
|
theToken.theText = "#### A Section\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD4, 1, "A Section", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "#### A Section\n\n"
|
|
|
|
# Comment
|
|
theToken.theText = "% A comment\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_COMMENT, 1, "A comment", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "\n"
|
|
|
|
theToken.setComments(True)
|
|
theToken.tokenizeText()
|
|
assert theToken.theMarkdown == "% A comment\n\n"
|
|
|
|
# Symopsis
|
|
theToken.theText = "%synopsis: The synopsis\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_SYNOPSIS, 1, "The synopsis", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
theToken.theText = "% synopsis: The synopsis\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_SYNOPSIS, 1, "The synopsis", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "\n"
|
|
|
|
theToken.setSynopsis(True)
|
|
theToken.tokenizeText()
|
|
assert theToken.theMarkdown == "% synopsis: The synopsis\n\n"
|
|
|
|
# Keyword
|
|
theToken.theText = "@char: Bod\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_KEYWORD, 1, "char: Bod", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "\n"
|
|
|
|
theToken.setKeywords(True)
|
|
theToken.tokenizeText()
|
|
assert theToken.theMarkdown == "@char: Bod\n\n"
|
|
|
|
# Text
|
|
theToken.theText = "Some plain text\non two lines\n\n\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_TEXT, 1, "Some plain text", [], Tokenizer.A_NONE),
|
|
(Tokenizer.T_TEXT, 2, "on two lines", [], Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "Some plain text\non two lines\n\n\n\n"
|
|
|
|
theToken.setBodyText(False)
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 4, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "\n\n\n"
|
|
theToken.setBodyText(True)
|
|
|
|
# Text Emphasis
|
|
theToken.theText = "Some **bolded text** on this lines\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(
|
|
Tokenizer.T_TEXT, 1,
|
|
"Some **bolded text** on this lines",
|
|
[
|
|
[5, 2, Tokenizer.FMT_B_B],
|
|
[18, 2, Tokenizer.FMT_B_E],
|
|
],
|
|
Tokenizer.A_NONE
|
|
),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "Some **bolded text** on this lines\n\n"
|
|
|
|
theToken.theText = "Some _italic text_ on this lines\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(
|
|
Tokenizer.T_TEXT, 1,
|
|
"Some _italic text_ on this lines",
|
|
[
|
|
[5, 1, Tokenizer.FMT_I_B],
|
|
[17, 1, Tokenizer.FMT_I_E],
|
|
],
|
|
Tokenizer.A_NONE
|
|
),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "Some _italic text_ on this lines\n\n"
|
|
|
|
theToken.theText = "Some **_bold italic text_** on this lines\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(
|
|
Tokenizer.T_TEXT, 1,
|
|
"Some **_bold italic text_** on this lines",
|
|
[
|
|
[5, 2, Tokenizer.FMT_B_B],
|
|
[7, 1, Tokenizer.FMT_I_B],
|
|
[24, 1, Tokenizer.FMT_I_E],
|
|
[25, 2, Tokenizer.FMT_B_E],
|
|
],
|
|
Tokenizer.A_NONE
|
|
),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "Some **_bold italic text_** on this lines\n\n"
|
|
|
|
theToken.theText = "Some ~~strikethrough text~~ on this lines\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(
|
|
Tokenizer.T_TEXT, 1,
|
|
"Some ~~strikethrough text~~ on this lines",
|
|
[
|
|
[5, 2, Tokenizer.FMT_D_B],
|
|
[25, 2, Tokenizer.FMT_D_E],
|
|
],
|
|
Tokenizer.A_NONE
|
|
),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == "Some ~~strikethrough text~~ on this lines\n\n"
|
|
|
|
theToken.theText = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
|
|
theToken.tokenizeText()
|
|
assert theToken.theTokens == [
|
|
(
|
|
Tokenizer.T_TEXT, 1,
|
|
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here",
|
|
[
|
|
[5, 2, Tokenizer.FMT_B_B],
|
|
[23, 1, Tokenizer.FMT_I_B],
|
|
[30, 1, Tokenizer.FMT_I_E],
|
|
[36, 2, Tokenizer.FMT_D_B],
|
|
[51, 2, Tokenizer.FMT_D_E],
|
|
[58, 2, Tokenizer.FMT_B_E],
|
|
],
|
|
Tokenizer.A_NONE
|
|
),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
assert theToken.theMarkdown == (
|
|
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
|
|
)
|
|
|
|
# Check the markdown function as well
|
|
assert theToken.getFilteredMarkdown() == (
|
|
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
|
|
)
|
|
|
|
# END Test testCoreToken_Tokenize
|
|
|
|
@pytest.mark.core
|
|
def testCoreToken_Headers(dummyGUI):
|
|
"""Test the header and page parser of the Tokenizer class.
|
|
"""
|
|
theProject = NWProject(dummyGUI)
|
|
theToken = Tokenizer(theProject, dummyGUI)
|
|
|
|
# Nothing
|
|
theToken.theText = "Some text ...\n"
|
|
assert theToken.doHeaders() is True
|
|
theToken.isNone = True
|
|
assert theToken.doHeaders() is False
|
|
theToken.isNone = False
|
|
assert theToken.doHeaders() is True
|
|
theToken.isNote = True
|
|
assert theToken.doHeaders() is False
|
|
theToken.isNote = False
|
|
|
|
##
|
|
# Novel
|
|
##
|
|
|
|
theToken.isNovel = True
|
|
|
|
# Titles
|
|
# ======
|
|
|
|
# H1: Title
|
|
theToken.theText = "# Novel Title\n"
|
|
theToken.setTitleFormat(r"T: %title%")
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD1, 1, "T: Novel Title", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# Chapters
|
|
# ========
|
|
|
|
# H2: Chapter
|
|
theToken.theText = "## Chapter One\n"
|
|
theToken.setChapterFormat(r"C: %title%")
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD2, 1, "C: Chapter One", None, Tokenizer.A_PBB),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H2: Unnumbered Chapter
|
|
theToken.theText = "## Chapter One\n"
|
|
theToken.setUnNumberedFormat(r"U: %title%")
|
|
theToken.isUnNum = True
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD2, 1, "U: Chapter One", None, Tokenizer.A_PBB),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H2: Unnumbered Chapter with Star
|
|
theToken.theText = "## *Prologue\n"
|
|
theToken.setUnNumberedFormat(r"U: %title%")
|
|
theToken.isUnNum = False
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD2, 1, "U: Prologue", None, Tokenizer.A_PBB),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H2: Chapter Word Number
|
|
theToken.theText = "## Chapter\n"
|
|
theToken.setChapterFormat(r"Chapter %chw%")
|
|
theToken.numChapter = 0
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H2: Chapter Roman Number Upper Case
|
|
theToken.theText = "## Chapter\n"
|
|
theToken.setChapterFormat(r"Chapter %chI%")
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD2, 1, "Chapter II", None, Tokenizer.A_PBB),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H2: Chapter Roman Number Lower Case
|
|
theToken.theText = "## Chapter\n"
|
|
theToken.setChapterFormat(r"Chapter %chi%")
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD2, 1, "Chapter iii", None, Tokenizer.A_PBB),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# Scenes
|
|
# ======
|
|
|
|
# H3: Scene w/Title
|
|
theToken.theText = "### Scene One\n"
|
|
theToken.setSceneFormat(r"S: %title%", False)
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD3, 1, "S: Scene One", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H3: Scene Hidden wo/Format
|
|
theToken.theText = "### Scene One\n"
|
|
theToken.setSceneFormat(r"", True)
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H3: Scene wo/Format, first
|
|
theToken.theText = "### Scene One\n"
|
|
theToken.setSceneFormat(r"", False)
|
|
theToken.firstScene = True
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H3: Scene wo/Format, not first
|
|
theToken.theText = "### Scene One\n"
|
|
theToken.setSceneFormat(r"", False)
|
|
theToken.firstScene = False
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H3: Scene Separator, first
|
|
theToken.theText = "### Scene One\n"
|
|
theToken.setSceneFormat(r"* * *", False)
|
|
theToken.firstScene = True
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H3: Scene Separator, not first
|
|
theToken.theText = "### Scene One\n"
|
|
theToken.setSceneFormat(r"* * *", False)
|
|
theToken.firstScene = False
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H3: Scene w/Absolute Number
|
|
theToken.theText = "### A Scene\n"
|
|
theToken.setSceneFormat(r"Scene %sca%", False)
|
|
theToken.numAbsScene = 0
|
|
theToken.numChScene = 0
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD3, 1, "Scene 1", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H3: Scene w/Chapter Number
|
|
theToken.theText = "### A Scene\n"
|
|
theToken.setSceneFormat(r"Scene %ch%.%sc%", False)
|
|
theToken.numAbsScene = 0
|
|
theToken.numChScene = 1
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD3, 1, "Scene 3.2", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# Sections
|
|
# ========
|
|
|
|
# H4: Section Hidden wo/Format
|
|
theToken.theText = "#### A Section\n"
|
|
theToken.setSectionFormat(r"", True)
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H4: Section Visible wo/Format
|
|
theToken.theText = "#### A Section\n"
|
|
theToken.setSectionFormat(r"", False)
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H4: Section w/Format
|
|
theToken.theText = "#### A Section\n"
|
|
theToken.setSectionFormat(r"X: %title%", False)
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD4, 1, "X: A Section", None, Tokenizer.A_NONE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# H4: Section Separator
|
|
theToken.theText = "#### A Section\n"
|
|
theToken.setSectionFormat(r"* * *", False)
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
|
|
]
|
|
|
|
# Check the first scene detector
|
|
assert theToken.firstScene is False
|
|
theToken.firstScene = True
|
|
assert theToken.firstScene is True
|
|
theToken.theText = "Some text ...\n"
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.firstScene is False
|
|
|
|
##
|
|
# Title or Partition
|
|
##
|
|
|
|
theToken.isNovel = False
|
|
|
|
# H1: Title
|
|
theToken.theText = "# Novel Title\n"
|
|
theToken.tokenizeText()
|
|
theToken.isTitle = True
|
|
theToken.isPart = False
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_TITLE, 1, "Novel Title", None, Tokenizer.A_PBB_NO | Tokenizer.A_CENTRE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_PBA | Tokenizer.A_CENTRE),
|
|
]
|
|
|
|
# H1: Partition
|
|
theToken.theText = "# Partition Title\n"
|
|
theToken.setTitleFormat(r"T: %title%")
|
|
theToken.tokenizeText()
|
|
theToken.isTitle = False
|
|
theToken.isPart = True
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_HEAD1, 1, "Partition Title", None, Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
|
(Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_PBA | Tokenizer.A_CENTRE),
|
|
]
|
|
|
|
##
|
|
# Page
|
|
##
|
|
|
|
theToken.isNovel = False
|
|
theToken.isTitle = False
|
|
theToken.isPart = False
|
|
theToken.isPage = True
|
|
|
|
# Some Page Text
|
|
theToken.theText = "Page text\n\nMore text\n"
|
|
theToken.tokenizeText()
|
|
theToken.doHeaders()
|
|
assert theToken.theTokens == [
|
|
(Tokenizer.T_TEXT, 1, "Page text", [], Tokenizer.A_PBB | Tokenizer.A_LEFT),
|
|
(Tokenizer.T_EMPTY, 2, "", None, Tokenizer.A_LEFT),
|
|
(Tokenizer.T_TEXT, 3, "More text", [], Tokenizer.A_LEFT),
|
|
(Tokenizer.T_EMPTY, 3, "", None, Tokenizer.A_LEFT),
|
|
]
|
|
|
|
# END Test testCoreToken_Headers
|