"""
novelWriter – Tokenizer Class Tester
====================================
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
"""
from __future__ import annotations
import pytest
from PyQt5.QtGui import QFont
from novelwriter import CONFIG
from novelwriter.constants import nwHeadFmt, nwStyles
from novelwriter.core.project import NWProject
from novelwriter.enum import nwComment
from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt, stripEscape
from novelwriter.formats.tokenizer import COMMENT_STYLE, HeadingFormatter, Tokenizer
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toraw import ToRaw
from tests.tools import C, buildTestProject
TMH = "0123456789abc"
TM1 = f"{TMH}:T0001"
TM2 = f"{TMH}:T0002"
class BareTokenizer(Tokenizer):
def doConvert(self):
super().doConvert() # type: ignore (deliberate check)
def saveDocument(self, path) -> None:
super().saveDocument(path) # type: ignore (deliberate check)
@pytest.mark.core
def testFmtToken_Abstracts(mockGUI, tstPaths):
"""Test all the abstract methods of the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
with pytest.raises(NotImplementedError):
tokens.doConvert()
with pytest.raises(NotImplementedError):
tokens.saveDocument(tstPaths)
@pytest.mark.core
def testFmtToken_Setters(mockGUI):
"""Test all the setters for the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
# Verify defaults
assert tokens._fmtPart == nwHeadFmt.TITLE
assert tokens._fmtChapter == nwHeadFmt.TITLE
assert tokens._fmtUnNum == nwHeadFmt.TITLE
assert tokens._fmtScene == nwHeadFmt.TITLE
assert tokens._fmtHScene == nwHeadFmt.TITLE
assert tokens._fmtSection == nwHeadFmt.TITLE
assert tokens._textFont == QFont("Serif", 11)
assert tokens._lineHeight == 1.15
assert tokens._blockIndent == 4.0
assert tokens._doJustify is False
assert tokens._marginTitle == nwStyles.T_MARGIN["H0"]
assert tokens._marginHead1 == nwStyles.T_MARGIN["H1"]
assert tokens._marginHead2 == nwStyles.T_MARGIN["H2"]
assert tokens._marginHead3 == nwStyles.T_MARGIN["H3"]
assert tokens._marginHead4 == nwStyles.T_MARGIN["H4"]
assert tokens._marginSep == nwStyles.T_MARGIN["SP"]
assert tokens._marginText == nwStyles.T_MARGIN["TT"]
assert tokens._marginMeta == nwStyles.T_MARGIN["MT"]
assert tokens._hidePart is False
assert tokens._hideChapter is False
assert tokens._hideUnNum is False
assert tokens._hideScene is False
assert tokens._hideHScene is False
assert tokens._hideSection is False
assert tokens._linkHeadings is False
assert tokens._doBodyText is True
assert tokens._doSynopsis is False
assert tokens._doComments is False
assert tokens._doKeywords is False
# Set new values
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}", True)
tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}", True)
tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}", True)
tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", True)
tokens.setHardSceneFormat(f"H: {nwHeadFmt.TITLE}", True)
tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", True)
tokens.setFont(QFont("Monospace", 10))
tokens.setLineHeight(2.0)
tokens.setBlockIndent(6.0)
tokens.setJustify(True)
tokens.setTitleMargins(2.0, 2.0)
tokens.setHead1Margins(2.0, 2.0)
tokens.setHead2Margins(2.0, 2.0)
tokens.setHead3Margins(2.0, 2.0)
tokens.setHead4Margins(2.0, 2.0)
tokens.setTextMargins(2.0, 2.0)
tokens.setMetaMargins(2.0, 2.0)
tokens.setSeparatorMargins(2.0, 2.0)
tokens.setLinkHeadings(True)
tokens.setBodyText(False)
tokens.setSynopsis(True)
tokens.setComments(True)
tokens.setKeywords(True)
# Check new values
assert tokens._fmtPart == f"T: {nwHeadFmt.TITLE}"
assert tokens._fmtChapter == f"C: {nwHeadFmt.TITLE}"
assert tokens._fmtUnNum == f"U: {nwHeadFmt.TITLE}"
assert tokens._fmtScene == f"S: {nwHeadFmt.TITLE}"
assert tokens._fmtHScene == f"H: {nwHeadFmt.TITLE}"
assert tokens._fmtSection == f"X: {nwHeadFmt.TITLE}"
assert tokens._textFont == QFont("Monospace", 10)
assert tokens._lineHeight == 2.0
assert tokens._blockIndent == 6.0
assert tokens._doJustify is True
assert tokens._marginTitle == (2.0, 2.0)
assert tokens._marginHead1 == (2.0, 2.0)
assert tokens._marginHead2 == (2.0, 2.0)
assert tokens._marginHead3 == (2.0, 2.0)
assert tokens._marginHead4 == (2.0, 2.0)
assert tokens._marginText == (2.0, 2.0)
assert tokens._marginMeta == (2.0, 2.0)
assert tokens._marginSep == (2.0, 2.0)
assert tokens._hidePart is True
assert tokens._hideChapter is True
assert tokens._hideUnNum is True
assert tokens._hideScene is True
assert tokens._hideHScene is True
assert tokens._hideSection is True
assert tokens._linkHeadings is True
assert tokens._doBodyText is False
assert tokens._doSynopsis is True
assert tokens._doComments is True
assert tokens._doKeywords is True
# Properties
assert tokens.textStats == {}
assert tokens.errData == []
# Check Limits
tokens.setLineHeight(0.0)
assert tokens._lineHeight == 0.5
tokens.setLineHeight(10.0)
assert tokens._lineHeight == 5.0
tokens.setBlockIndent(-6.0)
assert tokens._blockIndent == 0.0
tokens.setBlockIndent(60.0)
assert tokens._blockIndent == 10.0
@pytest.mark.core
def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
"""Test handling files and text in the Tokenizer class."""
project = NWProject()
mockRnd.reset()
buildTestProject(project, fncPath)
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = ToRaw(project)
# Set some content to work with
docText = (
"### Scene Six\n\n"
"This is text with _italic text_, some **bold text**, some ~~deleted text~~, "
"and some **_mixed text_** and **some _nested_ text**.\n\n"
"#### Replace\n\n"
"Also, replace and .\n\n"
)
docTextR = docText.replace("", "this").replace("", "that")
nDoc = project.storage.getDocument(C.hSceneDoc)
assert nDoc.writeDocument(docText)
project.data.setAutoReplace({"A": "this", "B": "that"})
assert project.saveProject()
# Root Heading
assert len(tokens._blocks) == 0
tokens.addRootHeading("stuff")
tokens.addRootHeading(C.hSceneDoc)
assert len(tokens._blocks) == 0
# First Page
tokens.addRootHeading(C.hPlotRoot)
assert tokens._raw[-1] == "#! Notes: Plot\n\n"
assert tokens._blocks[-1] == (
BlockTyp.TITLE, "0000000000009:T0001", "Notes: Plot", [], BlockFmt.CENTRE
)
# Not First Page
tokens.addRootHeading(C.hPlotRoot)
assert tokens._raw[-1] == "#! Notes: Plot\n\n"
assert tokens._blocks[-1] == (
BlockTyp.TITLE, "0000000000009:T0001", "Notes: Plot", [], BlockFmt.CENTRE | BlockFmt.PBB
)
# Set Text
tokens.setText("stuff")
assert tokens._text == ""
tokens.setText(C.hSceneDoc)
assert tokens._text == docText
tokens.setText(C.hSceneDoc, docText)
assert tokens._text == docText
assert tokens._isNovel is True
# Pre Processing
tokens.doPreProcessing()
assert tokens._text == docTextR
@pytest.mark.core
def testFmtToken_StripEscape():
"""Test the stripEscape helper function."""
text1 = "This is text with escapes: \\** \\~~ \\__"
text2 = "This is text with escapes: ** ~~ __"
assert stripEscape(text1) == "This is text with escapes: ** ~~ __"
assert stripEscape(text2) == "This is text with escapes: ** ~~ __"
@pytest.mark.core
def testFmtToken_HeaderFormat(mockGUI):
"""Test the tokenization of header formats in the Tokenizer class."""
project = NWProject()
tokens = ToRaw(project)
tokens._handle = TMH
# Title
# =====
# Story File
tokens._isNovel = True
tokens._isFirst = True
tokens._text = "#! Novel Title\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TITLE, TM1, "Novel Title", [], BlockFmt.CENTRE),
]
assert tokens._raw[-1] == "#! Novel Title\n\n"
# Note File
tokens._isNovel = False
tokens._isFirst = True
tokens._text = "#! Note Title\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TITLE, TM1, "Note Title", [], BlockFmt.CENTRE),
]
assert tokens._raw[-1] == "#! Note Title\n\n"
# Header 1
# ========
# Story File
tokens._isNovel = True
tokens._isFirst = True
tokens._text = "# Novel Title\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Novel Title", [], BlockFmt.CENTRE),
]
assert tokens._raw[-1] == "# Novel Title\n\n"
# Note File
tokens._isNovel = False
tokens._isFirst = True
tokens._text = "# Note Title\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Note Title", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "# Note Title\n\n"
# Header 2
# ========
# Story File
tokens._isNovel = True
tokens._text = "## Chapter One\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter One", [], BlockFmt.PBB),
]
assert tokens._raw[-1] == "## Chapter One\n\n"
# Note File
tokens._isNovel = False
tokens._text = "## Heading 2\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Heading 2", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "## Heading 2\n\n"
# Header 3
# ========
# Story File
tokens._isNovel = True
tokens._text = "### Scene One\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Scene One", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "### Scene One\n\n"
# Note File
tokens._isNovel = False
tokens._text = "### Heading 3\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Heading 3", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "### Heading 3\n\n"
# Header 4
# ========
# Story File
tokens._isNovel = True
tokens._text = "#### A Section\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD4, TM1, "A Section", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "#### A Section\n\n"
# Note File
tokens._isNovel = False
tokens._text = "#### Heading 4\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD4, TM1, "Heading 4", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "#### Heading 4\n\n"
# Title
# =====
# Story File
tokens._isNovel = True
tokens._isFirst = False
tokens._text = "#! Title\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TITLE, TM1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
assert tokens._raw[-1] == "#! Title\n\n"
# Note File
tokens._isNovel = False
tokens._isFirst = False
tokens._text = "#! Title\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TITLE, TM1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
assert tokens._raw[-1] == "#! Title\n\n"
# Unnumbered
# ==========
# Story File
tokens._isNovel = True
tokens._text = "##! Prologue\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Prologue", [], BlockFmt.PBB),
]
assert tokens._raw[-1] == "##! Prologue\n\n"
# Note File
tokens._isNovel = False
tokens._text = "##! Prologue\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Prologue", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "##! Prologue\n\n"
@pytest.mark.core
def testFmtToken_HeaderStyleNone(mockGUI):
"""Test header styling disabled."""
project = NWProject()
tokens = BareTokenizer(project)
def processStyle(text: str, first: bool) -> BlockFmt:
tokens._text = text
tokens._isFirst = first
tokens.tokenizeText()
return tokens._blocks[0][4]
tokens.setTitleStyle(False, False)
tokens.setPartitionStyle(False, False)
tokens.setChapterStyle(False, False)
tokens.setSceneStyle(False, False)
assert tokens._titleStyle == BlockFmt.NONE
assert tokens._partStyle == BlockFmt.NONE
assert tokens._chapterStyle == BlockFmt.NONE
assert tokens._sceneStyle == BlockFmt.NONE
# Novel Docs
tokens._isNovel = True
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.NONE
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.NONE
assert processStyle("## Chapter\n", True) == BlockFmt.NONE
assert processStyle("### Scene\n", True) == BlockFmt.NONE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.NONE
assert processStyle("##! Prologue\n", True) == BlockFmt.NONE
# Note Docs
tokens._isNovel = False
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.NONE
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.NONE
assert processStyle("## Chapter\n", True) == BlockFmt.NONE
assert processStyle("### Scene\n", True) == BlockFmt.NONE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.NONE
assert processStyle("##! Prologue\n", True) == BlockFmt.NONE
@pytest.mark.core
def testFmtToken_HeaderStyleCenter(mockGUI):
"""Test header styling centred."""
project = NWProject()
tokens = BareTokenizer(project)
def processStyle(text: str, first: bool) -> BlockFmt:
tokens._text = text
tokens._isFirst = first
tokens.tokenizeText()
return tokens._blocks[0][4]
tokens.setTitleStyle(True, False)
tokens.setPartitionStyle(True, False)
tokens.setChapterStyle(True, False)
tokens.setSceneStyle(True, False)
assert tokens._titleStyle == BlockFmt.CENTRE
assert tokens._partStyle == BlockFmt.CENTRE
assert tokens._chapterStyle == BlockFmt.CENTRE
assert tokens._sceneStyle == BlockFmt.CENTRE
# Novel Docs
tokens._isNovel = True
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.CENTRE
assert processStyle("## Chapter\n", False) == BlockFmt.CENTRE
assert processStyle("### Scene\n", False) == BlockFmt.CENTRE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.CENTRE
assert processStyle("##! Prologue\n", False) == BlockFmt.CENTRE
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.CENTRE
assert processStyle("## Chapter\n", True) == BlockFmt.CENTRE
assert processStyle("### Scene\n", True) == BlockFmt.CENTRE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.CENTRE
assert processStyle("##! Prologue\n", True) == BlockFmt.CENTRE
# Note Docs
tokens._isNovel = False
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.CENTRE
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.NONE
assert processStyle("## Chapter\n", True) == BlockFmt.NONE
assert processStyle("### Scene\n", True) == BlockFmt.NONE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.CENTRE
assert processStyle("##! Prologue\n", True) == BlockFmt.NONE
@pytest.mark.core
def testFmtToken_HeaderStylePageBreak(mockGUI):
"""Test header styling page break."""
project = NWProject()
tokens = BareTokenizer(project)
def processStyle(text: str, first: bool) -> BlockFmt:
tokens._text = text
tokens._isFirst = first
tokens.tokenizeText()
return tokens._blocks[0][4]
tokens.setTitleStyle(False, True)
tokens.setPartitionStyle(False, True)
tokens.setChapterStyle(False, True)
tokens.setSceneStyle(False, True)
assert tokens._titleStyle == BlockFmt.PBB
assert tokens._partStyle == BlockFmt.PBB
assert tokens._chapterStyle == BlockFmt.PBB
assert tokens._sceneStyle == BlockFmt.PBB
# Novel Docs
tokens._isNovel = True
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.PBB
assert processStyle("## Chapter\n", False) == BlockFmt.PBB
assert processStyle("### Scene\n", False) == BlockFmt.PBB
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.PBB
assert processStyle("##! Prologue\n", False) == BlockFmt.PBB
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.NONE
assert processStyle("## Chapter\n", True) == BlockFmt.NONE
assert processStyle("### Scene\n", True) == BlockFmt.NONE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.NONE
assert processStyle("##! Prologue\n", True) == BlockFmt.NONE
# Note Docs
tokens._isNovel = False
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.PBB
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.NONE
assert processStyle("## Chapter\n", True) == BlockFmt.NONE
assert processStyle("### Scene\n", True) == BlockFmt.NONE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.NONE
assert processStyle("##! Prologue\n", True) == BlockFmt.NONE
@pytest.mark.core
def testFmtToken_HeaderStylePageBreakCenter(mockGUI):
"""Test header styling page break and centred."""
project = NWProject()
tokens = BareTokenizer(project)
def processStyle(text: str, first: bool) -> BlockFmt:
tokens._text = text
tokens._isFirst = first
tokens.tokenizeText()
return tokens._blocks[0][4]
tokens.setTitleStyle(True, True)
tokens.setPartitionStyle(True, True)
tokens.setChapterStyle(True, True)
tokens.setSceneStyle(True, True)
assert tokens._titleStyle == BlockFmt.CENTRE | BlockFmt.PBB
assert tokens._partStyle == BlockFmt.CENTRE | BlockFmt.PBB
assert tokens._chapterStyle == BlockFmt.CENTRE | BlockFmt.PBB
assert tokens._sceneStyle == BlockFmt.CENTRE | BlockFmt.PBB
# Novel Docs
tokens._isNovel = True
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("## Chapter\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("### Scene\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("##! Prologue\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.CENTRE
assert processStyle("## Chapter\n", True) == BlockFmt.CENTRE
assert processStyle("### Scene\n", True) == BlockFmt.CENTRE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.CENTRE
assert processStyle("##! Prologue\n", True) == BlockFmt.CENTRE
# Note Docs
tokens._isNovel = False
# First Document is False
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
# First Document is True
assert processStyle("# Title\n", True) == BlockFmt.NONE
assert processStyle("## Chapter\n", True) == BlockFmt.NONE
assert processStyle("### Scene\n", True) == BlockFmt.NONE
assert processStyle("#### Section\n", True) == BlockFmt.NONE
assert processStyle("#! My Novel\n", True) == BlockFmt.CENTRE
assert processStyle("##! Prologue\n", True) == BlockFmt.NONE
@pytest.mark.core
def testFmtToken_HeaderStyleSeparation(mockGUI):
"""Test header styling separation."""
project = NWProject()
tokens = BareTokenizer(project)
def processStyle(text: str, first: bool) -> BlockFmt:
tokens._text = text
tokens._isFirst = first
tokens.tokenizeText()
return tokens._blocks[0][4]
tokens._isNovel = True
# Title Styles
tokens.setTitleStyle(True, True)
tokens.setPartitionStyle(False, False)
tokens.setChapterStyle(False, False)
tokens.setSceneStyle(False, False)
assert tokens._titleStyle == BlockFmt.CENTRE | BlockFmt.PBB
assert tokens._partStyle == BlockFmt.NONE
assert tokens._chapterStyle == BlockFmt.NONE
assert tokens._sceneStyle == BlockFmt.NONE
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
# Partition Styles
tokens.setTitleStyle(False, False)
tokens.setPartitionStyle(True, True)
tokens.setChapterStyle(False, False)
tokens.setSceneStyle(False, False)
assert tokens._titleStyle == BlockFmt.NONE
assert tokens._partStyle == BlockFmt.CENTRE | BlockFmt.PBB
assert tokens._chapterStyle == BlockFmt.NONE
assert tokens._sceneStyle == BlockFmt.NONE
assert processStyle("# Title\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.NONE
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
# Chapter Styles
tokens.setTitleStyle(False, False)
tokens.setPartitionStyle(False, False)
tokens.setChapterStyle(True, True)
tokens.setSceneStyle(False, False)
assert tokens._titleStyle == BlockFmt.NONE
assert tokens._partStyle == BlockFmt.NONE
assert tokens._chapterStyle == BlockFmt.CENTRE | BlockFmt.PBB
assert tokens._sceneStyle == BlockFmt.NONE
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("### Scene\n", False) == BlockFmt.NONE
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.NONE
assert processStyle("##! Prologue\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
# Scene Styles
tokens.setTitleStyle(False, False)
tokens.setPartitionStyle(False, False)
tokens.setChapterStyle(False, False)
tokens.setSceneStyle(True, True)
assert tokens._titleStyle == BlockFmt.NONE
assert tokens._partStyle == BlockFmt.NONE
assert tokens._chapterStyle == BlockFmt.NONE
assert tokens._sceneStyle == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("# Title\n", False) == BlockFmt.NONE
assert processStyle("## Chapter\n", False) == BlockFmt.NONE
assert processStyle("### Scene\n", False) == BlockFmt.CENTRE | BlockFmt.PBB
assert processStyle("#### Section\n", False) == BlockFmt.NONE
assert processStyle("#! My Novel\n", False) == BlockFmt.NONE
assert processStyle("##! Prologue\n", False) == BlockFmt.NONE
@pytest.mark.core
def testFmtToken_MetaFormat(mockGUI):
"""Test the tokenization of meta formats in the Tokenizer class."""
project = NWProject()
tokens = ToRaw(project)
tokens._handle = TMH
# Ignore Text
tokens._text = "%~ Some text\n"
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens._raw[-1] == "\n"
# Comment
tokens.setComments(False)
tokens._text = "% A comment\n"
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens._raw[-1] == "\n"
tokens.setComments(True)
tokens._text = "% A comment\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.COMMENT, "", "Comment: A comment", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"),
(8, TextFmt.COL_E, ""), (8, TextFmt.B_E, ""),
(9, TextFmt.COL_B, "comment"), (18, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "% A comment\n\n"
# Synopsis
tokens.setSynopsis(False)
tokens._text = "%synopsis: The synopsis\n"
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens._raw[-1] == "\n"
tokens.setSynopsis(True)
tokens._text = "% synopsis: The synopsis\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.COMMENT, "", "Synopsis: The synopsis", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"),
(9, TextFmt.COL_E, ""), (9, TextFmt.B_E, ""),
(10, TextFmt.COL_B, "synopsis"), (22, TextFmt.COL_E, "")
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "% synopsis: The synopsis\n\n"
# Short
tokens.setSynopsis(False)
tokens._text = "% short: A short description\n"
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens._raw[-1] == "\n"
tokens.setSynopsis(True)
tokens._text = "% short: A short description\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.COMMENT, "", "Short Description: A short description", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"),
(18, TextFmt.COL_E, ""), (18, TextFmt.B_E, ""),
(19, TextFmt.COL_B, "synopsis"), (38, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "% short: A short description\n\n"
# Keyword
tokens.setKeywords(False)
tokens._text = "@char: Bod\n"
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens._raw[-1] == "\n"
tokens.setKeywords(True)
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.KEYWORD, "char", "Characters: Bod", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(11, TextFmt.COL_E, ""), (11, TextFmt.B_E, ""),
(12, TextFmt.COL_B, "tag"), (12, TextFmt.HRF_B, "#tag_bod"),
(15, TextFmt.HRF_E, ""), (15, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "@char: Bod\n\n"
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.KEYWORD, "pov", "Point of View: Bod", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(14, TextFmt.COL_E, ""), (14, TextFmt.B_E, ""),
(15, TextFmt.COL_B, "tag"), (15, TextFmt.HRF_B, "#tag_bod"),
(18, TextFmt.HRF_E, ""), (18, TextFmt.COL_E, ""),
], BlockFmt.Z_BTM
), (
BlockTyp.KEYWORD, "plot", "Plot: Main", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(5, TextFmt.COL_E, ""), (5, TextFmt.B_E, ""),
(6, TextFmt.COL_B, "tag"), (6, TextFmt.HRF_B, "#tag_main"),
(10, TextFmt.HRF_E, ""), (10, TextFmt.COL_E, ""),
], BlockFmt.Z_TOP | BlockFmt.Z_BTM
), (
BlockTyp.KEYWORD, "location", "Locations: Europe", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(10, TextFmt.COL_E, ""), (10, TextFmt.B_E, ""),
(11, TextFmt.COL_B, "tag"), (11, TextFmt.HRF_B, "#tag_europe"),
(17, TextFmt.HRF_E, ""), (17, TextFmt.COL_E, ""),
], BlockFmt.Z_TOP
)]
assert tokens._raw[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n"
# Ignored keywords
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
tokens.setIgnoredKeywords("@plot, @location")
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.KEYWORD, "pov", "Point of View: Bod", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(14, TextFmt.COL_E, ""), (14, TextFmt.B_E, ""),
(15, TextFmt.COL_B, "tag"), (15, TextFmt.HRF_B, "#tag_bod"),
(18, TextFmt.HRF_E, ""), (18, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
@pytest.mark.core
def testFmtToken_MarginFormat(mockGUI):
"""Test the tokenization of margin formats in the Tokenizer class."""
project = NWProject()
tokens = ToRaw(project)
# Alignment and Indentation
dblIndent = BlockFmt.IND_L | BlockFmt.IND_R
rIndAlign = BlockFmt.RIGHT | BlockFmt.IND_R
tokens._text = (
"Some regular text\n\n"
"Some left-aligned text <<\n\n"
">> Some right-aligned text\n\n"
">> Some centered text <<\n\n"
"> Left-indented block\n\n"
"Right-indented block <\n\n"
"> Double-indented block <\n\n"
">> Right-indent, right-aligned <\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "Some regular text", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some left-aligned text", [], BlockFmt.LEFT),
(BlockTyp.TEXT, "", "Some right-aligned text", [], BlockFmt.RIGHT),
(BlockTyp.TEXT, "", "Some centered text", [], BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Left-indented block", [], BlockFmt.IND_L),
(BlockTyp.TEXT, "", "Right-indented block", [], BlockFmt.IND_R),
(BlockTyp.TEXT, "", "Double-indented block", [], dblIndent),
(BlockTyp.TEXT, "", "Right-indent, right-aligned", [], rIndAlign),
]
assert tokens._raw[-1] == (
"Some regular text\n\n"
"Some left-aligned text <<\n\n"
">> Some right-aligned text\n\n"
">> Some centered text <<\n\n"
"> Left-indented block\n\n"
"Right-indented block <\n\n"
"> Double-indented block <\n\n"
">> Right-indent, right-aligned <\n\n\n"
)
@pytest.mark.core
def testFmtToken_ExtractFormats(mockGUI):
"""Test the extraction of formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
# Markdown
# ========
# Plain bold
text, fmt = tokens._extractFormats("Text with **bold** in it.")
assert text == "Text with bold in it."
assert fmt == [(10, TextFmt.B_B, ""), (14, TextFmt.B_E, "")]
# Plain italics
text, fmt = tokens._extractFormats("Text with _italics_ in it.")
assert text == "Text with italics in it."
assert fmt == [(10, TextFmt.I_B, ""), (17, TextFmt.I_E, "")]
# Plain strikethrough
text, fmt = tokens._extractFormats("Text with ~~strikethrough~~ in it.")
assert text == "Text with strikethrough in it."
assert fmt == [(10, TextFmt.D_B, ""), (23, TextFmt.D_E, "")]
# Nested bold/italics
text, fmt = tokens._extractFormats("Text with **bold and _italics_** in it.")
assert text == "Text with bold and italics in it."
assert fmt == [
(10, TextFmt.B_B, ""), (19, TextFmt.I_B, ""),
(26, TextFmt.I_E, ""), (26, TextFmt.B_E, ""),
]
# Bold with overlapping italics
# Here, bold is ignored because it is not on word boundary
text, fmt = tokens._extractFormats("Text with **bold and overlapping _italics**_ in it.")
assert text == "Text with **bold and overlapping italics** in it."
assert fmt == [(33, TextFmt.I_B, ""), (42, TextFmt.I_E, "")]
# Shortcodes
# ==========
# Plain bold
text, fmt = tokens._extractFormats("Text with [b]bold[/b] in it.")
assert text == "Text with bold in it."
assert fmt == [(10, TextFmt.B_B, ""), (14, TextFmt.B_E, "")]
# Plain italics
text, fmt = tokens._extractFormats("Text with [i]italics[/i] in it.")
assert text == "Text with italics in it."
assert fmt == [(10, TextFmt.I_B, ""), (17, TextFmt.I_E, "")]
# Plain strikethrough
text, fmt = tokens._extractFormats("Text with [s]strikethrough[/s] in it.")
assert text == "Text with strikethrough in it."
assert fmt == [(10, TextFmt.D_B, ""), (23, TextFmt.D_E, "")]
# Plain underline
text, fmt = tokens._extractFormats("Text with [u]underline[/u] in it.")
assert text == "Text with underline in it."
assert fmt == [(10, TextFmt.U_B, ""), (19, TextFmt.U_E, "")]
# Plain mark
text, fmt = tokens._extractFormats("Text with [m]highlight[/m] in it.")
assert text == "Text with highlight in it."
assert fmt == [(10, TextFmt.M_B, ""), (19, TextFmt.M_E, "")]
# Plain superscript
text, fmt = tokens._extractFormats("Text with super[sup]script[/sup] in it.")
assert text == "Text with superscript in it."
assert fmt == [(15, TextFmt.SUP_B, ""), (21, TextFmt.SUP_E, "")]
# Plain subscript
text, fmt = tokens._extractFormats("Text with sub[sub]script[/sub] in it.")
assert text == "Text with subscript in it."
assert fmt == [(13, TextFmt.SUB_B, ""), (19, TextFmt.SUB_E, "")]
# Nested bold/italics
text, fmt = tokens._extractFormats("Text with [b]bold and [i]italics[/i][/b] in it.")
assert text == "Text with bold and italics in it."
assert fmt == [
(10, TextFmt.B_B, ""), (19, TextFmt.I_B, ""),
(26, TextFmt.I_E, ""), (26, TextFmt.B_E, ""),
]
# Bold with overlapping italics
# With shortcodes, this works
text, fmt = tokens._extractFormats(
"Text with [b]bold and overlapping [i]italics[/b][/i] in it."
)
assert text == "Text with bold and overlapping italics in it."
assert fmt == [
(10, TextFmt.B_B, ""), (31, TextFmt.I_B, ""),
(38, TextFmt.B_E, ""), (38, TextFmt.I_E, ""),
]
# So does this
text, fmt = tokens._extractFormats(
"Text with [b]bold and overlapping [i]italics[/b] in[/i] it."
)
assert text == "Text with bold and overlapping italics in it."
assert fmt == [
(10, TextFmt.B_B, ""), (31, TextFmt.I_B, ""),
(38, TextFmt.B_E, ""), (41, TextFmt.I_E, ""),
]
@pytest.mark.core
def testFmtToken_Paragraphs(mockGUI):
"""Test the splitting of paragraphs."""
project = NWProject()
tokens = BareTokenizer(project)
tokens._handle = TMH
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "First paragraph", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Second paragraph", [], BlockFmt.NONE),
]
# Combine multi-line paragraphs, keep breaks
tokens._text = "This is text\nspanning multiple\nlines"
tokens.setKeepLineBreaks(True)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "This is text\nspanning multiple\nlines", [], BlockFmt.NONE),
]
# Combine multi-line paragraphs, remove breaks
tokens._text = "This is text\nspanning multiple\nlines"
tokens.setKeepLineBreaks(False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "This is text spanning multiple lines", [], BlockFmt.NONE),
]
# Combine multi-line paragraphs, remove breaks, with formatting
tokens._text = "This **is text**\nspanning _multiple_\nlines"
tokens.setKeepLineBreaks(False)
tokens.tokenizeText()
assert tokens._blocks == [
(
BlockTyp.TEXT, "",
"This is text spanning multiple lines",
[
(5, TextFmt.B_B, ""),
(12, TextFmt.B_E, ""),
(22, TextFmt.I_B, ""),
(30, TextFmt.I_E, ""),
],
BlockFmt.NONE
),
]
# Make sure titles break a paragraph
tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n"
tokens.setKeepLineBreaks(False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title", [], BlockFmt.NONE),
(
BlockTyp.TEXT, "", "Text on two lines.", [
(5, TextFmt.I_B, ""),
(7, TextFmt.I_E, ""),
], BlockFmt.NONE
),
(BlockTyp.HEAD2, TM2, "Chapter", [], BlockFmt.NONE),
(
BlockTyp.TEXT, "", "More text here.", [
(5, TextFmt.B_B, ""),
(9, TextFmt.B_E, ""),
(10, TextFmt.I_B, ""),
(14, TextFmt.I_E, ""),
], BlockFmt.NONE
),
]
@pytest.mark.core
def testFmtToken_TextFormat(mockGUI):
"""Test the tokenization of text formats in the Tokenizer class."""
project = NWProject()
tokens = ToRaw(project)
tokens._handle = TMH
# Text
tokens._text = "Some plain text\non two lines\n\n\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "Some plain text\non two lines", [], BlockFmt.NONE),
]
assert tokens._raw[-1] == "Some plain text\non two lines\n\n\n\n"
tokens.setBodyText(False)
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens._raw[-1] == "\n\n\n"
tokens.setBodyText(True)
# Text Emphasis
tokens._text = "Some **bolded text** on this lines\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "", "Some bolded text on this lines", [
(5, TextFmt.B_B, ""),
(16, TextFmt.B_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "Some **bolded text** on this lines\n\n"
tokens._text = "Some _italic text_ on this lines\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "", "Some italic text on this lines", [
(5, TextFmt.I_B, ""),
(16, TextFmt.I_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "Some _italic text_ on this lines\n\n"
tokens._text = "Some **_bold italic text_** on this lines\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "", "Some bold italic text on this lines", [
(5, TextFmt.B_B, ""),
(5, TextFmt.I_B, ""),
(21, TextFmt.I_E, ""),
(21, TextFmt.B_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "Some **_bold italic text_** on this lines\n\n"
tokens._text = "Some ~~strikethrough text~~ on this lines\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "", "Some strikethrough text on this lines", [
(5, TextFmt.D_B, ""),
(23, TextFmt.D_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == "Some ~~strikethrough text~~ on this lines\n\n"
tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "", "Some nested bold and italic and strikethrough text here", [
(5, TextFmt.B_B, ""),
(21, TextFmt.I_B, ""),
(27, TextFmt.I_E, ""),
(32, TextFmt.D_B, ""),
(45, TextFmt.D_E, ""),
(50, TextFmt.B_E, ""),
], BlockFmt.NONE
)]
assert tokens._raw[-1] == (
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
)
@pytest.mark.core
def testFmtToken_LineBreak(mockGUI):
"""Test processing of forced line breaks in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens._handle = TMH
tokens.setComments(True)
# They are stripped in headers
tokens._text = "## Hello[br] World"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Hello World", [], BlockFmt.NONE)
]
# They are stripped in comments
tokens._text = "% Hello[br] World"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.COMMENT, "", "Comment: Hello World", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"),
(8, TextFmt.COL_E, ""), (8, TextFmt.B_E, ""),
(9, TextFmt.COL_B, "comment"), (20, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
# They are used in text, with breaks enabled
tokens.setKeepLineBreaks(True)
tokens._text = "Hello[br]\nWorld"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "Hello\nWorld", [], BlockFmt.NONE)
]
# They are used in text, with breaks disabled
tokens.setKeepLineBreaks(False)
tokens._text = "Hello[br]\nWorld"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "Hello\nWorld", [], BlockFmt.NONE)
]
# Without forced breaks, they are preserved with breaks enabled
tokens.setKeepLineBreaks(True)
tokens._text = "Hello\nWorld"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "Hello\nWorld", [], BlockFmt.NONE)
]
# Without forced breaks, they are not preserved with breaks disabled
tokens.setKeepLineBreaks(False)
tokens._text = "Hello\nWorld"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "Hello World", [], BlockFmt.NONE)
]
@pytest.mark.core
def testFmtToken_Dialogue(mockGUI):
"""Test the tokenization of dialogue in the Tokenizer class."""
CONFIG.fmtDQuoteOpen = "\u201c"
CONFIG.fmtDQuoteClose = "\u201d"
CONFIG.fmtSQuoteOpen = "\u2018"
CONFIG.fmtSQuoteClose = "\u2019"
CONFIG.dialogStyle = 3
CONFIG.altDialogOpen = "::"
CONFIG.altDialogClose = "::"
CONFIG.dialogLine = "\u2013"
CONFIG.narratorBreak = "\u2013"
project = NWProject()
tokens = BareTokenizer(project)
tokens.setDialogueHighlight(True)
tokens._handle = TMH
tokens._isNovel = True
# Single quotes
tokens._text = "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "",
"Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019",
[
(10, TextFmt.COL_B, "dialog"),
(25, TextFmt.COL_E, ""),
(30, TextFmt.COL_B, "dialog"),
(45, TextFmt.COL_E, ""),
],
BlockFmt.NONE
)]
# Double quotes
tokens._text = "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "",
"Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d",
[
(10, TextFmt.COL_B, "dialog"),
(25, TextFmt.COL_E, ""),
(30, TextFmt.COL_B, "dialog"),
(45, TextFmt.COL_E, ""),
],
BlockFmt.NONE
)]
# Alt quotes
tokens._text = "Text with ::dialogue one,:: and ::dialogue two.::\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "",
"Text with ::dialogue one,:: and ::dialogue two.::",
[
(10, TextFmt.COL_B, "altdialog"),
(27, TextFmt.COL_E, ""),
(32, TextFmt.COL_B, "altdialog"),
(49, TextFmt.COL_E, ""),
],
BlockFmt.NONE
)]
# Dialogue line with narrator break
tokens._text = "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "",
"\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?",
[
(0, TextFmt.COL_B, "dialog"),
(34, TextFmt.COL_E, ""),
(44, TextFmt.COL_B, "dialog"),
(49, TextFmt.COL_E, ""),
],
BlockFmt.NONE
)]
# Special Cases
# =============
# Dialogue + formatting on same index (Issue #2012)
tokens._text = "[i]\u201cDialogue text.\u201d[/i]\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.TEXT, "",
"\u201cDialogue text.\u201d",
[
(0, TextFmt.I_B, ""),
(0, TextFmt.COL_B, "dialog"),
(16, TextFmt.I_E, ""),
(16, TextFmt.COL_E, ""),
],
BlockFmt.NONE
)]
@pytest.mark.core
def testFmtToken_SpecialFormat(mockGUI):
"""Test the tokenization of special formats in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens._handle = TMH
tokens._isNovel = True
# New Page
# ========
correctResp = [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.CENTRE),
(BlockTyp.HEAD1, TM2, "Title Two", [], BlockFmt.CENTRE | BlockFmt.PBB),
]
# Command wo/Space
tokens._isFirst = True
tokens._text = (
"# Title One\n\n"
"[newpage]\n\n"
"# Title Two\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == correctResp
# Command w/Space
tokens._isFirst = True
tokens._text = (
"# Title One\n\n"
"[new page]\n\n"
"# Title Two\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == correctResp
# Trailing Spaces
tokens._isFirst = True
tokens._text = (
"# Title One\n\n"
"[new page] \t\n\n"
"# Title Two\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == correctResp
# Single Empty Paragraph
# ======================
tokens._text = (
"# Title One\n\n"
"[vspace] \n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Multiple Empty Paragraphs
# =========================
# One Skip
tokens._text = (
"# Title One\n\n"
"[vspace:1] \n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Three Skips
tokens._text = (
"# Title One\n\n"
"[vspace:3] \n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Malformed Command, Case 1
tokens._text = (
"# Title One\n\n"
"[vspace:3xa] \n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Malformed Command, Case 2
tokens._text = (
"# Title One\n\n"
"[vspace:3.5]\n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Malformed Command, Case 3
tokens._text = (
"# Title One\n\n"
"[vspace:-1]\n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Empty Paragraph and Page Break
# ==============================
# Single Skip
tokens._text = (
"# Title One\n\n"
"[new page]\n\n"
"[vspace]\n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.PBB),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Multiple Skip
tokens._text = (
"# Title One\n\n"
"[new page]\n\n"
"[vspace:3]\n\n"
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.PBB),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
@pytest.mark.core
def testFmtToken_TextIndent(mockGUI):
"""Test the handling of text indent in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setSynopsis(True)
tokens._handle = TMH
# No First Indent
tokens.setFirstLineIndent(True, 1.0, False)
assert tokens._noIndent is False
assert tokens._firstIndent is True
assert tokens._firstWidth == 1.0
assert tokens._indentFirst is False
# Page One
# Two paragraphs in the same scene
tokens._text = (
"# Title One\n\n"
"### Scene One\n\n"
"First paragraph.\n\n"
"Second paragraph.\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.NONE),
(BlockTyp.HEAD3, TM2, "Scene One", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "First paragraph.", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Second paragraph.", [], BlockFmt.IND_T),
]
assert tokens._noIndent is False
# Page Two
# New scene with only a synopsis
tokens._text = (
"### Scene Two\n\n"
"%Synopsis: Stuff happens.\n\n"
)
tokens.tokenizeText()
tFmt = [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"), (9, TextFmt.COL_E, ""),
(9, TextFmt.B_E, ""), (10, TextFmt.COL_B, "synopsis"), (24, TextFmt.COL_E, ""),
]
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Scene Two", [], BlockFmt.NONE),
(BlockTyp.COMMENT, "", "Synopsis: Stuff happens.", tFmt, BlockFmt.NONE),
]
assert tokens._noIndent is True
# Page Three
# Two paragraphs for the scene on the previous page
tokens._text = (
"First paragraph.\n\n"
"Second paragraph.\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.TEXT, "", "First paragraph.", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Second paragraph.", [], BlockFmt.IND_T),
]
assert tokens._noIndent is False
# First Indent
tokens.setFirstLineIndent(True, 1.0, True)
assert tokens._noIndent is False
assert tokens._firstIndent is True
assert tokens._firstWidth == 1.0
assert tokens._indentFirst is True
# Page Four
# Two paragraphs in the same scene
tokens._text = (
"# Title One\n\n"
"### Scene One\n\n"
"First paragraph.\n\n"
"Second paragraph.\n\n"
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.NONE),
(BlockTyp.HEAD3, TM2, "Scene One", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "First paragraph.", [], BlockFmt.IND_T),
(BlockTyp.TEXT, "", "Second paragraph.", [], BlockFmt.IND_T),
]
assert tokens._noIndent is False
@pytest.mark.core
def testFmtToken_ProcessHeaders(mockGUI):
"""Test the header and page parser of the Tokenizer class."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens._isNovel = True
tokens._handle = TMH
# Titles
# ======
# H1: Title, First Page
assert tokens._isFirst is True
tokens._text = "# Part One\n"
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "T: Part One", [], BlockFmt.CENTRE),
]
# H1: Title, Not First Page
assert tokens._isFirst is False
tokens._text = "# Part One\n"
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "T: Part One", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
# Chapters
# ========
# H2: Chapter
tokens._text = "## Chapter One\n"
tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "C: Chapter One", [], BlockFmt.PBB),
]
# H2: Unnumbered Chapter
tokens._text = "##! Prologue\n"
tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "U: Prologue", [], BlockFmt.PBB),
]
# H2: Chapter Word Number
tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}")
tokens._hFormatter._chCount = 0
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter One", [], BlockFmt.PBB),
]
# H2: Chapter Roman Number Upper Case
tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter II", [], BlockFmt.PBB),
]
# H2: Chapter Roman Number Lower Case
tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter iii", [], BlockFmt.PBB),
]
# Scenes
# ======
# H3: Scene w/Title
tokens._text = "### Scene One\n"
tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "S: Scene One", [], BlockFmt.NONE),
]
# H3: Scene Hidden wo/Format
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", True)
tokens.tokenizeText()
assert tokens._blocks == []
# H3: Scene wo/Format, first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._noSep = True
tokens.tokenizeText()
assert tokens._blocks == []
# H3: Scene wo/Format, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._noSep = False
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.SKIP, TM1, "", [], BlockFmt.NONE),
]
# H3: Scene Separator, first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False)
tokens._noSep = True
tokens.tokenizeText()
assert tokens._blocks == []
# H3: Scene Separator, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False)
tokens._noSep = False
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.SEP, TM1, "* * *", [], BlockFmt.CENTRE),
]
# H3: Scene w/Absolute Number
tokens._text = "### A Scene\n"
tokens.setSceneFormat(f"Scene {nwHeadFmt.SC_ABS}", False)
tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 0
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Scene 1", [], BlockFmt.NONE),
]
# H3: Scene w/Chapter Number
tokens._text = "### A Scene\n"
tokens.setSceneFormat(f"Scene {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM}", False)
tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 1
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Scene 3.2", [], BlockFmt.NONE),
]
# Sections
# ========
# H4: Section Hidden wo/Format
tokens._text = "#### A Section\n"
tokens.setSectionFormat("", True)
tokens.tokenizeText()
assert tokens._blocks == []
# H4: Section Visible wo/Format
tokens._text = "#### A Section\n"
tokens.setSectionFormat("", False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.SKIP, TM1, "", [], BlockFmt.NONE),
]
# H4: Section w/Format
tokens._text = "#### A Section\n"
tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD4, TM1, "X: A Section", [], BlockFmt.NONE),
]
# H4: Section Separator
tokens._text = "#### A Section\n"
tokens.setSectionFormat("* * *", False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.SEP, TM1, "* * *", [], BlockFmt.CENTRE),
]
# Check the first scene detector, plain text
tokens._noSep = True
tokens._text = "Some text ...\n"
tokens.tokenizeText()
assert tokens._noSep is True
# Check the first scene detector, text plus scene
tokens._noSep = True
tokens._text = "Some text ...\n\n### Scene\n\nText"
tokens.tokenizeText()
assert tokens._noSep is False
@pytest.mark.core
def testFmtToken_FormatComment(mockGUI):
"""Test note and comment formatting."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
# Comment, No Formatting
style = COMMENT_STYLE[nwComment.PLAIN]
assert tokens._formatComment(style, "", "Hello world!") == (
"Comment: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"),
(8, TextFmt.COL_E, ""), (8, TextFmt.B_E, ""),
(9, TextFmt.COL_B, "comment"), (21, TextFmt.COL_E, ""),
]
)
# Synopsis, No Formatting
style = COMMENT_STYLE[nwComment.SYNOPSIS]
assert tokens._formatComment(style, "", "Hello world!") == (
"Synopsis: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"),
(9, TextFmt.COL_E, ""), (9, TextFmt.B_E, ""),
(10, TextFmt.COL_B, "synopsis"), (22, TextFmt.COL_E, ""),
]
)
@pytest.mark.core
def testFmtToken_FormatMeta(mockGUI):
"""Test meta formatting."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
assert tokens._formatMeta("@tag: Jane | Jane Smith") == (
"@tag", "Tag: Jane | Jane Smith", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(4, TextFmt.COL_E, ""), (4, TextFmt.B_E, ""),
(5, TextFmt.COL_B, "tag"), (5, TextFmt.ANM_B, "tag_jane"),
(9, TextFmt.ANM_E, ""), (9, TextFmt.COL_E, ""),
(12, TextFmt.COL_B, "optional"), (22, TextFmt.COL_E, ""),
]
)
assert tokens._formatMeta("@char: Jane, John") == (
"@char", "Characters: Jane, John", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(11, TextFmt.COL_E, ""), (11, TextFmt.B_E, ""),
(12, TextFmt.COL_B, "tag"), (12, TextFmt.HRF_B, "#tag_jane"),
(16, TextFmt.HRF_E, ""), (16, TextFmt.COL_E, ""),
(18, TextFmt.COL_B, "tag"), (18, TextFmt.HRF_B, "#tag_john"),
(22, TextFmt.HRF_E, ""), (22, TextFmt.COL_E, ""),
]
)
@pytest.mark.core
def testFmtToken_BuildOutline(mockGUI, ipsumText):
"""Test stats counter of the Tokenizer class."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
# Novel
tokens._isNovel = True
tokens._handle = "0000000000000"
tokens._text = (
"#! My Novel\n\n"
"# Part One\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"###! Scene Four\n\n"
"Text\n\n"
"#### Section\n\n"
"Text\n\n"
)
tokens.tokenizeText()
tokens.buildOutline()
# Note
tokens._isNovel = False
tokens._handle = "0000000000001"
tokens._text = (
"#! My Notes\n\n"
"# Header 1\n\n"
"Text\n\n"
"## Header 2\n\n"
"Text\n\n"
"### Header 3\n\n"
"Text\n\n"
"#### Header 4\n\n"
"Text\n\n"
)
tokens.tokenizeText()
tokens.buildOutline()
# Check Outline
assert tokens.textOutline == {
"0000000000000:T0001": "TT|My Novel",
"0000000000000:T0002": "PT|Part One",
"0000000000000:T0003": "CH|Chapter One",
"0000000000000:T0004": "SC|Scene One",
"0000000000000:T0005": "SC|Scene Two",
"0000000000000:T0006": "CH|Chapter Two",
"0000000000000:T0007": "SC|Scene Three",
"0000000000000:T0008": "SC|Scene Four",
"0000000000001:T0001": "TT|My Notes",
"0000000000001:T0002": "H1|Header 1",
"0000000000001:T0003": "H2|Header 2",
"0000000000001:T0004": "H3|Header 3",
}
@pytest.mark.core
def testFmtToken_CountStats(mockGUI, ipsumText):
"""Test stats counter of the Tokenizer class."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
tokens._isNovel = True
# Short Text
# ==========
# Header wo/Format
tokens._text = "## A Chapter Title\n\n"
tokens._counts = {}
tokens.tokenizeText()
tokens.countStats()
assert tokens._blocks[0][2] == "A Chapter Title"
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 0,
"allWords": 3, "textWords": 0, "titleWords": 3,
"allChars": 15, "textChars": 0, "titleChars": 15,
"allWordChars": 13, "textWordChars": 0, "titleWordChars": 13
}
# Header w/Format
tokens._text = "## A Chapter Title\n\n"
tokens._counts = {}
tokens.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}")
tokens._hFormatter.resetAll()
tokens.tokenizeText()
tokens.countStats()
assert tokens._blocks[0][2] == "C 1: A Chapter Title"
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 0,
"allWords": 5, "textWords": 0, "titleWords": 5,
"allChars": 20, "textChars": 0, "titleChars": 20,
"allWordChars": 16, "textWordChars": 0, "titleWordChars": 16
}
# Two Paragraphs
# First break should be counted, the double breaks not.
tokens._text = "Some text\non two lines.\n\nWith a second paragraph.\n\n"
tokens._counts = {}
tokens.tokenizeText()
tokens.countStats()
assert tokens.textStats == {
"titleCount": 0, "paragraphCount": 2,
"allWords": 9, "textWords": 9, "titleWords": 0,
"allChars": 47, "textChars": 47, "titleChars": 0,
"allWordChars": 40, "textWordChars": 40, "titleWordChars": 0
}
# Two Scenes w/Separator
tokens._text = "## Chapter\n\n### Scene\n\nText\n\n### Scene\n\nText"
tokens._counts = {}
tokens.setChapterFormat(nwHeadFmt.TITLE)
tokens.setSceneFormat("* * *", False)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Text", "* * *", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 2,
"allWords": 6, "textWords": 2, "titleWords": 1,
"allChars": 20, "textChars": 8, "titleChars": 7,
"allWordChars": 18, "textWordChars": 8, "titleWordChars": 7
}
# Scene w/Synopsis
# Synopsis does not count as a paragraph, and counts as "Synopsis: Stuff"
tokens._text = "## Chapter\n\n### Scene\n\n%Synopsis: Stuff\n\nText"
tokens._counts = {}
tokens.setChapterFormat(nwHeadFmt.TITLE)
tokens.setSceneFormat("* * *", False)
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Synopsis: Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
"allChars": 26, "textChars": 4, "titleChars": 7,
"allWordChars": 25, "textWordChars": 4, "titleWordChars": 7
}
# Scene w/Short
# Short does not count as a paragraph, and counts as "Short Description: Stuff"
tokens._text = "## Chapter\n\n### Scene\n\n%Short: Stuff\n\nText"
tokens._counts = {}
tokens.setChapterFormat(nwHeadFmt.TITLE)
tokens.setSceneFormat("* * *", False)
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Short Description: Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 5, "textWords": 1, "titleWords": 1,
"allChars": 35, "textChars": 4, "titleChars": 7,
"allWordChars": 33, "textWordChars": 4, "titleWordChars": 7
}
# Scene w/Comment
# Comment does not count as a paragraph, and counts as "Comment: Stuff"
tokens._text = "## Chapter\n\n### Scene\n\n% Stuff\n\nText"
tokens._counts = {}
tokens.setChapterFormat(nwHeadFmt.TITLE)
tokens.setSceneFormat("* * *", False)
tokens.setComments(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Comment: Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
"allChars": 25, "textChars": 4, "titleChars": 7,
"allWordChars": 24, "textWordChars": 4, "titleWordChars": 7
}
# Scene w/Keyword
# Keyword does not count as a paragraph, and counts as "Point of View: Jane"
tokens._text = "## Chapter\n\n### Scene\n\n@pov: Jane\n\nText"
tokens._counts = {}
tokens.setChapterFormat(nwHeadFmt.TITLE)
tokens.setSceneFormat("* * *", False)
tokens.setKeywords(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Point of View: Jane", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 6, "textWords": 1, "titleWords": 1,
"allChars": 30, "textChars": 4, "titleChars": 7,
"allWordChars": 27, "textWordChars": 4, "titleWordChars": 7
}
# Long Text
# =========
tokens._text = (
"# Act One\n\n"
"## Chapter\n\n"
"### Scene\n\n"
"@pov: Jane\n"
"@char: Jane, John\n\n"
"%Synopsis: A scene\n\n"
f"{ipsumText[0]}.\n\n"
"### Scene\n\n"
"@pov: Jane\n"
"@char: Jane, John\n\n"
"%Synopsis: A scene\n\n"
f"{ipsumText[1]}.\n\n"
"## Chapter\n\n"
"### Scene\n\n"
"@pov: Jane\n"
"@char: Jane, John\n\n"
"%Synopsis: A scene\n\n"
f"{ipsumText[2]}.\n\n"
"### Scene\n\n"
"@pov: Jane\n"
"@char: Jane, John\n\n"
"%Synopsis: A scene\n\n"
f"{ipsumText[3]}.\n\n"
"## Chapter\n\n"
"### Scene\n\n"
"@pov: Jane\n"
"@char: Jane, John\n\n"
"%Synopsis: A scene\n\n"
f"{ipsumText[4]}.\n\n"
)
tokens._counts = {}
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}")
tokens.setSceneFormat("* * *", False)
tokens.setSynopsis(True)
tokens.setComments(True)
tokens.setKeywords(True)
tokens.tokenizeText()
tokens.countStats()
assert tokens.textStats == {
"titleCount": 4, "paragraphCount": 5,
"allWords": 596, "textWords": 528, "titleWords": 12,
"allChars": 3859, "textChars": 3513, "titleChars": 46,
"allWordChars": 3289, "textWordChars": 2990, "titleWordChars": 38
}
@pytest.mark.core
def testFmtToken_SceneSeparators(mockGUI):
"""Test the section and scene separators of the Tokenizer class."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
md = ToMarkdown(project, False)
md._isNovel = True
# Separator Handling, Titles
# ==========================
md._text = (
"# Title One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"# Title Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"###! Scene Four\n\n"
"Text\n\n"
)
md.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
md.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
md.setSectionFormat("", True)
# Static Separator
md.setSceneFormat("~", False)
md.setHardSceneFormat("* * *", False)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# T: Title One\n\n"
"Text\n\n"
"~\n\n"
"Text\n\n"
"# T: Title Two\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
)
# Scene Title Formatted
md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
md.setHardSceneFormat(f"H: {nwHeadFmt.TITLE}", False)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# T: Title One\n\n"
"### S: Scene One\n\n"
"Text\n\n"
"### S: Scene Two\n\n"
"Text\n\n"
"# T: Title Two\n\n"
"### S: Scene Three\n\n"
"Text\n\n"
"### H: Scene Four\n\n"
"Text\n\n"
)
# Separator Handling, Chapters
# ============================
md._text = (
"# Title One\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"###! Scene Four\n\n"
"Text\n\n"
)
md.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
md.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
md.setSectionFormat("", True)
# Static Separator
md.setSceneFormat("* * *", False)
md.setHardSceneFormat("* * *", False)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# T: Title One\n\n"
"## C: Chapter One\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
"## C: Chapter Two\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
)
# Scene Title Formatted
md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
md.setHardSceneFormat(f"H: {nwHeadFmt.TITLE}", False)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# T: Title One\n\n"
"## C: Chapter One\n\n"
"### S: Scene One\n\n"
"Text\n\n"
"### S: Scene Two\n\n"
"Text\n\n"
"## C: Chapter Two\n\n"
"### S: Scene Three\n\n"
"Text\n\n"
"### H: Scene Four\n\n"
"Text\n\n"
)
# Separators with Scenes Only
# ===========================
# Requires a fresh builder class
md = ToMarkdown(project, True)
md._isNovel = True
md._text = (
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"### Scene Three\n\n"
"Text\n\n"
"###! Scene Four\n\n"
"Text\n\n"
)
md.setSceneFormat("", False)
md.setHardSceneFormat("* * *", False)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"Text\n\n"
"\u205f\n\n"
"Text\n\n"
"\u205f\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
)
@pytest.mark.core
def testFmtToken_HeaderVisibility(mockGUI):
"""Test the heading visibility settings of the Tokenizer class."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
md = ToMarkdown(project, False)
md._text = (
"#! Novel\n\n"
"# Title One\n\n"
"##! Prologue\n\n"
"Text\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"#### Section Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"###! Scene Four\n\n"
"Text\n\n"
)
# Novel Files
# ===========
md._isNovel = True
# Show All
md.setPartitionFormat(nwHeadFmt.TITLE, False)
md.setChapterFormat(nwHeadFmt.TITLE, False)
md.setUnNumberedFormat(nwHeadFmt.TITLE, False)
md.setSceneFormat(nwHeadFmt.TITLE, False)
md.setHardSceneFormat(nwHeadFmt.TITLE, False)
md.setSectionFormat(nwHeadFmt.TITLE, False)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# Novel\n\n"
"# Title One\n\n"
"## Prologue\n\n"
"Text\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"#### Section Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"### Scene Four\n\n"
"Text\n\n"
)
# Hide All
md.setPartitionFormat(nwHeadFmt.TITLE, True)
md.setChapterFormat(nwHeadFmt.TITLE, True)
md.setUnNumberedFormat(nwHeadFmt.TITLE, True)
md.setSceneFormat(nwHeadFmt.TITLE, True)
md.setHardSceneFormat(nwHeadFmt.TITLE, True)
md.setSectionFormat(nwHeadFmt.TITLE, True)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# Novel\n\n"
"Text\n\n"
"Text\n\n"
"Text\n\n"
"Text\n\n"
"Text\n\n"
)
# Note Files
# ==========
md._isNovel = False
# Hide All
md.setPartitionFormat(nwHeadFmt.TITLE, True)
md.setChapterFormat(nwHeadFmt.TITLE, True)
md.setUnNumberedFormat(nwHeadFmt.TITLE, True)
md.setSceneFormat(nwHeadFmt.TITLE, True)
md.setHardSceneFormat(nwHeadFmt.TITLE, True)
md.setSectionFormat(nwHeadFmt.TITLE, True)
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# Novel\n\n"
"# Title One\n\n"
"## Prologue\n\n"
"Text\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"#### Section Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"### Scene Four\n\n"
"Text\n\n"
)
@pytest.mark.core
def testFmtToken_CounterHandling(mockGUI):
"""Test the heading counter of the Tokenizer class."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
md = ToMarkdown(project, False)
md._isNovel = True
# Counter Handling, Novel Titles
# ==============================
# This also checks that only numbered chapters bump the counter
md._text = (
"#! Novel One\n\n"
"##! Prologue\n\n"
"Text\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"###! Scene Four\n\n"
"Text\n\n"
"#! Novel Two\n\n"
"##! Prologue\n\n"
"Text\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"###! Scene Four\n\n"
"Text\n\n"
)
md.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
md.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}")
md.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
md.setSceneFormat(
f"S {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM} ({nwHeadFmt.SC_ABS}): {nwHeadFmt.TITLE}", False
)
md.setHardSceneFormat(
f"H {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM} ({nwHeadFmt.SC_ABS}): {nwHeadFmt.TITLE}", False
)
md.setSectionFormat("", True)
# Two Novel Format
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"# Novel One\n\n"
"## U: Prologue\n\n"
"Text\n\n"
"## C 1: Chapter One\n\n"
"### S 1.1 (1): Scene One\n\n"
"Text\n\n"
"### S 1.2 (2): Scene Two\n\n"
"Text\n\n"
"## C 2: Chapter Two\n\n"
"### S 2.1 (3): Scene Three\n\n"
"Text\n\n"
"### H 2.2 (4): Scene Four\n\n"
"Text\n\n"
"# Novel Two\n\n"
"## U: Prologue\n\n"
"Text\n\n"
"## C 1: Chapter One\n\n"
"### S 1.1 (1): Scene One\n\n"
"Text\n\n"
"### S 1.2 (2): Scene Two\n\n"
"Text\n\n"
"## C 2: Chapter Two\n\n"
"### S 2.1 (3): Scene Three\n\n"
"Text\n\n"
"### H 2.2 (4): Scene Four\n\n"
"Text\n\n"
)
@pytest.mark.core
def testFmtToken_HeadingFormatter(fncPath, mockGUI, mockRnd):
"""Check the HeadingFormatter class."""
project = NWProject()
project.setProjectLang("en_GB")
mockRnd.reset()
buildTestProject(project, fncPath)
nHandle = project.newFile("Hello", C.hNovelRoot)
cHandle = project.newFile("Jane", C.hCharRoot)
dHandle = project.newFile("John", C.hCharRoot)
assert isinstance(nHandle, str)
assert isinstance(cHandle, str)
assert isinstance(dHandle, str)
assert project.index.scanText(cHandle, (
"# Jane Smith\n"
"@tag: Jane | Jane Smith\n"
))
assert project.index.scanText(dHandle, (
"# John Smith\n"
"@tag: John | John Smith\n"
))
assert project.index.scanText(nHandle, (
"## Hi Bob\n"
"@pov: Jane\n"
"@focus: John\n"
"@char: Jane, John\n\n"
"This is a story about Jane Smith.\n\n"
"Well, not really.\n"
))
formatter = HeadingFormatter(project)
formatter.setHandle(nHandle)
# Counters
# ========
cFormat = (
f"Chapter {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM} - "
f"Scene {nwHeadFmt.SC_ABS} - {nwHeadFmt.TITLE}"
)
# Initial
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 0.0 - Scene 0 - Hi Bob"
# First Chapter and Scene
formatter.incChapter()
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.1 - Scene 1 - Hi Bob"
# Next Scenes
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.2 - Scene 2 - Hi Bob"
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.3 - Scene 3 - Hi Bob"
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.4 - Scene 4 - Hi Bob"
# Next Chapter and Scene
formatter.incChapter()
formatter.resetScene()
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 2.1 - Scene 5 - Hi Bob"
# New Main Title
formatter.resetAll()
formatter.incChapter()
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.1 - Scene 1 - Hi Bob"
# Special Formats
# ===============
formatter._chCount = 2
formatter._scChCount = 3
formatter._scAbsCount = 8
# Chapter Number Word
cFormat = f"Chapter {nwHeadFmt.CH_WORD}, Scene {nwHeadFmt.SC_NUM} - {nwHeadFmt.TITLE}"
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter Two, Scene 3 - Hi Bob"
# Chapter Upper Case Roman
cFormat = f"Chapter {nwHeadFmt.CH_ROMU}, Scene {nwHeadFmt.SC_NUM} - {nwHeadFmt.TITLE}"
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter II, Scene 3 - Hi Bob"
# Chapter Lower Case Roman
cFormat = f"Chapter {nwHeadFmt.CH_ROML}, Scene {nwHeadFmt.SC_NUM} - {nwHeadFmt.TITLE}"
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter ii, Scene 3 - Hi Bob"
# Chapter w/PoV Character
cFormat = f"Chapter {nwHeadFmt.CH_NUM}, Scene {nwHeadFmt.SC_NUM} - {nwHeadFmt.CHAR_POV}"
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 2, Scene 3 - Jane Smith"
# Chapter w/Focus Character
cFormat = f"Chapter {nwHeadFmt.CH_NUM}, Scene {nwHeadFmt.SC_NUM} - {nwHeadFmt.CHAR_FOCUS}"
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 2, Scene 3 - John Smith"
# Chapter w/Fallback PoV
cFormat = f"Chapter {nwHeadFmt.CH_NUM}, Scene {nwHeadFmt.SC_NUM} - {nwHeadFmt.CHAR_POV}"
assert formatter.apply(cFormat, "Hi Bob", 0) == "Chapter 2, Scene 3 - Point of View"
# Chapter w/Fallback Focus
cFormat = f"Chapter {nwHeadFmt.CH_NUM}, Scene {nwHeadFmt.SC_NUM} - {nwHeadFmt.CHAR_FOCUS}"
assert formatter.apply(cFormat, "Hi Bob", 0) == "Chapter 2, Scene 3 - Focus"