Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Separate ordinary headers and novel headers in the Tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2025-01-25 14:58:06 +01:00
parent 5be1e2c954
commit 13e6199af2
5 changed files with 129 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_formats/test_fmt_tokenizer.py
+79 -77
View File
@@ -291,7 +291,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Novel Title", [], BlockFmt.CENTRE),
(BlockTyp.PART, TM1, "Novel Title", [], BlockFmt.CENTRE),
]
# Note File
@@ -313,7 +313,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter One", [], BlockFmt.PBB),
(BlockTyp.CHAPTER, TM1, "Chapter One", [], BlockFmt.PBB),
]
# Note File
@@ -334,7 +334,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Scene One", [], BlockFmt.NONE),
(BlockTyp.SCENE, TM1, "Scene One", [], BlockFmt.NONE),
]
# Note File
@@ -355,7 +355,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD4, TM1, "A Section", [], BlockFmt.NONE),
(BlockTyp.SECTION, TM1, "A Section", [], BlockFmt.NONE),
]
# Note File
@@ -399,7 +399,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Prologue", [], BlockFmt.PBB),
(BlockTyp.CHAPTER, TM1, "Prologue", [], BlockFmt.PBB),
]
# Note File
@@ -1361,8 +1361,8 @@ def testFmtToken_SpecialFormat(mockGUI):
# ========
correctResp = [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.CENTRE),
(BlockTyp.HEAD1, TM2, "Title Two", [], BlockFmt.CENTRE | BlockFmt.PBB),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.CENTRE),
(BlockTyp.PART, TM2, "Title Two", [], BlockFmt.CENTRE | BlockFmt.PBB),
]
# Command wo/Space
@@ -1405,9 +1405,9 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Multiple Empty Paragraphs
@@ -1421,9 +1421,9 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Three Skips
@@ -1434,11 +1434,11 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Malformed Command, Case 1
@@ -1449,8 +1449,8 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Malformed Command, Case 2
@@ -1461,8 +1461,8 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Malformed Command, Case 3
@@ -1473,8 +1473,8 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Empty Paragraph and Page Break
@@ -1489,9 +1489,9 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.PBB),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.PBB),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
# Multiple Skip
@@ -1503,11 +1503,11 @@ def testFmtToken_SpecialFormat(mockGUI):
)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.PBB),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
(BlockTyp.PART, TM1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, "", "", [], BlockFmt.PBB),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.SKIP, "", "", [], BlockFmt.NONE),
(BlockTyp.TEXT, "", "Some text to go here ...", [], BlockFmt.NONE),
]
@@ -1619,7 +1619,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "T: Part One", [], BlockFmt.CENTRE),
(BlockTyp.PART, TM1, "T: Part One", [], BlockFmt.CENTRE),
]
# H1: Title, Not First Page
@@ -1628,7 +1628,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD1, TM1, "T: Part One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.PART, TM1, "T: Part One", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
# Chapters
@@ -1639,7 +1639,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "C: Chapter One", [], BlockFmt.PBB),
(BlockTyp.CHAPTER, TM1, "C: Chapter One", [], BlockFmt.PBB),
]
# H2: Unnumbered Chapter
@@ -1647,7 +1647,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "U: Prologue", [], BlockFmt.PBB),
(BlockTyp.CHAPTER, TM1, "U: Prologue", [], BlockFmt.PBB),
]
# H2: Chapter Word Number
@@ -1656,7 +1656,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._hFormatter._chCount = 0
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter One", [], BlockFmt.PBB),
(BlockTyp.CHAPTER, TM1, "Chapter One", [], BlockFmt.PBB),
]
# H2: Chapter Roman Number Upper Case
@@ -1664,7 +1664,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter II", [], BlockFmt.PBB),
(BlockTyp.CHAPTER, TM1, "Chapter II", [], BlockFmt.PBB),
]
# H2: Chapter Roman Number Lower Case
@@ -1672,7 +1672,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD2, TM1, "Chapter iii", [], BlockFmt.PBB),
(BlockTyp.CHAPTER, TM1, "Chapter iii", [], BlockFmt.PBB),
]
# Scenes
@@ -1683,7 +1683,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "S: Scene One", [], BlockFmt.NONE),
(BlockTyp.SCENE, TM1, "S: Scene One", [], BlockFmt.NONE),
]
# H3: Scene Hidden wo/Format
@@ -1731,7 +1731,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scChCount = 0
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Scene 1", [], BlockFmt.NONE),
(BlockTyp.SCENE, TM1, "Scene 1", [], BlockFmt.NONE),
]
# H3: Scene w/Chapter Number
@@ -1741,7 +1741,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scChCount = 1
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD3, TM1, "Scene 3.2", [], BlockFmt.NONE),
(BlockTyp.SCENE, TM1, "Scene 3.2", [], BlockFmt.NONE),
]
# Sections
@@ -1766,7 +1766,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.HEAD4, TM1, "X: A Section", [], BlockFmt.NONE),
(BlockTyp.SECTION, TM1, "X: A Section", [], BlockFmt.NONE),
]
# H4: Section Separator
@@ -2160,14 +2160,14 @@ def testFmtToken_SceneSeparators(mockGUI):
md.doConvert()
assert md._pages[-1] == (
"# T: Title One\n\n"
"### S: Scene One\n\n"
"## S: Scene One\n\n"
"Text\n\n"
"### S: Scene Two\n\n"
"## S: Scene Two\n\n"
"Text\n\n"
"# T: Title Two\n\n"
"### S: Scene Three\n\n"
"## S: Scene Three\n\n"
"Text\n\n"
"### H: Scene Four\n\n"
"## H: Scene Four\n\n"
"Text\n\n"
)
@@ -2198,11 +2198,11 @@ def testFmtToken_SceneSeparators(mockGUI):
md.doConvert()
assert md._pages[-1] == (
"# T: Title One\n\n"
"## C: Chapter One\n\n"
"# C: Chapter One\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
"## C: Chapter Two\n\n"
"# C: Chapter Two\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
@@ -2215,15 +2215,15 @@ def testFmtToken_SceneSeparators(mockGUI):
md.doConvert()
assert md._pages[-1] == (
"# T: Title One\n\n"
"## C: Chapter One\n\n"
"### S: Scene One\n\n"
"# C: Chapter One\n\n"
"## S: Scene One\n\n"
"Text\n\n"
"### S: Scene Two\n\n"
"## S: Scene Two\n\n"
"Text\n\n"
"## C: Chapter Two\n\n"
"### S: Scene Three\n\n"
"# C: Chapter Two\n\n"
"## S: Scene Three\n\n"
"Text\n\n"
"### H: Scene Four\n\n"
"## H: Scene Four\n\n"
"Text\n\n"
)
@@ -2287,6 +2287,7 @@ def testFmtToken_HeaderVisibility(mockGUI):
# Novel Files
# ===========
# Headers are promoted one level
md._isNovel = True
@@ -2303,18 +2304,18 @@ def testFmtToken_HeaderVisibility(mockGUI):
assert md._pages[-1] == (
"# Novel\n\n"
"# Title One\n\n"
"## Prologue\n\n"
"# Prologue\n\n"
"Text\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"# Chapter One\n\n"
"## Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"#### Section Two\n\n"
"## Scene Two\n\n"
"### Section Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"# Chapter Two\n\n"
"## Scene Three\n\n"
"Text\n\n"
"### Scene Four\n\n"
"## Scene Four\n\n"
"Text\n\n"
)
@@ -2339,6 +2340,7 @@ def testFmtToken_HeaderVisibility(mockGUI):
# Note Files
# ==========
# Headers are exported as-is
md._isNovel = False
@@ -2428,30 +2430,30 @@ def testFmtToken_CounterHandling(mockGUI):
md.doConvert()
assert md._pages[-1] == (
"# Novel One\n\n"
"## U: Prologue\n\n"
"# U: Prologue\n\n"
"Text\n\n"
"## C 1: Chapter One\n\n"
"### S 1.1 (1): Scene One\n\n"
"# C 1: Chapter One\n\n"
"## S 1.1 (1): Scene One\n\n"
"Text\n\n"
"### S 1.2 (2): Scene Two\n\n"
"## S 1.2 (2): Scene Two\n\n"
"Text\n\n"
"## C 2: Chapter Two\n\n"
"### S 2.1 (3): Scene Three\n\n"
"# C 2: Chapter Two\n\n"
"## S 2.1 (3): Scene Three\n\n"
"Text\n\n"
"### H 2.2 (4): Scene Four\n\n"
"## H 2.2 (4): Scene Four\n\n"
"Text\n\n"
"# Novel Two\n\n"
"## U: Prologue\n\n"
"# U: Prologue\n\n"
"Text\n\n"
"## C 1: Chapter One\n\n"
"### S 1.1 (1): Scene One\n\n"
"# C 1: Chapter One\n\n"
"## S 1.1 (1): Scene One\n\n"
"Text\n\n"
"### S 1.2 (2): Scene Two\n\n"
"## S 1.2 (2): Scene Two\n\n"
"Text\n\n"
"## C 2: Chapter Two\n\n"
"### S 2.1 (3): Scene Three\n\n"
"# C 2: Chapter Two\n\n"
"## S 2.1 (3): Scene Three\n\n"
"Text\n\n"
"### H 2.2 (4): Scene Four\n\n"
"## H 2.2 (4): Scene Four\n\n"
"Text\n\n"
)
tests/test_formats/test_fmt_tomarkdown.py
+12 -12
View File
@@ -49,20 +49,20 @@ def testFmtToMarkdown_ConvertHeaders(mockGUI):
md.setChapterFormat(f"Chapter {nwHeadFmt.CH_NUM}{nwHeadFmt.BR}{nwHeadFmt.TITLE}")
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == "## Chapter 1 - Title\n\n"
assert md._pages[-1] == "# Chapter 1 - Title\n\n"
# Header 3
md._text = "### Title\n"
md.setSceneFormat(f"Scene {nwHeadFmt.SC_ABS}{nwHeadFmt.BR}{nwHeadFmt.TITLE}")
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == "### Scene 1 - Title\n\n"
assert md._pages[-1] == "## Scene 1 - Title\n\n"
# Header 4
md._text = "#### Section Title\n"
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == "#### Section Title\n\n"
assert md._pages[-1] == "### Section Title\n\n"
# Title
md._text = "#! Title\n"
@@ -74,7 +74,7 @@ def testFmtToMarkdown_ConvertHeaders(mockGUI):
md._text = "##! Prologue\n"
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == "## Prologue\n\n"
assert md._pages[-1] == "# Prologue\n\n"
@pytest.mark.core
@@ -190,7 +190,7 @@ def testFmtToMarkdown_ConvertParagraphs(mockGUI):
md.tokenizeText()
md.doConvert()
assert md._pages[-1] == (
"## Chapter\n\n"
"# Chapter\n\n"
"**Point of View:** Bod \n"
"**Plot:** Main \n"
"**Locations:** Europe\n\n"
@@ -271,12 +271,12 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
]
resText = [
"# My Novel\n\n**By Jane Doh**\n\n",
"## Chapter 1\n\nThe text of chapter one.\n\n",
"### Scene 1\n\nThe text of scene one.\n\n",
"#### A Section\n\nMore text in scene one.\n\n",
"## Chapter 2\n\nThe text of chapter two.\n\n",
"### Scene 2\n\nThe text of scene two.\n\n",
"#### A Section\n\n\tMore text in scene two.\n\n",
"# Chapter 1\n\nThe text of chapter one.\n\n",
"## Scene 1\n\nThe text of scene one.\n\n",
"### A Section\n\nMore text in scene one.\n\n",
"# Chapter 2\n\nThe text of chapter two.\n\n",
"## Scene 2\n\nThe text of scene two.\n\n",
"### A Section\n\n\tMore text in scene two.\n\n",
]
for i in range(len(docText)):
@@ -289,7 +289,7 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
assert md.getFullResultSize() == len("".join(resText))
md.replaceTabs(nSpaces=4, spaceChar=" ")
resText[6] = "#### A Section\n\n More text in scene two.\n\n"
resText[6] = "### A Section\n\n More text in scene two.\n\n"
assert md._pages == resText
# Check File