Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix scene separators and chapter and scene counters (#1711)

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-02-24 15:32:50 +01:00
committed by GitHub
parent 9f6a1b4bc0 6ef646acb2
commit b0614e8c25
2 changed files with 269 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tokenizer.py
+66 -63
View File
@@ -89,7 +89,7 @@ class Tokenizer(ABC):
T_UNNUM = 7 # Unnumbered
T_HEAD1 = 8 # Header 1
T_HEAD2 = 9 # Header 2
T_HEAD3 = 10 # Header 3
T_HEAD3 = 10 # Header 3
T_HEAD4 = 11 # Header 4
T_TEXT = 12 # Text line
T_SEP = 13 # Scene separator
@@ -158,7 +158,7 @@ class Tokenizer(ABC):
# Instance Variables
self._hFormatter = HeadingFormatter(self._project)
self._firstScene = False # Flag to indicate that the first scene of the chapter
self._allowSeparator = False # Flag to indicate that the first scene of the chapter
# This File
self._isNone = False # Document has unknown layout
@@ -615,9 +615,9 @@ class Tokenizer(ABC):
# Make sure the token array doesn't start with a page break
# on the very first page, adding a blank first page.
if self._tokens[0][4] & self.A_PBB:
tToken = self._tokens[0]
token = self._tokens[0]
self._tokens[0] = (
tToken[0], tToken[1], tToken[2], tToken[3], tToken[4] & ~self.A_PBB
token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
)
# Always add an empty line at the end of the file
@@ -637,22 +637,20 @@ class Tokenizer(ABC):
pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
tCount = len(self._tokens)
for n, tToken in enumerate(self._tokens):
for n, token in enumerate(self._tokens):
if n > 0:
pToken = self._tokens[n-1]
if n < tCount - 1:
nToken = self._tokens[n+1]
if tToken[0] == self.T_KEYWORD:
aStyle = tToken[4]
if token[0] == self.T_KEYWORD:
aStyle = token[4]
if pToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_TOPMRG
if nToken[0] == self.T_KEYWORD:
aStyle |= self.A_Z_BTMMRG
self._tokens[n] = (
tToken[0], tToken[1], tToken[2], tToken[3], aStyle
)
self._tokens[n] = (token[0], token[1], token[2], token[3], aStyle)
return
@@ -665,93 +663,91 @@ class Tokenizer(ABC):
self._hFormatter.setHandle(self._nwItem.itemHandle if self._nwItem else None)
for n, tToken in enumerate(self._tokens):
for n, token in enumerate(self._tokens):
# In case we see text before a scene, we reset the flag
if tToken[0] == self.T_TEXT:
self._firstScene = False
if token[0] == self.T_TEXT:
# If we see text before a scene, we consider it a "scene"
self._allowSeparator = False
elif tToken[0] == self.T_HEAD1:
# Partition
elif token[0] == self.T_TITLE: # Title
# For new titles, we reset all counters
self._allowSeparator = True
self._hFormatter.resetAll()
tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1])
elif token[0] == self.T_HEAD1: # Partition
tTemp = self._hFormatter.apply(self._fmtTitle, token[2], token[1])
self._tokens[n] = (
tToken[0], tToken[1], tTemp, [], tToken[4]
)
elif tToken[0] in (self.T_HEAD2, self.T_UNNUM):
# Chapter
# Numbered or Unnumbered
if tToken[0] == self.T_UNNUM:
tTemp = self._hFormatter.apply(self._fmtUnNum, tToken[2], tToken[1])
else:
self._hFormatter.incChapter()
tTemp = self._hFormatter.apply(self._fmtChapter, tToken[2], tToken[1])
# Format the chapter header
self._tokens[n] = (
tToken[0], tToken[1], tTemp, [], tToken[4]
token[0], token[1], tTemp, [], token[4]
)
# Set scene variables
self._firstScene = True
self._allowSeparator = True
self._hFormatter.resetScene()
elif tToken[0] == self.T_HEAD3:
# Scene
elif token[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter
# Numbered or Unnumbered
if token[0] == self.T_UNNUM:
tTemp = self._hFormatter.apply(self._fmtUnNum, token[2], token[1])
else:
self._hFormatter.incChapter()
tTemp = self._hFormatter.apply(self._fmtChapter, token[2], token[1])
# Format the chapter header
self._tokens[n] = (
token[0], token[1], tTemp, [], token[4]
)
# Set scene variables
self._allowSeparator = True
self._hFormatter.resetScene()
elif token[0] == self.T_HEAD3: # Scene
self._hFormatter.incScene()
tTemp = self._hFormatter.apply(self._fmtScene, tToken[2], tToken[1])
tTemp = self._hFormatter.apply(self._fmtScene, token[2], token[1])
if tTemp == "" and self._hideScene:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", [], self.A_NONE
self.T_EMPTY, token[1], "", [], self.A_NONE
)
elif tTemp == "" and not self._hideScene:
if self._firstScene:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", [], self.A_NONE
)
else:
self._tokens[n] = (
self.T_SKIP, tToken[1], "", [], tToken[4]
)
self._tokens[n] = (
self.T_EMPTY if self._allowSeparator else self.T_SKIP, token[1],
"", [], self.A_NONE if self._allowSeparator else token[4]
)
elif tTemp == self._fmtScene:
if self._firstScene:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", [], self.A_NONE
)
else:
self._tokens[n] = (
self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE
)
self._tokens[n] = (
self.T_EMPTY if self._allowSeparator else self.T_SEP, token[1],
"" if self._allowSeparator else tTemp, [],
self.A_NONE if self._allowSeparator else (token[4] | self.A_CENTRE)
)
else:
self._tokens[n] = (
tToken[0], tToken[1], tTemp, [], tToken[4]
token[0], token[1], tTemp, [], token[4]
)
self._firstScene = False
self._allowSeparator = False
elif tToken[0] == self.T_HEAD4:
# Section
elif token[0] == self.T_HEAD4: # Section
tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1])
tTemp = self._hFormatter.apply(self._fmtSection, token[2], token[1])
if tTemp == "" and self._hideSection:
self._tokens[n] = (
self.T_EMPTY, tToken[1], "", [], self.A_NONE
self.T_EMPTY, token[1], "", [], self.A_NONE
)
elif tTemp == "" and not self._hideSection:
self._tokens[n] = (
self.T_SKIP, tToken[1], "", [], tToken[4]
self.T_SKIP, token[1], "", [], token[4]
)
elif tTemp == self._fmtSection:
self._tokens[n] = (
self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE
self.T_SEP, token[1], tTemp, [], token[4] | self.A_CENTRE
)
else:
self._tokens[n] = (
tToken[0], tToken[1], tTemp, [], tToken[4]
token[0], token[1], tTemp, [], token[4]
)
return True
@@ -849,6 +845,13 @@ class HeadingFormatter:
self._scAbsCount += 1
return
def resetAll(self) -> None:
"""Reset all counters."""
self._chCount = 0
self._scChCount = 0
self._scAbsCount = 0
return
def resetScene(self) -> None:
"""Reset the chapter scene counter."""
self._scChCount = 0
tests/test_core/test_core_tokenizer.py
+203 -9
View File
@@ -26,6 +26,7 @@ import pytest
from tools import C, buildTestProject, readFile
from novelwriter.constants import nwHeadFmt
from novelwriter.core.tomd import ToMarkdown
from novelwriter.core.project import NWProject
from novelwriter.core.tokenizer import HeadingFormatter, Tokenizer, stripEscape
@@ -1010,7 +1011,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
tokens._isNote = False
##
# Story FIles
# Story Files
##
tokens._isNone = False
@@ -1122,7 +1123,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene wo/Format, first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._firstScene = True
tokens._allowSeparator = True
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
@@ -1133,7 +1134,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene wo/Format, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._firstScene = False
tokens._allowSeparator = False
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
@@ -1144,7 +1145,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene Separator, first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False)
tokens._firstScene = True
tokens._allowSeparator = True
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
@@ -1155,7 +1156,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H3: Scene Separator, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False)
tokens._firstScene = False
tokens._allowSeparator = False
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
@@ -1192,7 +1193,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
# H4: Section Hidden wo/Format
tokens._text = "#### A Section\n"
tokens.setSectionFormat(r"", True)
tokens.setSectionFormat("", True)
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._tokens == [
@@ -1231,16 +1232,203 @@ def testCoreToken_ProcessHeaders(mockGUI):
]
# Check the first scene detector
assert tokens._firstScene is False
tokens._firstScene = True
assert tokens._allowSeparator is False
tokens._allowSeparator = True
tokens._text = "Some text ...\n"
tokens.tokenizeText()
tokens.doHeaders()
assert tokens._firstScene is False
assert tokens._allowSeparator is False
# END Test testCoreToken_ProcessHeaders
@pytest.mark.core
def testCoreToken_HeaderCounterAndVisibility(mockGUI):
"""Test the header counter and visibility of the Tokenizer class.
This is a special test to cover issue #1704.
"""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
md = ToMarkdown(project)
md._isNone = False
md._isNote = False
md._isNovel = True
# Separator Handling, Titles
# ==========================
md._text = (
"# Title One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"# Title Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"### Scene Four\n\n"
"Text\n\n"
)
md.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
md.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
md.setSectionFormat("", True)
# Static Separator
md.setSceneFormat("* * *", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
"# T: Title Two\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
)
# Scene Title Formatted
md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
"### S: Scene One\n\n"
"Text\n\n"
"### S: Scene Two\n\n"
"Text\n\n"
"# T: Title Two\n\n"
"### S: Scene Three\n\n"
"Text\n\n"
"### S: Scene Four\n\n"
"Text\n\n"
)
# Separator Handling, Chapters
# ============================
md._text = (
"# Title One\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"### Scene Four\n\n"
"Text\n\n"
)
md.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
md.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
md.setSectionFormat("", True)
# Static Separator
md.setSceneFormat("* * *", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
"## C: Chapter One\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
"## C: Chapter Two\n\n"
"Text\n\n"
"* * *\n\n"
"Text\n\n"
)
# Scene Title Formatted
md.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# T: Title One\n\n"
"## C: Chapter One\n\n"
"### S: Scene One\n\n"
"Text\n\n"
"### S: Scene Two\n\n"
"Text\n\n"
"## C: Chapter Two\n\n"
"### S: Scene Three\n\n"
"Text\n\n"
"### S: Scene Four\n\n"
"Text\n\n"
)
# Counter Handling, Novel Titles
# ==============================
md._text = (
"#! Novel One\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"### Scene Four\n\n"
"Text\n\n"
"#! Novel Two\n\n"
"## Chapter One\n\n"
"### Scene One\n\n"
"Text\n\n"
"### Scene Two\n\n"
"Text\n\n"
"## Chapter Two\n\n"
"### Scene Three\n\n"
"Text\n\n"
"### Scene Four\n\n"
"Text\n\n"
)
md.setTitleFormat(f"T: {nwHeadFmt.TITLE}")
md.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}")
md.setSceneFormat(f"S {nwHeadFmt.CH_NUM}.{nwHeadFmt.SC_NUM} ({nwHeadFmt.SC_ABS}): "
f"{nwHeadFmt.TITLE}", False)
md.setSectionFormat("", True)
# Two Novel Format
md.tokenizeText()
md.doHeaders()
md.doConvert()
assert md.result == (
"# Novel One\n\n"
"## C 1: Chapter One\n\n"
"### S 1.1 (1): Scene One\n\n"
"Text\n\n"
"### S 1.2 (2): Scene Two\n\n"
"Text\n\n"
"## C 2: Chapter Two\n\n"
"### S 2.1 (3): Scene Three\n\n"
"Text\n\n"
"### S 2.2 (4): Scene Four\n\n"
"Text\n\n"
"# Novel Two\n\n"
"## C 1: Chapter One\n\n"
"### S 1.1 (1): Scene One\n\n"
"Text\n\n"
"### S 1.2 (2): Scene Two\n\n"
"Text\n\n"
"## C 2: Chapter Two\n\n"
"### S 2.1 (3): Scene Three\n\n"
"Text\n\n"
"### S 2.2 (4): Scene Four\n\n"
"Text\n\n"
)
# END Test testCoreToken_HeaderCounterAndVisibility
@pytest.mark.core
def testCoreIndex_HeadingFormatter(fncPath, mockRnd):
"""Check the HeadingFormatter class."""
@@ -1306,6 +1494,12 @@ def testCoreIndex_HeadingFormatter(fncPath, mockRnd):
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 2.1 - Scene 5 - Hi Bob"
# New Main Title
formatter.resetAll()
formatter.incChapter()
formatter.incScene()
assert formatter.apply(cFormat, "Hi Bob", 1) == "Chapter 1.1 - Scene 1 - Hi Bob"
# Special Formats
# ===============
formatter._chCount = 2