Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve page break handling in the tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-11-07 17:53:19 +01:00
parent fc6e0a577f
commit d04a21b70f
1 changed files with 39 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/formats/tokenizer.py
+39 -34
View File
@@ -166,6 +166,7 @@ class Tokenizer(ABC):
self._hFormatter = HeadingFormatter(self._project) self._hFormatter = HeadingFormatter(self._project)
self._noSep = True # Flag to indicate that we don't want a scene separator self._noSep = True # Flag to indicate that we don't want a scene separator
self._noIndent = False # Flag to disable text indent on next paragraph self._noIndent = False # Flag to disable text indent on next paragraph
self._forcePage = False # Force a page break on next token
# This File # This File
self._isNovel = False # Document is a novel document self._isNovel = False # Document is a novel document
@@ -444,6 +445,11 @@ class Tokenizer(ABC):
self._classes["optional"] = self._theme.optional self._classes["optional"] = self._theme.optional
return return
def setBreakNext(self) -> None:
"""Set a page break for next block."""
self._forcePage = True
return
def addRootHeading(self, tHandle: str) -> None: def addRootHeading(self, tHandle: str) -> None:
"""Add a heading at the start of a new root folder.""" """Add a heading at the start of a new root folder."""
self._text = "" self._text = ""
@@ -531,7 +537,6 @@ class Tokenizer(ABC):
text = REGEX_PATTERNS.lineBreak.sub("\uffff", self._text) text = REGEX_PATTERNS.lineBreak.sub("\uffff", self._text)
nHead = 0 nHead = 0
breakNext = False
rawText = [] rawText = []
tHandle = self._handle or "" tHandle = self._handle or ""
tBlocks: list[T_Block] = [B_EMPTY] tBlocks: list[T_Block] = [B_EMPTY]
@@ -546,11 +551,11 @@ class Tokenizer(ABC):
rawText.append("\n") rawText.append("\n")
continue continue
if breakNext: if self._forcePage:
sAlign = BlockFmt.PBB tStyle = BlockFmt.PBB
breakNext = False self._forcePage = False
else: else:
sAlign = BlockFmt.NONE tStyle = BlockFmt.NONE
# Check Line Format # Check Line Format
# ================= # =================
@@ -563,12 +568,12 @@ class Tokenizer(ABC):
# therefore proceed to check other formats. # therefore proceed to check other formats.
if sLine in ("[newpage]", "[new page]"): if sLine in ("[newpage]", "[new page]"):
breakNext = True self._forcePage = True
continue continue
elif sLine == "[vspace]": elif sLine == "[vspace]":
tBlocks.append( tBlocks.append(
(BlockTyp.SKIP, "", "", [], sAlign) (BlockTyp.SKIP, "", "", [], tStyle)
) )
continue continue
@@ -576,7 +581,7 @@ class Tokenizer(ABC):
nSkip = checkInt(sLine[8:-1], 0) nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1: if nSkip >= 1:
tBlocks.append( tBlocks.append(
(BlockTyp.SKIP, "", "", [], sAlign) (BlockTyp.SKIP, "", "", [], tStyle)
) )
if nSkip > 1: if nSkip > 1:
tBlocks += (nSkip - 1) * [ tBlocks += (nSkip - 1) * [
@@ -599,14 +604,14 @@ class Tokenizer(ABC):
if cStyle == nwComment.PLAIN and not self._doComments: if cStyle == nwComment.PLAIN and not self._doComments:
continue continue
if doJustify and not sAlign & BlockFmt.ALIGNED: if doJustify and not tStyle & BlockFmt.ALIGNED:
sAlign |= BlockFmt.JUSTIFY tStyle |= BlockFmt.JUSTIFY
if cStyle in (nwComment.SYNOPSIS, nwComment.SHORT, nwComment.PLAIN): if cStyle in (nwComment.SYNOPSIS, nwComment.SHORT, nwComment.PLAIN):
bStyle = COMMENT_STYLE[cStyle] bStyle = COMMENT_STYLE[cStyle]
tLine, tFmt = self._formatComment(bStyle, cKey, cText) tLine, tFmt = self._formatComment(bStyle, cKey, cText)
tBlocks.append(( tBlocks.append((
BlockTyp.COMMENT, "", tLine, tFmt, sAlign BlockTyp.COMMENT, "", tLine, tFmt, tStyle
)) ))
if keepRaw: if keepRaw:
rawText.append(f"{aLine}\n") rawText.append(f"{aLine}\n")
@@ -627,7 +632,7 @@ class Tokenizer(ABC):
tTag, tLine, tFmt = self._formatMeta(aLine) tTag, tLine, tFmt = self._formatMeta(aLine)
if tLine: if tLine:
tBlocks.append(( tBlocks.append((
BlockTyp.KEYWORD, tTag[1:], tLine, tFmt, sAlign BlockTyp.KEYWORD, tTag[1:], tLine, tFmt, tStyle
)) ))
if keepRaw: if keepRaw:
rawText.append(f"{aLine}\n") rawText.append(f"{aLine}\n")
@@ -646,16 +651,16 @@ class Tokenizer(ABC):
nHead += 1 nHead += 1
tText = aLine[2:].strip() tText = aLine[2:].strip()
tType = BlockTyp.HEAD1 if isPlain else BlockTyp.TITLE tType = BlockTyp.HEAD1 if isPlain else BlockTyp.TITLE
tStyle = BlockFmt.NONE if isPlain else self._titleStyle
sHide = self._hidePart if isPlain else False sHide = self._hidePart if isPlain else False
if not (isPlain or isNovel and sHide):
tStyle |= self._titleStyle
if isNovel: if isNovel:
if sHide: if sHide:
tText = "" tText = ""
tType = BlockTyp.EMPTY tType = BlockTyp.EMPTY
tStyle = BlockFmt.NONE
elif isPlain: elif isPlain:
tText = self._hFormatter.apply(self._fmtPart, tText, nHead) tText = self._hFormatter.apply(self._fmtPart, tText, nHead)
tStyle = self._partStyle tStyle |= self._partStyle
if isPlain: if isPlain:
self._hFormatter.resetScene() self._hFormatter.resetScene()
else: else:
@@ -682,7 +687,6 @@ class Tokenizer(ABC):
nHead += 1 nHead += 1
tText = aLine[3:].strip() tText = aLine[3:].strip()
tType = BlockTyp.HEAD2 tType = BlockTyp.HEAD2
tStyle = BlockFmt.NONE
sHide = self._hideChapter if isPlain else self._hideUnNum sHide = self._hideChapter if isPlain else self._hideUnNum
tFormat = self._fmtChapter if isPlain else self._fmtUnNum tFormat = self._fmtChapter if isPlain else self._fmtUnNum
if isNovel: if isNovel:
@@ -693,7 +697,7 @@ class Tokenizer(ABC):
tType = BlockTyp.EMPTY tType = BlockTyp.EMPTY
else: else:
tText = self._hFormatter.apply(tFormat, tText, nHead) tText = self._hFormatter.apply(tFormat, tText, nHead)
tStyle = self._chapterStyle tStyle |= self._chapterStyle
self._hFormatter.resetScene() self._hFormatter.resetScene()
self._noSep = True self._noSep = True
@@ -719,7 +723,6 @@ class Tokenizer(ABC):
nHead += 1 nHead += 1
tText = aLine[4:].strip() tText = aLine[4:].strip()
tType = BlockTyp.HEAD3 tType = BlockTyp.HEAD3
tStyle = BlockFmt.NONE
sHide = self._hideScene if isPlain else self._hideHScene sHide = self._hideScene if isPlain else self._hideHScene
tFormat = self._fmtScene if isPlain else self._fmtHScene tFormat = self._fmtScene if isPlain else self._fmtHScene
if isNovel: if isNovel:
@@ -729,13 +732,13 @@ class Tokenizer(ABC):
tType = BlockTyp.EMPTY tType = BlockTyp.EMPTY
else: else:
tText = self._hFormatter.apply(tFormat, tText, nHead) tText = self._hFormatter.apply(tFormat, tText, nHead)
tStyle = self._sceneStyle tStyle |= self._sceneStyle
if tText == "": # Empty Format if tText == "": # Empty Format
tType = BlockTyp.EMPTY if self._noSep else BlockTyp.SKIP tType = BlockTyp.EMPTY if self._noSep else BlockTyp.SKIP
elif tText == tFormat: # Static Format elif tText == tFormat: # Static Format
tText = "" if self._noSep else tText tText = "" if self._noSep else tText
tType = BlockTyp.EMPTY if self._noSep else BlockTyp.SEP tType = BlockTyp.EMPTY if self._noSep else BlockTyp.SEP
tStyle = BlockFmt.NONE if self._noSep else BlockFmt.CENTRE tStyle |= BlockFmt.NONE if self._noSep else BlockFmt.CENTRE
self._noSep = False self._noSep = False
tBlocks.append(( tBlocks.append((
@@ -755,7 +758,6 @@ class Tokenizer(ABC):
nHead += 1 nHead += 1
tText = aLine[5:].strip() tText = aLine[5:].strip()
tType = BlockTyp.HEAD4 tType = BlockTyp.HEAD4
tStyle = BlockFmt.NONE
if isNovel: if isNovel:
if self._hideSection: if self._hideSection:
tText = "" tText = ""
@@ -766,7 +768,7 @@ class Tokenizer(ABC):
tType = BlockTyp.SKIP tType = BlockTyp.SKIP
elif tText == self._fmtSection: # Static Format elif tText == self._fmtSection: # Static Format
tType = BlockTyp.SEP tType = BlockTyp.SEP
tStyle = BlockFmt.CENTRE tStyle |= BlockFmt.CENTRE
tBlocks.append(( tBlocks.append((
tType, f"{tHandle}:T{nHead:04d}", tText, [], tStyle tType, f"{tHandle}:T{nHead:04d}", tText, [], tStyle
@@ -803,35 +805,38 @@ class Tokenizer(ABC):
bLine = bLine[:-1].rstrip(" ") bLine = bLine[:-1].rstrip(" ")
if alnLeft and alnRight: if alnLeft and alnRight:
sAlign |= BlockFmt.CENTRE tStyle |= BlockFmt.CENTRE
elif alnLeft: elif alnLeft:
sAlign |= BlockFmt.LEFT tStyle |= BlockFmt.LEFT
elif alnRight: elif alnRight:
sAlign |= BlockFmt.RIGHT tStyle |= BlockFmt.RIGHT
if indLeft: if indLeft:
sAlign |= BlockFmt.IND_L tStyle |= BlockFmt.IND_L
if indRight: if indRight:
sAlign |= BlockFmt.IND_R tStyle |= BlockFmt.IND_R
# Process formats # Process formats
tLine, tFmt = self._extractFormats(bLine, hDialog=isNovel) tLine, tFmt = self._extractFormats(bLine, hDialog=isNovel)
tBlocks.append(( tBlocks.append((
BlockTyp.TEXT, "", tLine, tFmt, sAlign BlockTyp.TEXT, "", tLine, tFmt, tStyle
)) ))
if keepRaw: if keepRaw:
rawText.append(f"{aLine}\n") rawText.append(f"{aLine}\n")
# If we have content, turn off the first page flag # If we have content, turn off the first page flag
if self._isFirst and len(tBlocks) > 1: if self._isFirst and tBlocks:
self._isFirst = False # First document has been processed self._isFirst = False # First document has been processed
# Make sure the blocks array doesn't start with a page break # Make sure the blocks array doesn't start with a page break
# on the very first page, adding a blank first page. # on the very first block, adding a blank first page.
if (cBlock := tBlocks[1])[4] & BlockFmt.PBB: for n, cBlock in enumerate(tBlocks):
tBlocks[1] = ( if cBlock[0] != BlockTyp.EMPTY:
cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB if cBlock[4] & BlockFmt.PBB:
) tBlocks[n] = (
cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB
)
break
# Always add an empty line at the end of the file # Always add an empty line at the end of the file
tBlocks.append(B_EMPTY) tBlocks.append(B_EMPTY)