Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up heading processing in tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-03-12 17:28:02 +01:00
parent 8f25be8cb2
commit c5c9726794
1 changed files with 44 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tokenizer.py
+44 -68
View File
@@ -492,7 +492,7 @@ class Tokenizer(ABC):
# Check Line Format # Check Line Format
# ================= # =================
if aLine[0] == "[": if aLine.startswith("["):
# Special Formats # Special Formats
# =============== # ===============
# Parse special formatting line. This must be a separate if # Parse special formatting line. This must be a separate if
@@ -521,13 +521,13 @@ class Tokenizer(ABC):
] ]
continue continue
if aLine[0] == "%": if aLine.startswith("%"):
# Comments # Comments
# ======== # ========
# All style comments are processed and the exact type exact # All style comments are processed and the exact type exact
# style extracted. Ignored comments on the '%~' format are # style extracted. Ignored comments on the '%~' format are
# skipped completely. # skipped completely.
if aLine[1] == "~": if aLine.startswith("%~"):
continue continue
cStyle, cText, _ = processComment(aLine) cStyle, cText, _ = processComment(aLine)
@@ -550,7 +550,7 @@ class Tokenizer(ABC):
if self._doComments and self._keepMarkdown: if self._doComments and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif aLine[0] == "@": elif aLine.startswith("@"):
# Keywords # Keywords
# ======== # ========
# Only valid keyword lines are parsed, and any ignored keywords # Only valid keyword lines are parsed, and any ignored keywords
@@ -564,56 +564,71 @@ class Tokenizer(ABC):
if self._doKeywords and self._keepMarkdown: if self._doKeywords and self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif aLine[:2] == "# ": elif aLine.startswith(("# ", "#! ")):
# Partition Headings # Title or Partition Headings
# ================== # ===========================
# Main titles are allowed in any document, and they are always
# centred and start on a new page. For novel documents, we also
# reset all counters when such a title is encountered.
# Partition headings are only formatted in novel documents, and # Partition headings are only formatted in novel documents, and
# otherwise unchanged. Scene separators are disabled # otherwise unchanged. Scene separators are disabled
# immediately after partitions, and scene numbers are reset. # immediately after partitions, and scene numbers are reset.
isPart = aLine.startswith("# ")
nHead += 1 nHead += 1
tText = aLine[2:].strip() nSkip = 2 if isPart else 3
tStyle = self.A_NONE tText = aLine[nSkip:].strip()
if self._isNovel: tType = self.T_HEAD1 if isPart else self.T_TITLE
tStyle = self.A_NONE if isPart else (self.A_PBB | self.A_CENTRE)
if self._isNovel and isPart:
tText = self._hFormatter.apply(self._fmtTitle, tText, nHead) tText = self._hFormatter.apply(self._fmtTitle, tText, nHead)
tStyle = self._titleStyle tStyle = self._titleStyle
self._noSep = True
self._hFormatter.resetScene() self._hFormatter.resetScene()
self._noSep = True
elif self._isNovel and not isPart:
self._hFormatter.resetAll()
self._noSep = True
self._tokens.append(( self._tokens.append((
self.T_HEAD1, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif aLine[:3] == "## ": elif aLine.startswith(("## ", "##! ")):
# Chapter Headings # (Unnumbered) Chapter Headings
# ================ # =============================
# Chapter headings are only formatted in novel documents, and # Chapter headings are only formatted in novel documents, and
# otherwise unchanged. Chapter numbers are bumped before the # otherwise unchanged. Chapter numbers are bumped before the
# heading is formatted. Scene separators are disabled # heading is formatted. Scene separators are disabled
# immediately after chapter headings, and scene numbers are # immediately after chapter headings, and scene numbers are
# reset. # reset. Unnumbered chapters are only meaningful in Novel docs,
# so if we're in a note, we keep them as level 2 headings.
isUnNum = aLine.startswith("##! ")
nHead += 1 nHead += 1
tText = aLine[3:].strip() nSkip = 4 if isUnNum else 3
tText = aLine[nSkip:].strip()
tType = self.T_HEAD2
tStyle = self.A_NONE tStyle = self.A_NONE
tFormat = self._fmtUnNum if isUnNum else self._fmtChapter
if self._isNovel: if self._isNovel:
self._hFormatter.incChapter() self._hFormatter.incChapter()
tText = self._hFormatter.apply(self._fmtChapter, tText, nHead) tText = self._hFormatter.apply(tFormat, tText, nHead)
tType = self.T_UNNUM if isUnNum else tType
tStyle = self._chapterStyle tStyle = self._chapterStyle
self._noSep = True self._noSep = True
self._hFormatter.resetScene() self._hFormatter.resetScene()
self._tokens.append(( self._tokens.append((
self.T_HEAD2, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif aLine[:4] == "### " or aLine[:5] == "###! ": elif aLine.startswith(("### ", "###! ")):
# Scene Headings # (Hard) Scene Headings
# ============== # =====================
# Scene headings in novel documents are treated as centred # Scene headings in novel documents are treated as centred
# separators if the formatting does not change the text. If the # separators if the formatting does not change the text. If the
# format is empty, the scene can be hidden or a blank paragraph # format is empty, the scene can be hidden or a blank paragraph
@@ -622,22 +637,23 @@ class Tokenizer(ABC):
# separators immediately after other titles. Scene numbers are # separators immediately after other titles. Scene numbers are
# always incremented before formatting. For notes, the heading # always incremented before formatting. For notes, the heading
# is unchanged. # is unchanged.
isHard = aLine.startswith("###! ")
nHead += 1 nHead += 1
tText = aLine[4:].strip() nSkip = 5 if isHard else 4
tText = aLine[nSkip:].strip()
tType = self.T_HEAD3 tType = self.T_HEAD3
tStyle = self.A_NONE tStyle = self.A_NONE
isHard = aLine[:5] == "###! "
sHide = self._hideHScene if isHard else self._hideScene sHide = self._hideHScene if isHard else self._hideScene
fScene = self._fmtHScene if isHard else self._fmtScene tFormat = self._fmtHScene if isHard else self._fmtScene
if self._isNovel: if self._isNovel:
self._hFormatter.incScene() self._hFormatter.incScene()
tText = self._hFormatter.apply(fScene, tText, nHead) tText = self._hFormatter.apply(tFormat, tText, nHead)
tStyle = self._sceneStyle tStyle = self._sceneStyle
if tText == "": if tText == "":
tType = self.T_EMPTY if self._noSep or sHide else self.T_SKIP tType = self.T_EMPTY if self._noSep or sHide else self.T_SKIP
tStyle = self.A_NONE tStyle = self.A_NONE
elif tText == self._fmtScene: elif tText == tFormat:
tText = "" if self._noSep else tText tText = "" if self._noSep else tText
tType = self.T_EMPTY if self._noSep else self.T_SEP tType = self.T_EMPTY if self._noSep else self.T_SEP
tStyle = self.A_NONE if self._noSep else self.A_CENTRE tStyle = self.A_NONE if self._noSep else self.A_CENTRE
@@ -649,7 +665,7 @@ class Tokenizer(ABC):
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif aLine[:5] == "#### ": elif aLine.startswith("#### "):
# Section Headings # Section Headings
# ================= # =================
# Section headings in novel docs are treated as centred # Section headings in novel docs are treated as centred
@@ -675,46 +691,6 @@ class Tokenizer(ABC):
if self._keepMarkdown: if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif aLine[:3] == "#! ":
# Main Title
# ==========
# Main titles are allowed in any document, and they are always
# centred and start on a new page. For novel documents, we also
# reset all counters when such a title is encountered.
nHead += 1
self._tokens.append((
self.T_TITLE, nHead, aLine[3:].strip(), [], self.A_PBB | self.A_CENTRE
))
if self._isNovel:
self._noSep = True
self._hFormatter.resetAll()
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
elif aLine[:4] == "##! ":
# Unnumbered Chapter Headings
# ===========================
# Unnumbered chapters are only meaningful in Novel docs, so if
# we're in a note, we convert them to a plain level 2 heading.
nHead += 1
tText = aLine[4:].strip()
tType = self.T_HEAD2
tStyle = self.A_NONE
if self._isNovel:
tText = self._hFormatter.apply(self._fmtUnNum, tText, nHead)
tType = self.T_UNNUM
tStyle = self._chapterStyle
self._noSep = True
self._hFormatter.resetScene()
self._tokens.append((
tType, nHead, tText, [], tStyle
))
if self._keepMarkdown:
tmpMarkdown.append(f"{aLine}\n")
else: else:
# Text Lines # Text Lines
# ========== # ==========