Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up the tokenizer code a little

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-02-24 14:28:12 +01:00
parent 9f6a1b4bc0
commit eb105951ac
1 changed files with 15 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tokenizer.py
+15 -24
View File
@@ -671,16 +671,18 @@ class Tokenizer(ABC):
if tToken[0] == self.T_TEXT: if tToken[0] == self.T_TEXT:
self._firstScene = False self._firstScene = False
elif tToken[0] == self.T_HEAD1: elif tToken[0] == self.T_HEAD1: # Partition
# Partition
tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1]) tTemp = self._hFormatter.apply(self._fmtTitle, tToken[2], tToken[1])
self._tokens[n] = ( self._tokens[n] = (
tToken[0], tToken[1], tTemp, [], tToken[4] tToken[0], tToken[1], tTemp, [], tToken[4]
) )
elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): # Set scene variables
# Chapter # self._firstScene = True
# self._hFormatter.resetScene()
elif tToken[0] in (self.T_HEAD2, self.T_UNNUM): # Chapter
# Numbered or Unnumbered # Numbered or Unnumbered
if tToken[0] == self.T_UNNUM: if tToken[0] == self.T_UNNUM:
@@ -698,8 +700,7 @@ class Tokenizer(ABC):
self._firstScene = True self._firstScene = True
self._hFormatter.resetScene() self._hFormatter.resetScene()
elif tToken[0] == self.T_HEAD3: elif tToken[0] == self.T_HEAD3: # Scene
# Scene
self._hFormatter.incScene() self._hFormatter.incScene()
@@ -709,23 +710,14 @@ class Tokenizer(ABC):
self.T_EMPTY, tToken[1], "", [], self.A_NONE self.T_EMPTY, tToken[1], "", [], self.A_NONE
) )
elif tTemp == "" and not self._hideScene: elif tTemp == "" and not self._hideScene:
if self._firstScene: t1 = self.T_EMPTY if self._firstScene else self.T_SKIP
self._tokens[n] = ( t4 = self.A_NONE if self._firstScene else tToken[4]
self.T_EMPTY, tToken[1], "", [], self.A_NONE self._tokens[n] = (t1, tToken[1], "", [], t4)
)
else:
self._tokens[n] = (
self.T_SKIP, tToken[1], "", [], tToken[4]
)
elif tTemp == self._fmtScene: elif tTemp == self._fmtScene:
if self._firstScene: t1 = self.T_EMPTY if self._firstScene else self.T_SEP
self._tokens[n] = ( t2 = "" if self._firstScene else tTemp
self.T_EMPTY, tToken[1], "", [], self.A_NONE t4 = self.A_NONE if self._firstScene else (tToken[4] | self.A_CENTRE)
) self._tokens[n] = (t1, tToken[1], t2, [], t4)
else:
self._tokens[n] = (
self.T_SEP, tToken[1], tTemp, [], tToken[4] | self.A_CENTRE
)
else: else:
self._tokens[n] = ( self._tokens[n] = (
tToken[0], tToken[1], tTemp, [], tToken[4] tToken[0], tToken[1], tTemp, [], tToken[4]
@@ -733,8 +725,7 @@ class Tokenizer(ABC):
self._firstScene = False self._firstScene = False
elif tToken[0] == self.T_HEAD4: elif tToken[0] == self.T_HEAD4: # Section
# Section
tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1]) tTemp = self._hFormatter.apply(self._fmtSection, tToken[2], tToken[1])
if tTemp == "" and self._hideSection: if tTemp == "" and self._hideSection: