Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved the tokenizer class a bit

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2020-05-28 20:19:55 +02:00
parent 65f74e6fdc
commit 2bbde2c445
3 changed files with 141 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/core/tokenizer.py
+136 -54
View File
@@ -320,7 +320,9 @@ class Tokenizer():
# Tag lines starting with specific characters # Tag lines starting with specific characters
if len(aLine.strip()) == 0: if len(aLine.strip()) == 0:
self.theTokens.append(( self.theTokens.append((
self.T_EMPTY, nLine, "", None, self.A_NONE self.T_EMPTY, nLine,
"", None,
self.A_NONE
)) ))
tmpMarkdown.append("\n") tmpMarkdown.append("\n")
@@ -328,45 +330,59 @@ class Tokenizer():
cLine = aLine[1:].strip() cLine = aLine[1:].strip()
if cLine.lower().startswith("synopsis:"): if cLine.lower().startswith("synopsis:"):
self.theTokens.append(( self.theTokens.append((
self.T_SYNOPSIS, nLine, cLine[9:].strip(), None, self.A_NONE self.T_SYNOPSIS, nLine,
cLine[9:].strip(), None,
self.A_NONE
)) ))
if self.doSynopsis: if self.doSynopsis:
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
else: else:
self.theTokens.append(( self.theTokens.append((
self.T_COMMENT, nLine, aLine[1:].strip(), None, self.A_NONE self.T_COMMENT, nLine,
aLine[1:].strip(), None,
self.A_NONE
)) ))
if self.doComments: if self.doComments:
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
elif aLine[0] == "@": elif aLine[0] == "@":
self.theTokens.append(( self.theTokens.append((
self.T_KEYWORD, nLine, aLine[1:].strip(), None, self.A_NONE self.T_KEYWORD, nLine,
aLine[1:].strip(), None,
self.A_NONE
)) ))
if self.doKeywords: if self.doKeywords:
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
elif aLine[:2] == "# ": elif aLine[:2] == "# ":
self.theTokens.append(( self.theTokens.append((
self.T_HEAD1, nLine, aLine[2:].strip(), None, self.A_NONE self.T_HEAD1, nLine,
aLine[2:].strip(), None,
self.A_NONE
)) ))
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
elif aLine[:3] == "## ": elif aLine[:3] == "## ":
self.theTokens.append(( self.theTokens.append((
self.T_HEAD2, nLine, aLine[3:].strip(), None, self.A_NONE self.T_HEAD2, nLine,
aLine[3:].strip(), None,
self.A_NONE
)) ))
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
elif aLine[:4] == "### ": elif aLine[:4] == "### ":
self.theTokens.append(( self.theTokens.append((
self.T_HEAD3, nLine, aLine[4:].strip(), None, self.A_NONE self.T_HEAD3, nLine,
aLine[4:].strip(), None,
self.A_NONE
)) ))
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
elif aLine[:5] == "#### ": elif aLine[:5] == "#### ":
self.theTokens.append(( self.theTokens.append((
self.T_HEAD4, nLine, aLine[5:].strip(), None, self.A_NONE self.T_HEAD4, nLine,
aLine[5:].strip(), None,
self.A_NONE
)) ))
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
@@ -391,13 +407,17 @@ class Tokenizer():
# sorted by position # sorted by position
fmtPos = sorted(fmtPos, key=itemgetter(0)) fmtPos = sorted(fmtPos, key=itemgetter(0))
self.theTokens.append(( self.theTokens.append((
self.T_TEXT, nLine, aLine, fmtPos, self.A_NONE self.T_TEXT, nLine,
aLine, fmtPos,
self.A_NONE
)) ))
tmpMarkdown.append("%s\n" % aLine) tmpMarkdown.append("%s\n" % aLine)
# Always add an empty line at the end # Always add an empty line at the end
self.theTokens.append(( self.theTokens.append((
self.T_EMPTY, nLine, "", None, self.A_NONE self.T_EMPTY, nLine,
"", None,
self.A_NONE
)) ))
tmpMarkdown.append("\n") tmpMarkdown.append("\n")
@@ -421,101 +441,146 @@ class Tokenizer():
for n in range(len(self.theTokens)): for n in range(len(self.theTokens)):
tToken = self.theTokens[n] tToken = self.theTokens[n]
tType = tToken[0]
tLine = tToken[1]
tText = tToken[2]
# In case we see text before a scene, we reset the flag # In case we see text before a scene, we reset the flag
if tType == self.T_TEXT: if tToken[0] == self.T_TEXT:
self.firstScene = False self.firstScene = False
elif tType == self.T_HEAD1: elif tToken[0] == self.T_HEAD1:
# Main Title # Main Title
# ========== # ==========
tText = self._formatHeading(self.fmtTitle, tText) tTemp = self._formatHeading(self.fmtTitle, tToken[2])
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tText, None, self.A_NONE tToken[0],
tToken[1],
tTemp,
None,
self.A_NONE
) )
elif tType == self.T_HEAD2: elif tToken[0] == self.T_HEAD2:
# Novel Chapter # Novel Chapter
# ============= # =============
# Numbered or Unnumbered # Numbered or Unnumbered
if self.isUnNum: if self.isUnNum:
tText = self._formatHeading(self.fmtUnNum, tText) tTemp = self._formatHeading(self.fmtUnNum, tToken[2])
else: else:
self.numChapter += 1 self.numChapter += 1
tText = self._formatHeading(self.fmtChapter, tText) tTemp = self._formatHeading(self.fmtChapter, tToken[2])
# Format the chapter header # Format the chapter header
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tText, None, self.A_PBB tToken[0],
tToken[1],
tTemp,
None,
self.A_PBB
) )
# Set scene variables # Set scene variables
self.firstScene = True self.firstScene = True
self.numChScene = 0 self.numChScene = 0
elif tType == self.T_HEAD3: elif tToken[0] == self.T_HEAD3:
# Novel Scene # Novel Scene
# =========== # ===========
self.numChScene += 1 self.numChScene += 1
self.numAbsScene += 1 self.numAbsScene += 1
tTemp = self._formatHeading(self.fmtScene, tText) tTemp = self._formatHeading(self.fmtScene, tToken[2])
if tTemp == "" and self.hideScene: if tTemp == "" and self.hideScene:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
) )
elif tTemp == "" and not self.hideScene: elif tTemp == "" and not self.hideScene:
if self.firstScene: if self.firstScene:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
) )
else: else:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_SKIP, tLine, "", None, self.A_NONE self.T_SKIP,
tToken[1],
"",
None,
self.A_NONE
) )
elif tTemp == self.fmtScene: elif tTemp == self.fmtScene:
if self.firstScene: if self.firstScene:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
) )
else: else:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_SEP, tLine, tTemp, None, self.A_CENTRE self.T_SEP,
tToken[1],
tTemp,
None,
self.A_CENTRE
) )
else: else:
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tTemp, None, self.A_NONE tToken[0],
tToken[1],
tTemp,
None,
self.A_NONE
) )
# Definitely no longer the first scene # Definitely no longer the first scene
self.firstScene = False self.firstScene = False
elif tType == self.T_HEAD4: elif tToken[0] == self.T_HEAD4:
# Novel Section # Novel Section
# ============= # =============
tTemp = self._formatHeading(self.fmtSection, tText) tTemp = self._formatHeading(self.fmtSection, tToken[2])
if tTemp == "" and self.hideSection: if tTemp == "" and self.hideSection:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
) )
elif tTemp == "" and not self.hideSection: elif tTemp == "" and not self.hideSection:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_SKIP, tLine, "", None, self.A_NONE self.T_SKIP,
tToken[1],
"",
None,
self.A_NONE
) )
elif tTemp == self.fmtSection: elif tTemp == self.fmtSection:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_SEP, tLine, tTemp, None, self.A_CENTRE self.T_SEP,
tToken[1],
tTemp,
None,
self.A_CENTRE
) )
else: else:
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tTemp, None, self.A_NONE tToken[0],
tToken[1],
tTemp,
None,
self.A_NONE
) )
# For title page and partitions, we need to centre all text. # For title page and partitions, we need to centre all text.
@@ -524,22 +589,30 @@ class Tokenizer():
# We also swap header level 1 with a title type instead. # We also swap header level 1 with a title type instead.
if self.isTitle or self.isPart: if self.isTitle or self.isPart:
for n, tToken in enumerate(self.theTokens): for n, tToken in enumerate(self.theTokens):
tType = tToken[0] if tToken[0] == self.T_HEAD1:
tLine = tToken[1]
tText = tToken[2]
tFormat = tToken[3]
if tType == self.T_HEAD1:
if self.isTitle: if self.isTitle:
self.theTokens[n] = ( self.theTokens[n] = (
self.T_TITLE, tLine, tText, tFormat, self.A_PBB_NO | self.A_CENTRE self.T_TITLE,
tToken[1],
tToken[2],
tToken[3],
self.A_PBB_NO | self.A_CENTRE
) )
else: else:
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_PBB | self.A_CENTRE tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_PBB | self.A_CENTRE
) )
else: else:
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_CENTRE tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_CENTRE
) )
# Add a page break after the last entry # Add a page break after the last entry
@@ -547,24 +620,32 @@ class Tokenizer():
if n >= 0: if n >= 0:
tToken = self.theTokens[n] tToken = self.theTokens[n]
self.theTokens[n] = ( self.theTokens[n] = (
tToken[0], tToken[1], tToken[2], tToken[3], tToken[4] | self.A_PBA tToken[0],
tToken[1],
tToken[2],
tToken[3],
tToken[4] | self.A_PBA
) )
# A single page is always left-aligned and starts on a fresh # A single page is always left-aligned and starts on a fresh
# page, unless it's empty. # page, unless it's empty.
if self.isPage: if self.isPage:
for n, tToken in enumerate(self.theTokens): for n, tToken in enumerate(self.theTokens):
tType = tToken[0]
tLine = tToken[1]
tText = tToken[2]
tFormat = tToken[3]
if n == 0: if n == 0:
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_LEFT | self.A_PBB tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_LEFT | self.A_PBB
) )
else: else:
self.theTokens[n] = ( self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_LEFT tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_LEFT
) )
return return
@@ -577,10 +658,11 @@ class Tokenizer():
"""Replaces the %keyword% strings. """Replaces the %keyword% strings.
""" """
theTitle = theTitle.replace(r"%title%", theText) theTitle = theTitle.replace(r"%title%", theText)
theTitle = theTitle.replace(r"%ch%", str(self.numChapter)) theTitle = theTitle.replace(r"%ch%", str(self.numChapter))
theTitle = theTitle.replace(r"%sc%", str(self.numChScene)) theTitle = theTitle.replace(r"%sc%", str(self.numChScene))
theTitle = theTitle.replace(r"%sca%", str(self.numAbsScene)) theTitle = theTitle.replace(r"%sca%", str(self.numAbsScene))
theTitle = theTitle.replace(r"%chw%", numberToWord(self.numChapter,"en")) if r"%chw%" in theTitle:
theTitle = theTitle.replace(r"%chw%", numberToWord(self.numChapter,"en"))
return theTitle return theTitle
# END Class Tokenizer # END Class Tokenizer
nw/gui/build.py
-3
View File
@@ -294,9 +294,6 @@ class GuiBuildNovel(QDialog):
# ============== # ==============
self.buttonForm = QGridLayout() self.buttonForm = QGridLayout()
self.btnHelp = QPushButton("Help")
self.btnHelp.clicked.connect(self._showHelp)
self.btnPrint = QPushButton("Print") self.btnPrint = QPushButton("Print")
self.btnPrint.clicked.connect(self._printDocument) self.btnPrint.clicked.connect(self._printDocument)
sample/nwProject.nwx
+5 -16
View File
@@ -1,5 +1,5 @@
<?xml version='1.0' encoding='utf-8'?> <?xml version='1.0' encoding='utf-8'?>
<novelWriterXML appVersion="0.6.1" hexVersion="0x000601f0" fileVersion="1.1" saveCount="189" autoCount="28" timeStamp="2020-05-28 19:12:56"> <novelWriterXML appVersion="0.6.2" hexVersion="0x000602f0" fileVersion="1.1" saveCount="189" autoCount="30" timeStamp="2020-05-28 20:19:28">
<project> <project>
<name>Sample Project</name> <name>Sample Project</name>
<title>Sample Project</title> <title>Sample Project</title>
@@ -11,8 +11,8 @@
<spellCheck>True</spellCheck> <spellCheck>True</spellCheck>
<autoOutline>True</autoOutline> <autoOutline>True</autoOutline>
<lastEdited>636b6aa9b697b</lastEdited> <lastEdited>636b6aa9b697b</lastEdited>
<lastViewed>ba8a28a246524</lastViewed> <lastViewed>bc0cbd2a407f3</lastViewed>
<lastWordCount>914</lastWordCount> <lastWordCount>941</lastWordCount>
<autoReplace> <autoReplace>
<A>B</A> <A>B</A>
<B>E</B> <B>E</B>
@@ -122,7 +122,7 @@
<charCount>1199</charCount> <charCount>1199</charCount>
<wordCount>216</wordCount> <wordCount>216</wordCount>
<paraCount>7</paraCount> <paraCount>7</paraCount>
<cursorPos>825</cursorPos> <cursorPos>1066</cursorPos>
</item> </item>
<item handle="bc0cbd2a407f3" order="2" parent="e7ded148d6e4a"> <item handle="bc0cbd2a407f3" order="2" parent="e7ded148d6e4a">
<name>Another Scene</name> <name>Another Scene</name>
@@ -174,11 +174,7 @@
<charCount>139</charCount> <charCount>139</charCount>
<wordCount>28</wordCount> <wordCount>28</wordCount>
<paraCount>1</paraCount> <paraCount>1</paraCount>
<<<<<<< HEAD
<cursorPos>237</cursorPos>
=======
<cursorPos>242</cursorPos> <cursorPos>242</cursorPos>
>>>>>>> dev
</item> </item>
<item handle="ae7339df26ded" order="6" parent="e7ded148d6e4a"> <item handle="ae7339df26ded" order="6" parent="e7ded148d6e4a">
<name>We Found John!</name> <name>We Found John!</name>
@@ -191,7 +187,7 @@
<charCount>189</charCount> <charCount>189</charCount>
<wordCount>37</wordCount> <wordCount>37</wordCount>
<paraCount>1</paraCount> <paraCount>1</paraCount>
<cursorPos>236</cursorPos> <cursorPos>224</cursorPos>
</item> </item>
<item handle="f6622b4617424" order="1" parent="None"> <item handle="f6622b4617424" order="1" parent="None">
<name>Characters</name> <name>Characters</name>
@@ -261,17 +257,10 @@
<expanded>False</expanded> <expanded>False</expanded>
<exported>True</exported> <exported>True</exported>
<layout>NOTE</layout> <layout>NOTE</layout>
<<<<<<< HEAD
<charCount>241</charCount> <charCount>241</charCount>
<wordCount>51</wordCount> <wordCount>51</wordCount>
<paraCount>3</paraCount> <paraCount>3</paraCount>
<cursorPos>286</cursorPos>
=======
<charCount>115</charCount>
<wordCount>24</wordCount>
<paraCount>1</paraCount>
<cursorPos>135</cursorPos> <cursorPos>135</cursorPos>
>>>>>>> dev
</item> </item>
<item handle="5eaea4e8cdee8" order="2" parent="15c4492bd5107"> <item handle="5eaea4e8cdee8" order="2" parent="15c4492bd5107">
<name>Mars</name> <name>Mars</name>