Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved the tokenizer class a bit

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2020-05-28 20:19:55 +02:00
parent 65f74e6fdc
commit 2bbde2c445
3 changed files with 141 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/core/tokenizer.py
+136 -54
View File
@@ -320,7 +320,9 @@ class Tokenizer():
# Tag lines starting with specific characters
if len(aLine.strip()) == 0:
self.theTokens.append((
self.T_EMPTY, nLine, "", None, self.A_NONE
self.T_EMPTY, nLine,
"", None,
self.A_NONE
))
tmpMarkdown.append("\n")
@@ -328,45 +330,59 @@ class Tokenizer():
cLine = aLine[1:].strip()
if cLine.lower().startswith("synopsis:"):
self.theTokens.append((
self.T_SYNOPSIS, nLine, cLine[9:].strip(), None, self.A_NONE
self.T_SYNOPSIS, nLine,
cLine[9:].strip(), None,
self.A_NONE
))
if self.doSynopsis:
tmpMarkdown.append("%s\n" % aLine)
else:
self.theTokens.append((
self.T_COMMENT, nLine, aLine[1:].strip(), None, self.A_NONE
self.T_COMMENT, nLine,
aLine[1:].strip(), None,
self.A_NONE
))
if self.doComments:
tmpMarkdown.append("%s\n" % aLine)
elif aLine[0] == "@":
self.theTokens.append((
self.T_KEYWORD, nLine, aLine[1:].strip(), None, self.A_NONE
self.T_KEYWORD, nLine,
aLine[1:].strip(), None,
self.A_NONE
))
if self.doKeywords:
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:2] == "# ":
self.theTokens.append((
self.T_HEAD1, nLine, aLine[2:].strip(), None, self.A_NONE
self.T_HEAD1, nLine,
aLine[2:].strip(), None,
self.A_NONE
))
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:3] == "## ":
self.theTokens.append((
self.T_HEAD2, nLine, aLine[3:].strip(), None, self.A_NONE
self.T_HEAD2, nLine,
aLine[3:].strip(), None,
self.A_NONE
))
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:4] == "### ":
self.theTokens.append((
self.T_HEAD3, nLine, aLine[4:].strip(), None, self.A_NONE
self.T_HEAD3, nLine,
aLine[4:].strip(), None,
self.A_NONE
))
tmpMarkdown.append("%s\n" % aLine)
elif aLine[:5] == "#### ":
self.theTokens.append((
self.T_HEAD4, nLine, aLine[5:].strip(), None, self.A_NONE
self.T_HEAD4, nLine,
aLine[5:].strip(), None,
self.A_NONE
))
tmpMarkdown.append("%s\n" % aLine)
@@ -391,13 +407,17 @@ class Tokenizer():
# sorted by position
fmtPos = sorted(fmtPos, key=itemgetter(0))
self.theTokens.append((
self.T_TEXT, nLine, aLine, fmtPos, self.A_NONE
self.T_TEXT, nLine,
aLine, fmtPos,
self.A_NONE
))
tmpMarkdown.append("%s\n" % aLine)
# Always add an empty line at the end
self.theTokens.append((
self.T_EMPTY, nLine, "", None, self.A_NONE
self.T_EMPTY, nLine,
"", None,
self.A_NONE
))
tmpMarkdown.append("\n")
@@ -421,101 +441,146 @@ class Tokenizer():
for n in range(len(self.theTokens)):
tToken = self.theTokens[n]
tType = tToken[0]
tLine = tToken[1]
tText = tToken[2]
# In case we see text before a scene, we reset the flag
if tType == self.T_TEXT:
if tToken[0] == self.T_TEXT:
self.firstScene = False
elif tType == self.T_HEAD1:
elif tToken[0] == self.T_HEAD1:
# Main Title
# ==========
tText = self._formatHeading(self.fmtTitle, tText)
tTemp = self._formatHeading(self.fmtTitle, tToken[2])
self.theTokens[n] = (
tType, tLine, tText, None, self.A_NONE
tToken[0],
tToken[1],
tTemp,
None,
self.A_NONE
)
elif tType == self.T_HEAD2:
elif tToken[0] == self.T_HEAD2:
# Novel Chapter
# =============
# Numbered or Unnumbered
if self.isUnNum:
tText = self._formatHeading(self.fmtUnNum, tText)
tTemp = self._formatHeading(self.fmtUnNum, tToken[2])
else:
self.numChapter += 1
tText = self._formatHeading(self.fmtChapter, tText)
tTemp = self._formatHeading(self.fmtChapter, tToken[2])
# Format the chapter header
self.theTokens[n] = (
tType, tLine, tText, None, self.A_PBB
tToken[0],
tToken[1],
tTemp,
None,
self.A_PBB
)
# Set scene variables
self.firstScene = True
self.numChScene = 0
elif tType == self.T_HEAD3:
elif tToken[0] == self.T_HEAD3:
# Novel Scene
# ===========
self.numChScene += 1
self.numAbsScene += 1
tTemp = self._formatHeading(self.fmtScene, tText)
tTemp = self._formatHeading(self.fmtScene, tToken[2])
if tTemp == "" and self.hideScene:
self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE
self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
)
elif tTemp == "" and not self.hideScene:
if self.firstScene:
self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE
self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
)
else:
self.theTokens[n] = (
self.T_SKIP, tLine, "", None, self.A_NONE
self.T_SKIP,
tToken[1],
"",
None,
self.A_NONE
)
elif tTemp == self.fmtScene:
if self.firstScene:
self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE
self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
)
else:
self.theTokens[n] = (
self.T_SEP, tLine, tTemp, None, self.A_CENTRE
self.T_SEP,
tToken[1],
tTemp,
None,
self.A_CENTRE
)
else:
self.theTokens[n] = (
tType, tLine, tTemp, None, self.A_NONE
tToken[0],
tToken[1],
tTemp,
None,
self.A_NONE
)
# Definitely no longer the first scene
self.firstScene = False
elif tType == self.T_HEAD4:
elif tToken[0] == self.T_HEAD4:
# Novel Section
# =============
tTemp = self._formatHeading(self.fmtSection, tText)
tTemp = self._formatHeading(self.fmtSection, tToken[2])
if tTemp == "" and self.hideSection:
self.theTokens[n] = (
self.T_EMPTY, tLine, "", None, self.A_NONE
self.T_EMPTY,
tToken[1],
"",
None,
self.A_NONE
)
elif tTemp == "" and not self.hideSection:
self.theTokens[n] = (
self.T_SKIP, tLine, "", None, self.A_NONE
self.T_SKIP,
tToken[1],
"",
None,
self.A_NONE
)
elif tTemp == self.fmtSection:
self.theTokens[n] = (
self.T_SEP, tLine, tTemp, None, self.A_CENTRE
self.T_SEP,
tToken[1],
tTemp,
None,
self.A_CENTRE
)
else:
self.theTokens[n] = (
tType, tLine, tTemp, None, self.A_NONE
tToken[0],
tToken[1],
tTemp,
None,
self.A_NONE
)
# For title page and partitions, we need to centre all text.
@@ -524,22 +589,30 @@ class Tokenizer():
# We also swap header level 1 with a title type instead.
if self.isTitle or self.isPart:
for n, tToken in enumerate(self.theTokens):
tType = tToken[0]
tLine = tToken[1]
tText = tToken[2]
tFormat = tToken[3]
if tType == self.T_HEAD1:
if tToken[0] == self.T_HEAD1:
if self.isTitle:
self.theTokens[n] = (
self.T_TITLE, tLine, tText, tFormat, self.A_PBB_NO | self.A_CENTRE
self.T_TITLE,
tToken[1],
tToken[2],
tToken[3],
self.A_PBB_NO | self.A_CENTRE
)
else:
self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_PBB | self.A_CENTRE
tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_PBB | self.A_CENTRE
)
else:
self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_CENTRE
tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_CENTRE
)
# Add a page break after the last entry
@@ -547,24 +620,32 @@ class Tokenizer():
if n >= 0:
tToken = self.theTokens[n]
self.theTokens[n] = (
tToken[0], tToken[1], tToken[2], tToken[3], tToken[4] | self.A_PBA
tToken[0],
tToken[1],
tToken[2],
tToken[3],
tToken[4] | self.A_PBA
)
# A single page is always left-aligned and starts on a fresh
# page, unless it's empty.
if self.isPage:
for n, tToken in enumerate(self.theTokens):
tType = tToken[0]
tLine = tToken[1]
tText = tToken[2]
tFormat = tToken[3]
if n == 0:
self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_LEFT | self.A_PBB
tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_LEFT | self.A_PBB
)
else:
self.theTokens[n] = (
tType, tLine, tText, tFormat, self.A_LEFT
tToken[0],
tToken[1],
tToken[2],
tToken[3],
self.A_LEFT
)
return
@@ -577,10 +658,11 @@ class Tokenizer():
"""Replaces the %keyword% strings.
"""
theTitle = theTitle.replace(r"%title%", theText)
theTitle = theTitle.replace(r"%ch%", str(self.numChapter))
theTitle = theTitle.replace(r"%sc%", str(self.numChScene))
theTitle = theTitle.replace(r"%sca%", str(self.numAbsScene))
theTitle = theTitle.replace(r"%chw%", numberToWord(self.numChapter,"en"))
theTitle = theTitle.replace(r"%ch%", str(self.numChapter))
theTitle = theTitle.replace(r"%sc%", str(self.numChScene))
theTitle = theTitle.replace(r"%sca%", str(self.numAbsScene))
if r"%chw%" in theTitle:
theTitle = theTitle.replace(r"%chw%", numberToWord(self.numChapter,"en"))
return theTitle
# END Class Tokenizer
nw/gui/build.py
-3
View File
@@ -294,9 +294,6 @@ class GuiBuildNovel(QDialog):
# ==============
self.buttonForm = QGridLayout()
self.btnHelp = QPushButton("Help")
self.btnHelp.clicked.connect(self._showHelp)
self.btnPrint = QPushButton("Print")
self.btnPrint.clicked.connect(self._printDocument)
sample/nwProject.nwx
+5 -16
View File
@@ -1,5 +1,5 @@
<?xml version='1.0' encoding='utf-8'?>
<novelWriterXML appVersion="0.6.1" hexVersion="0x000601f0" fileVersion="1.1" saveCount="189" autoCount="28" timeStamp="2020-05-28 19:12:56">
<novelWriterXML appVersion="0.6.2" hexVersion="0x000602f0" fileVersion="1.1" saveCount="189" autoCount="30" timeStamp="2020-05-28 20:19:28">
<project>
<name>Sample Project</name>
<title>Sample Project</title>
@@ -11,8 +11,8 @@
<spellCheck>True</spellCheck>
<autoOutline>True</autoOutline>
<lastEdited>636b6aa9b697b</lastEdited>
<lastViewed>ba8a28a246524</lastViewed>
<lastWordCount>914</lastWordCount>
<lastViewed>bc0cbd2a407f3</lastViewed>
<lastWordCount>941</lastWordCount>
<autoReplace>
<A>B</A>
<B>E</B>
@@ -122,7 +122,7 @@
<charCount>1199</charCount>
<wordCount>216</wordCount>
<paraCount>7</paraCount>
<cursorPos>825</cursorPos>
<cursorPos>1066</cursorPos>
</item>
<item handle="bc0cbd2a407f3" order="2" parent="e7ded148d6e4a">
<name>Another Scene</name>
@@ -174,11 +174,7 @@
<charCount>139</charCount>
<wordCount>28</wordCount>
<paraCount>1</paraCount>
<<<<<<< HEAD
<cursorPos>237</cursorPos>
=======
<cursorPos>242</cursorPos>
>>>>>>> dev
</item>
<item handle="ae7339df26ded" order="6" parent="e7ded148d6e4a">
<name>We Found John!</name>
@@ -191,7 +187,7 @@
<charCount>189</charCount>
<wordCount>37</wordCount>
<paraCount>1</paraCount>
<cursorPos>236</cursorPos>
<cursorPos>224</cursorPos>
</item>
<item handle="f6622b4617424" order="1" parent="None">
<name>Characters</name>
@@ -261,17 +257,10 @@
<expanded>False</expanded>
<exported>True</exported>
<layout>NOTE</layout>
<<<<<<< HEAD
<charCount>241</charCount>
<wordCount>51</wordCount>
<paraCount>3</paraCount>
<cursorPos>286</cursorPos>
=======
<charCount>115</charCount>
<wordCount>24</wordCount>
<paraCount>1</paraCount>
<cursorPos>135</cursorPos>
>>>>>>> dev
</item>
<item handle="5eaea4e8cdee8" order="2" parent="15c4492bd5107">
<name>Mars</name>