Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop empty tokens from the tokenizer data

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-05-21 22:45:01 +02:00
parent adf0550571
commit 7a13b14c15
5 changed files with 31 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tohtml.py
+2 -5
View File
@@ -238,8 +238,8 @@ class ToHtml(Tokenizer):
aNm = ""
# Process Text Type
if tType == self.T_EMPTY:
pass
if tType == self.T_TEXT:
lines.append(f"<p{hStyle}>{self._formatText(tText, tFormat, hTags)}</p>\n")
elif tType == self.T_TITLE:
tHead = tText.replace(nwHeadFmt.BR, "<br>")
@@ -267,9 +267,6 @@ class ToHtml(Tokenizer):
elif tType == self.T_SKIP:
lines.append(f"<p class='skip'{hStyle}>&nbsp;</p>\n")
elif tType == self.T_TEXT:
lines.append(f"<p{hStyle}>{self._formatText(tText, tFormat, hTags)}</p>\n")
elif tType == self.T_SYNOPSIS and self._doSynopsis:
lines.append(self._formatSynopsis(self._formatText(tText, tFormat, hTags), True))
novelwriter/core/tokenizer.py
+15 -23
View File
@@ -840,9 +840,8 @@ class Tokenizer(ABC):
nToken = tokens[n+1] # Look ahead
if cToken[0] == self.T_EMPTY:
# Strip multiple empty
if pToken[0] != self.T_EMPTY:
self._tokens.append(cToken)
# We don't need to keep the empty lines after this pass
pass
elif cToken[0] == self.T_KEYWORD:
# Adjust margins for lines in a list of keyword lines
@@ -918,7 +917,6 @@ class Tokenizer(ABC):
textWordChars = self._counts.get("textWordChars", 0)
titleWordChars = self._counts.get("titleWordChars", 0)
para = []
for tType, _, tText, _, _ in self._tokens:
tText = tText.replace(nwUnicode.U_ENDASH, " ")
tText = tText.replace(nwUnicode.U_EMDASH, " ")
@@ -928,24 +926,7 @@ class Tokenizer(ABC):
nChars = len(tText)
nWChars = len("".join(tWords))
if tType == self.T_EMPTY:
if len(para) > 0:
tTemp = "\n".join(para)
tPWords = tTemp.split()
nPWords = len(tPWords)
nPChars = len(tTemp)
nPWChars = len("".join(tPWords))
paragraphCount += 1
allWords += nPWords
textWords += nPWords
allChars += nPChars
textChars += nPChars
allWordChars += nPWChars
textWordChars += nPWChars
para = []
elif tType in self.L_HEADINGS:
if tType in self.L_HEADINGS:
titleCount += 1
allWords += nWords
titleWords += nWords
@@ -960,7 +941,18 @@ class Tokenizer(ABC):
allWordChars += nWChars
elif tType == self.T_TEXT:
para.append(tText.rstrip())
tPWords = tText.split()
nPWords = len(tPWords)
nPChars = len(tText)
nPWChars = len("".join(tPWords))
paragraphCount += 1
allWords += nPWords
textWords += nPWords
allChars += nPChars
textChars += nPChars
allWordChars += nPWChars
textWordChars += nPWChars
elif tType == self.T_SYNOPSIS and self._doSynopsis:
text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
novelwriter/core/tomarkdown.py
+3 -6
View File
@@ -142,8 +142,9 @@ class ToMarkdown(Tokenizer):
lines = []
for tType, _, tText, tFormat, tStyle in self._tokens:
if tType == self.T_EMPTY:
pass
if tType == self.T_TEXT:
tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n")
lines.append(f"{tTemp}\n\n")
elif tType == self.T_TITLE:
tHead = tText.replace(nwHeadFmt.BR, "\n")
@@ -171,10 +172,6 @@ class ToMarkdown(Tokenizer):
elif tType == self.T_SKIP:
lines.append(f"{cSkip}\n\n")
elif tType == self.T_TEXT:
tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n")
lines.append(f"{tTemp}\n\n")
elif tType == self.T_SYNOPSIS and self._doSynopsis:
label = self._localLookup("Synopsis")
lines.append(f"**{label}:** {self._formatText(tText, tFormat, mTags)}\n\n")
novelwriter/core/toodt.py
+6 -9
View File
@@ -454,8 +454,12 @@ class ToOdt(Tokenizer):
pIndent = False
# Process Text Types
if tType == self.T_EMPTY:
pass
if tType == self.T_TEXT:
if self._firstIndent and pIndent and oStyle.isUnaligned():
self._addTextPar(xText, S_FIND, oStyle, tText, tFmt=tFormat)
else:
self._addTextPar(xText, S_TEXT, oStyle, tText, tFmt=tFormat)
pIndent = True
elif tType == self.T_TITLE:
# Title must be text:p
@@ -484,13 +488,6 @@ class ToOdt(Tokenizer):
elif tType == self.T_SKIP:
self._addTextPar(xText, S_SEP, oStyle, "")
elif tType == self.T_TEXT:
if self._firstIndent and pIndent and oStyle.isUnaligned():
self._addTextPar(xText, S_FIND, oStyle, tText, tFmt=tFormat)
else:
self._addTextPar(xText, S_TEXT, oStyle, tText, tFmt=tFormat)
pIndent = True
elif tType == self.T_SYNOPSIS and self._doSynopsis:
tTemp, tFmt = self._formatSynopsis(tText, tFormat, True)
self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt)