Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Some cleanup of the tokenizer and tohtml classes

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2020-05-10 14:44:44 +02:00
parent 33f4626aec
commit 2eaa707f4e
3 changed files with 80 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/convert/file/text.py
+1 -1
View File
@@ -143,7 +143,7 @@ class TextFile():
self.theConv.setText(tHandle)
self.theConv.doAutoReplace()
self.theConv.tokenizeText()
self.theConv.doHeaders()
self.theConv.formatHeaders()
self.theConv.doConvert()
self.theConv.doPostProcessing()
nw/core/tohtml.py
+20 -2
View File
@@ -41,6 +41,10 @@ class ToHtml(Tokenizer):
self.forPreview = False
return
##
# Setters
##
def setPreview(self, forPreview, doComments):
"""If we're using this class to generate markdown preview, we
need to make a few changes to formatting, which is selected by
@@ -52,7 +56,14 @@ class ToHtml(Tokenizer):
self.doComments = doComments
return
##
# Class Methods
##
def doAutoReplace(self):
"""Extend the auto-replace to also properly encode some unicode
characters into their respective HTML entities.
"""
Tokenizer.doAutoReplace(self)
if self.forPreview:
@@ -76,6 +87,9 @@ class ToHtml(Tokenizer):
return
def doConvert(self):
"""Convert the list of text tokens into a HTML document saved
to theResult.
"""
htmlTags = {
self.FMT_B_B : "<strong>",
@@ -136,7 +150,7 @@ class ToHtml(Tokenizer):
self.theResult += self._formatComments(tText)
elif tType == self.T_KEYWORD and self.doKeywords:
self.theResult += self._formatTags(tText)
self.theResult += self._formatKeywords(tText)
return
@@ -144,7 +158,9 @@ class ToHtml(Tokenizer):
# Internal Functions
##
def _formatTags(self, tText):
def _formatKeywords(self, tText):
"""Apply HTML formatting to keywords.
"""
if not self.forPreview:
return "<pre>@%s</pre>\n" % tText
@@ -167,6 +183,8 @@ class ToHtml(Tokenizer):
return "<div>%s</div>" % retText
def _formatComments(self, tText):
"""Apply HTML formatting to comments.
"""
if not self.forPreview:
return "<div class='comment'>%s</div>\n" % tText
nw/core/tokenizer.py
+59 -39
View File
@@ -70,30 +70,45 @@ class Tokenizer():
self.theProject = theProject
self.theParent = theParent
self.theText = None
self.theHandle = None
self.theItem = None
self.theTokens = None
self.theResult = None
# Data Variables
self.theText = None # The raw text to be tokenized
self.theHandle = None # The handle associated with the text
self.theItem = None # The NWItem associated with the handle
self.theTokens = None # The list of the processed tokens
self.theResult = None # The result text after conversion
self.wordWrap = 0
self.doComments = False
self.doKeywords = False
# User Settings
self.doComments = False # Also process comments
self.doKeywords = False # Also process keywords like tags and references
self.fmtTitle = "%title%"
self.fmtChapter = "%title%"
self.fmtUnNum = "%title%"
self.fmtScene = "%title%"
self.fmtSection = "%title%"
self.fmtTitle = "%title%" # Formatting for titles
self.fmtChapter = "%title%" # Formatting for numbered chapters
self.fmtUnNum = "%title%" # Formatting for unnumbered chapters
self.fmtScene = "%title%" # Formatting for scenes
self.fmtSection = "%title%" # Formatting for sections
self.hideScene = False
self.hideSection = False
self.hideScene = False # Do not include scene headers
self.hideSection = False # Do not include section headers
self.numChapter = 0
self.firstScene = False
# Instance Variables
self.numChapter = 0 # Counter for chapter numbers
self.firstScene = False # Flag to indicate that the first scene of the chapter
return
def clearData(self):
"""Clear the data arrays and variables, but not settings, so the class
can be reused for multiple documents.
"""
self.theText = None
self.theHandle = None
self.theItem = None
self.theTokens = None
self.theResult = None
self.numChapter = 0
self.firstScene = False
return
##
# Setters
##
@@ -106,13 +121,6 @@ class Tokenizer():
self.doKeywords = doKeywords
return
def setWordWrap(self, wordWrap):
if wordWrap >= 0:
self.wordWrap = wordWrap
else:
self.wordWrap = 0
return
def setTitleFormat(self, fmtTitle):
self.fmtTitle = fmtTitle
return
@@ -140,6 +148,9 @@ class Tokenizer():
##
def setText(self, theHandle, theText=None):
"""Set the text for the tokenizer from a handle. If theText is
not set, load it from the file.
"""
self.theHandle = theHandle
self.theItem = self.theProject.projTree[theHandle]
@@ -155,12 +166,16 @@ class Tokenizer():
return
def doAutoReplace(self):
"""Run through the user's auto-replace dictionary.
"""
if len(self.theProject.autoReplace) > 0:
repDict = {}
for aKey, aVal in self.theProject.autoReplace.items():
repDict["<%s>" % aKey] = aVal
xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
self.theText = xRep.sub(lambda x: repDict[x.group(0)], self.theText)
return
def doPostProcessing(self):
@@ -229,6 +244,9 @@ class Tokenizer():
return
def doHeaders(self):
"""Apply formatting to the text headers according to document
layout and user settings.
"""
isNone = self.theItem.itemLayout == nwItemLayout.NO_LAYOUT
isTitle = self.theItem.itemLayout == nwItemLayout.TITLE
@@ -241,8 +259,8 @@ class Tokenizer():
isNote = self.theItem.itemLayout == nwItemLayout.NOTE
# No special header formatting for notes and no-layout files
if isNone: return
if isNote: return
if isNone or isNote:
return
# For novel files, we need to handle chapter numbering and scene
# breaks
@@ -259,12 +277,12 @@ class Tokenizer():
elif tType == self.T_HEAD2:
if not isUnNum:
self.numChapter += 1
tText = self._doFormatChapter(tText,isUnNum)
tText = self._formatChapter(tText,isUnNum)
self.theTokens[n] = (tType,tText,None,self.A_LEFT)
self.firstScene = True
elif tType == self.T_HEAD3:
tTemp = self._doFormatScene(tText)
tTemp = self._formatScene(tText)
if tTemp == "" and self.hideScene:
self.theTokens[n] = (self.T_EMPTY,"",None,self.A_LEFT)
elif tTemp == "" and not self.hideScene:
@@ -282,7 +300,7 @@ class Tokenizer():
self.firstScene = False
elif tType == self.T_HEAD4:
tTemp = self._doFormatSection(tText)
tTemp = self._formatSection(tText)
if tTemp == "" and self.hideSection:
self.theTokens[n] = (self.T_EMPTY,"",None,self.A_LEFT)
elif tTemp == "" and not self.hideSection:
@@ -310,12 +328,16 @@ class Tokenizer():
# Internal Functions
##
def _doFormatTitle(self, theText):
def _formatTitle(self, theText):
"""Replace tokens for headers level 1.
"""
theTitle = self.fmtTitle
theTitle = theTitle.replace("%title%", theText)
return theTitle
def _doFormatChapter(self, theText, noNum):
def _formatChapter(self, theText, noNum):
"""Replace tokens for headers level 2.
"""
if noNum:
theTitle = self.fmtUnNum
theTitle = theTitle.replace("%title%", theText)
@@ -326,20 +348,18 @@ class Tokenizer():
theTitle = theTitle.replace("%numword%", numberToWord(self.numChapter,"en"))
return theTitle
def _doFormatScene(self, theText):
def _formatScene(self, theText):
"""Replace tokens for headers level 3.
"""
theTitle = self.fmtScene
theTitle = theTitle.replace("%title%", theText)
return theTitle
def _doFormatSection(self, theText):
def _formatSection(self, theText):
"""Replace tokens for headers level 4.
"""
theTitle = self.fmtSection
theTitle = theTitle.replace("%title%", theText)
return theTitle
def _centreText(self, theText, theWidth):
tLen = len(theText)
if tLen < theWidth:
return " "*int((theWidth-tLen)/2) + theText
return theText
# END Class Tokenizer