Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added header formatting to tokenizer

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2019-10-19 01:17:19 +02:00
parent d4e74848bc
commit 287d94369f
5 changed files with 208 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/convert/tokenizer.py
+153 -13
View File
@@ -17,7 +17,10 @@ import nw
from operator import itemgetter
from PyQt5.QtCore import QRegularExpression
from nw.project.document import NWDoc
from nw.tools.translate import numberToWord
from nw.enum import nwItemLayout
logger = logging.getLogger(__name__)
@@ -38,6 +41,12 @@ class Tokenizer():
T_HEAD3 = 6 # Header 3 (scene)
T_HEAD4 = 7 # Header 4
T_TEXT = 8 # Text line
T_SEP = 9 # Scene separator
A_LEFT = 1 # Left aligned
A_RIGHT = 2 # Right aligned
A_CENTRE = 3 # Centred
A_JUSTIFY = 4 # Justified
def __init__(self, theProject, theParent):
@@ -55,6 +64,17 @@ class Tokenizer():
self.doComments = False
self.doCommands = False
self.fmtTitle = "%title%"
self.fmtUnNum = "%title%"
self.fmtChapter = "Chapter %numword%: %title%"
self.fmtScene = "* * *"
self.fmtSection = "%title%"
self.noSection = True
self.numChapter = 0
self.firstScene = False
return
##
@@ -130,19 +150,19 @@ class Tokenizer():
# Tag lines starting with specific characters
if len(aLine) == 0:
self.theTokens.append((self.T_EMPTY,"",None))
self.theTokens.append((self.T_EMPTY,"",None,self.A_LEFT))
elif aLine[0] == "%":
self.theTokens.append((self.T_COMMENT,aLine[1:].strip(),None))
self.theTokens.append((self.T_COMMENT,aLine[1:].strip(),None,self.A_LEFT))
elif aLine[0] == "@":
self.theTokens.append((self.T_COMMENT,aLine[1:].strip(),None))
self.theTokens.append((self.T_COMMAND,aLine[1:].strip(),None,self.A_LEFT))
elif aLine[:2] == "# ":
self.theTokens.append((self.T_HEAD1,aLine[2:].strip(),None))
self.theTokens.append((self.T_HEAD1,aLine[2:].strip(),None,self.A_LEFT))
elif aLine[:3] == "## ":
self.theTokens.append((self.T_HEAD2,aLine[3:].strip(),None))
self.theTokens.append((self.T_HEAD2,aLine[3:].strip(),None,self.A_LEFT))
elif aLine[:4] == "### ":
self.theTokens.append((self.T_HEAD3,aLine[4:].strip(),None))
self.theTokens.append((self.T_HEAD3,aLine[4:].strip(),None,self.A_LEFT))
elif aLine[:5] == "#### ":
self.theTokens.append((self.T_HEAD4,aLine[5:].strip(),None))
self.theTokens.append((self.T_HEAD4,aLine[5:].strip(),None,self.A_LEFT))
else:
# Otherwise we use RegEx to find formatting tags within a line of text
fmtPos = []
@@ -158,10 +178,73 @@ class Tokenizer():
# Save the line as is, but append the array of formatting locations sorted by position
fmtPos = sorted(fmtPos,key=itemgetter(0))
self.theTokens.append((self.T_TEXT,aLine,fmtPos))
self.theTokens.append((self.T_TEXT,aLine,fmtPos,self.A_LEFT))
# Always add an empty line at the end
self.theTokens.append((self.T_EMPTY,"",None))
self.theTokens.append((self.T_EMPTY,"",None,self.A_LEFT))
return
def doHeaders(self):
isNone = self.theItem.itemLayout == nwItemLayout.NO_LAYOUT
isTitle = self.theItem.itemLayout == nwItemLayout.TITLE
isBook = self.theItem.itemLayout == nwItemLayout.BOOK
isPage = self.theItem.itemLayout == nwItemLayout.PAGE
isPart = self.theItem.itemLayout == nwItemLayout.PARTITION
isUnNum = self.theItem.itemLayout == nwItemLayout.UNNUMBERED
isChap = self.theItem.itemLayout == nwItemLayout.CHAPTER
isScene = self.theItem.itemLayout == nwItemLayout.SCENE
isNote = self.theItem.itemLayout == nwItemLayout.NOTE
# No special header formatting for notes and no layout files
if isNone: return
if isNote: return
# For novel files, we need to handle chapter numbering and scene breaks
if isBook or isUnNum or isChap or isScene:
for n in range(len(self.theTokens)):
tToken = self.theTokens[n]
tType = tToken[0]
tText = tToken[1]
if tType == self.T_TEXT:
self.firstScene = False
elif tType == self.T_HEAD2:
if not isUnNum:
self.numChapter += 1
tText = self._doFormatChapter(tText,isUnNum)
self.theTokens[n] = (tType,tText,None,self.A_LEFT)
self.firstScene = True
elif tType == self.T_HEAD3:
tTemp = self._doFormatScene(tText)
if tTemp == self.fmtScene:
if self.firstScene:
self.theTokens[n] = (self.T_EMPTY,"",None,self.A_LEFT)
else:
self.theTokens[n] = (self.T_SEP,tTemp,None,self.A_LEFT)
else:
self.theTokens[n] = (tType,tTemp,None,self.A_LEFT)
self.firstScene = False
elif tType == self.T_HEAD4:
if self.noSection:
self.theTokens[n] = (self.T_EMPTY,"",None,self.A_LEFT)
else:
tTemp = self._doFormatSection(tText)
self.theTokens[n] = (self.T_SEP,tTemp,None,self.A_LEFT)
# For title page and partitions, we need to centre all text
if isTitle or isPart:
for n in range(len(self.theTokens)):
tToken = self.theTokens[n]
tType = tToken[0]
tText = tToken[1]
tFormat = tToken[2]
self.theTokens[n] = (tType,tText,tFormat,self.A_CENTRE)
return
@@ -186,11 +269,11 @@ class Tokenizer():
self.theResult = ""
thisPar = []
for tType, tText, tFormat in self.theTokens:
for tType, tText, tFormat, tAlign in self.theTokens:
# First check if we have a comment or plain text, as they need some
# extra replacing before we proceed to wrapping and final formatting.
if tType == self.T_COMMAND:
if tType == self.T_COMMENT:
tText = "[%s]" % tText
elif tType == self.T_TEXT:
@@ -202,8 +285,18 @@ class Tokenizer():
tLen = len(tText)
# The text can now be word wrapped, if we have requested this and it's needed.
if self.wordWrap > 0 and tLen > self.wordWrap:
tText = tWrap.fill(tText)
if tAlign == self.A_CENTRE:
if self.wordWrap > 0:
if tLen > self.wordWrap:
aText = tWrap.wrap(tText)
for n in range(len(aText)):
aText[n] = self._centreText(aText[n],self.wordWrap)
tText = "\n".join(aText)
else:
tText = self._centreText(tText,self.wordWrap)
else:
if self.wordWrap > 0 and tLen > self.wordWrap:
tText = tWrap.fill(tText)
# Then the text can receive final formatting before we append it to the results.
# We also store text lines in a buffer and merge them only when we find an empty line,
@@ -215,6 +308,8 @@ class Tokenizer():
elif tType == self.T_HEAD1:
uLine = "="*min(tLen,self.wordWrap)
if tAlign == self.A_CENTRE:
uLine = self._centreText(uLine,self.wordWrap)
self.theResult += "%s\n%s\n\n" % (tText,uLine)
elif tType == self.T_HEAD2:
@@ -228,6 +323,11 @@ class Tokenizer():
elif tType == self.T_HEAD4:
self.theResult += "%s\n\n" % tText
elif tType == self.T_SEP:
if self.wordWrap > 0 and tLen < self.wordWrap:
tText = self._centreText(tText,self.wordWrap)
self.theResult += "%s\n\n" % tText
elif tType == self.T_TEXT:
thisPar.append(tText)
@@ -239,4 +339,44 @@ class Tokenizer():
return
def windowsEndings(self):
self.theResult = self.theResult.replace("\n","\r\n")
return
##
# Internal Functions
##
def _doFormatTitle(self, theText):
theTitle = self.fmtTitle
theTitle = theTitle.replace("%title%", theText)
return theTitle
def _doFormatChapter(self, theText, noNum):
if noNum:
theTitle = self.fmtUnNum
theTitle = theTitle.replace("%title%", theText)
else:
theTitle = self.fmtChapter
theTitle = theTitle.replace("%title%", theText)
theTitle = theTitle.replace("%num%", str(self.numChapter))
theTitle = theTitle.replace("%numword%", numberToWord(self.numChapter,"en"))
return theTitle
def _doFormatScene(self, theText):
theTitle = self.fmtScene
theTitle = theTitle.replace("%title%", theText)
return theTitle
def _doFormatSection(self, theText):
theTitle = self.fmtSection
theTitle = theTitle.replace("%title%", theText)
return theTitle
def _centreText(self, theText, theWidth):
tLen = len(theText)
if tLen < theWidth:
return " "*int((theWidth-tLen)/2) + theText
return theText
# END Class Tokenizer