Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added left/right indentation to Tokenizer class

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2021-06-10 00:04:54 +02:00
parent e0de3ff191
commit 80c4826253
3 changed files with 82 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/core/tokenizer.py
+22 -1
View File
@@ -75,6 +75,8 @@ class Tokenizer():
A_PBA_AUT = 0x0080 # Page break after auto A_PBA_AUT = 0x0080 # Page break after auto
A_Z_TOPMRG = 0x0100 # Zero top margin A_Z_TOPMRG = 0x0100 # Zero top margin
A_Z_BTMMRG = 0x0200 # Zero bottom margin A_Z_BTMMRG = 0x0200 # Zero bottom margin
A_IND_L = 0x0400 # Left indentation
A_IND_R = 0x0800 # Right indentation
def __init__(self, theProject): def __init__(self, theProject):
@@ -442,15 +444,23 @@ class Tokenizer():
# Skip all body text # Skip all body text
continue continue
# Check Alignment # Check Alignment and Indentation
tagLeft = False tagLeft = False
tagRight = False tagRight = False
indLeft = False
indRight = False
if aLine.startswith(">>"): if aLine.startswith(">>"):
tagRight = True tagRight = True
aLine = aLine[2:].lstrip() aLine = aLine[2:].lstrip()
elif aLine.startswith(">>"): elif aLine.startswith(">>"):
tagRight = True tagRight = True
aLine = aLine[8:].lstrip() aLine = aLine[8:].lstrip()
elif aLine.startswith(">"):
indLeft = True
aLine = aLine[1:].lstrip()
elif aLine.startswith(">"):
indLeft = True
aLine = aLine[4:].lstrip()
if aLine.endswith("<<"): if aLine.endswith("<<"):
tagLeft = True tagLeft = True
@@ -458,6 +468,12 @@ class Tokenizer():
elif aLine.endswith("&lt;&lt;"): elif aLine.endswith("&lt;&lt;"):
tagLeft = True tagLeft = True
aLine = aLine[:-8].rstrip() aLine = aLine[:-8].rstrip()
elif aLine.endswith("<"):
indRight = True
aLine = aLine[:-1].rstrip()
elif aLine.endswith("&lt;"):
indRight = True
aLine = aLine[:-4].rstrip()
textAlign = defAlign textAlign = defAlign
if tagLeft and tagRight: if tagLeft and tagRight:
@@ -467,6 +483,11 @@ class Tokenizer():
elif tagRight: elif tagRight:
textAlign = self.A_RIGHT textAlign = self.A_RIGHT
if indLeft:
textAlign |= self.A_IND_L
if indRight:
textAlign |= self.A_IND_R
# Otherwise we use RegEx to find formatting tags within a line of text # Otherwise we use RegEx to find formatting tags within a line of text
fmtPos = [] fmtPos = []
for theRX, theKeys in rxFormats: for theRX, theKeys in rxFormats:
nw/gui/dochighlight.py
+21 -21
View File
@@ -106,27 +106,27 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self.colEmph = QColor(*self.theTheme.colEmph) self.colEmph = QColor(*self.theTheme.colEmph)
self.hStyles = { self.hStyles = {
"header1" : self._makeFormat(self.colHead, "bold", 1.8), "header1" : self._makeFormat(self.colHead, "bold", 1.8),
"header2" : self._makeFormat(self.colHead, "bold", 1.6), "header2" : self._makeFormat(self.colHead, "bold", 1.6),
"header3" : self._makeFormat(self.colHead, "bold", 1.4), "header3" : self._makeFormat(self.colHead, "bold", 1.4),
"header4" : self._makeFormat(self.colHead, "bold", 1.2), "header4" : self._makeFormat(self.colHead, "bold", 1.2),
"header1h" : self._makeFormat(self.colHeadH, "bold", 1.8), "header1h" : self._makeFormat(self.colHeadH, "bold", 1.8),
"header2h" : self._makeFormat(self.colHeadH, "bold", 1.6), "header2h" : self._makeFormat(self.colHeadH, "bold", 1.6),
"header3h" : self._makeFormat(self.colHeadH, "bold", 1.4), "header3h" : self._makeFormat(self.colHeadH, "bold", 1.4),
"header4h" : self._makeFormat(self.colHeadH, "bold", 1.2), "header4h" : self._makeFormat(self.colHeadH, "bold", 1.2),
"bold" : self._makeFormat(self.colEmph, "bold"), "bold" : self._makeFormat(self.colEmph, "bold"),
"italic" : self._makeFormat(self.colEmph, "italic"), "italic" : self._makeFormat(self.colEmph, "italic"),
"strike" : self._makeFormat(self.colHidden, "strike"), "strike" : self._makeFormat(self.colHidden, "strike"),
"mspaces" : self._makeFormat(self.colError, "errline"), "mspaces" : self._makeFormat(self.colError, "errline"),
"nobreak" : self._makeFormat(self.colBreak, "background"), "nobreak" : self._makeFormat(self.colBreak, "background"),
"dialogue1" : self._makeFormat(self.colDialN), "dialogue1" : self._makeFormat(self.colDialN),
"dialogue2" : self._makeFormat(self.colDialD), "dialogue2" : self._makeFormat(self.colDialD),
"dialogue3" : self._makeFormat(self.colDialS), "dialogue3" : self._makeFormat(self.colDialS),
"replace" : self._makeFormat(self.colRepTag), "replace" : self._makeFormat(self.colRepTag),
"hidden" : self._makeFormat(self.colHidden), "hidden" : self._makeFormat(self.colHidden),
"keyword" : self._makeFormat(self.colKey), "keyword" : self._makeFormat(self.colKey),
"modifier" : self._makeFormat(self.colMod), "modifier" : self._makeFormat(self.colMod),
"value" : self._makeFormat(self.colVal, "underline"), "value" : self._makeFormat(self.colVal, "underline"),
} }
self.hRules = [] self.hRules = []
tests/test_core/test_core_tokenizer.py
+39 -5
View File
@@ -406,12 +406,18 @@ def testCoreToken_Tokenize(dummyGUI):
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n" "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
) )
# Alignment # Alignment and Indentation
dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
rIndAlign = Tokenizer.A_RIGHT | Tokenizer.A_IND_R
theToken.theText = ( theToken.theText = (
"Some regular text\n\n" "Some regular text\n\n"
"Some left-aligned text <<\n\n" "Some left-aligned text <<\n\n"
">> Some right-aligned text\n\n" ">> Some right-aligned text\n\n"
">> Some centered text <<\n\n" ">> Some centered text <<\n\n"
"> Left-indented block\n\n"
"Right-indented block <\n\n"
"> Double-indented block <\n\n"
">> Right-indent, right-aligned <\n\n"
) )
theToken.tokenizeText() theToken.tokenizeText()
assert theToken.theTokens == [ assert theToken.theTokens == [
@@ -423,13 +429,25 @@ def testCoreToken_Tokenize(dummyGUI):
(Tokenizer.T_EMPTY, 6, "", None, Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 6, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 7, "Some centered text", [], Tokenizer.A_CENTRE), (Tokenizer.T_TEXT, 7, "Some centered text", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE), (Tokenizer.T_TEXT, 9, "Left-indented block", [], Tokenizer.A_IND_L),
(Tokenizer.T_EMPTY, 10, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 11, "Right-indented block", [], Tokenizer.A_IND_R),
(Tokenizer.T_EMPTY, 12, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 13, "Double-indented block", [], dblIndent),
(Tokenizer.T_EMPTY, 14, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 15, "Right-indent, right-aligned", [], rIndAlign),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
] ]
assert theToken.theMarkdown[-1] == ( assert theToken.theMarkdown[-1] == (
"Some regular text\n\n" "Some regular text\n\n"
"Some left-aligned text\n\n" "Some left-aligned text\n\n"
"Some right-aligned text\n\n" "Some right-aligned text\n\n"
"Some centered text\n\n\n" "Some centered text\n\n"
"Left-indented block\n\n"
"Right-indented block\n\n"
"Double-indented block\n\n"
"Right-indent, right-aligned\n\n\n"
) )
# Alignment w/HTML Codes # Alignment w/HTML Codes
@@ -438,6 +456,10 @@ def testCoreToken_Tokenize(dummyGUI):
"Some left-aligned text &lt;&lt;\n\n" "Some left-aligned text &lt;&lt;\n\n"
"&gt;&gt; Some right-aligned text\n\n" "&gt;&gt; Some right-aligned text\n\n"
"&gt;&gt; Some centered text &lt;&lt;\n\n" "&gt;&gt; Some centered text &lt;&lt;\n\n"
"&gt; Left-indented block\n\n"
"Right-indented block &lt;\n\n"
"&gt; Double-indented block &lt;\n\n"
"&gt;&gt; Right-indent, right-aligned &lt;\n\n"
) )
theToken.tokenizeText() theToken.tokenizeText()
assert theToken.theTokens == [ assert theToken.theTokens == [
@@ -449,13 +471,25 @@ def testCoreToken_Tokenize(dummyGUI):
(Tokenizer.T_EMPTY, 6, "", None, Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 6, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 7, "Some centered text", [], Tokenizer.A_CENTRE), (Tokenizer.T_TEXT, 7, "Some centered text", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE), (Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE), (Tokenizer.T_TEXT, 9, "Left-indented block", [], Tokenizer.A_IND_L),
(Tokenizer.T_EMPTY, 10, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 11, "Right-indented block", [], Tokenizer.A_IND_R),
(Tokenizer.T_EMPTY, 12, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 13, "Double-indented block", [], dblIndent),
(Tokenizer.T_EMPTY, 14, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 15, "Right-indent, right-aligned", [], rIndAlign),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
] ]
assert theToken.theMarkdown[-1] == ( assert theToken.theMarkdown[-1] == (
"Some regular text\n\n" "Some regular text\n\n"
"Some left-aligned text\n\n" "Some left-aligned text\n\n"
"Some right-aligned text\n\n" "Some right-aligned text\n\n"
"Some centered text\n\n\n" "Some centered text\n\n"
"Left-indented block\n\n"
"Right-indented block\n\n"
"Double-indented block\n\n"
"Right-indent, right-aligned\n\n\n"
) )
# END Test testCoreToken_Tokenize # END Test testCoreToken_Tokenize