Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added left/right indentation to Tokenizer class

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2021-06-10 00:04:54 +02:00
parent e0de3ff191
commit 80c4826253
3 changed files with 82 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/core/tokenizer.py
+22 -1
View File
@@ -75,6 +75,8 @@ class Tokenizer():
A_PBA_AUT = 0x0080 # Page break after auto
A_Z_TOPMRG = 0x0100 # Zero top margin
A_Z_BTMMRG = 0x0200 # Zero bottom margin
A_IND_L = 0x0400 # Left indentation
A_IND_R = 0x0800 # Right indentation
def __init__(self, theProject):
@@ -442,15 +444,23 @@ class Tokenizer():
# Skip all body text
continue
# Check Alignment
# Check Alignment and Indentation
tagLeft = False
tagRight = False
indLeft = False
indRight = False
if aLine.startswith(">>"):
tagRight = True
aLine = aLine[2:].lstrip()
elif aLine.startswith(">>"):
tagRight = True
aLine = aLine[8:].lstrip()
elif aLine.startswith(">"):
indLeft = True
aLine = aLine[1:].lstrip()
elif aLine.startswith(">"):
indLeft = True
aLine = aLine[4:].lstrip()
if aLine.endswith("<<"):
tagLeft = True
@@ -458,6 +468,12 @@ class Tokenizer():
elif aLine.endswith("&lt;&lt;"):
tagLeft = True
aLine = aLine[:-8].rstrip()
elif aLine.endswith("<"):
indRight = True
aLine = aLine[:-1].rstrip()
elif aLine.endswith("&lt;"):
indRight = True
aLine = aLine[:-4].rstrip()
textAlign = defAlign
if tagLeft and tagRight:
@@ -467,6 +483,11 @@ class Tokenizer():
elif tagRight:
textAlign = self.A_RIGHT
if indLeft:
textAlign |= self.A_IND_L
if indRight:
textAlign |= self.A_IND_R
# Otherwise we use RegEx to find formatting tags within a line of text
fmtPos = []
for theRX, theKeys in rxFormats:
nw/gui/dochighlight.py
+21 -21
View File
@@ -106,27 +106,27 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self.colEmph = QColor(*self.theTheme.colEmph)
self.hStyles = {
"header1" : self._makeFormat(self.colHead, "bold", 1.8),
"header2" : self._makeFormat(self.colHead, "bold", 1.6),
"header3" : self._makeFormat(self.colHead, "bold", 1.4),
"header4" : self._makeFormat(self.colHead, "bold", 1.2),
"header1h" : self._makeFormat(self.colHeadH, "bold", 1.8),
"header2h" : self._makeFormat(self.colHeadH, "bold", 1.6),
"header3h" : self._makeFormat(self.colHeadH, "bold", 1.4),
"header4h" : self._makeFormat(self.colHeadH, "bold", 1.2),
"bold" : self._makeFormat(self.colEmph, "bold"),
"italic" : self._makeFormat(self.colEmph, "italic"),
"strike" : self._makeFormat(self.colHidden, "strike"),
"mspaces" : self._makeFormat(self.colError, "errline"),
"nobreak" : self._makeFormat(self.colBreak, "background"),
"dialogue1" : self._makeFormat(self.colDialN),
"dialogue2" : self._makeFormat(self.colDialD),
"dialogue3" : self._makeFormat(self.colDialS),
"replace" : self._makeFormat(self.colRepTag),
"hidden" : self._makeFormat(self.colHidden),
"keyword" : self._makeFormat(self.colKey),
"modifier" : self._makeFormat(self.colMod),
"value" : self._makeFormat(self.colVal, "underline"),
"header1" : self._makeFormat(self.colHead, "bold", 1.8),
"header2" : self._makeFormat(self.colHead, "bold", 1.6),
"header3" : self._makeFormat(self.colHead, "bold", 1.4),
"header4" : self._makeFormat(self.colHead, "bold", 1.2),
"header1h" : self._makeFormat(self.colHeadH, "bold", 1.8),
"header2h" : self._makeFormat(self.colHeadH, "bold", 1.6),
"header3h" : self._makeFormat(self.colHeadH, "bold", 1.4),
"header4h" : self._makeFormat(self.colHeadH, "bold", 1.2),
"bold" : self._makeFormat(self.colEmph, "bold"),
"italic" : self._makeFormat(self.colEmph, "italic"),
"strike" : self._makeFormat(self.colHidden, "strike"),
"mspaces" : self._makeFormat(self.colError, "errline"),
"nobreak" : self._makeFormat(self.colBreak, "background"),
"dialogue1" : self._makeFormat(self.colDialN),
"dialogue2" : self._makeFormat(self.colDialD),
"dialogue3" : self._makeFormat(self.colDialS),
"replace" : self._makeFormat(self.colRepTag),
"hidden" : self._makeFormat(self.colHidden),
"keyword" : self._makeFormat(self.colKey),
"modifier" : self._makeFormat(self.colMod),
"value" : self._makeFormat(self.colVal, "underline"),
}
self.hRules = []
tests/test_core/test_core_tokenizer.py
+39 -5
View File
@@ -406,12 +406,18 @@ def testCoreToken_Tokenize(dummyGUI):
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
)
# Alignment
# Alignment and Indentation
dblIndent = Tokenizer.A_IND_L | Tokenizer.A_IND_R
rIndAlign = Tokenizer.A_RIGHT | Tokenizer.A_IND_R
theToken.theText = (
"Some regular text\n\n"
"Some left-aligned text <<\n\n"
">> Some right-aligned text\n\n"
">> Some centered text <<\n\n"
"> Left-indented block\n\n"
"Right-indented block <\n\n"
"> Double-indented block <\n\n"
">> Right-indent, right-aligned <\n\n"
)
theToken.tokenizeText()
assert theToken.theTokens == [
@@ -423,13 +429,25 @@ def testCoreToken_Tokenize(dummyGUI):
(Tokenizer.T_EMPTY, 6, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 7, "Some centered text", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 9, "Left-indented block", [], Tokenizer.A_IND_L),
(Tokenizer.T_EMPTY, 10, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 11, "Right-indented block", [], Tokenizer.A_IND_R),
(Tokenizer.T_EMPTY, 12, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 13, "Double-indented block", [], dblIndent),
(Tokenizer.T_EMPTY, 14, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 15, "Right-indent, right-aligned", [], rIndAlign),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
]
assert theToken.theMarkdown[-1] == (
"Some regular text\n\n"
"Some left-aligned text\n\n"
"Some right-aligned text\n\n"
"Some centered text\n\n\n"
"Some centered text\n\n"
"Left-indented block\n\n"
"Right-indented block\n\n"
"Double-indented block\n\n"
"Right-indent, right-aligned\n\n\n"
)
# Alignment w/HTML Codes
@@ -438,6 +456,10 @@ def testCoreToken_Tokenize(dummyGUI):
"Some left-aligned text &lt;&lt;\n\n"
"&gt;&gt; Some right-aligned text\n\n"
"&gt;&gt; Some centered text &lt;&lt;\n\n"
"&gt; Left-indented block\n\n"
"Right-indented block &lt;\n\n"
"&gt; Double-indented block &lt;\n\n"
"&gt;&gt; Right-indent, right-aligned &lt;\n\n"
)
theToken.tokenizeText()
assert theToken.theTokens == [
@@ -449,13 +471,25 @@ def testCoreToken_Tokenize(dummyGUI):
(Tokenizer.T_EMPTY, 6, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 7, "Some centered text", [], Tokenizer.A_CENTRE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 8, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 9, "Left-indented block", [], Tokenizer.A_IND_L),
(Tokenizer.T_EMPTY, 10, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 11, "Right-indented block", [], Tokenizer.A_IND_R),
(Tokenizer.T_EMPTY, 12, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 13, "Double-indented block", [], dblIndent),
(Tokenizer.T_EMPTY, 14, "", None, Tokenizer.A_NONE),
(Tokenizer.T_TEXT, 15, "Right-indent, right-aligned", [], rIndAlign),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 16, "", None, Tokenizer.A_NONE),
]
assert theToken.theMarkdown[-1] == (
"Some regular text\n\n"
"Some left-aligned text\n\n"
"Some right-aligned text\n\n"
"Some centered text\n\n\n"
"Some centered text\n\n"
"Left-indented block\n\n"
"Right-indented block\n\n"
"Double-indented block\n\n"
"Right-indent, right-aligned\n\n\n"
)
# END Test testCoreToken_Tokenize