Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Process keyword flag in Tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-22 19:33:05 +02:00
parent 63f9cd0bac
commit c7554f2b8d
7 changed files with 33 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/formats/todocx.py
+1 -1
View File
@@ -291,7 +291,7 @@ class ToDocX(Tokenizer):
elif tType == BlockTyp.COMMENT:
self._processFragments(par, S_META, tText, tFormat)
elif tType == BlockTyp.KEYWORD and self._doKeywords:
elif tType == BlockTyp.KEYWORD:
tTemp, tFmt = self._formatKeywords(tText)
self._processFragments(par, S_META, tTemp, tFmt)
novelwriter/formats/tohtml.py
+1 -1
View File
@@ -252,7 +252,7 @@ class ToHtml(Tokenizer):
elif tType == BlockTyp.COMMENT:
lines.append(f"<p class='comment'>{self._formatText(tText, tFormat)}</p>\n")
elif tType == BlockTyp.KEYWORD and self._doKeywords:
elif tType == BlockTyp.KEYWORD:
tag, text = self._formatKeywords(tText)
kClass = f" class='meta meta-{tag}'" if tag else ""
tTemp = f"<p{kClass}{hStyle}>{text}</p>\n"
novelwriter/formats/tokenizer.py
+21 -4
View File
@@ -613,7 +613,7 @@ class Tokenizer(ABC):
if cStyle in (nwComment.SYNOPSIS, nwComment.SHORT, nwComment.PLAIN):
bStyle = COMMENT_STYLE[cStyle]
tLine, tFmt = self._formatNote(bStyle, cKey, cText)
tLine, tFmt = self._formatComment(bStyle, cKey, cText)
blocks.append((BlockTyp.COMMENT, nHead, tLine, tFmt, sAlign))
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
@@ -630,6 +630,9 @@ class Tokenizer(ABC):
# Only valid keyword lines are parsed, and any ignored keywords
# are automatically skipped.
if not self._doKeywords:
continue
valid, bits, _ = self._project.index.scanThis(aLine)
if (
valid and bits and bits[0] in nwLabels.KEY_NAME
@@ -638,9 +641,23 @@ class Tokenizer(ABC):
blocks.append((
BlockTyp.KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepRaw:
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
# valid, bits, _ = self._project.index.scanThis("@"+text)
# if not valid or not bits or bits[0] not in nwLabels.KEY_NAME:
# return "", []
# rTxt = f"{self._localLookup(nwLabels.KEY_NAME[bits[0]])}: "
# rFmt: T_Formats = [(0, TextFmt.B_B, ""), (len(rTxt) - 1, TextFmt.B_E, "")]
# if len(bits) > 1:
# if bits[0] == nwKeyWords.TAG_KEY:
# rTxt += bits[1]
# else:
# rTxt += ", ".join(bits[1:])
# return rTxt, rFmt
elif aLine.startswith(("# ", "#! ")):
# Title or Partition Headings
# ===========================
@@ -1022,7 +1039,7 @@ class Tokenizer(ABC):
allChars += len(tText)
allWordChars += len("".join(words))
elif tType == BlockTyp.KEYWORD and self._doKeywords:
elif tType == BlockTyp.KEYWORD:
valid, bits, _ = self._project.index.scanThis("@"+tText)
if valid and bits:
key = self._localLookup(nwLabels.KEY_NAME[bits[0]])
@@ -1080,7 +1097,7 @@ class Tokenizer(ABC):
# Internal Functions
##
def _formatNote(self, style: ComStyle, key: str, text: str) -> tuple[str, T_Formats]:
def _formatComment(self, style: ComStyle, key: str, text: str) -> tuple[str, T_Formats]:
"""Apply formatting to comments and notes."""
tTxt, tFmt = self._extractFormats(text)
tFmt.insert(0, (0, TextFmt.COL_B, style.textClass))
novelwriter/formats/tomarkdown.py
+1 -1
View File
@@ -153,7 +153,7 @@ class ToMarkdown(Tokenizer):
elif tType == BlockTyp.COMMENT:
lines.append(f"{self._formatText(tText, tFormat, mTags)}\n\n")
elif tType == BlockTyp.KEYWORD and self._doKeywords:
elif tType == BlockTyp.KEYWORD:
lines.append(self._formatKeywords(tText, tStyle))
self._result = "".join(lines)
novelwriter/formats/toodt.py
+1 -1
View File
@@ -483,7 +483,7 @@ class ToOdt(Tokenizer):
elif tType == BlockTyp.COMMENT:
self._addTextPar(xText, S_META, oStyle, tText, tFmt=tFormat)
elif tType == BlockTyp.KEYWORD and self._doKeywords:
elif tType == BlockTyp.KEYWORD:
tTemp, tFmt = self._formatKeywords(tText)
self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt)
novelwriter/formats/toqdoc.py
+1 -1
View File
@@ -244,7 +244,7 @@ class ToQTextDocument(Tokenizer):
newBlock(cursor, bFmt)
self._insertFragments(tText, tFormat, cursor, self._cText)
elif tType == BlockTyp.KEYWORD and self._doKeywords:
elif tType == BlockTyp.KEYWORD:
newBlock(cursor, bFmt)
self._insertKeywords(tText, cursor)
tests/test_formats/test_fmt_tokenizer.py
+7 -5
View File
@@ -762,15 +762,17 @@ def testFmtToken_MetaFormat(mockGUI):
assert tokens.allMarkdown[-1] == "% short: A short description\n\n"
# Keyword
tokens.setKeywords(False)
tokens._text = "@char: Bod\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
]
assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n"
tokens.setKeywords(True)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "@char: Bod\n\n"
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
@@ -1650,7 +1652,7 @@ def testFmtToken_FormatNote(mockGUI, ipsumText):
# Comment, No Formatting
style = COMMENT_STYLE[nwComment.PLAIN]
assert tokens._formatNote(style, "", "Hello world!") == (
assert tokens._formatComment(style, "", "Hello world!") == (
"Comment: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"),
(8, TextFmt.B_E, ""), (8, TextFmt.COL_E, ""),
@@ -1660,7 +1662,7 @@ def testFmtToken_FormatNote(mockGUI, ipsumText):
# Synopsis, No Formatting
style = COMMENT_STYLE[nwComment.SYNOPSIS]
assert tokens._formatNote(style, "", "Hello world!") == (
assert tokens._formatComment(style, "", "Hello world!") == (
"Synopsis: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"),
(9, TextFmt.B_E, ""), (9, TextFmt.COL_E, ""),