From c7554f2b8d51c3916b4f325c3054a92dc13bcdbd Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Tue, 22 Oct 2024 19:33:05 +0200
Subject: [PATCH] Process keyword flag in Tokenizer
---
novelwriter/formats/todocx.py | 2 +-
novelwriter/formats/tohtml.py | 2 +-
novelwriter/formats/tokenizer.py | 25 ++++++++++++++++++++----
novelwriter/formats/tomarkdown.py | 2 +-
novelwriter/formats/toodt.py | 2 +-
novelwriter/formats/toqdoc.py | 2 +-
tests/test_formats/test_fmt_tokenizer.py | 12 +++++++-----
7 files changed, 33 insertions(+), 14 deletions(-)
diff --git a/novelwriter/formats/todocx.py b/novelwriter/formats/todocx.py
index 9dbf8966..ac867ede 100644
--- a/novelwriter/formats/todocx.py
+++ b/novelwriter/formats/todocx.py
@@ -291,7 +291,7 @@ class ToDocX(Tokenizer):
elif tType == BlockTyp.COMMENT:
self._processFragments(par, S_META, tText, tFormat)
- elif tType == BlockTyp.KEYWORD and self._doKeywords:
+ elif tType == BlockTyp.KEYWORD:
tTemp, tFmt = self._formatKeywords(tText)
self._processFragments(par, S_META, tTemp, tFmt)
diff --git a/novelwriter/formats/tohtml.py b/novelwriter/formats/tohtml.py
index 17547ca9..c467fedb 100644
--- a/novelwriter/formats/tohtml.py
+++ b/novelwriter/formats/tohtml.py
@@ -252,7 +252,7 @@ class ToHtml(Tokenizer):
elif tType == BlockTyp.COMMENT:
lines.append(f"\n")
- elif tType == BlockTyp.KEYWORD and self._doKeywords:
+ elif tType == BlockTyp.KEYWORD:
tag, text = self._formatKeywords(tText)
kClass = f" class='meta meta-{tag}'" if tag else ""
tTemp = f"{text}\n"
diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py
index a4850fbd..fb447df2 100644
--- a/novelwriter/formats/tokenizer.py
+++ b/novelwriter/formats/tokenizer.py
@@ -613,7 +613,7 @@ class Tokenizer(ABC):
if cStyle in (nwComment.SYNOPSIS, nwComment.SHORT, nwComment.PLAIN):
bStyle = COMMENT_STYLE[cStyle]
- tLine, tFmt = self._formatNote(bStyle, cKey, cText)
+ tLine, tFmt = self._formatComment(bStyle, cKey, cText)
blocks.append((BlockTyp.COMMENT, nHead, tLine, tFmt, sAlign))
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
@@ -630,6 +630,9 @@ class Tokenizer(ABC):
# Only valid keyword lines are parsed, and any ignored keywords
# are automatically skipped.
+ if not self._doKeywords:
+ continue
+
valid, bits, _ = self._project.index.scanThis(aLine)
if (
valid and bits and bits[0] in nwLabels.KEY_NAME
@@ -638,9 +641,23 @@ class Tokenizer(ABC):
blocks.append((
BlockTyp.KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
- if self._doKeywords and self._keepRaw:
+ if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
+ # valid, bits, _ = self._project.index.scanThis("@"+text)
+ # if not valid or not bits or bits[0] not in nwLabels.KEY_NAME:
+ # return "", []
+
+ # rTxt = f"{self._localLookup(nwLabels.KEY_NAME[bits[0]])}: "
+ # rFmt: T_Formats = [(0, TextFmt.B_B, ""), (len(rTxt) - 1, TextFmt.B_E, "")]
+ # if len(bits) > 1:
+ # if bits[0] == nwKeyWords.TAG_KEY:
+ # rTxt += bits[1]
+ # else:
+ # rTxt += ", ".join(bits[1:])
+
+ # return rTxt, rFmt
+
elif aLine.startswith(("# ", "#! ")):
# Title or Partition Headings
# ===========================
@@ -1022,7 +1039,7 @@ class Tokenizer(ABC):
allChars += len(tText)
allWordChars += len("".join(words))
- elif tType == BlockTyp.KEYWORD and self._doKeywords:
+ elif tType == BlockTyp.KEYWORD:
valid, bits, _ = self._project.index.scanThis("@"+tText)
if valid and bits:
key = self._localLookup(nwLabels.KEY_NAME[bits[0]])
@@ -1080,7 +1097,7 @@ class Tokenizer(ABC):
# Internal Functions
##
- def _formatNote(self, style: ComStyle, key: str, text: str) -> tuple[str, T_Formats]:
+ def _formatComment(self, style: ComStyle, key: str, text: str) -> tuple[str, T_Formats]:
"""Apply formatting to comments and notes."""
tTxt, tFmt = self._extractFormats(text)
tFmt.insert(0, (0, TextFmt.COL_B, style.textClass))
diff --git a/novelwriter/formats/tomarkdown.py b/novelwriter/formats/tomarkdown.py
index 599e7b0f..09e33eb0 100644
--- a/novelwriter/formats/tomarkdown.py
+++ b/novelwriter/formats/tomarkdown.py
@@ -153,7 +153,7 @@ class ToMarkdown(Tokenizer):
elif tType == BlockTyp.COMMENT:
lines.append(f"{self._formatText(tText, tFormat, mTags)}\n\n")
- elif tType == BlockTyp.KEYWORD and self._doKeywords:
+ elif tType == BlockTyp.KEYWORD:
lines.append(self._formatKeywords(tText, tStyle))
self._result = "".join(lines)
diff --git a/novelwriter/formats/toodt.py b/novelwriter/formats/toodt.py
index 85aefb03..9802ce4b 100644
--- a/novelwriter/formats/toodt.py
+++ b/novelwriter/formats/toodt.py
@@ -483,7 +483,7 @@ class ToOdt(Tokenizer):
elif tType == BlockTyp.COMMENT:
self._addTextPar(xText, S_META, oStyle, tText, tFmt=tFormat)
- elif tType == BlockTyp.KEYWORD and self._doKeywords:
+ elif tType == BlockTyp.KEYWORD:
tTemp, tFmt = self._formatKeywords(tText)
self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt)
diff --git a/novelwriter/formats/toqdoc.py b/novelwriter/formats/toqdoc.py
index ea4912eb..e0b1faf5 100644
--- a/novelwriter/formats/toqdoc.py
+++ b/novelwriter/formats/toqdoc.py
@@ -244,7 +244,7 @@ class ToQTextDocument(Tokenizer):
newBlock(cursor, bFmt)
self._insertFragments(tText, tFormat, cursor, self._cText)
- elif tType == BlockTyp.KEYWORD and self._doKeywords:
+ elif tType == BlockTyp.KEYWORD:
newBlock(cursor, bFmt)
self._insertKeywords(tText, cursor)
diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py
index 8f917924..cf236348 100644
--- a/tests/test_formats/test_fmt_tokenizer.py
+++ b/tests/test_formats/test_fmt_tokenizer.py
@@ -762,15 +762,17 @@ def testFmtToken_MetaFormat(mockGUI):
assert tokens.allMarkdown[-1] == "% short: A short description\n\n"
# Keyword
+ tokens.setKeywords(False)
tokens._text = "@char: Bod\n"
tokens.tokenizeText()
- assert tokens._blocks == [
- (BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
- ]
+ assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n"
tokens.setKeywords(True)
tokens.tokenizeText()
+ assert tokens._blocks == [
+ (BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
+ ]
assert tokens.allMarkdown[-1] == "@char: Bod\n\n"
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
@@ -1650,7 +1652,7 @@ def testFmtToken_FormatNote(mockGUI, ipsumText):
# Comment, No Formatting
style = COMMENT_STYLE[nwComment.PLAIN]
- assert tokens._formatNote(style, "", "Hello world!") == (
+ assert tokens._formatComment(style, "", "Hello world!") == (
"Comment: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"),
(8, TextFmt.B_E, ""), (8, TextFmt.COL_E, ""),
@@ -1660,7 +1662,7 @@ def testFmtToken_FormatNote(mockGUI, ipsumText):
# Synopsis, No Formatting
style = COMMENT_STYLE[nwComment.SYNOPSIS]
- assert tokens._formatNote(style, "", "Hello world!") == (
+ assert tokens._formatComment(style, "", "Hello world!") == (
"Synopsis: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"),
(9, TextFmt.B_E, ""), (9, TextFmt.COL_E, ""),