Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix highlighting of multiple dialogue sections

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-06-10 23:50:59 +02:00
parent 0b3488c265
commit 416c580a45
3 changed files with 129 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/tohtml.py
+1 -1
View File
@@ -457,7 +457,7 @@ class ToHtml(Tokenizer):
else:
html = "<sup>ERR</sup>"
else:
html = HTML5_TAGS.get(fmt, "ERR")
html = HTML5_TAGS.get(fmt, "")
temp = f"{temp[:pos]}{html}{temp[pos:]}"
temp = temp.replace("\n", "<br>")
return stripEscape(temp)
novelwriter/core/tokenizer.py
+3 -1
View File
@@ -1139,7 +1139,9 @@ class Tokenizer(ABC):
# Match Dialogue
if self._rxDialogue:
for regEx, fmtB, fmtE in self._rxDialogue:
if (rxMatch := regEx.match(text, 0)).hasMatch():
rxItt = regEx.globalMatch(text, 0)
while rxItt.hasNext():
rxMatch = rxItt.next()
temp.append((rxMatch.capturedStart(0), 0, fmtB, ""))
temp.append((rxMatch.capturedEnd(0), 0, fmtE, ""))
tests/test_core/test_core_tokenizer.py
+125 -66
View File
@@ -26,6 +26,7 @@ import pytest
from PyQt5.QtGui import QFont
from novelwriter import CONFIG
from novelwriter.constants import nwHeadFmt
from novelwriter.core.project import NWProject
from novelwriter.core.tokenizer import HeadingFormatter, Tokenizer, stripEscape
@@ -1016,88 +1017,151 @@ def testCoreToken_TextFormat(mockGUI):
# Text Emphasis
tokens._text = "Some **bolded text** on this lines\n"
tokens.tokenizeText()
assert tokens._tokens == [
(
Tokenizer.T_TEXT, 0,
"Some bolded text on this lines",
[
(5, Tokenizer.FMT_B_B, ""),
(16, Tokenizer.FMT_B_E, ""),
],
Tokenizer.A_NONE
),
]
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0, "Some bolded text on this lines",
[
(5, Tokenizer.FMT_B_B, ""),
(16, Tokenizer.FMT_B_E, ""),
],
Tokenizer.A_NONE
)]
assert tokens.allMarkdown[-1] == "Some **bolded text** on this lines\n\n"
tokens._text = "Some _italic text_ on this lines\n"
tokens.tokenizeText()
assert tokens._tokens == [
(
Tokenizer.T_TEXT, 0,
"Some italic text on this lines",
[
(5, Tokenizer.FMT_I_B, ""),
(16, Tokenizer.FMT_I_E, ""),
],
Tokenizer.A_NONE
),
]
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0, "Some italic text on this lines",
[
(5, Tokenizer.FMT_I_B, ""),
(16, Tokenizer.FMT_I_E, ""),
],
Tokenizer.A_NONE
)]
assert tokens.allMarkdown[-1] == "Some _italic text_ on this lines\n\n"
tokens._text = "Some **_bold italic text_** on this lines\n"
tokens.tokenizeText()
assert tokens._tokens == [
(
Tokenizer.T_TEXT, 0,
"Some bold italic text on this lines",
[
(5, Tokenizer.FMT_B_B, ""),
(5, Tokenizer.FMT_I_B, ""),
(21, Tokenizer.FMT_I_E, ""),
(21, Tokenizer.FMT_B_E, ""),
],
Tokenizer.A_NONE
),
]
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0, "Some bold italic text on this lines",
[
(5, Tokenizer.FMT_B_B, ""),
(5, Tokenizer.FMT_I_B, ""),
(21, Tokenizer.FMT_I_E, ""),
(21, Tokenizer.FMT_B_E, ""),
],
Tokenizer.A_NONE
)]
assert tokens.allMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n"
tokens._text = "Some ~~strikethrough text~~ on this lines\n"
tokens.tokenizeText()
assert tokens._tokens == [
(
Tokenizer.T_TEXT, 0,
"Some strikethrough text on this lines",
[
(5, Tokenizer.FMT_D_B, ""),
(23, Tokenizer.FMT_D_E, ""),
],
Tokenizer.A_NONE
),
]
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0, "Some strikethrough text on this lines",
[
(5, Tokenizer.FMT_D_B, ""),
(23, Tokenizer.FMT_D_E, ""),
],
Tokenizer.A_NONE
)]
assert tokens.allMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n"
tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
tokens.tokenizeText()
assert tokens._tokens == [
(
Tokenizer.T_TEXT, 0,
"Some nested bold and italic and strikethrough text here",
[
(5, Tokenizer.FMT_B_B, ""),
(21, Tokenizer.FMT_I_B, ""),
(27, Tokenizer.FMT_I_E, ""),
(32, Tokenizer.FMT_D_B, ""),
(45, Tokenizer.FMT_D_E, ""),
(50, Tokenizer.FMT_B_E, ""),
],
Tokenizer.A_NONE
),
]
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0, "Some nested bold and italic and strikethrough text here",
[
(5, Tokenizer.FMT_B_B, ""),
(21, Tokenizer.FMT_I_B, ""),
(27, Tokenizer.FMT_I_E, ""),
(32, Tokenizer.FMT_D_B, ""),
(45, Tokenizer.FMT_D_E, ""),
(50, Tokenizer.FMT_B_E, ""),
],
Tokenizer.A_NONE
)]
assert tokens.allMarkdown[-1] == (
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
)
@pytest.mark.core
def testCoreToken_Dialogue(mockGUI):
"""Test the tokenization of dialogue in the Tokenizer class."""
CONFIG.fmtDQuoteOpen = "\u201c"
CONFIG.fmtDQuoteClose = "\u201d"
CONFIG.fmtSQuoteOpen = "\u2018"
CONFIG.fmtSQuoteClose = "\u2019"
CONFIG.dialogStyle = 3
CONFIG.altDialogOpen = "::"
CONFIG.altDialogClose = "::"
CONFIG.dialogLine = "\u2013"
CONFIG.narratorBreak = "\u2013"
project = NWProject()
tokens = BareTokenizer(project)
tokens.setDialogueHighlight(True)
# Single quotes
tokens._text = "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019\n"
tokens.tokenizeText()
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0,
"Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019",
[
(10, Tokenizer.FMT_DL_B, ""),
(25, Tokenizer.FMT_DL_E, ""),
(30, Tokenizer.FMT_DL_B, ""),
(45, Tokenizer.FMT_DL_E, ""),
],
Tokenizer.A_NONE
)]
# Double quotes
tokens._text = "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d\n"
tokens.tokenizeText()
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0,
"Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d",
[
(10, Tokenizer.FMT_DL_B, ""),
(25, Tokenizer.FMT_DL_E, ""),
(30, Tokenizer.FMT_DL_B, ""),
(45, Tokenizer.FMT_DL_E, ""),
],
Tokenizer.A_NONE
)]
# Alt quotes
tokens._text = "Text with ::dialogue one,:: and ::dialogue two.::\n"
tokens.tokenizeText()
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0,
"Text with ::dialogue one,:: and ::dialogue two.::",
[
(10, Tokenizer.FMT_ADL_B, ""),
(27, Tokenizer.FMT_ADL_E, ""),
(32, Tokenizer.FMT_ADL_B, ""),
(49, Tokenizer.FMT_ADL_E, ""),
],
Tokenizer.A_NONE
)]
# Dialogue line with narrator break
tokens._text = "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?\n"
tokens.tokenizeText()
assert tokens._tokens == [(
Tokenizer.T_TEXT, 0,
"\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?",
[
(0, Tokenizer.FMT_DL_B, ""),
(34, Tokenizer.FMT_DL_E, ""),
(44, Tokenizer.FMT_DL_B, ""),
(49, Tokenizer.FMT_DL_E, ""),
],
Tokenizer.A_NONE
)]
@pytest.mark.core
def testCoreToken_SpecialFormat(mockGUI):
"""Test the tokenization of special formats in the Tokenizer class."""
@@ -1267,11 +1331,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
##
# Story Files
##
tokens._isNovel = True
# Titles