diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py index 5805f240..446abdca 100644 --- a/novelwriter/core/tohtml.py +++ b/novelwriter/core/tohtml.py @@ -457,7 +457,7 @@ class ToHtml(Tokenizer): else: html = "ERR" else: - html = HTML5_TAGS.get(fmt, "ERR") + html = HTML5_TAGS.get(fmt, "") temp = f"{temp[:pos]}{html}{temp[pos:]}" temp = temp.replace("\n", "
") return stripEscape(temp) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index b7bc1adc..a2238b73 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -1139,7 +1139,9 @@ class Tokenizer(ABC): # Match Dialogue if self._rxDialogue: for regEx, fmtB, fmtE in self._rxDialogue: - if (rxMatch := regEx.match(text, 0)).hasMatch(): + rxItt = regEx.globalMatch(text, 0) + while rxItt.hasNext(): + rxMatch = rxItt.next() temp.append((rxMatch.capturedStart(0), 0, fmtB, "")) temp.append((rxMatch.capturedEnd(0), 0, fmtE, "")) diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 814a27be..aa23ad1d 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -26,6 +26,7 @@ import pytest from PyQt5.QtGui import QFont +from novelwriter import CONFIG from novelwriter.constants import nwHeadFmt from novelwriter.core.project import NWProject from novelwriter.core.tokenizer import HeadingFormatter, Tokenizer, stripEscape @@ -1016,88 +1017,151 @@ def testCoreToken_TextFormat(mockGUI): # Text Emphasis tokens._text = "Some **bolded text** on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [ - ( - Tokenizer.T_TEXT, 0, - "Some bolded text on this lines", - [ - (5, Tokenizer.FMT_B_B, ""), - (16, Tokenizer.FMT_B_E, ""), - ], - Tokenizer.A_NONE - ), - ] + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, "Some bolded text on this lines", + [ + (5, Tokenizer.FMT_B_B, ""), + (16, Tokenizer.FMT_B_E, ""), + ], + Tokenizer.A_NONE + )] assert tokens.allMarkdown[-1] == "Some **bolded text** on this lines\n\n" tokens._text = "Some _italic text_ on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [ - ( - Tokenizer.T_TEXT, 0, - "Some italic text on this lines", - [ - (5, Tokenizer.FMT_I_B, ""), - (16, Tokenizer.FMT_I_E, ""), - ], - Tokenizer.A_NONE - ), - ] + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, "Some italic text on this lines", + [ + (5, Tokenizer.FMT_I_B, ""), + (16, Tokenizer.FMT_I_E, ""), + ], + Tokenizer.A_NONE + )] assert tokens.allMarkdown[-1] == "Some _italic text_ on this lines\n\n" tokens._text = "Some **_bold italic text_** on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [ - ( - Tokenizer.T_TEXT, 0, - "Some bold italic text on this lines", - [ - (5, Tokenizer.FMT_B_B, ""), - (5, Tokenizer.FMT_I_B, ""), - (21, Tokenizer.FMT_I_E, ""), - (21, Tokenizer.FMT_B_E, ""), - ], - Tokenizer.A_NONE - ), - ] + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, "Some bold italic text on this lines", + [ + (5, Tokenizer.FMT_B_B, ""), + (5, Tokenizer.FMT_I_B, ""), + (21, Tokenizer.FMT_I_E, ""), + (21, Tokenizer.FMT_B_E, ""), + ], + Tokenizer.A_NONE + )] assert tokens.allMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n" tokens._text = "Some ~~strikethrough text~~ on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [ - ( - Tokenizer.T_TEXT, 0, - "Some strikethrough text on this lines", - [ - (5, Tokenizer.FMT_D_B, ""), - (23, Tokenizer.FMT_D_E, ""), - ], - Tokenizer.A_NONE - ), - ] + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, "Some strikethrough text on this lines", + [ + (5, Tokenizer.FMT_D_B, ""), + (23, Tokenizer.FMT_D_E, ""), + ], + Tokenizer.A_NONE + )] assert tokens.allMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n" tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n" tokens.tokenizeText() - assert tokens._tokens == [ - ( - Tokenizer.T_TEXT, 0, - "Some nested bold and italic and strikethrough text here", - [ - (5, Tokenizer.FMT_B_B, ""), - (21, Tokenizer.FMT_I_B, ""), - (27, Tokenizer.FMT_I_E, ""), - (32, Tokenizer.FMT_D_B, ""), - (45, Tokenizer.FMT_D_E, ""), - (50, Tokenizer.FMT_B_E, ""), - ], - Tokenizer.A_NONE - ), - ] + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, "Some nested bold and italic and strikethrough text here", + [ + (5, Tokenizer.FMT_B_B, ""), + (21, Tokenizer.FMT_I_B, ""), + (27, Tokenizer.FMT_I_E, ""), + (32, Tokenizer.FMT_D_B, ""), + (45, Tokenizer.FMT_D_E, ""), + (50, Tokenizer.FMT_B_E, ""), + ], + Tokenizer.A_NONE + )] assert tokens.allMarkdown[-1] == ( "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n" ) +@pytest.mark.core +def testCoreToken_Dialogue(mockGUI): + """Test the tokenization of dialogue in the Tokenizer class.""" + CONFIG.fmtDQuoteOpen = "\u201c" + CONFIG.fmtDQuoteClose = "\u201d" + CONFIG.fmtSQuoteOpen = "\u2018" + CONFIG.fmtSQuoteClose = "\u2019" + CONFIG.dialogStyle = 3 + CONFIG.altDialogOpen = "::" + CONFIG.altDialogClose = "::" + CONFIG.dialogLine = "\u2013" + CONFIG.narratorBreak = "\u2013" + + project = NWProject() + tokens = BareTokenizer(project) + tokens.setDialogueHighlight(True) + + # Single quotes + tokens._text = "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019\n" + tokens.tokenizeText() + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, + "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019", + [ + (10, Tokenizer.FMT_DL_B, ""), + (25, Tokenizer.FMT_DL_E, ""), + (30, Tokenizer.FMT_DL_B, ""), + (45, Tokenizer.FMT_DL_E, ""), + ], + Tokenizer.A_NONE + )] + + # Double quotes + tokens._text = "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d\n" + tokens.tokenizeText() + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, + "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d", + [ + (10, Tokenizer.FMT_DL_B, ""), + (25, Tokenizer.FMT_DL_E, ""), + (30, Tokenizer.FMT_DL_B, ""), + (45, Tokenizer.FMT_DL_E, ""), + ], + Tokenizer.A_NONE + )] + + # Alt quotes + tokens._text = "Text with ::dialogue one,:: and ::dialogue two.::\n" + tokens.tokenizeText() + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, + "Text with ::dialogue one,:: and ::dialogue two.::", + [ + (10, Tokenizer.FMT_ADL_B, ""), + (27, Tokenizer.FMT_ADL_E, ""), + (32, Tokenizer.FMT_ADL_B, ""), + (49, Tokenizer.FMT_ADL_E, ""), + ], + Tokenizer.A_NONE + )] + + # Dialogue line with narrator break + tokens._text = "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?\n" + tokens.tokenizeText() + assert tokens._tokens == [( + Tokenizer.T_TEXT, 0, + "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?", + [ + (0, Tokenizer.FMT_DL_B, ""), + (34, Tokenizer.FMT_DL_E, ""), + (44, Tokenizer.FMT_DL_B, ""), + (49, Tokenizer.FMT_DL_E, ""), + ], + Tokenizer.A_NONE + )] + + @pytest.mark.core def testCoreToken_SpecialFormat(mockGUI): """Test the tokenization of special formats in the Tokenizer class.""" @@ -1267,11 +1331,6 @@ def testCoreToken_ProcessHeaders(mockGUI): project.data.setLanguage("en") project._loadProjectLocalisation() tokens = BareTokenizer(project) - - ## - # Story Files - ## - tokens._isNovel = True # Titles