From 887b4379b2ee921b9b8df8bd022ede270781d78a Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 25 Nov 2023 17:50:25 +0100 Subject: [PATCH 1/6] Add comment style processing to the index class --- novelwriter/core/index.py | 32 ++++++++++++++++++++++---------- novelwriter/enum.py | 9 +++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 3c833b2c..2c76c3ae 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -36,7 +36,7 @@ from typing import TYPE_CHECKING, ItemsView, Iterable, Iterator from pathlib import Path from novelwriter import SHARED -from novelwriter.enum import nwItemClass, nwItemType, nwItemLayout +from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout from novelwriter.error import logException from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode from novelwriter.constants import nwFiles, nwKeyWords, nwRegEx, nwUnicode, nwHeaders @@ -338,14 +338,9 @@ class NWIndex: elif line.startswith("%"): if cTitle != TT_NONE: - toCheck = line[1:].lstrip() - synTag = toCheck[:9].lower() - tLen = len(line) - cLen = len(toCheck) - cOff = tLen - cLen - if synTag == "synopsis:": - sText = line[cOff+9:].strip() - self._itemIndex.setHeadingSynopsis(tHandle, cTitle, sText) + cStyle, cText, _ = processComment(line) + if cStyle in (nwComment.BRIEF, nwComment.SYNOPSIS): + self._itemIndex.setHeadingSynopsis(tHandle, cTitle, cText) # Count words for remaining text after last heading if pTitle != TT_NONE: @@ -1269,9 +1264,26 @@ class IndexHeading: # =============================================================================================== # -# Simple Word Counter +# text Processing Functions # =============================================================================================== # +CLASSIFIERS = { + "brief": nwComment.BRIEF, + "synopsis": nwComment.SYNOPSIS, +} + + +def processComment(text: str) -> tuple[nwComment, str, int]: + """Extract comment style and text. Should only be called on text + starting with a %. + """ + check = text[1:].lstrip() + classifier, _, content = check.partition(":") + if content and (clean := classifier.strip().lower()) in CLASSIFIERS: + return CLASSIFIERS[clean], content.strip(), text.find(":") + 1 + return nwComment.PLAIN, check, 0 + + def countWords(text: str) -> tuple[int, int, int]: """Count words in a piece of text, skipping special syntax and comments. diff --git a/novelwriter/enum.py b/novelwriter/enum.py index c1d2aef4..1258e5e2 100644 --- a/novelwriter/enum.py +++ b/novelwriter/enum.py @@ -61,6 +61,15 @@ class nwItemLayout(Enum): # END Enum nwItemLayout +class nwComment(Enum): + + PLAIN = 0 + SYNOPSIS = 1 + BRIEF = 2 + +# END Enum nwComment + + class nwTrinary(Enum): NEGATIVE = -1 From 8ed85363e2162140c8790b419be6e4fe5b02ece1 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 25 Nov 2023 17:50:59 +0100 Subject: [PATCH 2/6] Add test coverage and update the highlighter --- novelwriter/gui/dochighlight.py | 16 ++++++-------- tests/test_core/test_core_index.py | 35 ++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index 6a77f267..b6850097 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -36,6 +36,8 @@ from PyQt5.QtGui import ( from novelwriter import CONFIG, SHARED from novelwriter.common import checkInt from novelwriter.constants import nwRegEx, nwUnicode +from novelwriter.core.index import processComment +from novelwriter.enum import nwComment logger = logging.getLogger(__name__) @@ -352,16 +354,12 @@ class GuiDocHighlighter(QSyntaxHighlighter): elif text.startswith("%"): # Comments self.setCurrentBlockState(self.BLOCK_TEXT) - toCheck = text[1:].lstrip() - synTag = toCheck[:9].lower() - tLen = len(text) - cLen = len(toCheck) - cOff = tLen - cLen - if synTag == "synopsis:": - self.setFormat(0, cOff+9, self._hStyles["modifier"]) - self.setFormat(cOff+9, tLen, self._hStyles["hidden"]) + cStyle, _, cPos = processComment(text) + if cStyle == nwComment.PLAIN: + self.setFormat(0, len(text), self._hStyles["hidden"]) else: - self.setFormat(0, tLen, self._hStyles["hidden"]) + self.setFormat(0, cPos, self._hStyles["modifier"]) + self.setFormat(cPos, len(text), self._hStyles["hidden"]) else: # Text Paragraph diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index 9e7856e8..d12ab318 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -29,9 +29,9 @@ from mocked import causeException from novelwriter.core.item import NWItem from tools import C, buildTestProject, cmpFiles, writeFile -from novelwriter.enum import nwItemClass, nwItemLayout +from novelwriter.enum import nwComment, nwItemClass, nwItemLayout from novelwriter.constants import nwFiles -from novelwriter.core.index import IndexItem, NWIndex, countWords, TagsIndex +from novelwriter.core.index import IndexItem, NWIndex, countWords, TagsIndex, processComment from novelwriter.core.project import NWProject @@ -1264,7 +1264,34 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd): @pytest.mark.core -def testCoreIndex_CountWords(): +def testCoreIndex_processComment(): + """Test the comment processing function.""" + # Regular comment + assert processComment("%Hi") == (nwComment.PLAIN, "Hi", 0) + assert processComment("% Hi") == (nwComment.PLAIN, "Hi", 0) + assert processComment("% Hi:you") == (nwComment.PLAIN, "Hi:you", 0) + + # Synopsis + assert processComment("%synopsis:") == (nwComment.PLAIN, "synopsis:", 0) + assert processComment("%synopsis: Hi") == (nwComment.SYNOPSIS, "Hi", 10) + assert processComment("% synopsis: Hi") == (nwComment.SYNOPSIS, "Hi", 11) + assert processComment("% synopsis : Hi") == (nwComment.SYNOPSIS, "Hi", 13) + assert processComment("% Synopsis : Hi") == (nwComment.SYNOPSIS, "Hi", 15) + assert processComment("% \t SYNOPSIS : Hi") == (nwComment.SYNOPSIS, "Hi", 16) + + # Brief + assert processComment("%brief:") == (nwComment.PLAIN, "brief:", 0) + assert processComment("%brief: Hi") == (nwComment.BRIEF, "Hi", 7) + assert processComment("% brief: Hi") == (nwComment.BRIEF, "Hi", 8) + assert processComment("% brief : Hi") == (nwComment.BRIEF, "Hi", 10) + assert processComment("% Brief : Hi") == (nwComment.BRIEF, "Hi", 12) + assert processComment("% \t BRIEF : Hi") == (nwComment.BRIEF, "Hi", 13) + +# END Test testCoreIndex_processComment + + +@pytest.mark.core +def testCoreIndex_countWords(): """Test the word counter and the exclusion filers.""" # Non-Text assert countWords(None) == (0, 0, 0) # type: ignore @@ -1362,4 +1389,4 @@ def testCoreIndex_CountWords(): assert wC == 14 assert pC == 2 -# END Test testCoreIndex_CountWords +# END Test testCoreIndex_countWords From 6a6b05c3380e60c82447685b85995643e273002a Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 25 Nov 2023 18:12:45 +0100 Subject: [PATCH 3/6] Update tokenizer and converters --- novelwriter/core/coretools.py | 7 ++++- novelwriter/core/tohtml.py | 9 ++++-- novelwriter/core/tokenizer.py | 41 +++++++++++++++----------- novelwriter/core/tomd.py | 4 +++ novelwriter/core/toodt.py | 10 +++++-- tests/test_core/test_core_tohtml.py | 20 +++++++++++-- tests/test_core/test_core_tokenizer.py | 14 +++++++++ tests/test_core/test_core_tomd.py | 8 ++++- tests/test_core/test_core_toodt.py | 14 ++++++--- 9 files changed, 95 insertions(+), 32 deletions(-) diff --git a/novelwriter/core/coretools.py b/novelwriter/core/coretools.py index 81803576..fd040461 100644 --- a/novelwriter/core/coretools.py +++ b/novelwriter/core/coretools.py @@ -406,6 +406,7 @@ class ProjectBuilder: chSynop = self.tr("Summary of the chapter.") scSynop = self.tr("Summary of the scene.") + bfNote = self.tr("A brief description.") # Create chapters if numChapters > 0: @@ -446,7 +447,11 @@ class ProjectBuilder: aHandle = project.newFile(noteTitles[newRoot], rHandle) ntTag = simplified(noteTitles[newRoot]).replace(" ", "") aDoc = project.storage.getDocument(aHandle) - aDoc.writeDocument(f"# {noteTitles[newRoot]}\n\n@tag: {ntTag}\n\n") + aDoc.writeDocument( + f"# {noteTitles[newRoot]}\n\n" + f"% Brief: {bfNote}\n\n" + f"@tag: {ntTag}\n\n" + ) # Also add the archive and trash folders project.newRoot(nwItemClass.ARCHIVE) diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py index 7443ed34..219f6032 100644 --- a/novelwriter/core/tohtml.py +++ b/novelwriter/core/tohtml.py @@ -287,7 +287,10 @@ class ToHtml(Tokenizer): para.append(stripEscape(tTemp.rstrip())) elif tType == self.T_SYNOPSIS and self._doSynopsis: - lines.append(self._formatSynopsis(tText)) + lines.append(self._formatSynopsis(tText, True)) + + elif tType == self.T_BRIEF and self._doSynopsis: + lines.append(self._formatSynopsis(tText, False)) elif tType == self.T_COMMENT and self._doComments: lines.append(self._formatComments(tText)) @@ -454,9 +457,9 @@ class ToHtml(Tokenizer): # Internal Functions ## - def _formatSynopsis(self, text: str) -> str: + def _formatSynopsis(self, text: str, synopsis: bool) -> str: """Apply HTML formatting to synopsis.""" - sSynop = self._localLookup("Synopsis") + sSynop = self._localLookup("Synopsis") if synopsis else self._localLookup("Brief") if self._genMode == self.M_PREVIEW: return f"
{sSynop}: {text}
\n" else: diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index a4e7f61a..b5d0fd18 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -34,8 +34,9 @@ from pathlib import Path from functools import partial from PyQt5.QtCore import QCoreApplication, QRegularExpression +from novelwriter.core.index import processComment -from novelwriter.enum import nwItemLayout +from novelwriter.enum import nwComment, nwItemLayout from novelwriter.common import formatTimeStamp, numberToRoman, checkInt from novelwriter.constants import nwHeadFmt, nwRegEx, nwShortcode, nwUnicode from novelwriter.core.project import NWProject @@ -79,17 +80,18 @@ class Tokenizer(ABC): # Block Type T_EMPTY = 1 # Empty line (new paragraph) T_SYNOPSIS = 2 # Synopsis comment - T_COMMENT = 3 # Comment line - T_KEYWORD = 4 # Command line - T_TITLE = 5 # Title - T_UNNUM = 6 # Unnumbered - T_HEAD1 = 7 # Header 1 - T_HEAD2 = 8 # Header 2 - T_HEAD3 = 9 # Header 3 - T_HEAD4 = 10 # Header 4 - T_TEXT = 11 # Text line - T_SEP = 12 # Scene separator - T_SKIP = 13 # Paragraph break + T_BRIEF = 3 # Brief comment + T_COMMENT = 4 # Comment line + T_KEYWORD = 5 # Command line + T_TITLE = 6 # Title + T_UNNUM = 7 # Unnumbered + T_HEAD1 = 8 # Header 1 + T_HEAD2 = 9 # Header 2 + T_HEAD3 = 10 # Header 3 + T_HEAD4 = 11 # Header 4 + T_TEXT = 12 # Text line + T_SEP = 13 # Scene separator + T_SKIP = 14 # Paragraph break # Block Style A_NONE = 0x0000 # No special style @@ -461,17 +463,22 @@ class Tokenizer(ABC): continue if aLine[0] == "%": - cLine = aLine[1:].lstrip() - synTag = cLine[:9].lower() - if synTag == "synopsis:": + cStyle, cText, _ = processComment(aLine) + if cStyle == nwComment.SYNOPSIS: self._tokens.append(( - self.T_SYNOPSIS, nHead, cLine[9:].strip(), None, sAlign + self.T_SYNOPSIS, nHead, cText, None, sAlign + )) + if self._doSynopsis and self._keepMarkdown: + tmpMarkdown.append("%s\n" % aLine) + elif cStyle == nwComment.BRIEF: + self._tokens.append(( + self.T_BRIEF, nHead, cText, None, sAlign )) if self._doSynopsis and self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) else: self._tokens.append(( - self.T_COMMENT, nHead, aLine[1:].strip(), None, sAlign + self.T_COMMENT, nHead, cText, None, sAlign )) if self._doComments and self._keepMarkdown: tmpMarkdown.append("%s\n" % aLine) diff --git a/novelwriter/core/tomd.py b/novelwriter/core/tomd.py index d13e3b18..3187258a 100644 --- a/novelwriter/core/tomd.py +++ b/novelwriter/core/tomd.py @@ -170,6 +170,10 @@ class ToMarkdown(Tokenizer): label = self._localLookup("Synopsis") lines.append(f"**{label}:** {tText}\n\n") + elif tType == self.T_BRIEF and self._doSynopsis: + label = self._localLookup("Brief") + lines.append(f"**{label}:** {tText}\n\n") + elif tType == self.T_COMMENT and self._doComments: label = self._localLookup("Comment") lines.append(f"**{label}:** {tText}\n\n") diff --git a/novelwriter/core/toodt.py b/novelwriter/core/toodt.py index aec5eefb..26cc76b2 100644 --- a/novelwriter/core/toodt.py +++ b/novelwriter/core/toodt.py @@ -481,7 +481,11 @@ class ToOdt(Tokenizer): pFmt.append(tFormat) elif tType == self.T_SYNOPSIS and self._doSynopsis: - tTemp, fTemp = self._formatSynopsis(tText) + tTemp, fTemp = self._formatSynopsis(tText, True) + self._addTextPar("Text_20_Meta", oStyle, tTemp, tFmt=fTemp) + + elif tType == self.T_BRIEF and self._doSynopsis: + tTemp, fTemp = self._formatSynopsis(tText, False) self._addTextPar("Text_20_Meta", oStyle, tTemp, tFmt=fTemp) elif tType == self.T_COMMENT and self._doComments: @@ -552,9 +556,9 @@ class ToOdt(Tokenizer): # Internal Functions ## - def _formatSynopsis(self, text: str) -> tuple[str, list[tuple[int, int]]]: + def _formatSynopsis(self, text: str, synopsis: bool) -> tuple[str, list[tuple[int, int]]]: """Apply formatting to synopsis lines.""" - name = self._localLookup("Synopsis") + name = self._localLookup("Synopsis") if synopsis else self._localLookup("Brief") rTxt = f"{name}: {text}" rFmt = [(0, self.FMT_B_B), (len(name) + 1, self.FMT_B_E)] return rTxt, rFmt diff --git a/tests/test_core/test_core_tohtml.py b/tests/test_core/test_core_tohtml.py index b0776ef6..7f616155 100644 --- a/tests/test_core/test_core_tohtml.py +++ b/tests/test_core/test_core_tohtml.py @@ -149,7 +149,7 @@ def testCoreToHtml_ConvertFormat(mockGUI): "Line one
Line two
Line three
Synopsis: The synopsis ...
\n" ) + html.setSynopsis(True) + html._text = "%brief: A description ...\n" + html.tokenizeText() + html.doConvert() + assert html.theResult == ( + "Brief: A description ...
\n" + ) + # Comment html._text = "% A comment ...\n" html.tokenizeText() @@ -610,9 +618,12 @@ def testCoreToHtml_Format(mockGUI): # Export Mode # =========== - assert html._formatSynopsis("synopsis text") == ( + assert html._formatSynopsis("synopsis text", True) == ( "Synopsis: synopsis text
\n" ) + assert html._formatSynopsis("brief text", False) == ( + "Brief: brief text
\n" + ) assert html._formatComments("comment text") == ( "Comment: comment text
\n" ) @@ -632,9 +643,12 @@ def testCoreToHtml_Format(mockGUI): html.setPreview(True, True) - assert html._formatSynopsis("synopsis text") == ( + assert html._formatSynopsis("synopsis text", True) == ( "Synopsis: synopsis text
\n" ) + assert html._formatSynopsis("brief text", False) == ( + "Brief: brief text
\n" + ) assert html._formatComments("comment text") == ( "comment text
\n" ) diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index c218b7d7..3dc513ce 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -471,6 +471,20 @@ def testCoreToken_MetaFormat(mockGUI): tokens.tokenizeText() assert tokens.theMarkdown[-1] == "% synopsis: The synopsis\n\n" + # Brief + tokens.setSynopsis(False) + tokens._text = "% brief: A description\n" + tokens.tokenizeText() + assert tokens._tokens == [ + (Tokenizer.T_BRIEF, 0, "A description", None, Tokenizer.A_NONE), + (Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE), + ] + assert tokens.theMarkdown[-1] == "\n" + + tokens.setSynopsis(True) + tokens.tokenizeText() + assert tokens.theMarkdown[-1] == "% brief: A description\n\n" + # Keyword tokens._text = "@char: Bod\n" tokens.tokenizeText() diff --git a/tests/test_core/test_core_tomd.py b/tests/test_core/test_core_tomd.py index c44447a4..78e94230 100644 --- a/tests/test_core/test_core_tomd.py +++ b/tests/test_core/test_core_tomd.py @@ -106,7 +106,7 @@ def testCoreToMarkdown_ConvertFormat(mockGUI): theMD.doConvert() assert theMD.theResult == "Line one \nLine two \nLine three\n\n" - # Synopsis + # Synopsis, Brief theMD._text = "%synopsis: The synopsis ...\n" theMD.tokenizeText() theMD.doConvert() @@ -118,6 +118,12 @@ def testCoreToMarkdown_ConvertFormat(mockGUI): theMD.doConvert() assert theMD.theResult == "**Synopsis:** The synopsis ...\n\n" + theMD.setSynopsis(True) + theMD._text = "%brief: A description ...\n" + theMD.tokenizeText() + theMD.doConvert() + assert theMD.theResult == "**Brief:** A description ...\n\n" + # Comment theMD._text = "% A comment ...\n" theMD.tokenizeText() diff --git a/tests/test_core/test_core_toodt.py b/tests/test_core/test_core_toodt.py index ab6bf41c..04d5180b 100644 --- a/tests/test_core/test_core_toodt.py +++ b/tests/test_core/test_core_toodt.py @@ -447,12 +447,13 @@ def testCoreToOdt_Convert(mockGUI): '' ) - # Synopsis, Comment, Keywords + # Synopsis, Brief, Comment, Keywords odt._text = ( "### Scene\n\n" "@pov: Jane\n\n" "% synopsis: So it begins\n\n" - "% a plain comment\n\n" + "% brief: Then what\n\n" + "% A plain comment\n\n" ) odt.setSynopsis(True) odt.setComments(True) @@ -470,7 +471,9 @@ def testCoreToOdt_Convert(mockGUI): '