Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update tokenizer and converters

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2023-11-25 18:12:45 +01:00
parent 8ed85363e2
commit 6a6b05c338
9 changed files with 95 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/coretools.py
+6 -1
View File
@@ -406,6 +406,7 @@ class ProjectBuilder:
chSynop = self.tr("Summary of the chapter.")
scSynop = self.tr("Summary of the scene.")
bfNote = self.tr("A brief description.")
# Create chapters
if numChapters > 0:
@@ -446,7 +447,11 @@ class ProjectBuilder:
aHandle = project.newFile(noteTitles[newRoot], rHandle)
ntTag = simplified(noteTitles[newRoot]).replace(" ", "")
aDoc = project.storage.getDocument(aHandle)
aDoc.writeDocument(f"# {noteTitles[newRoot]}\n\n@tag: {ntTag}\n\n")
aDoc.writeDocument(
f"# {noteTitles[newRoot]}\n\n"
f"% Brief: {bfNote}\n\n"
f"@tag: {ntTag}\n\n"
)
# Also add the archive and trash folders
project.newRoot(nwItemClass.ARCHIVE)
novelwriter/core/tohtml.py
+6 -3
View File
@@ -287,7 +287,10 @@ class ToHtml(Tokenizer):
para.append(stripEscape(tTemp.rstrip()))
elif tType == self.T_SYNOPSIS and self._doSynopsis:
lines.append(self._formatSynopsis(tText))
lines.append(self._formatSynopsis(tText, True))
elif tType == self.T_BRIEF and self._doSynopsis:
lines.append(self._formatSynopsis(tText, False))
elif tType == self.T_COMMENT and self._doComments:
lines.append(self._formatComments(tText))
@@ -454,9 +457,9 @@ class ToHtml(Tokenizer):
# Internal Functions
##
def _formatSynopsis(self, text: str) -> str:
def _formatSynopsis(self, text: str, synopsis: bool) -> str:
"""Apply HTML formatting to synopsis."""
sSynop = self._localLookup("Synopsis")
sSynop = self._localLookup("Synopsis") if synopsis else self._localLookup("Brief")
if self._genMode == self.M_PREVIEW:
return f"<p class='comment'><span class='synopsis'>{sSynop}:</span> {text}</p>\n"
else:
novelwriter/core/tokenizer.py
+24 -17
View File
@@ -34,8 +34,9 @@ from pathlib import Path
from functools import partial
from PyQt5.QtCore import QCoreApplication, QRegularExpression
from novelwriter.core.index import processComment
from novelwriter.enum import nwItemLayout
from novelwriter.enum import nwComment, nwItemLayout
from novelwriter.common import formatTimeStamp, numberToRoman, checkInt
from novelwriter.constants import nwHeadFmt, nwRegEx, nwShortcode, nwUnicode
from novelwriter.core.project import NWProject
@@ -79,17 +80,18 @@ class Tokenizer(ABC):
# Block Type
T_EMPTY = 1 # Empty line (new paragraph)
T_SYNOPSIS = 2 # Synopsis comment
T_COMMENT = 3 # Comment line
T_KEYWORD = 4 # Command line
T_TITLE = 5 # Title
T_UNNUM = 6 # Unnumbered
T_HEAD1 = 7 # Header 1
T_HEAD2 = 8 # Header 2
T_HEAD3 = 9 # Header 3
T_HEAD4 = 10 # Header 4
T_TEXT = 11 # Text line
T_SEP = 12 # Scene separator
T_SKIP = 13 # Paragraph break
T_BRIEF = 3 # Brief comment
T_COMMENT = 4 # Comment line
T_KEYWORD = 5 # Command line
T_TITLE = 6 # Title
T_UNNUM = 7 # Unnumbered
T_HEAD1 = 8 # Header 1
T_HEAD2 = 9 # Header 2
T_HEAD3 = 10 # Header 3
T_HEAD4 = 11 # Header 4
T_TEXT = 12 # Text line
T_SEP = 13 # Scene separator
T_SKIP = 14 # Paragraph break
# Block Style
A_NONE = 0x0000 # No special style
@@ -461,17 +463,22 @@ class Tokenizer(ABC):
continue
if aLine[0] == "%":
cLine = aLine[1:].lstrip()
synTag = cLine[:9].lower()
if synTag == "synopsis:":
cStyle, cText, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS:
self._tokens.append((
self.T_SYNOPSIS, nHead, cLine[9:].strip(), None, sAlign
self.T_SYNOPSIS, nHead, cText, None, sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
elif cStyle == nwComment.BRIEF:
self._tokens.append((
self.T_BRIEF, nHead, cText, None, sAlign
))
if self._doSynopsis and self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
else:
self._tokens.append((
self.T_COMMENT, nHead, aLine[1:].strip(), None, sAlign
self.T_COMMENT, nHead, cText, None, sAlign
))
if self._doComments and self._keepMarkdown:
tmpMarkdown.append("%s\n" % aLine)
novelwriter/core/tomd.py
+4
View File
@@ -170,6 +170,10 @@ class ToMarkdown(Tokenizer):
label = self._localLookup("Synopsis")
lines.append(f"**{label}:** {tText}\n\n")
elif tType == self.T_BRIEF and self._doSynopsis:
label = self._localLookup("Brief")
lines.append(f"**{label}:** {tText}\n\n")
elif tType == self.T_COMMENT and self._doComments:
label = self._localLookup("Comment")
lines.append(f"**{label}:** {tText}\n\n")
novelwriter/core/toodt.py
+7 -3
View File
@@ -481,7 +481,11 @@ class ToOdt(Tokenizer):
pFmt.append(tFormat)
elif tType == self.T_SYNOPSIS and self._doSynopsis:
tTemp, fTemp = self._formatSynopsis(tText)
tTemp, fTemp = self._formatSynopsis(tText, True)
self._addTextPar("Text_20_Meta", oStyle, tTemp, tFmt=fTemp)
elif tType == self.T_BRIEF and self._doSynopsis:
tTemp, fTemp = self._formatSynopsis(tText, False)
self._addTextPar("Text_20_Meta", oStyle, tTemp, tFmt=fTemp)
elif tType == self.T_COMMENT and self._doComments:
@@ -552,9 +556,9 @@ class ToOdt(Tokenizer):
# Internal Functions
##
def _formatSynopsis(self, text: str) -> tuple[str, list[tuple[int, int]]]:
def _formatSynopsis(self, text: str, synopsis: bool) -> tuple[str, list[tuple[int, int]]]:
"""Apply formatting to synopsis lines."""
name = self._localLookup("Synopsis")
name = self._localLookup("Synopsis") if synopsis else self._localLookup("Brief")
rTxt = f"{name}: {text}"
rFmt = [(0, self.FMT_B_B), (len(name) + 1, self.FMT_B_E)]
return rTxt, rFmt
tests/test_core/test_core_tohtml.py
+17 -3
View File
@@ -149,7 +149,7 @@ def testCoreToHtml_ConvertFormat(mockGUI):
"<p class='break'>Line one<br/>Line two<br/>Line three</p>\n"
)
# Synopsis
# Synopsis, Brief
html._text = "%synopsis: The synopsis ...\n"
html.tokenizeText()
html.doConvert()
@@ -163,6 +163,14 @@ def testCoreToHtml_ConvertFormat(mockGUI):
"<p class='synopsis'><strong>Synopsis:</strong> The synopsis ...</p>\n"
)
html.setSynopsis(True)
html._text = "%brief: A description ...\n"
html.tokenizeText()
html.doConvert()
assert html.theResult == (
"<p class='synopsis'><strong>Brief:</strong> A description ...</p>\n"
)
# Comment
html._text = "% A comment ...\n"
html.tokenizeText()
@@ -610,9 +618,12 @@ def testCoreToHtml_Format(mockGUI):
# Export Mode
# ===========
assert html._formatSynopsis("synopsis text") == (
assert html._formatSynopsis("synopsis text", True) == (
"<p class='synopsis'><strong>Synopsis:</strong> synopsis text</p>\n"
)
assert html._formatSynopsis("brief text", False) == (
"<p class='synopsis'><strong>Brief:</strong> brief text</p>\n"
)
assert html._formatComments("comment text") == (
"<p class='comment'><strong>Comment:</strong> comment text</p>\n"
)
@@ -632,9 +643,12 @@ def testCoreToHtml_Format(mockGUI):
html.setPreview(True, True)
assert html._formatSynopsis("synopsis text") == (
assert html._formatSynopsis("synopsis text", True) == (
"<p class='comment'><span class='synopsis'>Synopsis:</span> synopsis text</p>\n"
)
assert html._formatSynopsis("brief text", False) == (
"<p class='comment'><span class='synopsis'>Brief:</span> brief text</p>\n"
)
assert html._formatComments("comment text") == (
"<p class='comment'>comment text</p>\n"
)
tests/test_core/test_core_tokenizer.py
+14
View File
@@ -471,6 +471,20 @@ def testCoreToken_MetaFormat(mockGUI):
tokens.tokenizeText()
assert tokens.theMarkdown[-1] == "% synopsis: The synopsis\n\n"
# Brief
tokens.setSynopsis(False)
tokens._text = "% brief: A description\n"
tokens.tokenizeText()
assert tokens._tokens == [
(Tokenizer.T_BRIEF, 0, "A description", None, Tokenizer.A_NONE),
(Tokenizer.T_EMPTY, 0, "", None, Tokenizer.A_NONE),
]
assert tokens.theMarkdown[-1] == "\n"
tokens.setSynopsis(True)
tokens.tokenizeText()
assert tokens.theMarkdown[-1] == "% brief: A description\n\n"
# Keyword
tokens._text = "@char: Bod\n"
tokens.tokenizeText()
tests/test_core/test_core_tomd.py
+7 -1
View File
@@ -106,7 +106,7 @@ def testCoreToMarkdown_ConvertFormat(mockGUI):
theMD.doConvert()
assert theMD.theResult == "Line one \nLine two \nLine three\n\n"
# Synopsis
# Synopsis, Brief
theMD._text = "%synopsis: The synopsis ...\n"
theMD.tokenizeText()
theMD.doConvert()
@@ -118,6 +118,12 @@ def testCoreToMarkdown_ConvertFormat(mockGUI):
theMD.doConvert()
assert theMD.theResult == "**Synopsis:** The synopsis ...\n\n"
theMD.setSynopsis(True)
theMD._text = "%brief: A description ...\n"
theMD.tokenizeText()
theMD.doConvert()
assert theMD.theResult == "**Brief:** A description ...\n\n"
# Comment
theMD._text = "% A comment ...\n"
theMD.tokenizeText()
tests/test_core/test_core_toodt.py
+10 -4
View File
@@ -447,12 +447,13 @@ def testCoreToOdt_Convert(mockGUI):
'</office:text>'
)
# Synopsis, Comment, Keywords
# Synopsis, Brief, Comment, Keywords
odt._text = (
"### Scene\n\n"
"@pov: Jane\n\n"
"% synopsis: So it begins\n\n"
"% a plain comment\n\n"
"% brief: Then what\n\n"
"% A plain comment\n\n"
)
odt.setSynopsis(True)
odt.setComments(True)
@@ -470,7 +471,9 @@ def testCoreToOdt_Convert(mockGUI):
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T7">'
'Synopsis:</text:span> So it begins</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T7">'
'Comment:</text:span> a plain comment</text:p>'
'Brief:</text:span> Then what</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T7">'
'Comment:</text:span> A plain comment</text:p>'
'</office:text>'
)
@@ -804,9 +807,12 @@ def testCoreToOdt_Format(mockGUI):
project = NWProject()
odt = ToOdt(project, isFlat=True)
assert odt._formatSynopsis("synopsis text") == (
assert odt._formatSynopsis("synopsis text", True) == (
"Synopsis: synopsis text", [(0, ToOdt.FMT_B_B), (9, ToOdt.FMT_B_E)]
)
assert odt._formatSynopsis("brief text", False) == (
"Brief: brief text", [(0, ToOdt.FMT_B_B), (6, ToOdt.FMT_B_E)]
)
assert odt._formatComments("comment text") == (
"Comment: comment text", [(0, ToOdt.FMT_B_B), (8, ToOdt.FMT_B_E)]
)