Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Process comments in the Tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-22 17:37:00 +02:00
parent e759aa8d1b
commit c537d0f396
17 changed files with 392 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_formats/test_fmt_todocx.py
+15 -9
View File
@@ -150,37 +150,43 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Synopsis
xTest = ET.Element(_wTag("body"))
doc._blocks = [(BlockTyp.SYNOPSIS, 0, "Hello World", [], BlockFmt.NONE)]
doc._text = "%Synopsis: Hello World\n\n"
doc.tokenizeText()
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
f'<w:body><w:p><w:pPr><w:pStyle w:val="{S_META}" /></w:pPr>'
'<w:r><w:rPr><w:b /></w:rPr><w:t>Synopsis:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> Hello World</w:t></w:r>'
'<w:r><w:rPr><w:b /><w:color w:val="813709" /></w:rPr><w:t>Synopsis:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> </w:t></w:r>'
'<w:r><w:rPr><w:color w:val="813709" /></w:rPr><w:t>Hello World</w:t></w:r>'
'</w:p></w:body>'
)
# Short
xTest = ET.Element(_wTag("body"))
doc._blocks = [(BlockTyp.SHORT, 0, "Hello World", [], BlockFmt.NONE)]
doc._text = "%Short: Hello World\n\n"
doc.tokenizeText()
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
f'<w:body><w:p><w:pPr><w:pStyle w:val="{S_META}" /></w:pPr>'
'<w:r><w:rPr><w:b /></w:rPr><w:t>Short Description:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> Hello World</w:t></w:r>'
'<w:r><w:rPr><w:b /><w:color w:val="813709" /></w:rPr><w:t>Short Description:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> </w:t></w:r>'
'<w:r><w:rPr><w:color w:val="813709" /></w:rPr><w:t>Hello World</w:t></w:r>'
'</w:p></w:body>'
)
# Comment
xTest = ET.Element(_wTag("body"))
doc._blocks = [(BlockTyp.COMMENT, 0, "Hello World", [], BlockFmt.NONE)]
doc._text = "% Hello World\n\n"
doc.tokenizeText()
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
f'<w:body><w:p><w:pPr><w:pStyle w:val="{S_META}" /></w:pPr>'
'<w:r><w:rPr><w:b /></w:rPr><w:t>Comment:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> Hello World</w:t></w:r>'
'<w:r><w:rPr><w:b /><w:color w:val="646464" /></w:rPr><w:t>Comment:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> </w:t></w:r>'
'<w:r><w:rPr><w:color w:val="646464" /></w:rPr><w:t>Hello World</w:t></w:r>'
'</w:p></w:body>'
)
tests/test_formats/test_fmt_tohtml.py
-13
View File
@@ -708,19 +708,6 @@ def testFmtToHtml_Format(mockGUI):
project = NWProject()
html = ToHtml(project)
# Export Mode
# ===========
assert html._formatSynopsis("synopsis text", True) == (
"<p class='synopsis'><strong>Synopsis:</strong> synopsis text</p>\n"
)
assert html._formatSynopsis("short text", False) == (
"<p class='synopsis'><strong>Short Description:</strong> short text</p>\n"
)
assert html._formatComments("comment text") == (
"<p class='comment'><strong>Comment:</strong> comment text</p>\n"
)
assert html._formatKeywords("") == ("", "")
assert html._formatKeywords("tag: Jane") == (
"tag", "<span class='keyword'>Tag:</span> <a class='tag' name='tag_Jane'>Jane</a>"
tests/test_formats/test_fmt_tokenizer.py
+75 -28
View File
@@ -29,8 +29,9 @@ from PyQt5.QtGui import QFont
from novelwriter import CONFIG
from novelwriter.constants import nwHeadFmt, nwStyles
from novelwriter.core.project import NWProject
from novelwriter.enum import nwComment
from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt, stripEscape
from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer
from novelwriter.formats.tokenizer import COMMENT_STYLE, HeadingFormatter, Tokenizer
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toraw import ToRaw
@@ -700,52 +701,64 @@ def testFmtToken_MetaFormat(mockGUI):
project = NWProject()
tokens = ToRaw(project)
# Comment
tokens._text = "% A comment\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.COMMENT, 0, "A comment", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "\n"
tokens.setComments(True)
tokens.tokenizeText()
assert tokens.allMarkdown[-1] == "% A comment\n\n"
# Ignore Text
tokens._text = "%~ Some text\n"
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n"
# Comment
tokens.setComments(False)
tokens._text = "% A comment\n"
tokens.tokenizeText()
assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n"
tokens.setComments(True)
tokens._text = "% A comment\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.COMMENT, 0, "Comment: A comment", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"), (8, TextFmt.B_E, ""),
(8, TextFmt.COL_E, ""), (9, TextFmt.COL_B, "comment"), (18, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
assert tokens.allMarkdown[-1] == "% A comment\n\n"
# Synopsis
tokens.setSynopsis(False)
tokens._text = "%synopsis: The synopsis\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE),
]
tokens._text = "% synopsis: The synopsis\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE),
]
assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n"
tokens.setSynopsis(True)
tokens._text = "% synopsis: The synopsis\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.SUMMARY, 0, "Synopsis: The synopsis", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"), (9, TextFmt.B_E, ""),
(9, TextFmt.COL_E, ""), (10, TextFmt.COL_B, "synopsis"), (22, TextFmt.COL_E, "")
], BlockFmt.NONE
)]
assert tokens.allMarkdown[-1] == "% synopsis: The synopsis\n\n"
# Short
tokens.setSynopsis(False)
tokens._text = "% short: A short description\n"
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.SHORT, 0, "A short description", [], BlockFmt.NONE),
]
assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n"
tokens.setSynopsis(True)
tokens._text = "% short: A short description\n"
tokens.tokenizeText()
assert tokens._blocks == [(
BlockTyp.SUMMARY, 0, "Short Description: A short description", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"), (18, TextFmt.B_E, ""),
(18, TextFmt.COL_E, ""), (19, TextFmt.COL_B, "synopsis"), (38, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
assert tokens.allMarkdown[-1] == "% short: A short description\n\n"
# Keyword
@@ -1355,6 +1368,7 @@ def testFmtToken_TextIndent(mockGUI):
"""Test the handling of text indent in the Tokenizer class."""
project = NWProject()
tokens = BareTokenizer(project)
tokens.setSynopsis(True)
# No First Indent
tokens.setFirstLineIndent(True, 1.0, False)
@@ -1388,9 +1402,13 @@ def testFmtToken_TextIndent(mockGUI):
"%Synopsis: Stuff happens.\n\n"
)
tokens.tokenizeText()
tFmt = [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"), (9, TextFmt.B_E, ""),
(9, TextFmt.COL_E, ""), (10, TextFmt.COL_B, "synopsis"), (24, TextFmt.COL_E, ""),
]
assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene Two", [], BlockFmt.NONE),
(BlockTyp.SYNOPSIS, 1, "Stuff happens.", [], BlockFmt.NONE),
(BlockTyp.SUMMARY, 1, "Synopsis: Stuff happens.", tFmt, BlockFmt.NONE),
]
assert tokens._noIndent is True
@@ -1622,6 +1640,35 @@ def testFmtToken_ProcessHeaders(mockGUI):
assert tokens._noSep is False
@pytest.mark.core
def testFmtToken_FormatNote(mockGUI, ipsumText):
"""Test note and comment formatting."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
# Comment, No Formatting
style = COMMENT_STYLE[nwComment.PLAIN]
assert tokens._formatNote(style, "", "Hello world!") == (
"Comment: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "comment"),
(8, TextFmt.B_E, ""), (8, TextFmt.COL_E, ""),
(9, TextFmt.COL_B, "comment"), (21, TextFmt.COL_E, ""),
]
)
# Synopsis, No Formatting
style = COMMENT_STYLE[nwComment.SYNOPSIS]
assert tokens._formatNote(style, "", "Hello world!") == (
"Synopsis: Hello world!", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "modifier"),
(9, TextFmt.B_E, ""), (9, TextFmt.COL_E, ""),
(10, TextFmt.COL_B, "synopsis"), (22, TextFmt.COL_E, ""),
]
)
@pytest.mark.core
def testFmtToken_BuildOutline(mockGUI, ipsumText):
"""Test stats counter of the Tokenizer class."""
@@ -1763,7 +1810,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert [t[2] for t in tokens._blocks] == ["Chapter", "Synopsis: Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
@@ -1780,7 +1827,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert [t[2] for t in tokens._blocks] == ["Chapter", "Short Description: Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 5, "textWords": 1, "titleWords": 1,
@@ -1797,7 +1844,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setComments(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert [t[2] for t in tokens._blocks] == ["Chapter", "Comment: Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
tests/test_formats/test_fmt_toodt.py
+25 -40
View File
@@ -523,12 +523,15 @@ def testFmtToOdt_ConvertParagraphs(mockGUI):
'<text:h text:style-name="Heading_20_3" text:outline-level="3">Scene</text:h>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T9">'
'Point of View:</text:span> Jane</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T9">'
'Synopsis:</text:span> So it begins</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T9">'
'Short Description:</text:span> Then what</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T9">'
'Comment:</text:span> A plain comment</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T10">'
'Synopsis:</text:span> '
'<text:span text:style-name="T11">So it begins</text:span></text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T10">'
'Short Description:</text:span> '
'<text:span text:style-name="T11">Then what</text:span></text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T12">'
'Comment:</text:span> '
'<text:span text:style-name="T13">A plain comment</text:span></text:p>'
'</office:text>'
)
@@ -703,7 +706,7 @@ def testFmtToOdt_ConvertParagraphs(mockGUI):
assert odt.errData == []
assert xmlToText(odt._xText) == (
'<office:text>'
'<text:p text:style-name="Text_20_body">Test text **<text:span text:style-name="T10">'
'<text:p text:style-name="Text_20_body">Test text **<text:span text:style-name="T14">'
'bold</text:span>** and more.</text:p>'
'</office:text>'
)
@@ -897,22 +900,6 @@ def testFmtToOdt_SpecialFormats(mockGUI):
project = NWProject()
odt = ToOdt(project, isFlat=True)
assert odt._formatSynopsis("synopsis text", [(9, TextFmt.STRIP, "")], True) == (
"Synopsis: synopsis text", [
(0, TextFmt.B_B, ""), (9, TextFmt.B_E, ""), (19, TextFmt.STRIP, "")
]
)
assert odt._formatSynopsis("short text", [(6, TextFmt.STRIP, "")], False) == (
"Short Description: short text", [
(0, TextFmt.B_B, ""), (18, TextFmt.B_E, ""), (25, TextFmt.STRIP, "")
]
)
assert odt._formatComments("comment text", [(8, TextFmt.STRIP, "")]) == (
"Comment: comment text", [
(0, TextFmt.B_B, ""), (8, TextFmt.B_E, ""), (17, TextFmt.STRIP, "")
]
)
assert odt._formatKeywords("") == ("", [])
assert odt._formatKeywords("tag: Jane") == (
"Tag: Jane", [(0, TextFmt.B_B, ""), (4, TextFmt.B_E, "")]
@@ -1108,12 +1095,12 @@ def testFmtToOdt_ODTParagraphStyle():
assert parStyle._tAttr["color"] == ["fo", None]
assert parStyle._tAttr["opacity"] == ["loext", None]
parStyle.setColour(QColor(0, 0, 0, 128))
parStyle.setColor(QColor(0, 0, 0, 128))
assert parStyle._tAttr["color"] == ["fo", "#000000"]
assert parStyle._tAttr["opacity"] == ["loext", "50%"]
parStyle.setColour(None)
parStyle.setColor(None)
assert parStyle._tAttr["color"] == ["fo", None]
assert parStyle._tAttr["opacity"] == ["loext", None]
@@ -1157,8 +1144,8 @@ def testFmtToOdt_ODTParagraphStyle():
aStyle = ODTParagraphStyle("test")
oStyle = ODTParagraphStyle("test")
aStyle.setColour(QColor(0, 0, 0))
oStyle.setColour(QColor(42, 42, 42))
aStyle.setColor(QColor(0, 0, 0))
oStyle.setColor(QColor(42, 42, 42))
assert aStyle.checkNew(oStyle) is True
assert aStyle.getID() != oStyle.getID()
@@ -1212,24 +1199,22 @@ def testFmtToOdt_ODTTextStyle():
# Text Color
assert txtStyle._tAttr["color"] == ["fo", None]
txtStyle.setColour("stuff")
txtStyle.setColor("#012345") # type: ignore
assert txtStyle._tAttr["color"] == ["fo", None]
txtStyle.setColour("012345")
assert txtStyle._tAttr["color"] == ["fo", None]
txtStyle.setColour("#012345")
assert txtStyle._tAttr["color"] == ["fo", "#012345"]
txtStyle.setColour("stuff")
txtStyle.setColor(QColor(255, 128, 0))
assert txtStyle._tAttr["color"] == ["fo", "#ff8000"]
txtStyle.setColor(None)
assert txtStyle._tAttr["color"] == ["fo", None]
# Background Color
assert txtStyle._tAttr["background-color"] == ["fo", None]
txtStyle.setBackgroundColour("stuff")
txtStyle.setBackgroundColor("stuff")
assert txtStyle._tAttr["background-color"] == ["fo", None]
txtStyle.setBackgroundColour("012345")
txtStyle.setBackgroundColor("012345")
assert txtStyle._tAttr["background-color"] == ["fo", None]
txtStyle.setBackgroundColour("#012345")
txtStyle.setBackgroundColor("#012345")
assert txtStyle._tAttr["background-color"] == ["fo", "#012345"]
txtStyle.setBackgroundColour("stuff")
txtStyle.setBackgroundColor("stuff")
assert txtStyle._tAttr["background-color"] == ["fo", None]
# Text Position
@@ -1289,11 +1274,11 @@ def testFmtToOdt_ODTTextStyle():
# Underline Colour
assert txtStyle._tAttr["text-underline-color"] == ["style", None]
txtStyle.setUnderlineColour("stuff")
txtStyle.setUnderlineColor("stuff")
assert txtStyle._tAttr["text-underline-color"] == ["style", None]
txtStyle.setUnderlineColour("font-color")
txtStyle.setUnderlineColor("font-color")
assert txtStyle._tAttr["text-underline-color"] == ["style", "font-color"]
txtStyle.setUnderlineColour("stuff")
txtStyle.setUnderlineColor("stuff")
assert txtStyle._tAttr["text-underline-color"] == ["style", None]
# Pack XML