Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Process tags and references in Tokenizer

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-22 21:37:18 +02:00
parent c7554f2b8d
commit bd5442c9e5
16 changed files with 589 additions and 466 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_formats/test_fmt_todocx.py
+16 -11
View File
@@ -192,36 +192,41 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Tags and References (Single)
xTest = ET.Element(_wTag("body"))
doc._blocks = [(BlockTyp.KEYWORD, 0, "tag: Stuff", [], BlockFmt.NONE)]
doc._text = "@tag: Stuff"
doc.tokenizeText()
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
f'<w:body><w:p><w:pPr><w:pStyle w:val="{S_META}" /></w:pPr>'
'<w:r><w:rPr><w:b /></w:rPr><w:t>Tag:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> Stuff</w:t></w:r>'
'<w:r><w:rPr><w:b /><w:color w:val="f5871f" /></w:rPr><w:t>Tag:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> </w:t></w:r>'
'<w:r><w:rPr><w:color w:val="4271ae" /></w:rPr><w:t>Stuff</w:t></w:r>'
'</w:p></w:body>'
)
# Tags and References (Multiple)
xTest = ET.Element(_wTag("body"))
doc._blocks = [(BlockTyp.KEYWORD, 0, "char: Jane, John", [], BlockFmt.NONE)]
doc._text = "@char: Jane, John"
doc.tokenizeText()
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
f'<w:body><w:p><w:pPr><w:pStyle w:val="{S_META}" /></w:pPr>'
'<w:r><w:rPr><w:b /></w:rPr><w:t>Characters:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> Jane, John</w:t></w:r>'
'<w:r><w:rPr><w:b /><w:color w:val="f5871f" /></w:rPr><w:t>Characters:</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve"> </w:t></w:r>'
'<w:r><w:rPr><w:color w:val="4271ae" /></w:rPr><w:t>Jane</w:t></w:r>'
'<w:r><w:rPr /><w:t xml:space="preserve">, </w:t></w:r>'
'<w:r><w:rPr><w:color w:val="4271ae" /></w:rPr><w:t>John</w:t></w:r>'
'</w:p></w:body>'
)
# Tags and References (Invalid)
xTest = ET.Element(_wTag("body"))
doc._blocks = [(BlockTyp.KEYWORD, 0, "stuff: Stuff", [], BlockFmt.NONE)]
doc._text = "@stuff: Stuff"
doc._pars = []
doc.tokenizeText()
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
f'<w:body><w:p><w:pPr><w:pStyle w:val="{S_META}" /></w:pPr></w:p></w:body>'
)
assert doc._pars == []
@pytest.mark.core
tests/test_formats/test_fmt_tohtml.py
+17 -35
View File
@@ -232,9 +232,9 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html.tokenizeText()
html.doConvert()
assert html.result == (
"<p class='meta meta-char'><span class='keyword'>Characters:</span> "
"<a class='tag' href='#tag_Bod'>Bod</a>, "
"<a class='tag' href='#tag_Jane'>Jane</a></p>\n"
"<p class='meta'><strong><span style='color: #f5871f'>Characters:</strong></span> "
"<span style='color: #4271ae'><a href='#tag_bod'>Bod</a></span>, "
"<span style='color: #4271ae'><a href='#tag_jane'>Jane</a></span></p>\n"
)
# Tags
@@ -242,16 +242,17 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html.tokenizeText()
html.doConvert()
assert html.result == (
"<p class='meta meta-tag'><span class='keyword'>Tag:</span> "
"<a class='tag' name='tag_Bod'>Bod</a></p>\n"
"<p class='meta'><strong><span style='color: #f5871f'>Tag:</strong></span> "
"<span style='color: #4271ae'><a name='tag_bod'>Bod</a></span></p>\n"
)
html._text = "@tag: Bod | Nobody Owens\n"
html.tokenizeText()
html.doConvert()
assert html.result == (
"<p class='meta meta-tag'><span class='keyword'>Tag:</span> "
"<a class='tag' name='tag_Bod'>Bod</a> | <span class='optional'>Nobody Owens</a></p>\n"
"<p class='meta'><strong><span style='color: #f5871f'>Tag:</strong></span> "
"<span style='color: #4271ae'><a name='tag_bod'>Bod</a></span> | "
"<span style='color: #4271ae'>Nobody Owens</span></p>\n"
)
# Multiple Keywords
@@ -262,15 +263,15 @@ def testFmtToHtml_ConvertParagraphs(mockGUI):
html.doConvert()
assert html.result == (
"<h1 style='page-break-before: always;'>Chapter</h1>\n"
"<p class='meta meta-pov' style='margin-bottom: 0;'>"
"<span class='keyword'>Point of View:</span> "
"<a class='tag' href='#tag_Bod'>Bod</a></p>\n"
"<p class='meta meta-plot' style='margin-bottom: 0; margin-top: 0;'>"
"<span class='keyword'>Plot:</span> "
"<a class='tag' href='#tag_Main'>Main</a></p>\n"
"<p class='meta meta-location' style='margin-top: 0;'>"
"<span class='keyword'>Locations:</span> "
"<a class='tag' href='#tag_Europe'>Europe</a></p>\n"
"<p class='meta' style='margin-bottom: 0;'>"
"<strong><span style='color: #f5871f'>Point of View:</strong></span> "
"<span style='color: #4271ae'><a href='#tag_bod'>Bod</a></span></p>\n"
"<p class='meta' style='margin-bottom: 0; margin-top: 0;'>"
"<strong><span style='color: #f5871f'>Plot:</strong></span> "
"<span style='color: #4271ae'><a href='#tag_main'>Main</a></span></p>\n"
"<p class='meta' style='margin-top: 0;'>"
"<strong><span style='color: #f5871f'>Locations:</strong></span> "
"<span style='color: #4271ae'><a href='#tag_europe'>Europe</a></span></p>\n"
)
# Dialogue
@@ -718,22 +719,3 @@ def testFmtToHtml_Methods(mockGUI):
html.setStyles(False)
assert html.getStyleSheet() == []
@pytest.mark.core
def testFmtToHtml_Format(mockGUI):
"""Test all the formatters for the ToHtml class."""
project = NWProject()
html = ToHtml(project)
html.initDocument()
assert html._formatKeywords("") == ("", "")
assert html._formatKeywords("tag: Jane") == (
"tag", "<span class='keyword'>Tag:</span> <a class='tag' name='tag_Jane'>Jane</a>"
)
assert html._formatKeywords("char: Bod, Jane") == (
"char",
"<span class='keyword'>Characters:</span> "
"<a class='tag' href='#tag_Bod'>Bod</a>, "
"<a class='tag' href='#tag_Jane'>Jane</a>"
)
tests/test_formats/test_fmt_tokenizer.py
+70 -16
View File
@@ -770,30 +770,54 @@ def testFmtToken_MetaFormat(mockGUI):
tokens.setKeywords(True)
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
]
assert tokens._blocks == [(
BlockTyp.KEYWORD, 0, "Characters: Bod", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(11, TextFmt.B_E, ""), (11, TextFmt.COL_E, ""),
(12, TextFmt.COL_B, "tag"), (12, TextFmt.HRF_B, "#tag_bod"),
(15, TextFmt.HRF_E, ""), (15, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
assert tokens.allMarkdown[-1] == "@char: Bod\n\n"
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
tokens.tokenizeText()
styTop = BlockFmt.NONE | BlockFmt.Z_BTMMRG
styMid = BlockFmt.NONE | BlockFmt.Z_BTMMRG | BlockFmt.Z_TOPMRG
styBtm = BlockFmt.NONE | BlockFmt.Z_TOPMRG
assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "pov: Bod", [], styTop),
(BlockTyp.KEYWORD, 0, "plot: Main", [], styMid),
(BlockTyp.KEYWORD, 0, "location: Europe", [], styBtm),
]
assert tokens._blocks == [(
BlockTyp.KEYWORD, 0, "Point of View: Bod", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(14, TextFmt.B_E, ""), (14, TextFmt.COL_E, ""),
(15, TextFmt.COL_B, "tag"), (15, TextFmt.HRF_B, "#tag_bod"),
(18, TextFmt.HRF_E, ""), (18, TextFmt.COL_E, ""),
], BlockFmt.Z_BTMMRG
), (
BlockTyp.KEYWORD, 0, "Plot: Main", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(5, TextFmt.B_E, ""), (5, TextFmt.COL_E, ""),
(6, TextFmt.COL_B, "tag"), (6, TextFmt.HRF_B, "#tag_main"),
(10, TextFmt.HRF_E, ""), (10, TextFmt.COL_E, ""),
], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG
), (
BlockTyp.KEYWORD, 0, "Locations: Europe", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(10, TextFmt.B_E, ""), (10, TextFmt.COL_E, ""),
(11, TextFmt.COL_B, "tag"), (11, TextFmt.HRF_B, "#tag_europe"),
(17, TextFmt.HRF_E, ""), (17, TextFmt.COL_E, ""),
], BlockFmt.Z_TOPMRG
)]
assert tokens.allMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n"
# Ignored keywords
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
tokens.setIgnoredKeywords("@plot, @location")
tokens.tokenizeText()
assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "pov: Bod", [], BlockFmt.NONE),
]
assert tokens._blocks == [(
BlockTyp.KEYWORD, 0, "Point of View: Bod", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(14, TextFmt.B_E, ""), (14, TextFmt.COL_E, ""),
(15, TextFmt.COL_B, "tag"), (15, TextFmt.HRF_B, "#tag_bod"),
(18, TextFmt.HRF_E, ""), (18, TextFmt.COL_E, ""),
], BlockFmt.NONE
)]
@pytest.mark.core
@@ -1643,7 +1667,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
@pytest.mark.core
def testFmtToken_FormatNote(mockGUI, ipsumText):
def testFmtToken_FormatComment(mockGUI):
"""Test note and comment formatting."""
project = NWProject()
project.data.setLanguage("en")
@@ -1671,6 +1695,36 @@ def testFmtToken_FormatNote(mockGUI, ipsumText):
)
@pytest.mark.core
def testFmtToken_FormatMeta(mockGUI):
"""Test meta formatting."""
project = NWProject()
project.data.setLanguage("en")
project._loadProjectLocalisation()
tokens = BareTokenizer(project)
assert tokens._formatMeta("@tag: Jane | Jane Smith") == (
"Tag: Jane | Jane Smith", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(4, TextFmt.B_E, ""), (4, TextFmt.COL_E, ""),
(5, TextFmt.COL_B, "tag"), (5, TextFmt.ANM_B, "tag_jane"),
(9, TextFmt.ANM_E, ""), (9, TextFmt.COL_E, ""),
(12, TextFmt.COL_B, "optional"), (22, TextFmt.COL_E, ""),
]
)
assert tokens._formatMeta("@char: Jane, John") == (
"Characters: Jane, John", [
(0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
(11, TextFmt.B_E, ""), (11, TextFmt.COL_E, ""),
(12, TextFmt.COL_B, "tag"), (12, TextFmt.HRF_B, "#tag_jane"),
(16, TextFmt.HRF_E, ""), (16, TextFmt.COL_E, ""),
(18, TextFmt.COL_B, "tag"), (18, TextFmt.HRF_B, "#tag_john"),
(22, TextFmt.HRF_E, ""), (22, TextFmt.COL_E, ""),
]
)
@pytest.mark.core
def testFmtToken_BuildOutline(mockGUI, ipsumText):
"""Test stats counter of the Tokenizer class."""
@@ -1863,7 +1917,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setKeywords(True)
tokens.tokenizeText()
tokens.countStats()
assert [t[2] for t in tokens._blocks] == ["Chapter", "pov: Jane", "Text"]
assert [t[2] for t in tokens._blocks] == ["Chapter", "Point of View: Jane", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 6, "textWords": 1, "titleWords": 1,
tests/test_formats/test_fmt_tomarkdown.py
-12
View File
@@ -293,15 +293,3 @@ def testFmtToMarkdown_Save(mockGUI, fncPath):
saveFile = fncPath / "outFile.md"
toMD.saveDocument(saveFile)
assert saveFile.read_text(encoding="utf-8") == "".join(resText)
@pytest.mark.core
def testFmtToMarkdown_Format(mockGUI):
"""Test all the formatters for the ToMarkdown class."""
project = NWProject()
toMD = ToMarkdown(project, False)
assert toMD._formatKeywords("", BlockFmt.NONE) == ""
assert toMD._formatKeywords("tag: Jane", BlockFmt.NONE) == "**Tag:** Jane\n\n"
assert toMD._formatKeywords("tag: Jane, John", BlockFmt.NONE) == "**Tag:** Jane, John\n\n"
assert toMD._formatKeywords("tag: Jane", BlockFmt.Z_BTMMRG) == "**Tag:** Jane \n"
tests/test_formats/test_fmt_toodt.py
+19 -37
View File
@@ -202,7 +202,6 @@ def testFmtToOdt_TextFormatting(mockGUI):
fmt = [(2, TextFmt.B_B, ""), (5, TextFmt.B_E, ""), (7, 99999, "")]
xTest = ET.Element(_mkTag("office", "text"))
odt._addTextPar(xTest, "Standard", oStyle, text, tFmt=fmt)
assert odt.errData == ["Unknown format tag encountered"]
assert xmlToText(xTest) == (
'<office:text>'
'<text:p text:style-name="Standard">A <text:span text:style-name="T1">few</text:span> '
@@ -521,17 +520,17 @@ def testFmtToOdt_ConvertParagraphs(mockGUI):
assert xmlToText(odt._xText) == (
'<office:text>'
'<text:h text:style-name="Heading_20_3" text:outline-level="3">Scene</text:h>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T9">'
'Point of View:</text:span> Jane</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T10">'
'Synopsis:</text:span> '
'<text:span text:style-name="T11">So it begins</text:span></text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T10">'
'Short Description:</text:span> '
'<text:span text:style-name="T11">Then what</text:span></text:p>'
'Point of View:</text:span> <text:span text:style-name="T11">Jane</text:span></text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T12">'
'Synopsis:</text:span> '
'<text:span text:style-name="T13">So it begins</text:span></text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T12">'
'Short Description:</text:span> '
'<text:span text:style-name="T13">Then what</text:span></text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T14">'
'Comment:</text:span> '
'<text:span text:style-name="T13">A plain comment</text:span></text:p>'
'<text:span text:style-name="T15">A plain comment</text:span></text:p>'
'</office:text>'
)
@@ -591,12 +590,12 @@ def testFmtToOdt_ConvertParagraphs(mockGUI):
assert xmlToText(odt._xText) == (
'<office:text>'
'<text:h text:style-name="Heading_20_3" text:outline-level="3">Scene</text:h>'
'<text:p text:style-name="P1"><text:span text:style-name="T9">'
'Point of View:</text:span> Jane</text:p>'
'<text:p text:style-name="P2"><text:span text:style-name="T9">'
'Characters:</text:span> John</text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T9">'
'Plot:</text:span> Main</text:p>'
'<text:p text:style-name="P1"><text:span text:style-name="T10">'
'Point of View:</text:span> <text:span text:style-name="T11">Jane</text:span></text:p>'
'<text:p text:style-name="P2"><text:span text:style-name="T10">'
'Characters:</text:span> <text:span text:style-name="T11">John</text:span></text:p>'
'<text:p text:style-name="Text_20_Meta"><text:span text:style-name="T10">'
'Plot:</text:span> <text:span text:style-name="T11">Main</text:span></text:p>'
'<text:p text:style-name="P3">Right align</text:p>'
'<text:p text:style-name="Text_20_body">Left Align</text:p>'
'<text:p text:style-name="P4">Centered</text:p>'
@@ -706,7 +705,7 @@ def testFmtToOdt_ConvertParagraphs(mockGUI):
assert odt.errData == []
assert xmlToText(odt._xText) == (
'<office:text>'
'<text:p text:style-name="Text_20_body">Test text **<text:span text:style-name="T14">'
'<text:p text:style-name="Text_20_body">Test text **<text:span text:style-name="T16">'
'bold</text:span>** and more.</text:p>'
'</office:text>'
)
@@ -894,21 +893,6 @@ def testFmtToOdt_SaveFull(mockGUI, fncPath, tstPaths):
assert cmpFiles(stylFile, stylComp)
@pytest.mark.core
def testFmtToOdt_SpecialFormats(mockGUI):
"""Test the special formatters for the ToOdt class."""
project = NWProject()
odt = ToOdt(project, isFlat=True)
assert odt._formatKeywords("") == ("", [])
assert odt._formatKeywords("tag: Jane") == (
"Tag: Jane", [(0, TextFmt.B_B, ""), (4, TextFmt.B_E, "")]
)
assert odt._formatKeywords("char: Bod, Jane") == (
"Characters: Bod, Jane", [(0, TextFmt.B_B, ""), (11, TextFmt.B_E, "")]
)
@pytest.mark.core
def testFmtToOdt_ODTParagraphStyle():
"""Test the ODTParagraphStyle class."""
@@ -1208,13 +1192,11 @@ def testFmtToOdt_ODTTextStyle():
# Background Color
assert txtStyle._tAttr["background-color"] == ["fo", None]
txtStyle.setBackgroundColor("stuff")
txtStyle.setBackgroundColor("#012345") # type: ignore
assert txtStyle._tAttr["background-color"] == ["fo", None]
txtStyle.setBackgroundColor("012345")
assert txtStyle._tAttr["background-color"] == ["fo", None]
txtStyle.setBackgroundColor("#012345")
assert txtStyle._tAttr["background-color"] == ["fo", "#012345"]
txtStyle.setBackgroundColor("stuff")
txtStyle.setBackgroundColor(QColor(255, 128, 0))
assert txtStyle._tAttr["background-color"] == ["fo", "#ff8000"]
txtStyle.setBackgroundColor(None)
assert txtStyle._tAttr["background-color"] == ["fo", None]
# Text Position