From e9596e1bbc0e57da0f9e8dbb31482f23c772de4b Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 27 Apr 2024 19:05:13 +0200 Subject: [PATCH] Fix footnote formatting for ODT and update document builder tests --- novelwriter/core/toodt.py | 33 +++++--- tests/test_base/test_base_common.py | 39 ++++++--- tests/test_core/test_core_index.py | 89 +++++++++++++++++++-- tests/test_core/test_core_tohtml.py | 118 +++++++++++++++++++--------- tests/test_core/test_core_tomd.py | 37 +++++++-- tests/test_core/test_core_toodt.py | 37 +++++++++ 6 files changed, 278 insertions(+), 75 deletions(-) diff --git a/novelwriter/core/toodt.py b/novelwriter/core/toodt.py index 67867133..8e0cd41d 100644 --- a/novelwriter/core/toodt.py +++ b/novelwriter/core/toodt.py @@ -646,8 +646,14 @@ class ToOdt(Tokenizer): xFmt = 0x00 tFrag = "" fLast = 0 + xNode = None for fPos, fFmt, fData in tFmt: + # Add any extra nodes + if xNode: + parProc.appendNode(xNode) + xNode = None + # Add the text up to the current fragment if tFrag := tText[fLast:fPos]: if xFmt == 0x00: @@ -685,7 +691,7 @@ class ToOdt(Tokenizer): elif fFmt == self.FMT_SUB_E: xFmt &= M_SUB elif fFmt == self.FMT_FNOTE: - parProc.appendNode(self._generateFootnote(fData)) + xNode = self._generateFootnote(fData) elif fFmt == self.FMT_STRIP: pass else: @@ -693,6 +699,9 @@ class ToOdt(Tokenizer): fLast = fPos + if xNode: + parProc.appendNode(xNode) + if tFrag := tText[fLast:]: if xFmt == 0x00: parProc.appendText(tFrag) @@ -1575,18 +1584,16 @@ class XMLParagraph: return def appendNode(self, xNode: ET.Element | None) -> None: - """Append an XML node to the paragraph.""" - if xNode: - if self._nState in (X_ROOT_TEXT, X_ROOT_TAIL): - self._xRoot.append(xNode) - self._xTail = xNode - self._xTail.tail = "" - self._nState = X_ROOT_TAIL - elif self._nState in (X_SPAN_TEXT, X_SPAN_SING): - self._xTail.append(xNode) - self._xSing = xNode - self._xSing.tail = "" - self._nState = X_SPAN_SING + """Append an XML node to the paragraph. We only check for the + X_ROOT_TEXT and X_ROOT_TAIL states. X_SPAN_TEXT is not possible + at all, and X_SPAN_SING only happens internally in an appendSpan + call, returning us to an X_ROOT_TAIL state. + """ + if xNode and self._nState in (X_ROOT_TEXT, X_ROOT_TAIL): + self._xRoot.append(xNode) + self._xTail = xNode + self._xTail.tail = "" + self._nState = X_ROOT_TAIL return def checkError(self) -> tuple[int, str]: diff --git a/tests/test_base/test_base_common.py b/tests/test_base/test_base_common.py index 1b2f7f9e..b97d3f7f 100644 --- a/tests/test_base/test_base_common.py +++ b/tests/test_base/test_base_common.py @@ -21,24 +21,25 @@ along with this program. If not, see . from __future__ import annotations import time -import pytest from pathlib import Path from xml.etree import ElementTree as ET -from tools import writeFile -from mocked import causeOSError +import pytest -from PyQt5.QtGui import QColor, QDesktopServices +from mocked import causeOSError from PyQt5.QtCore import QUrl +from PyQt5.QtGui import QColor, QDesktopServices +from tools import writeFile from novelwriter.common import ( - checkBool, checkFloat, checkInt, checkIntTuple, checkPath, checkString, - checkStringNone, checkUuid, cssCol, formatFileFilter, formatInt, - formatTime, formatTimeStamp, formatVersion, fuzzyTime, getFileSize, - hexToInt, isHandle, isItemClass, isItemLayout, isItemType, isTitleTag, - jsonEncode, makeFileNameSafe, minmax, numberToRoman, NWConfigParser, - openExternalPath, readTextFile, simplified, transferCase, xmlIndent, yesNo + NWConfigParser, checkBool, checkFloat, checkInt, checkIntTuple, checkPath, + checkString, checkStringNone, checkUuid, cssCol, formatFileFilter, + formatInt, formatTime, formatTimeStamp, formatVersion, fuzzyTime, + getFileSize, hexToInt, isHandle, isItemClass, isItemLayout, isItemType, + isListInstance, isTitleTag, jsonEncode, makeFileNameSafe, minmax, + numberToRoman, openExternalPath, readTextFile, simplified, transferCase, + xmlIndent, yesNo ) @@ -272,6 +273,24 @@ def testBaseCommon_isItemLayout(): # END Test testBaseCommon_isItemLayout +@pytest.mark.base +def testBaseCommon_isListInstance(): + """Test the isListInstance function.""" + # String + assert isListInstance("stuff", str) is False + assert isListInstance(["stuff"], str) is True + + # Int + assert isListInstance(1, int) is False + assert isListInstance([1], int) is True + + # Mixed + assert isListInstance([1], str) is False + assert isListInstance(["stuff"], int) is False + +# END Test testBaseCommon_isListInstance + + @pytest.mark.base def testBaseCommon_hexToInt(): """Test the hexToInt function.""" diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index 09d7ff03..23e58431 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -307,7 +307,7 @@ def testCoreIndex_CheckThese(mockGUI, fncPath, mockRnd): @pytest.mark.core -def testCoreIndex_ScanText(mockGUI, fncPath, mockRnd): +def testCoreIndex_ScanText(monkeypatch, mockGUI, fncPath, mockRnd): """Check the index text scanner.""" project = NWProject() mockRnd.reset() @@ -377,12 +377,14 @@ def testCoreIndex_ScanText(mockGUI, fncPath, mockRnd): "@char: Jane\n\n" "% this is a comment\n\n" "This is a story about Jane Smith.\n\n" - "Well, not really.\n" + "Well, not really.[footnote:key]\n\n" + "%Footnote.key: Footnote text.\n\n" )) assert index._tagsIndex.tagHandle("Jane") == cHandle assert index._tagsIndex.tagHeading("Jane") == "T0001" assert index._tagsIndex.tagClass("Jane") == "CHARACTER" assert index.getItemHeading(nHandle, "T0001").title == "Hello World!" # type: ignore + assert index._itemIndex[nHandle].noteKeys("footnotes") == {"key"} # type: ignore # Title Indexing # ============== @@ -549,6 +551,45 @@ def testCoreIndex_ScanText(mockGUI, fncPath, mockRnd): # END Test testCoreIndex_ScanText +@pytest.mark.core +def testCoreIndex_CommentKeys(monkeypatch, mockGUI, fncPath, mockRnd): + """Check the index comment key generator.""" + project = NWProject() + mockRnd.reset() + buildTestProject(project, fncPath) + index = project.index + + nKeys = 1000 + + # Generate footnote keys + keys = set() + for _ in range(nKeys): + key = index.newCommentKey(C.hSceneDoc, nwComment.FOOTNOTE) + assert key not in keys + assert key != "err" + keys.add(key) + assert len(keys) == nKeys + + # Generate comment keys + keys = set() + for _ in range(nKeys): + key = index.newCommentKey(C.hSceneDoc, nwComment.COMMENT) + assert key not in keys + keys.add(key) + assert len(keys) == nKeys + + # Induce collision + with monkeypatch.context() as mp: + mp.setattr("random.choices", lambda *a, **k: "aaaa") + assert index.newCommentKey(C.hSceneDoc, nwComment.FOOTNOTE) == "faaaa" + assert index.newCommentKey(C.hSceneDoc, nwComment.FOOTNOTE) == "err" + + # Check invalid comment style + assert index.newCommentKey(C.hSceneDoc, None) == "err" # type: ignore + +# END Test testCoreIndex_CommentKeys + + @pytest.mark.core def testCoreIndex_ExtractData(mockGUI, fncPath, mockRnd): """Check the index data extraction functions.""" @@ -1250,12 +1291,14 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd): itemIndex.clear() # Data must be dictionary - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc: itemIndex.unpackData("stuff") # type: ignore + assert str(exc.value) == "itemIndex is not a dict" # Keys must be valid handles - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc: itemIndex.unpackData({"stuff": "more stuff"}) + assert str(exc.value) == "itemIndex keys must be handles" # Unknown keys should be skipped itemIndex.unpackData({C.hInvalid: {}}) @@ -1267,8 +1310,9 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd): assert itemIndex[nHandle].handle == nHandle # type: ignore # Title tags must be valid - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc: itemIndex.unpackData({cHandle: {"headings": {"TTTTTTT": {}}}}) + assert str(exc.value) == "The itemIndex contains an invalid title key" # Reference without a heading should be rejected itemIndex.unpackData({ @@ -1282,37 +1326,66 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd): itemIndex.clear() # Tag keys must be strings - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc: itemIndex.unpackData({ cHandle: { "headings": {"T0001": {}}, "references": {"T0001": {1234: "@pov"}}, + "notes": {"footnotes": [], "comments": []}, } }) + assert str(exc.value) == "itemIndex reference key must be a string" # Type must be strings - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc: itemIndex.unpackData({ cHandle: { "headings": {"T0001": {}}, "references": {"T0001": {"John": []}}, + "notes": {"footnotes": [], "comments": []}, } }) + assert str(exc.value) == "itemIndex reference type must be a string" # Types must be valid - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc: itemIndex.unpackData({ cHandle: { "headings": {"T0001": {}}, "references": {"T0001": {"John": "@pov,@char,@stuff"}}, + "notes": {"footnotes": [], "comments": []}, } }) + assert str(exc.value) == "The itemIndex contains an invalid reference type" + + # Note type must be valid + with pytest.raises(ValueError) as exc: + itemIndex.unpackData({ + cHandle: { + "headings": {"T0001": {}}, + "references": {"T0001": {"John": "@pov,@char"}}, + "notes": {"stuff": [], "comments": []}, + } + }) + assert str(exc.value) == "The notes style is invalid" + + # Note keys must be all strings + with pytest.raises(ValueError) as exc: + itemIndex.unpackData({ + cHandle: { + "headings": {"T0001": {}}, + "references": {"T0001": {"John": "@pov,@char"}}, + "notes": {"footnotes": ["fkey", 1], "comments": []}, + } + }) + assert str(exc.value) == "The notes keys must be a list of strings" # This should pass itemIndex.unpackData({ cHandle: { "headings": {"T0001": {}}, "references": {"T0001": {"John": "@pov,@char"}}, + "notes": {"footnotes": ["fkey"], "comments": ["ckey"]}, } }) diff --git a/tests/test_core/test_core_tohtml.py b/tests/test_core/test_core_tohtml.py index b9cee8e3..c2d6def1 100644 --- a/tests/test_core/test_core_tohtml.py +++ b/tests/test_core/test_core_tohtml.py @@ -20,9 +20,9 @@ along with this program. If not, see . """ from __future__ import annotations -import pytest +import json -from tools import readFile +import pytest from novelwriter.core.project import NWProject from novelwriter.core.tohtml import ToHtml @@ -225,6 +225,22 @@ def testCoreToHtml_ConvertParagraphs(mockGUI): "Bod, Jane

\n" ) + # Tags + html._text = "@tag: Bod\n" + html.tokenizeText() + html.doConvert() + assert html.result == ( + "

Tag: Bod

\n" + ) + + html._text = "@tag: Bod | Nobody Owens\n" + html.tokenizeText() + html.doConvert() + assert html.result == ( + "

Tag: Bod " + "| Nobody Owens

\n" + ) + # Multiple Keywords html._isFirst = False html.setKeywords(True) @@ -241,6 +257,30 @@ def testCoreToHtml_ConvertParagraphs(mockGUI): "Locations: Europe

\n" ) + # Footnotes + # ========= + + html._text = ( + "Text with one[footnote:fa] or two[footnote:fb] footnotes.\n\n" + "%footnote.fa: Footnote text A.\n\n" + ) + html.tokenizeText() + html.doConvert() + assert html.result == ( + "

Text with one1 " + "or twoERR footnotes.

\n" + ) + + html.appendFootnotes() + assert html.result == ( + "

Text with one1 " + "or twoERR footnotes.

\n" + "

Footnotes

\n" + "
    \n" + "
  1. Footnote text A.

  2. \n" + "
\n" + ) + # Preview Mode # ============ @@ -480,7 +520,7 @@ def testCoreToHtml_SpecialCases(mockGUI): @pytest.mark.core -def testCoreToHtml_Complex(mockGUI, fncPath): +def testCoreToHtml_Save(mockGUI, fncPath): """Test the save method of the ToHtml class.""" project = NWProject() html = ToHtml(project) @@ -498,36 +538,28 @@ def testCoreToHtml_Complex(mockGUI, fncPath): "### Scene 2\n\nThe text of scene two.\n", "#### A Section\n\n\tMore text in scene two.\n", ] - resText = [ - ( - "

My Novel

\n" - "

By Jane Doh

\n" - ), - ( - "

Chapter 1

\n" - "

The text of chapter one.

\n" - ), - ( - "

Scene 1

\n" - "

The text of scene one.

\n" - ), - ( - "

A Section

\n" - "

More text in scene one.

\n" - ), - ( - "

Chapter 2

\n" - "

The text of chapter two.

\n" - ), - ( - "

Scene 2

\n" - "

The text of scene two.

\n" - ), - ( - "

A Section

\n" - "

\tMore text in scene two.

\n" - ), - ] + resText = [( + "

My Novel

\n" + "

By Jane Doh

\n" + ), ( + "

Chapter 1

\n" + "

The text of chapter one.

\n" + ), ( + "

Scene 1

\n" + "

The text of scene one.

\n" + ), ( + "

A Section

\n" + "

More text in scene one.

\n" + ), ( + "

Chapter 2

\n" + "

The text of chapter two.

\n" + ), ( + "

Scene 2

\n" + "

The text of scene two.

\n" + ), ( + "

A Section

\n" + "

\tMore text in scene two.

\n" + )] for i in range(len(docText)): html._text = docText[i] @@ -541,9 +573,10 @@ def testCoreToHtml_Complex(mockGUI, fncPath): html.replaceTabs(nSpaces=2, spaceChar=" ") resText[6] = "

A Section

\n

  More text in scene two.

\n" - # Check File - # ========== + # Check Files + # =========== + # HTML hStyle = html.getStyleSheet() htmlDoc = ( "\n" @@ -568,9 +601,20 @@ def testCoreToHtml_Complex(mockGUI, fncPath): saveFile = fncPath / "outFile.htm" html.saveHtml5(saveFile) - assert readFile(saveFile) == htmlDoc + assert saveFile.read_text(encoding="utf-8") == htmlDoc -# END Test testCoreToHtml_Complex + # JSON + HTML + saveFile = fncPath / "outFile.json" + html.saveHtmlJson(saveFile) + data = json.loads(saveFile.read_text(encoding="utf-8")) + assert data["meta"]["projectName"] == "" + assert data["meta"]["novelAuthor"] == "" + assert data["meta"]["buildTime"] > 0 + assert data["meta"]["buildTimeStr"] != "" + assert data["text"]["css"] == hStyle + assert len(data["text"]["html"]) == len(resText) + +# END Test testCoreToHtml_Save @pytest.mark.core diff --git a/tests/test_core/test_core_tomd.py b/tests/test_core/test_core_tomd.py index 4767ebd4..db575262 100644 --- a/tests/test_core/test_core_tomd.py +++ b/tests/test_core/test_core_tomd.py @@ -22,10 +22,8 @@ from __future__ import annotations import pytest -from tools import readFile - -from novelwriter.core.tomd import ToMarkdown from novelwriter.core.project import NWProject +from novelwriter.core.tomd import ToMarkdown @pytest.mark.core @@ -134,6 +132,13 @@ def testCoreToMarkdown_ConvertParagraphs(mockGUI): toMD.doConvert() assert toMD.result == "Line one \nLine two \nLine three\n\n" + # Text wo/Hard Break + toMD._text = "Line one \nLine two \nLine three\n" + toMD.setPreserveBreaks(False) + toMD.tokenizeText() + toMD.doConvert() + assert toMD.result == "Line one Line two Line three\n\n" + # Synopsis, Short toMD._text = "%synopsis: The synopsis ...\n" toMD.tokenizeText() @@ -188,6 +193,22 @@ def testCoreToMarkdown_ConvertParagraphs(mockGUI): "**Locations:** Europe\n\n" ) + # Footnotes + toMD._text = ( + "Text with one[footnote:fa] or two[footnote:fb] footnotes.\n\n" + "%footnote.fa: Footnote text A.\n\n" + ) + toMD.tokenizeText() + toMD.doConvert() + assert toMD.result == "Text with one[1] or two[ERR] footnotes.\n\n" + + toMD.appendFootnotes() + assert toMD.result == ( + "Text with one[1] or two[ERR] footnotes.\n\n" + "### Footnotes\n\n" + "1. Footnote text A.\n\n" + ) + # END Test testCoreToMarkdown_ConvertParagraphs @@ -233,17 +254,18 @@ def testCoreToMarkdown_ConvertDirect(mockGUI): @pytest.mark.core -def testCoreToMarkdown_Complex(mockGUI, fncPath): +def testCoreToMarkdown_Save(mockGUI, fncPath): """Test the save method of the ToMarkdown class.""" project = NWProject() toMD = ToMarkdown(project) + toMD.setKeepMarkdown(True) toMD._isNovel = True # Build Project # ============= docText = [ - "# My Novel\n**By Jane Doh**\n", + "# My Novel\n\n**By Jane Doh**\n", "## Chapter 1\n\nThe text of chapter one.\n", "### Scene 1\n\nThe text of scene one.\n", "#### A Section\n\nMore text in scene one.\n", @@ -273,15 +295,16 @@ def testCoreToMarkdown_Complex(mockGUI, fncPath): toMD.replaceTabs(nSpaces=4, spaceChar=" ") resText[6] = "#### A Section\n\n More text in scene two.\n\n" + assert toMD.allMarkdown == resText # Check File # ========== saveFile = fncPath / "outFile.md" toMD.saveMarkdown(saveFile) - assert readFile(saveFile) == "".join(resText) + assert saveFile.read_text(encoding="utf-8") == "".join(resText) -# END Test testCoreToHtml_Complex +# END Test testCoreToMarkdown_Save @pytest.mark.core diff --git a/tests/test_core/test_core_toodt.py b/tests/test_core/test_core_toodt.py index 4efe8b44..2428a567 100644 --- a/tests/test_core/test_core_toodt.py +++ b/tests/test_core/test_core_toodt.py @@ -621,6 +621,43 @@ def testCoreToOdt_ConvertParagraphs(mockGUI): '' ) + # Footnotes + odt._text = ( + "Text with one[footnote:fa], **two**[footnote:fd], " + "or three[footnote:fb] footnotes.[footnote:fe]\n\n" + "%footnote.fa: Footnote text A.[footnote:fc]\n\n" + "%footnote.fc: This footnote is skipped.\n\n" + "%footnote.fd: Another footnote.\n\n" + "%footnote.fe: Again?\n\n" + ) + odt.tokenizeText() + odt.initDocument() + odt.doConvert() + odt.closeDocument() + assert xmlToText(odt._xText) == ( + '' + 'Text with one' + '' + '1' + '' + 'Footnote text A.' + '' + ', two' + '' + '2' + '' + 'Another footnote.' + '' + ', or three footnotes.' + '' + '3' + '' + 'Again?' + '' + '' + '' + ) + # Test for issue #1412 # ==================== # See: https://github.com/vkbo/novelWriter/issues/1412