Fix footnote formatting for ODT and update document builder tests

novelwriter/core/toodt.py

@@ -646,8 +646,14 @@ class ToOdt(Tokenizer):
         xFmt = 0x00
         tFrag = ""
         fLast = 0
+        xNode = None
         for fPos, fFmt, fData in tFmt:
 
+            # Add any extra nodes
+            if xNode:
+                parProc.appendNode(xNode)
+                xNode = None
+
             # Add the text up to the current fragment
             if tFrag := tText[fLast:fPos]:
                 if xFmt == 0x00:
@@ -685,7 +691,7 @@ class ToOdt(Tokenizer):
             elif fFmt == self.FMT_SUB_E:
                 xFmt &= M_SUB
             elif fFmt == self.FMT_FNOTE:
-                parProc.appendNode(self._generateFootnote(fData))
+                xNode = self._generateFootnote(fData)
             elif fFmt == self.FMT_STRIP:
                 pass
             else:
@@ -693,6 +699,9 @@ class ToOdt(Tokenizer):
 
             fLast = fPos
 
+        if xNode:
+            parProc.appendNode(xNode)
+
         if tFrag := tText[fLast:]:
             if xFmt == 0x00:
                 parProc.appendText(tFrag)
@@ -1575,18 +1584,16 @@ class XMLParagraph:
         return
 
     def appendNode(self, xNode: ET.Element | None) -> None:
-        """Append an XML node to the paragraph."""
+        """Append an XML node to the paragraph. We only check for the
-        if xNode:
+        X_ROOT_TEXT and X_ROOT_TAIL states. X_SPAN_TEXT is not possible
-            if self._nState in (X_ROOT_TEXT, X_ROOT_TAIL):
+        at all, and X_SPAN_SING only happens internally in an appendSpan
-                self._xRoot.append(xNode)
+        call, returning us to an X_ROOT_TAIL state.
-                self._xTail = xNode
+        """
-                self._xTail.tail = ""
+        if xNode and self._nState in (X_ROOT_TEXT, X_ROOT_TAIL):
-                self._nState = X_ROOT_TAIL
+            self._xRoot.append(xNode)
-            elif self._nState in (X_SPAN_TEXT, X_SPAN_SING):
+            self._xTail = xNode
-                self._xTail.append(xNode)
+            self._xTail.tail = ""
-                self._xSing = xNode
+            self._nState = X_ROOT_TAIL
-                self._xSing.tail = ""
-                self._nState = X_SPAN_SING
         return
 
     def checkError(self) -> tuple[int, str]:

tests/test_base/test_base_common.py

@@ -21,24 +21,25 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
 import time
-import pytest
 
 from pathlib import Path
 from xml.etree import ElementTree as ET
 
-from tools import writeFile
+import pytest
-from mocked import causeOSError
 
-from PyQt5.QtGui import QColor, QDesktopServices
+from mocked import causeOSError
 from PyQt5.QtCore import QUrl
+from PyQt5.QtGui import QColor, QDesktopServices
+from tools import writeFile
 
 from novelwriter.common import (
-    checkBool, checkFloat, checkInt, checkIntTuple, checkPath, checkString,
+    NWConfigParser, checkBool, checkFloat, checkInt, checkIntTuple, checkPath,
-    checkStringNone, checkUuid, cssCol, formatFileFilter, formatInt,
+    checkString, checkStringNone, checkUuid, cssCol, formatFileFilter,
-    formatTime, formatTimeStamp, formatVersion, fuzzyTime, getFileSize,
+    formatInt, formatTime, formatTimeStamp, formatVersion, fuzzyTime,
-    hexToInt, isHandle, isItemClass, isItemLayout, isItemType, isTitleTag,
+    getFileSize, hexToInt, isHandle, isItemClass, isItemLayout, isItemType,
-    jsonEncode, makeFileNameSafe, minmax, numberToRoman, NWConfigParser,
+    isListInstance, isTitleTag, jsonEncode, makeFileNameSafe, minmax,
-    openExternalPath, readTextFile, simplified, transferCase, xmlIndent, yesNo
+    numberToRoman, openExternalPath, readTextFile, simplified, transferCase,
+    xmlIndent, yesNo
 )
 
 
@@ -272,6 +273,24 @@ def testBaseCommon_isItemLayout():
 # END Test testBaseCommon_isItemLayout
 
 
+@pytest.mark.base
+def testBaseCommon_isListInstance():
+    """Test the isListInstance function."""
+    # String
+    assert isListInstance("stuff", str) is False
+    assert isListInstance(["stuff"], str) is True
+
+    # Int
+    assert isListInstance(1, int) is False
+    assert isListInstance([1], int) is True
+
+    # Mixed
+    assert isListInstance([1], str) is False
+    assert isListInstance(["stuff"], int) is False
+
+# END Test testBaseCommon_isListInstance
+
+
 @pytest.mark.base
 def testBaseCommon_hexToInt():
     """Test the hexToInt function."""

tests/test_core/test_core_index.py

@@ -307,7 +307,7 @@ def testCoreIndex_CheckThese(mockGUI, fncPath, mockRnd):
 
 
 @pytest.mark.core
-def testCoreIndex_ScanText(mockGUI, fncPath, mockRnd):
+def testCoreIndex_ScanText(monkeypatch, mockGUI, fncPath, mockRnd):
     """Check the index text scanner."""
     project = NWProject()
     mockRnd.reset()
@@ -377,12 +377,14 @@ def testCoreIndex_ScanText(mockGUI, fncPath, mockRnd):
         "@char: Jane\n\n"
         "% this is a comment\n\n"
         "This is a story about Jane Smith.\n\n"
-        "Well, not really.\n"
+        "Well, not really.[footnote:key]\n\n"
+        "%Footnote.key: Footnote text.\n\n"
     ))
     assert index._tagsIndex.tagHandle("Jane") == cHandle
     assert index._tagsIndex.tagHeading("Jane") == "T0001"
     assert index._tagsIndex.tagClass("Jane") == "CHARACTER"
     assert index.getItemHeading(nHandle, "T0001").title == "Hello World!"  # type: ignore
+    assert index._itemIndex[nHandle].noteKeys("footnotes") == {"key"}  # type: ignore
 
     # Title Indexing
     # ==============
@@ -549,6 +551,45 @@ def testCoreIndex_ScanText(mockGUI, fncPath, mockRnd):
 # END Test testCoreIndex_ScanText
 
 
+@pytest.mark.core
+def testCoreIndex_CommentKeys(monkeypatch, mockGUI, fncPath, mockRnd):
+    """Check the index comment key generator."""
+    project = NWProject()
+    mockRnd.reset()
+    buildTestProject(project, fncPath)
+    index = project.index
+
+    nKeys = 1000
+
+    # Generate footnote keys
+    keys = set()
+    for _ in range(nKeys):
+        key = index.newCommentKey(C.hSceneDoc, nwComment.FOOTNOTE)
+        assert key not in keys
+        assert key != "err"
+        keys.add(key)
+    assert len(keys) == nKeys
+
+    # Generate comment keys
+    keys = set()
+    for _ in range(nKeys):
+        key = index.newCommentKey(C.hSceneDoc, nwComment.COMMENT)
+        assert key not in keys
+        keys.add(key)
+    assert len(keys) == nKeys
+
+    # Induce collision
+    with monkeypatch.context() as mp:
+        mp.setattr("random.choices", lambda *a, **k: "aaaa")
+        assert index.newCommentKey(C.hSceneDoc, nwComment.FOOTNOTE) == "faaaa"
+        assert index.newCommentKey(C.hSceneDoc, nwComment.FOOTNOTE) == "err"
+
+    # Check invalid comment style
+    assert index.newCommentKey(C.hSceneDoc, None) == "err"  # type: ignore
+
+# END Test testCoreIndex_CommentKeys
+
+
 @pytest.mark.core
 def testCoreIndex_ExtractData(mockGUI, fncPath, mockRnd):
     """Check the index data extraction functions."""
@@ -1250,12 +1291,14 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd):
     itemIndex.clear()
 
     # Data must be dictionary
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc:
         itemIndex.unpackData("stuff")  # type: ignore
+    assert str(exc.value) == "itemIndex is not a dict"
 
     # Keys must be valid handles
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc:
         itemIndex.unpackData({"stuff": "more stuff"})
+    assert str(exc.value) == "itemIndex keys must be handles"
 
     # Unknown keys should be skipped
     itemIndex.unpackData({C.hInvalid: {}})
@@ -1267,8 +1310,9 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd):
     assert itemIndex[nHandle].handle == nHandle  # type: ignore
 
     # Title tags must be valid
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc:
         itemIndex.unpackData({cHandle: {"headings": {"TTTTTTT": {}}}})
+    assert str(exc.value) == "The itemIndex contains an invalid title key"
 
     # Reference without a heading should be rejected
     itemIndex.unpackData({
@@ -1282,37 +1326,66 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd):
     itemIndex.clear()
 
     # Tag keys must be strings
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc:
         itemIndex.unpackData({
             cHandle: {
                 "headings": {"T0001": {}},
                 "references": {"T0001": {1234: "@pov"}},
+                "notes": {"footnotes": [], "comments": []},
             }
         })
+    assert str(exc.value) == "itemIndex reference key must be a string"
 
     # Type must be strings
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc:
         itemIndex.unpackData({
             cHandle: {
                 "headings": {"T0001": {}},
                 "references": {"T0001": {"John": []}},
+                "notes": {"footnotes": [], "comments": []},
             }
         })
+    assert str(exc.value) == "itemIndex reference type must be a string"
 
     # Types must be valid
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc:
         itemIndex.unpackData({
             cHandle: {
                 "headings": {"T0001": {}},
                 "references": {"T0001": {"John": "@pov,@char,@stuff"}},
+                "notes": {"footnotes": [], "comments": []},
             }
         })
+    assert str(exc.value) == "The itemIndex contains an invalid reference type"
+
+    # Note type must be valid
+    with pytest.raises(ValueError) as exc:
+        itemIndex.unpackData({
+            cHandle: {
+                "headings": {"T0001": {}},
+                "references": {"T0001": {"John": "@pov,@char"}},
+                "notes": {"stuff": [], "comments": []},
+            }
+        })
+    assert str(exc.value) == "The notes style is invalid"
+
+    # Note keys must be all strings
+    with pytest.raises(ValueError) as exc:
+        itemIndex.unpackData({
+            cHandle: {
+                "headings": {"T0001": {}},
+                "references": {"T0001": {"John": "@pov,@char"}},
+                "notes": {"footnotes": ["fkey", 1], "comments": []},
+            }
+        })
+    assert str(exc.value) == "The notes keys must be a list of strings"
 
     # This should pass
     itemIndex.unpackData({
         cHandle: {
             "headings": {"T0001": {}},
             "references": {"T0001": {"John": "@pov,@char"}},
+            "notes": {"footnotes": ["fkey"], "comments": ["ckey"]},
         }
     })
 

tests/test_core/test_core_tohtml.py

@@ -20,9 +20,9 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
 from __future__ import annotations
 
-import pytest
+import json
 
-from tools import readFile
+import pytest
 
 from novelwriter.core.project import NWProject
 from novelwriter.core.tohtml import ToHtml
@@ -225,6 +225,22 @@ def testCoreToHtml_ConvertParagraphs(mockGUI):
         "<a href='#tag_Bod'>Bod</a>, <a href='#tag_Jane'>Jane</a></p>\n"
     )
 
+    # Tags
+    html._text = "@tag: Bod\n"
+    html.tokenizeText()
+    html.doConvert()
+    assert html.result == (
+        "<p class='meta meta-tag'><span class='tags'>Tag:</span> <a name='tag_Bod'>Bod</a></p>\n"
+    )
+
+    html._text = "@tag: Bod | Nobody Owens\n"
+    html.tokenizeText()
+    html.doConvert()
+    assert html.result == (
+        "<p class='meta meta-tag'><span class='tags'>Tag:</span> <a name='tag_Bod'>Bod</a> "
+        "| <span class='optional'>Nobody Owens</a></p>\n"
+    )
+
     # Multiple Keywords
     html._isFirst = False
     html.setKeywords(True)
@@ -241,6 +257,30 @@ def testCoreToHtml_ConvertParagraphs(mockGUI):
         "<span class='tags'>Locations:</span> <a href='#tag_Europe'>Europe</a></p>\n"
     )
 
+    # Footnotes
+    # =========
+
+    html._text = (
+        "Text with one[footnote:fa] or two[footnote:fb] footnotes.\n\n"
+        "%footnote.fa: Footnote text A.\n\n"
+    )
+    html.tokenizeText()
+    html.doConvert()
+    assert html.result == (
+        "<p>Text with one<sup><a href='#footnote_1'>1</a></sup> "
+        "or two<sup>ERR</sup> footnotes.</p>\n"
+    )
+
+    html.appendFootnotes()
+    assert html.result == (
+        "<p>Text with one<sup><a href='#footnote_1'>1</a></sup> "
+        "or two<sup>ERR</sup> footnotes.</p>\n"
+        "<h3>Footnotes</h3>\n"
+        "<ol>\n"
+        "<li id='footnote_1'><p>Footnote text A.</p></li>\n"
+        "</ol>\n"
+    )
+
     # Preview Mode
     # ============
 
@@ -480,7 +520,7 @@ def testCoreToHtml_SpecialCases(mockGUI):
 
 
 @pytest.mark.core
-def testCoreToHtml_Complex(mockGUI, fncPath):
+def testCoreToHtml_Save(mockGUI, fncPath):
     """Test the save method of the ToHtml class."""
     project = NWProject()
     html = ToHtml(project)
@@ -498,36 +538,28 @@ def testCoreToHtml_Complex(mockGUI, fncPath):
         "### Scene 2\n\nThe text of scene two.\n",
         "#### A Section\n\n\tMore text in scene two.\n",
     ]
-    resText = [
+    resText = [(
-        (
+        "<h1 class='title' style='text-align: center;'>My Novel</h1>\n"
-            "<h1 class='title' style='text-align: center;'>My Novel</h1>\n"
+        "<p><strong>By Jane Doh</strong></p>\n"
-            "<p><strong>By Jane Doh</strong></p>\n"
+    ), (
-        ),
+        "<h1 style='page-break-before: always;'>Chapter 1</h1>\n"
-        (
+        "<p>The text of chapter one.</p>\n"
-            "<h1 style='page-break-before: always;'>Chapter 1</h1>\n"
+    ), (
-            "<p>The text of chapter one.</p>\n"
+        "<h2>Scene 1</h2>\n"
-        ),
+        "<p>The text of scene one.</p>\n"
-        (
+    ), (
-            "<h2>Scene 1</h2>\n"
+        "<h3>A Section</h3>\n"
-            "<p>The text of scene one.</p>\n"
+        "<p>More text in scene one.</p>\n"
-        ),
+    ), (
-        (
+        "<h1 style='page-break-before: always;'>Chapter 2</h1>\n"
-            "<h3>A Section</h3>\n"
+        "<p>The text of chapter two.</p>\n"
-            "<p>More text in scene one.</p>\n"
+    ),  (
-        ),
+        "<h2>Scene 2</h2>\n"
-        (
+        "<p>The text of scene two.</p>\n"
-            "<h1 style='page-break-before: always;'>Chapter 2</h1>\n"
+    ),  (
-            "<p>The text of chapter two.</p>\n"
+        "<h3>A Section</h3>\n"
-        ),
+        "<p>\tMore text in scene two.</p>\n"
-        (
+    )]
-            "<h2>Scene 2</h2>\n"
-            "<p>The text of scene two.</p>\n"
-        ),
-        (
-            "<h3>A Section</h3>\n"
-            "<p>\tMore text in scene two.</p>\n"
-        ),
-    ]
 
     for i in range(len(docText)):
         html._text = docText[i]
@@ -541,9 +573,10 @@ def testCoreToHtml_Complex(mockGUI, fncPath):
     html.replaceTabs(nSpaces=2, spaceChar="&nbsp;")
     resText[6] = "<h3>A Section</h3>\n<p>&nbsp;&nbsp;More text in scene two.</p>\n"
 
-    # Check File
+    # Check Files
-    # ==========
+    # ===========
 
+    # HTML
     hStyle = html.getStyleSheet()
     htmlDoc = (
         "<!DOCTYPE html>\n"
@@ -568,9 +601,20 @@ def testCoreToHtml_Complex(mockGUI, fncPath):
 
     saveFile = fncPath / "outFile.htm"
     html.saveHtml5(saveFile)
-    assert readFile(saveFile) == htmlDoc
+    assert saveFile.read_text(encoding="utf-8") == htmlDoc
 
-# END Test testCoreToHtml_Complex
+    # JSON + HTML
+    saveFile = fncPath / "outFile.json"
+    html.saveHtmlJson(saveFile)
+    data = json.loads(saveFile.read_text(encoding="utf-8"))
+    assert data["meta"]["projectName"] == ""
+    assert data["meta"]["novelAuthor"] == ""
+    assert data["meta"]["buildTime"] > 0
+    assert data["meta"]["buildTimeStr"] != ""
+    assert data["text"]["css"] == hStyle
+    assert len(data["text"]["html"]) == len(resText)
+
+# END Test testCoreToHtml_Save
 
 
 @pytest.mark.core

tests/test_core/test_core_tomd.py

@@ -22,10 +22,8 @@ from __future__ import annotations
 
 import pytest
 
-from tools import readFile
-
-from novelwriter.core.tomd import ToMarkdown
 from novelwriter.core.project import NWProject
+from novelwriter.core.tomd import ToMarkdown
 
 
 @pytest.mark.core
@@ -134,6 +132,13 @@ def testCoreToMarkdown_ConvertParagraphs(mockGUI):
     toMD.doConvert()
     assert toMD.result == "Line one  \nLine two  \nLine three\n\n"
 
+    # Text wo/Hard Break
+    toMD._text = "Line one  \nLine two  \nLine three\n"
+    toMD.setPreserveBreaks(False)
+    toMD.tokenizeText()
+    toMD.doConvert()
+    assert toMD.result == "Line one Line two Line three\n\n"
+
     # Synopsis, Short
     toMD._text = "%synopsis: The synopsis ...\n"
     toMD.tokenizeText()
@@ -188,6 +193,22 @@ def testCoreToMarkdown_ConvertParagraphs(mockGUI):
         "**Locations:** Europe\n\n"
     )
 
+    # Footnotes
+    toMD._text = (
+        "Text with one[footnote:fa] or two[footnote:fb] footnotes.\n\n"
+        "%footnote.fa: Footnote text A.\n\n"
+    )
+    toMD.tokenizeText()
+    toMD.doConvert()
+    assert toMD.result == "Text with one[1] or two[ERR] footnotes.\n\n"
+
+    toMD.appendFootnotes()
+    assert toMD.result == (
+        "Text with one[1] or two[ERR] footnotes.\n\n"
+        "### Footnotes\n\n"
+        "1. Footnote text A.\n\n"
+    )
+
 # END Test testCoreToMarkdown_ConvertParagraphs
 
 
@@ -233,17 +254,18 @@ def testCoreToMarkdown_ConvertDirect(mockGUI):
 
 
 @pytest.mark.core
-def testCoreToMarkdown_Complex(mockGUI, fncPath):
+def testCoreToMarkdown_Save(mockGUI, fncPath):
     """Test the save method of the ToMarkdown class."""
     project = NWProject()
     toMD = ToMarkdown(project)
+    toMD.setKeepMarkdown(True)
     toMD._isNovel = True
 
     # Build Project
     # =============
 
     docText = [
-        "# My Novel\n**By Jane Doh**\n",
+        "# My Novel\n\n**By Jane Doh**\n",
         "## Chapter 1\n\nThe text of chapter one.\n",
         "### Scene 1\n\nThe text of scene one.\n",
         "#### A Section\n\nMore text in scene one.\n",
@@ -273,15 +295,16 @@ def testCoreToMarkdown_Complex(mockGUI, fncPath):
 
     toMD.replaceTabs(nSpaces=4, spaceChar=" ")
     resText[6] = "#### A Section\n\n    More text in scene two.\n\n"
+    assert toMD.allMarkdown == resText
 
     # Check File
     # ==========
 
     saveFile = fncPath / "outFile.md"
     toMD.saveMarkdown(saveFile)
-    assert readFile(saveFile) == "".join(resText)
+    assert saveFile.read_text(encoding="utf-8") == "".join(resText)
 
-# END Test testCoreToHtml_Complex
+# END Test testCoreToMarkdown_Save
 
 
 @pytest.mark.core

tests/test_core/test_core_toodt.py

@@ -621,6 +621,43 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
         '</office:text>'
     )
 
+    # Footnotes
+    odt._text = (
+        "Text with one[footnote:fa], **two**[footnote:fd], "
+        "or three[footnote:fb] footnotes.[footnote:fe]\n\n"
+        "%footnote.fa: Footnote text A.[footnote:fc]\n\n"
+        "%footnote.fc: This footnote is skipped.\n\n"
+        "%footnote.fd: Another footnote.\n\n"
+        "%footnote.fe: Again?\n\n"
+    )
+    odt.tokenizeText()
+    odt.initDocument()
+    odt.doConvert()
+    odt.closeDocument()
+    assert xmlToText(odt._xText) == (
+        '<office:text>'
+        '<text:p text:style-name="Text_20_body">Text with one'
+        '<text:note text:id="ftn1" text:note-class="footnote">'
+        '<text:note-citation>1</text:note-citation>'
+        '<text:note-body>'
+        '<text:p text:style-name="Footnote">Footnote text A.</text:p>'
+        '</text:note-body>'
+        '</text:note>, <text:span text:style-name="T9">two</text:span>'
+        '<text:note text:id="ftn2" text:note-class="footnote">'
+        '<text:note-citation>2</text:note-citation>'
+        '<text:note-body>'
+        '<text:p text:style-name="Footnote">Another footnote.</text:p>'
+        '</text:note-body>'
+        '</text:note>, or three footnotes.'
+        '<text:note text:id="ftn3" text:note-class="footnote">'
+        '<text:note-citation>3</text:note-citation>'
+        '<text:note-body>'
+        '<text:p text:style-name="Footnote">Again?</text:p>'
+        '</text:note-body>'
+        '</text:note></text:p>'
+        '</office:text>'
+    )
+
     # Test for issue #1412
     # ====================
     # See: https://github.com/vkbo/novelWriter/issues/1412