Reduce footnote index storage to keys only

novelwriter/common.py

@@ -24,30 +24,32 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
 import json
-import uuid
 import logging
 import unicodedata
+import uuid
 import xml.etree.ElementTree as ET
 
-from typing import TYPE_CHECKING, Any, Literal
-from pathlib import Path
-from datetime import datetime
 from configparser import ConfigParser
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal, TypeVar
 from urllib.parse import urljoin
 from urllib.request import pathname2url
 
-from PyQt5.QtGui import QColor, QDesktopServices
 from PyQt5.QtCore import QCoreApplication, QUrl
+from PyQt5.QtGui import QColor, QDesktopServices
 
+from novelwriter.constants import nwConst, nwLabels, nwUnicode, trConst
 from novelwriter.enum import nwItemClass, nwItemType, nwItemLayout
 from novelwriter.error import logException
-from novelwriter.constants import nwConst, nwLabels, nwUnicode, trConst
 
 if TYPE_CHECKING:  # pragma: no cover
     from typing import TypeGuard  # Requires Python 3.10
 
 logger = logging.getLogger(__name__)
 
+_Type = TypeVar("_Type")
+
 
 ##
 #  Checker Functions
@@ -172,6 +174,11 @@ def isItemLayout(value: Any) -> TypeGuard[str]:
     return isinstance(value, str) and value in nwItemLayout.__members__
 
 
+def isListInstance(data: Any, check: type[_Type]) -> TypeGuard[list[_Type]]:
+    """Check that all items of a list is of a given type."""
+    return isinstance(data, list) and all(isinstance(item, check) for item in data)
+
+
 def hexToInt(value: Any, default: int = 0) -> int:
     """Convert a hex string to an integer."""
     if isinstance(value, str):

novelwriter/core/index.py

@@ -30,17 +30,19 @@ from __future__ import annotations
 import json
 import logging
 
+from collections.abc import ItemsView, Iterable
+from pathlib import Path
 from random import randint
 from time import time
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
-from pathlib import Path
-from collections.abc import ItemsView, Iterable
 
 from novelwriter import SHARED
+from novelwriter.common import (
+    checkInt, isHandle, isItemClass, isListInstance, isTitleTag, jsonEncode
+)
+from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
 from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
 from novelwriter.error import logException
-from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode
-from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
 from novelwriter.text.counting import standardCounter
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -49,7 +51,11 @@ if TYPE_CHECKING:  # pragma: no cover
 
 logger = logging.getLogger(__name__)
 
+T_NoteTypes = Literal["footnotes", "comments"]
+
 TT_NONE = "T0000"
+KEY_SOURCE = "0123456789bcdfghjklmnopqrstvwxyz"
+NOTE_TYPES: list[T_NoteTypes] = ["footnotes", "comments"]
 
 
 class NWIndex:
@@ -86,7 +92,6 @@ class NWIndex:
         # Storage and State
         self._tagsIndex = TagsIndex()
         self._itemIndex = ItemIndex(project)
-        self._textIndex = TextIndex()
         self._indexBroken = False
 
         # TimeStamps
@@ -114,7 +119,6 @@ class NWIndex:
         """Clear the index dictionaries and time stamps."""
         self._tagsIndex.clear()
         self._itemIndex.clear()
-        self._textIndex.clear()
         self._indexChange = 0.0
         self._rootChange = {}
         SHARED.indexSignalProxy({"event": "clearIndex"})
@@ -138,7 +142,6 @@ class NWIndex:
         for tTag in delTags:
             del self._tagsIndex[tTag]
         del self._itemIndex[tHandle]
-        self._textIndex.removeHandle(tHandle)
         SHARED.indexSignalProxy({
             "event": "updateTags",
             "deleted": delTags,
@@ -193,7 +196,6 @@ class NWIndex:
             try:
                 self._tagsIndex.unpackData(data["novelWriter.tagsIndex"])
                 self._itemIndex.unpackData(data["novelWriter.itemIndex"])
-                self._textIndex.unpackData(data["novelWriter.textIndex"])
             except Exception:
                 logger.error("The index content is invalid")
                 logException()
@@ -229,12 +231,10 @@ class NWIndex:
         try:
             tagsIndex = jsonEncode(self._tagsIndex.packData(), n=1, nmax=2)
             itemIndex = jsonEncode(self._itemIndex.packData(), n=1, nmax=4)
-            textIndex = jsonEncode(self._textIndex.packData(), n=1, nmax=3)
             with open(indexFile, mode="w+", encoding="utf-8") as outFile:
                 outFile.write("{\n")
                 outFile.write(f'  "novelWriter.tagsIndex": {tagsIndex},\n')
-                outFile.write(f'  "novelWriter.itemIndex": {itemIndex},\n')
+                outFile.write(f'  "novelWriter.itemIndex": {itemIndex}\n')
-                outFile.write(f'  "novelWriter.textIndex": {textIndex}\n')
                 outFile.write("}\n")
 
         except Exception:
@@ -346,7 +346,7 @@ class NWIndex:
                 if cStyle in (nwComment.SYNOPSIS, nwComment.SHORT):
                     self._itemIndex.setHeadingSynopsis(tHandle, cTitle, cText)
                 elif cStyle == nwComment.FOOTNOTE:
-                    self._textIndex.footnotes.add(cKey, tHandle, cText)
+                    self._itemIndex.addNoteKey(tHandle, "footnotes", cKey)
 
         # Count words for remaining text after last heading
         if pTitle != TT_NONE:
@@ -514,13 +514,13 @@ class NWIndex:
         name, _, display = text.partition("|")
         return name.rstrip(), display.lstrip()
 
-    def newCommentKey(self, style: nwComment) -> str | None:
+    def newCommentKey(self, tHandle: str, style: nwComment) -> str:
         """Generate a new key for a comment style."""
         if style == nwComment.FOOTNOTE:
-            return self._textIndex.footnotes.newKey()
+            return self._itemIndex.genNewNoteKey(tHandle, "footnotes")
         elif style == nwComment.COMMENT:
-            return self._textIndex.comments.newKey()
+            return self._itemIndex.genNewNoteKey(tHandle, "comments")
-        return None
+        return "err"
 
     ##
     #  Extract Data
@@ -966,6 +966,25 @@ class ItemIndex:
             self._items[tHandle].addHeadingRef(sTitle, tagKeys, refType)
         return
 
+    def addNoteKey(self, tHandle: str, style: T_NoteTypes, key: str) -> None:
+        """Set notes key for a given item."""
+        if tHandle in self._items:
+            self._items[tHandle].addNoteKey(style, key)
+        return
+
+    def genNewNoteKey(self, tHandle: str, style: T_NoteTypes) -> str:
+        """Set notes key for a given item."""
+        keys = set()
+        for item in self._items.values():
+            keys.update(item.noteKeys(style))
+        if style in NOTE_TYPES and (item := self._items.get(tHandle)):
+            for _ in range(1000):
+                key = style[:1] + "".join([KEY_SOURCE[randint(0, 31)] for _ in range(4)])
+                if key not in keys:
+                    item.addNoteKey(style, key)
+                    return key
+        return "err"
+
     ##
     #  Pack/Unpack
     ##
@@ -1007,12 +1026,13 @@ class IndexItem:
     must be reset each time the item is re-indexed.
     """
 
-    __slots__ = ("_handle", "_item", "_headings", "_count")
+    __slots__ = ("_handle", "_item", "_headings", "_count", "_notes")
 
     def __init__(self, tHandle: str, nwItem: NWItem) -> None:
         self._handle = tHandle
         self._item = nwItem
         self._headings: dict[str, IndexHeading] = {TT_NONE: IndexHeading(TT_NONE)}
+        self._notes: dict[str, set[str]] = {}
         self._count = 0
         return
 
@@ -1080,6 +1100,13 @@ class IndexItem:
                 self._headings[sTitle].addReference(tagKey, refType)
         return
 
+    def addNoteKey(self, style: T_NoteTypes, key: str) -> None:
+        """Add a note key to the index."""
+        if style not in self._notes:
+            self._notes[style] = set()
+        self._notes[style].add(key)
+        return
+
     ##
     #  Data Methods
     ##
@@ -1101,6 +1128,10 @@ class IndexItem:
         self._count += 1
         return f"T{self._count:04d}"
 
+    def noteKeys(self, style: T_NoteTypes) -> set[str]:
+        """Return a set of all note keys."""
+        return self._notes.get(style, set())
+
     ##
     #  Pack/Unpack
     ##
@@ -1119,6 +1150,8 @@ class IndexItem:
         data["headings"] = heads
         if refs:
             data["references"] = refs
+        if self._notes:
+            data["notes"] = {style: list(keys) for style, keys in self._notes.items()}
 
         return data
 
@@ -1132,6 +1165,14 @@ class IndexItem:
             tHeading.unpackData(hData)
             tHeading.unpackReferences(references.get(sTitle, {}))
             self.addHeading(tHeading)
+
+        for style, keys in data.get("notes", {}).items():
+            if style not in NOTE_TYPES:
+                raise ValueError("The notes style is invalid")
+            if not isListInstance(keys, str):
+                raise ValueError("The notes keys must be a list of strings")
+            self._notes[style] = set(keys)
+
         return
 
 # END Class IndexItem
@@ -1314,162 +1355,6 @@ class IndexHeading:
 # END Class IndexHeading
 
 
-# =============================================================================================== #
-#  The Text Index Object
-# =============================================================================================== #
-
-KEY_SOURCE = "0123456789bcdfghjklmnopqrstvwxyz"
-
-
-class TextIndex:
-    """Core: Text Index Wrapper Class
-
-    A wrapper class that holds various global text entries.
-    """
-
-    __slots__ = ("_comments", "_footnotes")
-
-    def __init__(self) -> None:
-        self._comments = TextRegistry("c")
-        self._footnotes = TextRegistry("f")
-        return
-
-    @property
-    def comments(self) -> TextRegistry:
-        """Return the comments text registry."""
-        return self._comments
-
-    @property
-    def footnotes(self) -> TextRegistry:
-        """Return the footnotes text registry."""
-        return self._footnotes
-
-    ##
-    #  Methods
-    ##
-
-    def clear(self) -> None:
-        """Clear the index."""
-        self._comments.clear()
-        self._footnotes.clear()
-        return
-
-    def removeHandle(self, handle: str) -> None:
-        """Remove all entries for a given handle."""
-        self._comments.removeHandle(handle)
-        self._footnotes.removeHandle(handle)
-        return
-
-    ##
-    #  Pack/Unpack
-    ##
-
-    def packData(self) -> dict[str, dict]:
-        """Pack all the text comments into a single dictionary."""
-        return {
-            "comments": self._comments.packData(),
-            "footnotes": self._footnotes.packData(),
-        }
-
-    def unpackData(self, data: dict) -> None:
-        """Unpack the text comments index."""
-        self._comments.unpackData(data.get("comments", {}))
-        self._footnotes.unpackData(data.get("footnotes", {}))
-        return
-
-# END Class TextIndex
-
-
-class TextRegistry:
-    """Core: Text Registry Index Wrapper Class
-
-    A wrapper class that holds a category of text entries.
-    """
-
-    __slots__ = ("_map", "_text", "_prefix")
-
-    def __init__(self, prefix: str) -> None:
-        self._map: dict[str, str] = {}
-        self._text: dict[str, str] = {}
-        self._prefix = prefix
-        return
-
-    def __len__(self) -> int:
-        return len(self._text)
-
-    def __getitem__(self, key: str) -> str | None:
-        return self._text.get(key, (0, None))[1]
-
-    def __contains__(self, key: str) -> bool:
-        return key in self._text
-
-    ##
-    #  Methods
-    ##
-
-    def clear(self) -> None:
-        """Clear the index."""
-        self._map.clear()
-        self._text.clear()
-        return
-
-    def add(self, key: str, handle: str, text: str) -> None:
-        """Add a new text entry."""
-        self._map[key] = handle
-        self._text[key] = text
-        return
-
-    def keysForHandle(self, handle: str) -> list[str]:
-        """Return all keys for a given handle."""
-        return [k for k, v in self._map.items() if v == handle]
-
-    def removeHandle(self, handle: str) -> None:
-        """Iterate through the data and remove entries for a handle."""
-        for key in [k for k, v in self._map.items() if v == handle]:
-            del self._text[key]
-        return
-
-    def newKey(self) -> str:
-        """Generate a new key."""
-        key = self._prefix + "".join([KEY_SOURCE[randint(0, 31)] for _ in range(4)])
-        if key in self._text:
-            key = self.newKey()
-        return key
-
-    ##
-    #  Pack/Unpack
-    ##
-
-    def packData(self) -> dict[str, dict[str, str]]:
-        """Pack all the text entries into a dictionary."""
-        return {k: {"handle": self._map[k], "text": v} for k, v in self._text.items()}
-
-    def unpackData(self, data: dict) -> None:
-        """Unpack text entries from a dictionary."""
-        self.clear()
-        if not isinstance(data, dict):
-            raise ValueError("textEntry is not a dict")
-
-        for key, entry in data.items():
-            if not isinstance(key, str):
-                raise ValueError("textEntry key must be a string")
-            if not isinstance(entry, dict):
-                raise ValueError("textEntry entry is not a dict")
-
-            handle = entry.get("handle")
-            text = entry.get("text")
-            if not isHandle(handle):
-                raise ValueError("textEntry handle must be a handle")
-            if not isinstance(text, str):
-                raise ValueError("textEntry text is not a string")
-
-            self.add(key, handle, text)
-
-        return
-
-# END Class TextEntry
-
-
 # =============================================================================================== #
 #  Text Processing Functions
 # =============================================================================================== #

novelwriter/core/status.py

@@ -29,7 +29,7 @@ import logging
 import random
 
 from collections.abc import Iterable
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 from PyQt5.QtCore import QPointF, Qt
 from PyQt5.QtGui import QIcon, QPainter, QPainterPath, QPixmap, QColor, QPolygonF
@@ -75,7 +75,7 @@ class NWStatus:
 
     __slots__ = ("_store", "_default", "_prefix", "_height")
 
-    def __init__(self, prefix: str) -> None:
+    def __init__(self, prefix: Literal["s", "i"]) -> None:
         self._store: dict[str, StatusEntry] = {}
         self._default = None
         self._prefix = prefix[:1]

novelwriter/gui/doceditor.py

@@ -1855,8 +1855,9 @@ class GuiDocEditor(QPlainTextEdit):
 
     def _insertCommentStructure(self, style: nwComment) -> None:
         """Insert a shortcut/comment combo."""
-        if style == nwComment.FOOTNOTE:
+        if self._docHandle and style == nwComment.FOOTNOTE:
-            key = SHARED.project.index.newCommentKey(style)
+            self.saveText()  # Index must be up to date
+            key = SHARED.project.index.newCommentKey(self._docHandle, style)
             code = nwShortcode.COMMENT_STYLES[nwComment.FOOTNOTE]
 
             cursor = self.textCursor()
@@ -1868,14 +1869,12 @@ class GuiDocEditor(QPlainTextEdit):
 
             cursor.beginEditBlock()
             cursor.insertText(code.format(key))
-            cursor.setPosition(block.position() + block.length())
+            cursor.setPosition(block.position() + block.length() - 1)
+            cursor.insertBlock()
             cursor.insertBlock()
             cursor.insertText(f"%Footnote.{key}: ")
-            cursor.insertBlock()
             cursor.endEditBlock()
 
-            cursor.setPosition(cursor.position() - 1)
-
             self.setTextCursor(cursor)
 
         return

novelwriter/gui/dochighlight.py

@@ -93,19 +93,19 @@ class GuiDocHighlighter(QSyntaxHighlighter):
         colBreak.setAlpha(64)
 
         # Create Character Formats
-        self._addCharFormat("header1",  SHARED.theme.colHead, "bold", 1.8)
+        self._addCharFormat("header1",  SHARED.theme.colHead, "b", 1.8)
-        self._addCharFormat("header2",  SHARED.theme.colHead, "bold", 1.6)
+        self._addCharFormat("header2",  SHARED.theme.colHead, "b", 1.6)
-        self._addCharFormat("header3",  SHARED.theme.colHead, "bold", 1.4)
+        self._addCharFormat("header3",  SHARED.theme.colHead, "b", 1.4)
-        self._addCharFormat("header4",  SHARED.theme.colHead, "bold", 1.2)
+        self._addCharFormat("header4",  SHARED.theme.colHead, "b", 1.2)
-        self._addCharFormat("head1h",   SHARED.theme.colHeadH, "bold", 1.8)
+        self._addCharFormat("head1h",   SHARED.theme.colHeadH, "b", 1.8)
-        self._addCharFormat("head2h",   SHARED.theme.colHeadH, "bold", 1.6)
+        self._addCharFormat("head2h",   SHARED.theme.colHeadH, "b", 1.6)
-        self._addCharFormat("head3h",   SHARED.theme.colHeadH, "bold", 1.4)
+        self._addCharFormat("head3h",   SHARED.theme.colHeadH, "b", 1.4)
-        self._addCharFormat("head4h",   SHARED.theme.colHeadH, "bold", 1.2)
+        self._addCharFormat("head4h",   SHARED.theme.colHeadH, "b", 1.2)
-        self._addCharFormat("bold",     colEmph, "bold")
+        self._addCharFormat("bold",     colEmph, "b")
-        self._addCharFormat("italic",   colEmph, "italic")
+        self._addCharFormat("italic",   colEmph, "i")
-        self._addCharFormat("strike",   SHARED.theme.colHidden, "strike")
+        self._addCharFormat("strike",   SHARED.theme.colHidden, "s")
-        self._addCharFormat("mspaces",  SHARED.theme.colError, "errline")
+        self._addCharFormat("mspaces",  SHARED.theme.colError, "err")
-        self._addCharFormat("nobreak",  colBreak, "background")
+        self._addCharFormat("nobreak",  colBreak, "bg")
         self._addCharFormat("dialog1",  SHARED.theme.colDialN)
         self._addCharFormat("dialog2",  SHARED.theme.colDialD)
         self._addCharFormat("dialog3",  SHARED.theme.colDialS)
@@ -117,7 +117,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
         self._addCharFormat("modifier", SHARED.theme.colMod)
         self._addCharFormat("value",    SHARED.theme.colVal)
         self._addCharFormat("optional", SHARED.theme.colOpt)
-        self._addCharFormat("invalid",  None, "errline")
+        self._addCharFormat("invalid",  None, "err")
 
         # Cache Spell Error Format
         self._spellErr = QTextCharFormat()
@@ -442,16 +442,16 @@ class GuiDocHighlighter(QSyntaxHighlighter):
 
         if style:
             styles = style.split(",")
-            if "bold" in styles:
+            if "b" in styles:
                 charFormat.setFontWeight(QFont.Weight.Bold)
-            if "italic" in styles:
+            if "i" in styles:
                 charFormat.setFontItalic(True)
-            if "strike" in styles:
+            if "s" in styles:
                 charFormat.setFontStrikeOut(True)
-            if "errline" in styles:
+            if "err" in styles:
                 charFormat.setUnderlineColor(SHARED.theme.colError)
                 charFormat.setUnderlineStyle(QTextCharFormat.UnderlineStyle.SpellCheckUnderline)
-            if "background" in styles and color is not None:
+            if "bg" in styles and color is not None:
                 charFormat.setBackground(QBrush(color, Qt.BrushStyle.SolidPattern))
 
         if size:

sample/content/636b6aa9b697b.nwd

@@ -1,8 +1,8 @@
 %%~name: Making a Scene
 %%~path: 6a2d6d5f4f401/636b6aa9b697b
 %%~kind: NOVEL/DOCUMENT
-%%~hash: 06b80d830f3f4d5c703eff82067d4335b8b98151
+%%~hash: c7e664867218b3a9aac5c12119ef0ec63da2e5cc
-%%~date: Unknown/2024-04-14 23:28:43
+%%~date: Unknown/2024-04-18 17:56:30
 ### Making a Scene
 
 @pov: Jane
@@ -21,9 +21,9 @@ If you have the need for it, you can also add text that can be automatically rep
 
 The editor also supports non breaking spaces, and the spell checker accepts long dashes—like this—as valid word separators. Regular dashes are also supported – and can be automatically inserted when typing two hyphens.
 
-Thin spaces and thin non-breaking spaces are also supported from the Insert menu, and can be used to separate numbers from their units, like: 25 kg.[footnote:fq2ms]
+Thin spaces and thin non-breaking spaces are also supported from the Insert menu, and can be used to separate numbers from their units, like: 25 kg.[footnote:f4xr5]
 
-%Footnote.fq2ms: This is a footnote about non-breaking spaces.
+%Footnote.f4xr5: Using a non-breaking space is the correct way to separate a number from its unit. This ensures that line wrapping does not split the two.
 
 #### Some Section Here
 

sample/nwProject.nwx

@@ -1,6 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<novelWriterXML appVersion="2.5a1" hexVersion="0x020500a1" fileVersion="1.5" fileRevision="4" timeStamp="2024-04-14 23:31:44">
+<novelWriterXML appVersion="2.5a2" hexVersion="0x020500a1" fileVersion="1.5" fileRevision="4" timeStamp="2024-04-18 17:57:37">
-  <project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="1879" autoCount="272" editTime="86979">
+  <project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="1916" autoCount="274" editTime="87487">
     <name>Sample Project</name>
     <author>Jane Smith</author>
   </project>
@@ -36,7 +36,7 @@
       <entry key="i56be10" count="1" red="220" green="138" blue="221" shape="BLOCK_4">Main</entry>
     </importance>
   </settings>
-  <content items="31" novelWords="998" notesWords="416">
+  <content items="31" novelWords="996" notesWords="416">
     <item handle="7031beac91f75" parent="None" root="7031beac91f75" order="0" type="ROOT" class="NOVEL">
       <meta expanded="yes" />
       <name status="sc24b8f" import="ia857f0">Novel</name>
@@ -46,7 +46,7 @@
       <name status="sc24b8f" import="ia857f0" active="yes">Title Page</name>
     </item>
     <item handle="974e400180a99" parent="7031beac91f75" root="7031beac91f75" order="1" type="FILE" class="NOVEL" layout="DOCUMENT">
-      <meta expanded="no" heading="H0" charCount="250" wordCount="49" paraCount="2" cursorPos="275" />
+      <meta expanded="no" heading="H0" charCount="233" wordCount="47" paraCount="2" cursorPos="275" />
       <name status="sf12341" import="ia857f0" active="yes">Page</name>
     </item>
     <item handle="edca4be2fcaf8" parent="7031beac91f75" root="7031beac91f75" order="2" type="FILE" class="NOVEL" layout="DOCUMENT">
@@ -58,11 +58,11 @@
       <name status="sf24ce6" import="ia857f0" active="yes">Chapter One</name>
     </item>
     <item handle="636b6aa9b697b" parent="6a2d6d5f4f401" root="7031beac91f75" order="0" type="FILE" class="NOVEL" layout="DOCUMENT">
-      <meta expanded="no" heading="H3" charCount="2953" wordCount="520" paraCount="15" cursorPos="2049" />
+      <meta expanded="no" heading="H3" charCount="2953" wordCount="520" paraCount="15" cursorPos="2155" />
       <name status="s90e6c9" import="ia857f0" active="yes">Making a Scene</name>
     </item>
     <item handle="bc0cbd2a407f3" parent="6a2d6d5f4f401" root="7031beac91f75" order="1" type="FILE" class="NOVEL" layout="DOCUMENT">
-      <meta expanded="no" heading="H3" charCount="548" wordCount="108" paraCount="3" cursorPos="531" />
+      <meta expanded="no" heading="H3" charCount="563" wordCount="108" paraCount="3" cursorPos="691" />
       <name status="s90e6c9" import="ia857f0" active="yes">Another Scene</name>
     </item>
     <item handle="ba8a28a246524" parent="7031beac91f75" root="7031beac91f75" order="4" type="FILE" class="NOVEL" layout="DOCUMENT">