From a2d8aefb219b3f05664c0cfd8b02c152f0006475 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:07:17 +0200 Subject: [PATCH] Reduce footnote index storage to keys only --- novelwriter/common.py | 19 ++- novelwriter/core/index.py | 233 ++++++++----------------------- novelwriter/core/status.py | 4 +- novelwriter/gui/doceditor.py | 11 +- novelwriter/gui/dochighlight.py | 38 ++--- sample/content/636b6aa9b697b.nwd | 8 +- sample/nwProject.nwx | 12 +- 7 files changed, 108 insertions(+), 217 deletions(-) diff --git a/novelwriter/common.py b/novelwriter/common.py index ff36bae2..8f0ffcff 100644 --- a/novelwriter/common.py +++ b/novelwriter/common.py @@ -24,30 +24,32 @@ along with this program. If not, see . from __future__ import annotations import json -import uuid import logging import unicodedata +import uuid import xml.etree.ElementTree as ET -from typing import TYPE_CHECKING, Any, Literal -from pathlib import Path -from datetime import datetime from configparser import ConfigParser +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal, TypeVar from urllib.parse import urljoin from urllib.request import pathname2url -from PyQt5.QtGui import QColor, QDesktopServices from PyQt5.QtCore import QCoreApplication, QUrl +from PyQt5.QtGui import QColor, QDesktopServices +from novelwriter.constants import nwConst, nwLabels, nwUnicode, trConst from novelwriter.enum import nwItemClass, nwItemType, nwItemLayout from novelwriter.error import logException -from novelwriter.constants import nwConst, nwLabels, nwUnicode, trConst if TYPE_CHECKING: # pragma: no cover from typing import TypeGuard # Requires Python 3.10 logger = logging.getLogger(__name__) +_Type = TypeVar("_Type") + ## # Checker Functions @@ -172,6 +174,11 @@ def isItemLayout(value: Any) -> TypeGuard[str]: return isinstance(value, str) and value in nwItemLayout.__members__ +def isListInstance(data: Any, check: type[_Type]) -> TypeGuard[list[_Type]]: + """Check that all items of a list is of a given type.""" + return isinstance(data, list) and all(isinstance(item, check) for item in data) + + def hexToInt(value: Any, default: int = 0) -> int: """Convert a hex string to an integer.""" if isinstance(value, str): diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 13f75407..230582bf 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -30,17 +30,19 @@ from __future__ import annotations import json import logging +from collections.abc import ItemsView, Iterable +from pathlib import Path from random import randint from time import time -from typing import TYPE_CHECKING -from pathlib import Path -from collections.abc import ItemsView, Iterable +from typing import TYPE_CHECKING, Literal from novelwriter import SHARED +from novelwriter.common import ( + checkInt, isHandle, isItemClass, isListInstance, isTitleTag, jsonEncode +) +from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout from novelwriter.error import logException -from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode -from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders from novelwriter.text.counting import standardCounter if TYPE_CHECKING: # pragma: no cover @@ -49,7 +51,11 @@ if TYPE_CHECKING: # pragma: no cover logger = logging.getLogger(__name__) +T_NoteTypes = Literal["footnotes", "comments"] + TT_NONE = "T0000" +KEY_SOURCE = "0123456789bcdfghjklmnopqrstvwxyz" +NOTE_TYPES: list[T_NoteTypes] = ["footnotes", "comments"] class NWIndex: @@ -86,7 +92,6 @@ class NWIndex: # Storage and State self._tagsIndex = TagsIndex() self._itemIndex = ItemIndex(project) - self._textIndex = TextIndex() self._indexBroken = False # TimeStamps @@ -114,7 +119,6 @@ class NWIndex: """Clear the index dictionaries and time stamps.""" self._tagsIndex.clear() self._itemIndex.clear() - self._textIndex.clear() self._indexChange = 0.0 self._rootChange = {} SHARED.indexSignalProxy({"event": "clearIndex"}) @@ -138,7 +142,6 @@ class NWIndex: for tTag in delTags: del self._tagsIndex[tTag] del self._itemIndex[tHandle] - self._textIndex.removeHandle(tHandle) SHARED.indexSignalProxy({ "event": "updateTags", "deleted": delTags, @@ -193,7 +196,6 @@ class NWIndex: try: self._tagsIndex.unpackData(data["novelWriter.tagsIndex"]) self._itemIndex.unpackData(data["novelWriter.itemIndex"]) - self._textIndex.unpackData(data["novelWriter.textIndex"]) except Exception: logger.error("The index content is invalid") logException() @@ -229,12 +231,10 @@ class NWIndex: try: tagsIndex = jsonEncode(self._tagsIndex.packData(), n=1, nmax=2) itemIndex = jsonEncode(self._itemIndex.packData(), n=1, nmax=4) - textIndex = jsonEncode(self._textIndex.packData(), n=1, nmax=3) with open(indexFile, mode="w+", encoding="utf-8") as outFile: outFile.write("{\n") outFile.write(f' "novelWriter.tagsIndex": {tagsIndex},\n') - outFile.write(f' "novelWriter.itemIndex": {itemIndex},\n') - outFile.write(f' "novelWriter.textIndex": {textIndex}\n') + outFile.write(f' "novelWriter.itemIndex": {itemIndex}\n') outFile.write("}\n") except Exception: @@ -346,7 +346,7 @@ class NWIndex: if cStyle in (nwComment.SYNOPSIS, nwComment.SHORT): self._itemIndex.setHeadingSynopsis(tHandle, cTitle, cText) elif cStyle == nwComment.FOOTNOTE: - self._textIndex.footnotes.add(cKey, tHandle, cText) + self._itemIndex.addNoteKey(tHandle, "footnotes", cKey) # Count words for remaining text after last heading if pTitle != TT_NONE: @@ -514,13 +514,13 @@ class NWIndex: name, _, display = text.partition("|") return name.rstrip(), display.lstrip() - def newCommentKey(self, style: nwComment) -> str | None: + def newCommentKey(self, tHandle: str, style: nwComment) -> str: """Generate a new key for a comment style.""" if style == nwComment.FOOTNOTE: - return self._textIndex.footnotes.newKey() + return self._itemIndex.genNewNoteKey(tHandle, "footnotes") elif style == nwComment.COMMENT: - return self._textIndex.comments.newKey() - return None + return self._itemIndex.genNewNoteKey(tHandle, "comments") + return "err" ## # Extract Data @@ -966,6 +966,25 @@ class ItemIndex: self._items[tHandle].addHeadingRef(sTitle, tagKeys, refType) return + def addNoteKey(self, tHandle: str, style: T_NoteTypes, key: str) -> None: + """Set notes key for a given item.""" + if tHandle in self._items: + self._items[tHandle].addNoteKey(style, key) + return + + def genNewNoteKey(self, tHandle: str, style: T_NoteTypes) -> str: + """Set notes key for a given item.""" + keys = set() + for item in self._items.values(): + keys.update(item.noteKeys(style)) + if style in NOTE_TYPES and (item := self._items.get(tHandle)): + for _ in range(1000): + key = style[:1] + "".join([KEY_SOURCE[randint(0, 31)] for _ in range(4)]) + if key not in keys: + item.addNoteKey(style, key) + return key + return "err" + ## # Pack/Unpack ## @@ -1007,12 +1026,13 @@ class IndexItem: must be reset each time the item is re-indexed. """ - __slots__ = ("_handle", "_item", "_headings", "_count") + __slots__ = ("_handle", "_item", "_headings", "_count", "_notes") def __init__(self, tHandle: str, nwItem: NWItem) -> None: self._handle = tHandle self._item = nwItem self._headings: dict[str, IndexHeading] = {TT_NONE: IndexHeading(TT_NONE)} + self._notes: dict[str, set[str]] = {} self._count = 0 return @@ -1080,6 +1100,13 @@ class IndexItem: self._headings[sTitle].addReference(tagKey, refType) return + def addNoteKey(self, style: T_NoteTypes, key: str) -> None: + """Add a note key to the index.""" + if style not in self._notes: + self._notes[style] = set() + self._notes[style].add(key) + return + ## # Data Methods ## @@ -1101,6 +1128,10 @@ class IndexItem: self._count += 1 return f"T{self._count:04d}" + def noteKeys(self, style: T_NoteTypes) -> set[str]: + """Return a set of all note keys.""" + return self._notes.get(style, set()) + ## # Pack/Unpack ## @@ -1119,6 +1150,8 @@ class IndexItem: data["headings"] = heads if refs: data["references"] = refs + if self._notes: + data["notes"] = {style: list(keys) for style, keys in self._notes.items()} return data @@ -1132,6 +1165,14 @@ class IndexItem: tHeading.unpackData(hData) tHeading.unpackReferences(references.get(sTitle, {})) self.addHeading(tHeading) + + for style, keys in data.get("notes", {}).items(): + if style not in NOTE_TYPES: + raise ValueError("The notes style is invalid") + if not isListInstance(keys, str): + raise ValueError("The notes keys must be a list of strings") + self._notes[style] = set(keys) + return # END Class IndexItem @@ -1314,162 +1355,6 @@ class IndexHeading: # END Class IndexHeading -# =============================================================================================== # -# The Text Index Object -# =============================================================================================== # - -KEY_SOURCE = "0123456789bcdfghjklmnopqrstvwxyz" - - -class TextIndex: - """Core: Text Index Wrapper Class - - A wrapper class that holds various global text entries. - """ - - __slots__ = ("_comments", "_footnotes") - - def __init__(self) -> None: - self._comments = TextRegistry("c") - self._footnotes = TextRegistry("f") - return - - @property - def comments(self) -> TextRegistry: - """Return the comments text registry.""" - return self._comments - - @property - def footnotes(self) -> TextRegistry: - """Return the footnotes text registry.""" - return self._footnotes - - ## - # Methods - ## - - def clear(self) -> None: - """Clear the index.""" - self._comments.clear() - self._footnotes.clear() - return - - def removeHandle(self, handle: str) -> None: - """Remove all entries for a given handle.""" - self._comments.removeHandle(handle) - self._footnotes.removeHandle(handle) - return - - ## - # Pack/Unpack - ## - - def packData(self) -> dict[str, dict]: - """Pack all the text comments into a single dictionary.""" - return { - "comments": self._comments.packData(), - "footnotes": self._footnotes.packData(), - } - - def unpackData(self, data: dict) -> None: - """Unpack the text comments index.""" - self._comments.unpackData(data.get("comments", {})) - self._footnotes.unpackData(data.get("footnotes", {})) - return - -# END Class TextIndex - - -class TextRegistry: - """Core: Text Registry Index Wrapper Class - - A wrapper class that holds a category of text entries. - """ - - __slots__ = ("_map", "_text", "_prefix") - - def __init__(self, prefix: str) -> None: - self._map: dict[str, str] = {} - self._text: dict[str, str] = {} - self._prefix = prefix - return - - def __len__(self) -> int: - return len(self._text) - - def __getitem__(self, key: str) -> str | None: - return self._text.get(key, (0, None))[1] - - def __contains__(self, key: str) -> bool: - return key in self._text - - ## - # Methods - ## - - def clear(self) -> None: - """Clear the index.""" - self._map.clear() - self._text.clear() - return - - def add(self, key: str, handle: str, text: str) -> None: - """Add a new text entry.""" - self._map[key] = handle - self._text[key] = text - return - - def keysForHandle(self, handle: str) -> list[str]: - """Return all keys for a given handle.""" - return [k for k, v in self._map.items() if v == handle] - - def removeHandle(self, handle: str) -> None: - """Iterate through the data and remove entries for a handle.""" - for key in [k for k, v in self._map.items() if v == handle]: - del self._text[key] - return - - def newKey(self) -> str: - """Generate a new key.""" - key = self._prefix + "".join([KEY_SOURCE[randint(0, 31)] for _ in range(4)]) - if key in self._text: - key = self.newKey() - return key - - ## - # Pack/Unpack - ## - - def packData(self) -> dict[str, dict[str, str]]: - """Pack all the text entries into a dictionary.""" - return {k: {"handle": self._map[k], "text": v} for k, v in self._text.items()} - - def unpackData(self, data: dict) -> None: - """Unpack text entries from a dictionary.""" - self.clear() - if not isinstance(data, dict): - raise ValueError("textEntry is not a dict") - - for key, entry in data.items(): - if not isinstance(key, str): - raise ValueError("textEntry key must be a string") - if not isinstance(entry, dict): - raise ValueError("textEntry entry is not a dict") - - handle = entry.get("handle") - text = entry.get("text") - if not isHandle(handle): - raise ValueError("textEntry handle must be a handle") - if not isinstance(text, str): - raise ValueError("textEntry text is not a string") - - self.add(key, handle, text) - - return - -# END Class TextEntry - - # =============================================================================================== # # Text Processing Functions # =============================================================================================== # diff --git a/novelwriter/core/status.py b/novelwriter/core/status.py index 6b85ccec..50f7e1d2 100644 --- a/novelwriter/core/status.py +++ b/novelwriter/core/status.py @@ -29,7 +29,7 @@ import logging import random from collections.abc import Iterable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from PyQt5.QtCore import QPointF, Qt from PyQt5.QtGui import QIcon, QPainter, QPainterPath, QPixmap, QColor, QPolygonF @@ -75,7 +75,7 @@ class NWStatus: __slots__ = ("_store", "_default", "_prefix", "_height") - def __init__(self, prefix: str) -> None: + def __init__(self, prefix: Literal["s", "i"]) -> None: self._store: dict[str, StatusEntry] = {} self._default = None self._prefix = prefix[:1] diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index d5012b59..b916cb98 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -1855,8 +1855,9 @@ class GuiDocEditor(QPlainTextEdit): def _insertCommentStructure(self, style: nwComment) -> None: """Insert a shortcut/comment combo.""" - if style == nwComment.FOOTNOTE: - key = SHARED.project.index.newCommentKey(style) + if self._docHandle and style == nwComment.FOOTNOTE: + self.saveText() # Index must be up to date + key = SHARED.project.index.newCommentKey(self._docHandle, style) code = nwShortcode.COMMENT_STYLES[nwComment.FOOTNOTE] cursor = self.textCursor() @@ -1868,14 +1869,12 @@ class GuiDocEditor(QPlainTextEdit): cursor.beginEditBlock() cursor.insertText(code.format(key)) - cursor.setPosition(block.position() + block.length()) + cursor.setPosition(block.position() + block.length() - 1) + cursor.insertBlock() cursor.insertBlock() cursor.insertText(f"%Footnote.{key}: ") - cursor.insertBlock() cursor.endEditBlock() - cursor.setPosition(cursor.position() - 1) - self.setTextCursor(cursor) return diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index bba81e55..57a29349 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -93,19 +93,19 @@ class GuiDocHighlighter(QSyntaxHighlighter): colBreak.setAlpha(64) # Create Character Formats - self._addCharFormat("header1", SHARED.theme.colHead, "bold", 1.8) - self._addCharFormat("header2", SHARED.theme.colHead, "bold", 1.6) - self._addCharFormat("header3", SHARED.theme.colHead, "bold", 1.4) - self._addCharFormat("header4", SHARED.theme.colHead, "bold", 1.2) - self._addCharFormat("head1h", SHARED.theme.colHeadH, "bold", 1.8) - self._addCharFormat("head2h", SHARED.theme.colHeadH, "bold", 1.6) - self._addCharFormat("head3h", SHARED.theme.colHeadH, "bold", 1.4) - self._addCharFormat("head4h", SHARED.theme.colHeadH, "bold", 1.2) - self._addCharFormat("bold", colEmph, "bold") - self._addCharFormat("italic", colEmph, "italic") - self._addCharFormat("strike", SHARED.theme.colHidden, "strike") - self._addCharFormat("mspaces", SHARED.theme.colError, "errline") - self._addCharFormat("nobreak", colBreak, "background") + self._addCharFormat("header1", SHARED.theme.colHead, "b", 1.8) + self._addCharFormat("header2", SHARED.theme.colHead, "b", 1.6) + self._addCharFormat("header3", SHARED.theme.colHead, "b", 1.4) + self._addCharFormat("header4", SHARED.theme.colHead, "b", 1.2) + self._addCharFormat("head1h", SHARED.theme.colHeadH, "b", 1.8) + self._addCharFormat("head2h", SHARED.theme.colHeadH, "b", 1.6) + self._addCharFormat("head3h", SHARED.theme.colHeadH, "b", 1.4) + self._addCharFormat("head4h", SHARED.theme.colHeadH, "b", 1.2) + self._addCharFormat("bold", colEmph, "b") + self._addCharFormat("italic", colEmph, "i") + self._addCharFormat("strike", SHARED.theme.colHidden, "s") + self._addCharFormat("mspaces", SHARED.theme.colError, "err") + self._addCharFormat("nobreak", colBreak, "bg") self._addCharFormat("dialog1", SHARED.theme.colDialN) self._addCharFormat("dialog2", SHARED.theme.colDialD) self._addCharFormat("dialog3", SHARED.theme.colDialS) @@ -117,7 +117,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self._addCharFormat("modifier", SHARED.theme.colMod) self._addCharFormat("value", SHARED.theme.colVal) self._addCharFormat("optional", SHARED.theme.colOpt) - self._addCharFormat("invalid", None, "errline") + self._addCharFormat("invalid", None, "err") # Cache Spell Error Format self._spellErr = QTextCharFormat() @@ -442,16 +442,16 @@ class GuiDocHighlighter(QSyntaxHighlighter): if style: styles = style.split(",") - if "bold" in styles: + if "b" in styles: charFormat.setFontWeight(QFont.Weight.Bold) - if "italic" in styles: + if "i" in styles: charFormat.setFontItalic(True) - if "strike" in styles: + if "s" in styles: charFormat.setFontStrikeOut(True) - if "errline" in styles: + if "err" in styles: charFormat.setUnderlineColor(SHARED.theme.colError) charFormat.setUnderlineStyle(QTextCharFormat.UnderlineStyle.SpellCheckUnderline) - if "background" in styles and color is not None: + if "bg" in styles and color is not None: charFormat.setBackground(QBrush(color, Qt.BrushStyle.SolidPattern)) if size: diff --git a/sample/content/636b6aa9b697b.nwd b/sample/content/636b6aa9b697b.nwd index 7532c006..e1f5955e 100644 --- a/sample/content/636b6aa9b697b.nwd +++ b/sample/content/636b6aa9b697b.nwd @@ -1,8 +1,8 @@ %%~name: Making a Scene %%~path: 6a2d6d5f4f401/636b6aa9b697b %%~kind: NOVEL/DOCUMENT -%%~hash: 06b80d830f3f4d5c703eff82067d4335b8b98151 -%%~date: Unknown/2024-04-14 23:28:43 +%%~hash: c7e664867218b3a9aac5c12119ef0ec63da2e5cc +%%~date: Unknown/2024-04-18 17:56:30 ### Making a Scene @pov: Jane @@ -21,9 +21,9 @@ If you have the need for it, you can also add text that can be automatically rep The editor also supports non breaking spaces, and the spell checker accepts long dashes—like this—as valid word separators. Regular dashes are also supported – and can be automatically inserted when typing two hyphens. -Thin spaces and thin non-breaking spaces are also supported from the Insert menu, and can be used to separate numbers from their units, like: 25 kg.[footnote:fq2ms] +Thin spaces and thin non-breaking spaces are also supported from the Insert menu, and can be used to separate numbers from their units, like: 25 kg.[footnote:f4xr5] -%Footnote.fq2ms: This is a footnote about non-breaking spaces. +%Footnote.f4xr5: Using a non-breaking space is the correct way to separate a number from its unit. This ensures that line wrapping does not split the two. #### Some Section Here diff --git a/sample/nwProject.nwx b/sample/nwProject.nwx index 3e5cec69..37feacec 100644 --- a/sample/nwProject.nwx +++ b/sample/nwProject.nwx @@ -1,6 +1,6 @@ - - + + Sample Project Jane Smith @@ -36,7 +36,7 @@ Main - + Novel @@ -46,7 +46,7 @@ Title Page - + Page @@ -58,11 +58,11 @@ Chapter One - + Making a Scene - + Another Scene