From 655c9d94397e9c4a4c86068ee11bbce76d67d7a0 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Thu, 20 Feb 2025 23:28:50 +0100 Subject: [PATCH] Restructure index, and add support for story structure comments --- novelwriter/core/index.py | 177 +++++++++++++---------------- novelwriter/gui/docviewerpanel.py | 4 +- tests/test_core/test_core_index.py | 18 +-- 3 files changed, 89 insertions(+), 110 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index d8e8bfb2..fe15bf4d 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -4,7 +4,7 @@ novelWriter – Project Index File History: Created: 2019-05-27 [0.1.4] NWIndex -Created: 2022-05-28 [2.0rc1] IndexItem +Created: 2022-05-28 [2.0rc1] IndexNode Created: 2022-05-28 [2.0rc1] IndexHeading Created: 2022-05-29 [2.0rc1] TagsIndex Created: 2022-05-29 [2.0rc1] ItemIndex @@ -31,7 +31,7 @@ import json import logging import random -from collections.abc import ItemsView, Iterable +from collections.abc import ItemsView, Iterable, Sequence from pathlib import Path from time import time from typing import TYPE_CHECKING, Literal @@ -72,8 +72,8 @@ class NWIndex: this data is set directly by the indexer class in the NWItem object. The primary index data is contained in a single instance of the - ItemIndex class. This object contains an IndexItem representing each - NWItem of the project. Each IndexItem holds an IndexHeading object + ItemIndex class. This object contains an IndexNode representing each + NWItem of the project. Each IndexNode holds an IndexHeading object for each heading of the item's text. A reverse index of all tags is contained in a single instance of the @@ -523,7 +523,7 @@ class NWIndex: # Extract Data ## - def getItemData(self, tHandle: str) -> IndexItem | None: + def getItemData(self, tHandle: str) -> IndexNode | None: """Get the index data for a given item.""" return self._itemIndex[tHandle] @@ -572,7 +572,7 @@ class NWIndex: def getHandleHeaderCount(self, tHandle: str) -> int: """Get the number of headers in an item.""" tItem = self._itemIndex[tHandle] - if isinstance(tItem, IndexItem): + if isinstance(tItem, IndexNode): return len(tItem) return 0 @@ -684,7 +684,7 @@ class NWIndex: def getTagsData( self, activeOnly: bool = True - ) -> Iterable[tuple[str, str, str, IndexItem | None, IndexHeading | None]]: + ) -> Iterable[tuple[str, str, str, IndexNode | None, IndexHeading | None]]: """Return all known tags.""" for tag, data in self._tagsIndex.items(): iItem = self._itemIndex[data.get("handle")] @@ -693,7 +693,7 @@ class NWIndex: yield tag, data.get("name", ""), data.get("class", ""), iItem, hItem return - def getSingleTag(self, tagKey: str) -> tuple[str, str, IndexItem | None, IndexHeading | None]: + def getSingleTag(self, tagKey: str) -> tuple[str, str, IndexNode | None, IndexHeading | None]: """Return tag data for a specific tag.""" tName = self._tagsIndex.tagName(tagKey) tClass = self._tagsIndex.tagClass(tagKey) @@ -841,7 +841,7 @@ class ItemIndex: A wrapper object holding the indexed items. This is a wrapper class around a single storage dictionary with a set of utility functions for setting and accessing the index data. Each indexed - item is stored in an IndexItem object, which again holds an + item is stored in an IndexNode object, which again holds an IndexHeading object for each heading of the text. """ @@ -849,7 +849,7 @@ class ItemIndex: def __init__(self, project: NWProject) -> None: self._project = project - self._items: dict[str, IndexItem] = {} + self._items: dict[str, IndexNode] = {} return def __contains__(self, tHandle: str) -> bool: @@ -859,7 +859,7 @@ class ItemIndex: self._items.pop(tHandle, None) return - def __getitem__(self, tHandle: str) -> IndexItem | None: + def __getitem__(self, tHandle: str) -> IndexNode | None: return self._items.get(tHandle, None) ## @@ -875,7 +875,7 @@ class ItemIndex: """Add a new item to the index. This will overwrite the item if it already exists. """ - self._items[tHandle] = IndexItem(tHandle, nwItem) + self._items[tHandle] = IndexNode(tHandle, nwItem) return def allItemTags(self, tHandle: str) -> list[str]: @@ -1004,14 +1004,14 @@ class ItemIndex: nwItem = self._project.tree[tHandle] if nwItem is not None: - tItem = IndexItem(tHandle, nwItem) + tItem = IndexNode(tHandle, nwItem) tItem.unpackData(tData) self._items[tHandle] = tItem return -class IndexItem: +class IndexNode: """Core: Single Index Item Class This object represents the index data of a project item (NWItem). @@ -1032,7 +1032,7 @@ class IndexItem: return def __repr__(self) -> str: - return f"" + return f"" def __len__(self) -> int: return len(self._headings) @@ -1073,7 +1073,7 @@ class IndexItem: def setHeadingCounts(self, sTitle: str, cCount: int, wCount: int, pCount: int) -> None: """Set the character, word and paragraph count of a heading.""" if sTitle in self._headings: - self._headings[sTitle].setCounts(cCount, wCount, pCount) + self._headings[sTitle].setCounts([cCount, wCount, pCount]) return def setHeadingSynopsis(self, sTitle: str, text: str) -> None: @@ -1133,41 +1133,29 @@ class IndexItem: def packData(self) -> dict: """Pack the indexed item's data into a dictionary.""" - heads = {} - refs = {} - for sTitle, hItem in self._headings.items(): - heads[sTitle] = hItem.packData() - hRefs = hItem.packReferences() - if hRefs: - refs[sTitle] = hRefs - data = {} - data["headings"] = heads - if refs: - data["references"] = refs + for sTitle, hItem in self._headings.items(): + data[sTitle] = hItem.packData() if self._notes: - data["notes"] = {style: list(keys) for style, keys in self._notes.items()} - + data["document"] = {style: list(keys) for style, keys in self._notes.items()} return data def unpackData(self, data: dict) -> None: """Unpack an item entry from the data.""" - references = data.get("references", {}) - for sTitle, hData in data.get("headings", {}).items(): - if not isTitleTag(sTitle): + for key, entry in data.items(): + if isTitleTag(key): + heading = IndexHeading(key) + heading.unpackData(entry) + self.addHeading(heading) + elif key == "document": + for style, keys in entry.items(): + if style not in NOTE_TYPES: + raise ValueError("The notes style is invalid") + if not isListInstance(keys, str): + raise ValueError("The notes keys must be a list of strings") + self._notes[style] = set(keys) + else: raise ValueError("The itemIndex contains an invalid title key") - tHeading = IndexHeading(sTitle) - tHeading.unpackData(hData) - tHeading.unpackReferences(references.get(sTitle, {})) - self.addHeading(tHeading) - - for style, keys in data.get("notes", {}).items(): - if style not in NOTE_TYPES: - raise ValueError("The notes style is invalid") - if not isListInstance(keys, str): - raise ValueError("The notes keys must be a list of strings") - self._notes[style] = set(keys) - return @@ -1180,8 +1168,8 @@ class IndexHeading: """ __slots__ = ( - "_key", "_line", "_level", "_title", "_charCount", "_wordCount", - "_paraCount", "_synopsis", "_tag", "_refs", + "_key", "_line", "_level", "_title", "_counts", + "_tag", "_refs", "_comments", ) def __init__(self, key: str, line: int = 0, level: str = "H0", title: str = "") -> None: @@ -1189,15 +1177,10 @@ class IndexHeading: self._line = line self._level = level self._title = title - - self._charCount = 0 - self._wordCount = 0 - self._paraCount = 0 - self._synopsis = "" - + self._counts: tuple[int, int, int] = (0, 0, 0) self._tag = "" self._refs: dict[str, set[str]] = {} - + self._comments: dict[str, str] = {} return def __repr__(self) -> str: @@ -1225,19 +1208,19 @@ class IndexHeading: @property def charCount(self) -> int: - return self._charCount + return self._counts[0] @property def wordCount(self) -> int: - return self._wordCount + return self._counts[1] @property def paraCount(self) -> int: - return self._paraCount + return self._counts[2] @property def synopsis(self) -> str: - return self._synopsis + return self._comments.get("summary", "") @property def tag(self) -> str: @@ -1262,18 +1245,21 @@ class IndexHeading: self._line = max(0, checkInt(line, 0)) return - def setCounts(self, charCount: int, wordCount: int, paraCount: int) -> None: + def setCounts(self, counts: Sequence[int]) -> None: """Set the character, word and paragraph count. Make sure the value is an integer and is not smaller than 0. """ - self._charCount = max(0, checkInt(charCount, 0)) - self._wordCount = max(0, checkInt(wordCount, 0)) - self._paraCount = max(0, checkInt(paraCount, 0)) + if len(counts) == 3: + self._counts = ( + max(0, checkInt(counts[0], 0)), + max(0, checkInt(counts[1], 0)), + max(0, checkInt(counts[2], 0)), + ) return def setSynopsis(self, text: str) -> None: """Set the synopsis text and make sure it is a string.""" - self._synopsis = str(text) + self._comments["summary"] = str(text) return def setTag(self, tagKey: str) -> None: @@ -1298,49 +1284,42 @@ class IndexHeading: def packData(self) -> dict: """Pack the values into a dictionary for saving to cache.""" - return { + data = {} + data["meta"] = { "level": self._level, "title": self._title, "line": self._line, "tag": self._tag, - "cCount": self._charCount, - "wCount": self._wordCount, - "pCount": self._paraCount, - "synopsis": self._synopsis, + "counts": self._counts, } - - def packReferences(self) -> dict[str, str]: - """Pack references into a dictionary for saving to cache. - Multiple types are packed into a sorted, comma separated string. - It is sorted to prevent creating unnecessary diffs as the order - of a set is not guaranteed. - """ - return {key: ",".join(sorted(list(value))) for key, value in self._refs.items()} + if self._refs: + data["refs"] = {k: ",".join(sorted(list(v))) for k, v in self._refs.items()} + if self._comments: + data.update(self._comments) + return data def unpackData(self, data: dict) -> None: """Unpack a heading entry from a dictionary.""" - self.setLevel(data.get("level", "H0")) - self._title = str(data.get("title", "")) - self._tag = str(data.get("tag", "")) - self.setLine(data.get("line", 0)) - self.setCounts( - data.get("cCount", 0), - data.get("wCount", 0), - data.get("pCount", 0), - ) - self._synopsis = str(data.get("synopsis", "")) - return - - def unpackReferences(self, data: dict) -> None: - """Unpack a set of references from a dictionary.""" - for tagKey, refTypes in data.items(): - if not isinstance(tagKey, str): - raise ValueError("itemIndex reference key must be a string") - if not isinstance(refTypes, str): - raise ValueError("itemIndex reference type must be a string") - for refType in refTypes.split(","): - if refType in nwKeyWords.VALID_KEYS: - self.addReference(tagKey, refType) - else: - raise ValueError("The itemIndex contains an invalid reference type") + for key, entry in data.items(): + if key == "meta": + self.setLevel(entry.get("level", "H0")) + self._title = str(entry.get("title", "")) + self._tag = str(entry.get("tag", "")) + self.setLine(entry.get("line", 0)) + self.setCounts(entry.get("counts", [0, 0, 0])) + elif key == "refs": + for key, types in entry.items(): + if not isinstance(key, str): + raise ValueError("itemIndex reference key must be a string") + if not isinstance(types, str): + raise ValueError("itemIndex reference type must be a string") + for refType in types.split(","): + if refType in nwKeyWords.VALID_KEYS: + self.addReference(key, refType) + else: + raise ValueError("The itemIndex contains an invalid reference type") + elif key == "summary" or key.startswith("story"): + self._comments[str(key)] = str(entry) + else: + raise KeyError("Unknown key in itemIndex") return diff --git a/novelwriter/gui/docviewerpanel.py b/novelwriter/gui/docviewerpanel.py index 54284d10..1a9c6e04 100644 --- a/novelwriter/gui/docviewerpanel.py +++ b/novelwriter/gui/docviewerpanel.py @@ -36,7 +36,7 @@ from PyQt6.QtWidgets import ( from novelwriter import SHARED from novelwriter.common import checkInt, qtAddAction from novelwriter.constants import nwLabels, nwLists, nwStyles, trConst -from novelwriter.core.index import IndexHeading, IndexItem +from novelwriter.core.index import IndexHeading, IndexNode from novelwriter.enum import nwChange, nwDocMode, nwItemClass from novelwriter.extensions.modified import NIconToolButton from novelwriter.gui.theme import STYLES_FLAT_TABS, STYLES_MIN_TOOLBUTTON @@ -434,7 +434,7 @@ class _ViewPanelKeyWords(QTreeWidget): self.clear() return - def addUpdateEntry(self, tag: str, name: str, iItem: IndexItem, hItem: IndexHeading) -> None: + def addUpdateEntry(self, tag: str, name: str, iItem: IndexNode, hItem: IndexHeading) -> None: """Add a new entry, or update an existing one.""" nwItem = iItem.item impLabel, impIcon = nwItem.getImportStatus() diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index baa1870e..ea140a44 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -28,7 +28,7 @@ import pytest from novelwriter import SHARED from novelwriter.constants import nwFiles -from novelwriter.core.index import IndexItem, NWIndex, TagsIndex +from novelwriter.core.index import IndexNode, NWIndex, TagsIndex from novelwriter.core.item import NWItem from novelwriter.core.project import NWProject from novelwriter.enum import nwComment, nwItemClass, nwItemLayout @@ -712,7 +712,7 @@ def testCoreIndex_ExtractData(mockGUI, fncPath, mockRnd): # ================================== item = index.getItemData(nHandle) - assert isinstance(item, IndexItem) + assert isinstance(item, IndexNode) assert item.headings() == ["T0001"] assert index.getHandleHeaderCount(nHandle) == 1 assert index.getHandleHeaderCount("foo") == 0 @@ -1197,14 +1197,14 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd): itemIndex.addHeadingRef(cHandle, "T0001", ["Jane", "John"], "@char") idxData = itemIndex.packData() - assert idxData[cHandle]["headings"]["T0001"] == { + assert idxData[cHandle]["T0001"]["meta"] == { "level": "H2", "line": 1, "title": "Chapter One", "tag": "one", - "cCount": 60, "wCount": 10, "pCount": 2, "synopsis": "In the beginning ...", + "counts": (60, 10, 2), "summary": "In the beginning ...", } - assert "@pov" in idxData[cHandle]["references"]["T0001"]["jane"] - assert "@focus" in idxData[cHandle]["references"]["T0001"]["jane"] - assert "@char" in idxData[cHandle]["references"]["T0001"]["jane"] - assert "@char" in idxData[cHandle]["references"]["T0001"]["john"] + assert "@pov" in idxData[cHandle]["T0001"]["refs"]["jane"] + assert "@focus" in idxData[cHandle]["T0001"]["refs"]["jane"] + assert "@char" in idxData[cHandle]["T0001"]["refs"]["jane"] + assert "@char" in idxData[cHandle]["T0001"]["refs"]["john"] # Add the other two files itemIndex.add(nHandle, project.tree[nHandle]) # type: ignore @@ -1216,7 +1216,7 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd): # ==================================== # Check repr strings - assert repr(itemIndex[nHandle]) == f"" + assert repr(itemIndex[nHandle]) == f"" assert repr(itemIndex[nHandle]["T0001"]) == "" # type: ignore # Check content of a single item