Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restructure index, and add support for story structure comments

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2025-02-20 23:28:50 +01:00
parent 05feaf7d2d
commit 655c9d9439
3 changed files with 89 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/index.py
+78 -99
View File
@@ -4,7 +4,7 @@ novelWriter – Project Index
File History: File History:
Created: 2019-05-27 [0.1.4] NWIndex Created: 2019-05-27 [0.1.4] NWIndex
Created: 2022-05-28 [2.0rc1] IndexItem Created: 2022-05-28 [2.0rc1] IndexNode
Created: 2022-05-28 [2.0rc1] IndexHeading Created: 2022-05-28 [2.0rc1] IndexHeading
Created: 2022-05-29 [2.0rc1] TagsIndex Created: 2022-05-29 [2.0rc1] TagsIndex
Created: 2022-05-29 [2.0rc1] ItemIndex Created: 2022-05-29 [2.0rc1] ItemIndex
@@ -31,7 +31,7 @@ import json
import logging import logging
import random import random
from collections.abc import ItemsView, Iterable from collections.abc import ItemsView, Iterable, Sequence
from pathlib import Path from pathlib import Path
from time import time from time import time
from typing import TYPE_CHECKING, Literal from typing import TYPE_CHECKING, Literal
@@ -72,8 +72,8 @@ class NWIndex:
this data is set directly by the indexer class in the NWItem object. this data is set directly by the indexer class in the NWItem object.
The primary index data is contained in a single instance of the The primary index data is contained in a single instance of the
ItemIndex class. This object contains an IndexItem representing each ItemIndex class. This object contains an IndexNode representing each
NWItem of the project. Each IndexItem holds an IndexHeading object NWItem of the project. Each IndexNode holds an IndexHeading object
for each heading of the item's text. for each heading of the item's text.
A reverse index of all tags is contained in a single instance of the A reverse index of all tags is contained in a single instance of the
@@ -523,7 +523,7 @@ class NWIndex:
# Extract Data # Extract Data
## ##
def getItemData(self, tHandle: str) -> IndexItem | None: def getItemData(self, tHandle: str) -> IndexNode | None:
"""Get the index data for a given item.""" """Get the index data for a given item."""
return self._itemIndex[tHandle] return self._itemIndex[tHandle]
@@ -572,7 +572,7 @@ class NWIndex:
def getHandleHeaderCount(self, tHandle: str) -> int: def getHandleHeaderCount(self, tHandle: str) -> int:
"""Get the number of headers in an item.""" """Get the number of headers in an item."""
tItem = self._itemIndex[tHandle] tItem = self._itemIndex[tHandle]
if isinstance(tItem, IndexItem): if isinstance(tItem, IndexNode):
return len(tItem) return len(tItem)
return 0 return 0
@@ -684,7 +684,7 @@ class NWIndex:
def getTagsData( def getTagsData(
self, activeOnly: bool = True self, activeOnly: bool = True
) -> Iterable[tuple[str, str, str, IndexItem | None, IndexHeading | None]]: ) -> Iterable[tuple[str, str, str, IndexNode | None, IndexHeading | None]]:
"""Return all known tags.""" """Return all known tags."""
for tag, data in self._tagsIndex.items(): for tag, data in self._tagsIndex.items():
iItem = self._itemIndex[data.get("handle")] iItem = self._itemIndex[data.get("handle")]
@@ -693,7 +693,7 @@ class NWIndex:
yield tag, data.get("name", ""), data.get("class", ""), iItem, hItem yield tag, data.get("name", ""), data.get("class", ""), iItem, hItem
return return
def getSingleTag(self, tagKey: str) -> tuple[str, str, IndexItem | None, IndexHeading | None]: def getSingleTag(self, tagKey: str) -> tuple[str, str, IndexNode | None, IndexHeading | None]:
"""Return tag data for a specific tag.""" """Return tag data for a specific tag."""
tName = self._tagsIndex.tagName(tagKey) tName = self._tagsIndex.tagName(tagKey)
tClass = self._tagsIndex.tagClass(tagKey) tClass = self._tagsIndex.tagClass(tagKey)
@@ -841,7 +841,7 @@ class ItemIndex:
A wrapper object holding the indexed items. This is a wrapper A wrapper object holding the indexed items. This is a wrapper
class around a single storage dictionary with a set of utility class around a single storage dictionary with a set of utility
functions for setting and accessing the index data. Each indexed functions for setting and accessing the index data. Each indexed
item is stored in an IndexItem object, which again holds an item is stored in an IndexNode object, which again holds an
IndexHeading object for each heading of the text. IndexHeading object for each heading of the text.
""" """
@@ -849,7 +849,7 @@ class ItemIndex:
def __init__(self, project: NWProject) -> None: def __init__(self, project: NWProject) -> None:
self._project = project self._project = project
self._items: dict[str, IndexItem] = {} self._items: dict[str, IndexNode] = {}
return return
def __contains__(self, tHandle: str) -> bool: def __contains__(self, tHandle: str) -> bool:
@@ -859,7 +859,7 @@ class ItemIndex:
self._items.pop(tHandle, None) self._items.pop(tHandle, None)
return return
def __getitem__(self, tHandle: str) -> IndexItem | None: def __getitem__(self, tHandle: str) -> IndexNode | None:
return self._items.get(tHandle, None) return self._items.get(tHandle, None)
## ##
@@ -875,7 +875,7 @@ class ItemIndex:
"""Add a new item to the index. This will overwrite the item if """Add a new item to the index. This will overwrite the item if
it already exists. it already exists.
""" """
self._items[tHandle] = IndexItem(tHandle, nwItem) self._items[tHandle] = IndexNode(tHandle, nwItem)
return return
def allItemTags(self, tHandle: str) -> list[str]: def allItemTags(self, tHandle: str) -> list[str]:
@@ -1004,14 +1004,14 @@ class ItemIndex:
nwItem = self._project.tree[tHandle] nwItem = self._project.tree[tHandle]
if nwItem is not None: if nwItem is not None:
tItem = IndexItem(tHandle, nwItem) tItem = IndexNode(tHandle, nwItem)
tItem.unpackData(tData) tItem.unpackData(tData)
self._items[tHandle] = tItem self._items[tHandle] = tItem
return return
class IndexItem: class IndexNode:
"""Core: Single Index Item Class """Core: Single Index Item Class
This object represents the index data of a project item (NWItem). This object represents the index data of a project item (NWItem).
@@ -1032,7 +1032,7 @@ class IndexItem:
return return
def __repr__(self) -> str: def __repr__(self) -> str:
return f"<IndexItem handle='{self._handle}'>" return f"<IndexNode handle='{self._handle}'>"
def __len__(self) -> int: def __len__(self) -> int:
return len(self._headings) return len(self._headings)
@@ -1073,7 +1073,7 @@ class IndexItem:
def setHeadingCounts(self, sTitle: str, cCount: int, wCount: int, pCount: int) -> None: def setHeadingCounts(self, sTitle: str, cCount: int, wCount: int, pCount: int) -> None:
"""Set the character, word and paragraph count of a heading.""" """Set the character, word and paragraph count of a heading."""
if sTitle in self._headings: if sTitle in self._headings:
self._headings[sTitle].setCounts(cCount, wCount, pCount) self._headings[sTitle].setCounts([cCount, wCount, pCount])
return return
def setHeadingSynopsis(self, sTitle: str, text: str) -> None: def setHeadingSynopsis(self, sTitle: str, text: str) -> None:
@@ -1133,41 +1133,29 @@ class IndexItem:
def packData(self) -> dict: def packData(self) -> dict:
"""Pack the indexed item's data into a dictionary.""" """Pack the indexed item's data into a dictionary."""
heads = {}
refs = {}
for sTitle, hItem in self._headings.items():
heads[sTitle] = hItem.packData()
hRefs = hItem.packReferences()
if hRefs:
refs[sTitle] = hRefs
data = {} data = {}
data["headings"] = heads for sTitle, hItem in self._headings.items():
if refs: data[sTitle] = hItem.packData()
data["references"] = refs
if self._notes: if self._notes:
data["notes"] = {style: list(keys) for style, keys in self._notes.items()} data["document"] = {style: list(keys) for style, keys in self._notes.items()}
return data return data
def unpackData(self, data: dict) -> None: def unpackData(self, data: dict) -> None:
"""Unpack an item entry from the data.""" """Unpack an item entry from the data."""
references = data.get("references", {}) for key, entry in data.items():
for sTitle, hData in data.get("headings", {}).items(): if isTitleTag(key):
if not isTitleTag(sTitle): heading = IndexHeading(key)
heading.unpackData(entry)
self.addHeading(heading)
elif key == "document":
for style, keys in entry.items():
if style not in NOTE_TYPES:
raise ValueError("The notes style is invalid")
if not isListInstance(keys, str):
raise ValueError("The notes keys must be a list of strings")
self._notes[style] = set(keys)
else:
raise ValueError("The itemIndex contains an invalid title key") raise ValueError("The itemIndex contains an invalid title key")
tHeading = IndexHeading(sTitle)
tHeading.unpackData(hData)
tHeading.unpackReferences(references.get(sTitle, {}))
self.addHeading(tHeading)
for style, keys in data.get("notes", {}).items():
if style not in NOTE_TYPES:
raise ValueError("The notes style is invalid")
if not isListInstance(keys, str):
raise ValueError("The notes keys must be a list of strings")
self._notes[style] = set(keys)
return return
@@ -1180,8 +1168,8 @@ class IndexHeading:
""" """
__slots__ = ( __slots__ = (
"_key", "_line", "_level", "_title", "_charCount", "_wordCount", "_key", "_line", "_level", "_title", "_counts",
"_paraCount", "_synopsis", "_tag", "_refs", "_tag", "_refs", "_comments",
) )
def __init__(self, key: str, line: int = 0, level: str = "H0", title: str = "") -> None: def __init__(self, key: str, line: int = 0, level: str = "H0", title: str = "") -> None:
@@ -1189,15 +1177,10 @@ class IndexHeading:
self._line = line self._line = line
self._level = level self._level = level
self._title = title self._title = title
self._counts: tuple[int, int, int] = (0, 0, 0)
self._charCount = 0
self._wordCount = 0
self._paraCount = 0
self._synopsis = ""
self._tag = "" self._tag = ""
self._refs: dict[str, set[str]] = {} self._refs: dict[str, set[str]] = {}
self._comments: dict[str, str] = {}
return return
def __repr__(self) -> str: def __repr__(self) -> str:
@@ -1225,19 +1208,19 @@ class IndexHeading:
@property @property
def charCount(self) -> int: def charCount(self) -> int:
return self._charCount return self._counts[0]
@property @property
def wordCount(self) -> int: def wordCount(self) -> int:
return self._wordCount return self._counts[1]
@property @property
def paraCount(self) -> int: def paraCount(self) -> int:
return self._paraCount return self._counts[2]
@property @property
def synopsis(self) -> str: def synopsis(self) -> str:
return self._synopsis return self._comments.get("summary", "")
@property @property
def tag(self) -> str: def tag(self) -> str:
@@ -1262,18 +1245,21 @@ class IndexHeading:
self._line = max(0, checkInt(line, 0)) self._line = max(0, checkInt(line, 0))
return return
def setCounts(self, charCount: int, wordCount: int, paraCount: int) -> None: def setCounts(self, counts: Sequence[int]) -> None:
"""Set the character, word and paragraph count. Make sure the """Set the character, word and paragraph count. Make sure the
value is an integer and is not smaller than 0. value is an integer and is not smaller than 0.
""" """
self._charCount = max(0, checkInt(charCount, 0)) if len(counts) == 3:
self._wordCount = max(0, checkInt(wordCount, 0)) self._counts = (
self._paraCount = max(0, checkInt(paraCount, 0)) max(0, checkInt(counts[0], 0)),
max(0, checkInt(counts[1], 0)),
max(0, checkInt(counts[2], 0)),
)
return return
def setSynopsis(self, text: str) -> None: def setSynopsis(self, text: str) -> None:
"""Set the synopsis text and make sure it is a string.""" """Set the synopsis text and make sure it is a string."""
self._synopsis = str(text) self._comments["summary"] = str(text)
return return
def setTag(self, tagKey: str) -> None: def setTag(self, tagKey: str) -> None:
@@ -1298,49 +1284,42 @@ class IndexHeading:
def packData(self) -> dict: def packData(self) -> dict:
"""Pack the values into a dictionary for saving to cache.""" """Pack the values into a dictionary for saving to cache."""
return { data = {}
data["meta"] = {
"level": self._level, "level": self._level,
"title": self._title, "title": self._title,
"line": self._line, "line": self._line,
"tag": self._tag, "tag": self._tag,
"cCount": self._charCount, "counts": self._counts,
"wCount": self._wordCount,
"pCount": self._paraCount,
"synopsis": self._synopsis,
} }
if self._refs:
def packReferences(self) -> dict[str, str]: data["refs"] = {k: ",".join(sorted(list(v))) for k, v in self._refs.items()}
"""Pack references into a dictionary for saving to cache. if self._comments:
Multiple types are packed into a sorted, comma separated string. data.update(self._comments)
It is sorted to prevent creating unnecessary diffs as the order return data
of a set is not guaranteed.
"""
return {key: ",".join(sorted(list(value))) for key, value in self._refs.items()}
def unpackData(self, data: dict) -> None: def unpackData(self, data: dict) -> None:
"""Unpack a heading entry from a dictionary.""" """Unpack a heading entry from a dictionary."""
self.setLevel(data.get("level", "H0")) for key, entry in data.items():
self._title = str(data.get("title", "")) if key == "meta":
self._tag = str(data.get("tag", "")) self.setLevel(entry.get("level", "H0"))
self.setLine(data.get("line", 0)) self._title = str(entry.get("title", ""))
self.setCounts( self._tag = str(entry.get("tag", ""))
data.get("cCount", 0), self.setLine(entry.get("line", 0))
data.get("wCount", 0), self.setCounts(entry.get("counts", [0, 0, 0]))
data.get("pCount", 0), elif key == "refs":
) for key, types in entry.items():
self._synopsis = str(data.get("synopsis", "")) if not isinstance(key, str):
return raise ValueError("itemIndex reference key must be a string")
if not isinstance(types, str):
def unpackReferences(self, data: dict) -> None: raise ValueError("itemIndex reference type must be a string")
"""Unpack a set of references from a dictionary.""" for refType in types.split(","):
for tagKey, refTypes in data.items(): if refType in nwKeyWords.VALID_KEYS:
if not isinstance(tagKey, str): self.addReference(key, refType)
raise ValueError("itemIndex reference key must be a string") else:
if not isinstance(refTypes, str): raise ValueError("The itemIndex contains an invalid reference type")
raise ValueError("itemIndex reference type must be a string") elif key == "summary" or key.startswith("story"):
for refType in refTypes.split(","): self._comments[str(key)] = str(entry)
if refType in nwKeyWords.VALID_KEYS: else:
self.addReference(tagKey, refType) raise KeyError("Unknown key in itemIndex")
else:
raise ValueError("The itemIndex contains an invalid reference type")
return return
novelwriter/gui/docviewerpanel.py
+2 -2
View File
@@ -36,7 +36,7 @@ from PyQt6.QtWidgets import (
from novelwriter import SHARED from novelwriter import SHARED
from novelwriter.common import checkInt, qtAddAction from novelwriter.common import checkInt, qtAddAction
from novelwriter.constants import nwLabels, nwLists, nwStyles, trConst from novelwriter.constants import nwLabels, nwLists, nwStyles, trConst
from novelwriter.core.index import IndexHeading, IndexItem from novelwriter.core.index import IndexHeading, IndexNode
from novelwriter.enum import nwChange, nwDocMode, nwItemClass from novelwriter.enum import nwChange, nwDocMode, nwItemClass
from novelwriter.extensions.modified import NIconToolButton from novelwriter.extensions.modified import NIconToolButton
from novelwriter.gui.theme import STYLES_FLAT_TABS, STYLES_MIN_TOOLBUTTON from novelwriter.gui.theme import STYLES_FLAT_TABS, STYLES_MIN_TOOLBUTTON
@@ -434,7 +434,7 @@ class _ViewPanelKeyWords(QTreeWidget):
self.clear() self.clear()
return return
def addUpdateEntry(self, tag: str, name: str, iItem: IndexItem, hItem: IndexHeading) -> None: def addUpdateEntry(self, tag: str, name: str, iItem: IndexNode, hItem: IndexHeading) -> None:
"""Add a new entry, or update an existing one.""" """Add a new entry, or update an existing one."""
nwItem = iItem.item nwItem = iItem.item
impLabel, impIcon = nwItem.getImportStatus() impLabel, impIcon = nwItem.getImportStatus()
tests/test_core/test_core_index.py
+9 -9
View File
@@ -28,7 +28,7 @@ import pytest
from novelwriter import SHARED from novelwriter import SHARED
from novelwriter.constants import nwFiles from novelwriter.constants import nwFiles
from novelwriter.core.index import IndexItem, NWIndex, TagsIndex from novelwriter.core.index import IndexNode, NWIndex, TagsIndex
from novelwriter.core.item import NWItem from novelwriter.core.item import NWItem
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.enum import nwComment, nwItemClass, nwItemLayout from novelwriter.enum import nwComment, nwItemClass, nwItemLayout
@@ -712,7 +712,7 @@ def testCoreIndex_ExtractData(mockGUI, fncPath, mockRnd):
# ================================== # ==================================
item = index.getItemData(nHandle) item = index.getItemData(nHandle)
assert isinstance(item, IndexItem) assert isinstance(item, IndexNode)
assert item.headings() == ["T0001"] assert item.headings() == ["T0001"]
assert index.getHandleHeaderCount(nHandle) == 1 assert index.getHandleHeaderCount(nHandle) == 1
assert index.getHandleHeaderCount("foo") == 0 assert index.getHandleHeaderCount("foo") == 0
@@ -1197,14 +1197,14 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd):
itemIndex.addHeadingRef(cHandle, "T0001", ["Jane", "John"], "@char") itemIndex.addHeadingRef(cHandle, "T0001", ["Jane", "John"], "@char")
idxData = itemIndex.packData() idxData = itemIndex.packData()
assert idxData[cHandle]["headings"]["T0001"] == { assert idxData[cHandle]["T0001"]["meta"] == {
"level": "H2", "line": 1, "title": "Chapter One", "tag": "one", "level": "H2", "line": 1, "title": "Chapter One", "tag": "one",
"cCount": 60, "wCount": 10, "pCount": 2, "synopsis": "In the beginning ...", "counts": (60, 10, 2), "summary": "In the beginning ...",
} }
assert "@pov" in idxData[cHandle]["references"]["T0001"]["jane"] assert "@pov" in idxData[cHandle]["T0001"]["refs"]["jane"]
assert "@focus" in idxData[cHandle]["references"]["T0001"]["jane"] assert "@focus" in idxData[cHandle]["T0001"]["refs"]["jane"]
assert "@char" in idxData[cHandle]["references"]["T0001"]["jane"] assert "@char" in idxData[cHandle]["T0001"]["refs"]["jane"]
assert "@char" in idxData[cHandle]["references"]["T0001"]["john"] assert "@char" in idxData[cHandle]["T0001"]["refs"]["john"]
# Add the other two files # Add the other two files
itemIndex.add(nHandle, project.tree[nHandle]) # type: ignore itemIndex.add(nHandle, project.tree[nHandle]) # type: ignore
@@ -1216,7 +1216,7 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd):
# ==================================== # ====================================
# Check repr strings # Check repr strings
assert repr(itemIndex[nHandle]) == f"<IndexItem handle='{nHandle}'>" assert repr(itemIndex[nHandle]) == f"<IndexNode handle='{nHandle}'>"
assert repr(itemIndex[nHandle]["T0001"]) == "<IndexHeading key='T0001'>" # type: ignore assert repr(itemIndex[nHandle]["T0001"]) == "<IndexHeading key='T0001'>" # type: ignore
# Check content of a single item # Check content of a single item