Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce footnote index storage to keys only

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-04-18 18:07:17 +02:00
parent ae893be79b
commit a2d8aefb21
7 changed files with 108 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/common.py
+13 -6
View File
@@ -24,30 +24,32 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
from __future__ import annotations
import json
import uuid
import logging
import unicodedata
import uuid
import xml.etree.ElementTree as ET
from typing import TYPE_CHECKING, Any, Literal
from pathlib import Path
from datetime import datetime
from configparser import ConfigParser
from datetime import datetime
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, TypeVar
from urllib.parse import urljoin
from urllib.request import pathname2url
from PyQt5.QtGui import QColor, QDesktopServices
from PyQt5.QtCore import QCoreApplication, QUrl
from PyQt5.QtGui import QColor, QDesktopServices
from novelwriter.constants import nwConst, nwLabels, nwUnicode, trConst
from novelwriter.enum import nwItemClass, nwItemType, nwItemLayout
from novelwriter.error import logException
from novelwriter.constants import nwConst, nwLabels, nwUnicode, trConst
if TYPE_CHECKING: # pragma: no cover
from typing import TypeGuard # Requires Python 3.10
logger = logging.getLogger(__name__)
_Type = TypeVar("_Type")
##
# Checker Functions
@@ -172,6 +174,11 @@ def isItemLayout(value: Any) -> TypeGuard[str]:
return isinstance(value, str) and value in nwItemLayout.__members__
def isListInstance(data: Any, check: type[_Type]) -> TypeGuard[list[_Type]]:
"""Check that all items of a list is of a given type."""
return isinstance(data, list) and all(isinstance(item, check) for item in data)
def hexToInt(value: Any, default: int = 0) -> int:
"""Convert a hex string to an integer."""
if isinstance(value, str):
novelwriter/core/index.py
+59 -174
View File
@@ -30,17 +30,19 @@ from __future__ import annotations
import json
import logging
from collections.abc import ItemsView, Iterable
from pathlib import Path
from random import randint
from time import time
from typing import TYPE_CHECKING
from pathlib import Path
from collections.abc import ItemsView, Iterable
from typing import TYPE_CHECKING, Literal
from novelwriter import SHARED
from novelwriter.common import (
checkInt, isHandle, isItemClass, isListInstance, isTitleTag, jsonEncode
)
from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
from novelwriter.error import logException
from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode
from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
from novelwriter.text.counting import standardCounter
if TYPE_CHECKING: # pragma: no cover
@@ -49,7 +51,11 @@ if TYPE_CHECKING: # pragma: no cover
logger = logging.getLogger(__name__)
T_NoteTypes = Literal["footnotes", "comments"]
TT_NONE = "T0000"
KEY_SOURCE = "0123456789bcdfghjklmnopqrstvwxyz"
NOTE_TYPES: list[T_NoteTypes] = ["footnotes", "comments"]
class NWIndex:
@@ -86,7 +92,6 @@ class NWIndex:
# Storage and State
self._tagsIndex = TagsIndex()
self._itemIndex = ItemIndex(project)
self._textIndex = TextIndex()
self._indexBroken = False
# TimeStamps
@@ -114,7 +119,6 @@ class NWIndex:
"""Clear the index dictionaries and time stamps."""
self._tagsIndex.clear()
self._itemIndex.clear()
self._textIndex.clear()
self._indexChange = 0.0
self._rootChange = {}
SHARED.indexSignalProxy({"event": "clearIndex"})
@@ -138,7 +142,6 @@ class NWIndex:
for tTag in delTags:
del self._tagsIndex[tTag]
del self._itemIndex[tHandle]
self._textIndex.removeHandle(tHandle)
SHARED.indexSignalProxy({
"event": "updateTags",
"deleted": delTags,
@@ -193,7 +196,6 @@ class NWIndex:
try:
self._tagsIndex.unpackData(data["novelWriter.tagsIndex"])
self._itemIndex.unpackData(data["novelWriter.itemIndex"])
self._textIndex.unpackData(data["novelWriter.textIndex"])
except Exception:
logger.error("The index content is invalid")
logException()
@@ -229,12 +231,10 @@ class NWIndex:
try:
tagsIndex = jsonEncode(self._tagsIndex.packData(), n=1, nmax=2)
itemIndex = jsonEncode(self._itemIndex.packData(), n=1, nmax=4)
textIndex = jsonEncode(self._textIndex.packData(), n=1, nmax=3)
with open(indexFile, mode="w+", encoding="utf-8") as outFile:
outFile.write("{\n")
outFile.write(f' "novelWriter.tagsIndex": {tagsIndex},\n')
outFile.write(f' "novelWriter.itemIndex": {itemIndex},\n')
outFile.write(f' "novelWriter.textIndex": {textIndex}\n')
outFile.write(f' "novelWriter.itemIndex": {itemIndex}\n')
outFile.write("}\n")
except Exception:
@@ -346,7 +346,7 @@ class NWIndex:
if cStyle in (nwComment.SYNOPSIS, nwComment.SHORT):
self._itemIndex.setHeadingSynopsis(tHandle, cTitle, cText)
elif cStyle == nwComment.FOOTNOTE:
self._textIndex.footnotes.add(cKey, tHandle, cText)
self._itemIndex.addNoteKey(tHandle, "footnotes", cKey)
# Count words for remaining text after last heading
if pTitle != TT_NONE:
@@ -514,13 +514,13 @@ class NWIndex:
name, _, display = text.partition("|")
return name.rstrip(), display.lstrip()
def newCommentKey(self, style: nwComment) -> str | None:
def newCommentKey(self, tHandle: str, style: nwComment) -> str:
"""Generate a new key for a comment style."""
if style == nwComment.FOOTNOTE:
return self._textIndex.footnotes.newKey()
return self._itemIndex.genNewNoteKey(tHandle, "footnotes")
elif style == nwComment.COMMENT:
return self._textIndex.comments.newKey()
return None
return self._itemIndex.genNewNoteKey(tHandle, "comments")
return "err"
##
# Extract Data
@@ -966,6 +966,25 @@ class ItemIndex:
self._items[tHandle].addHeadingRef(sTitle, tagKeys, refType)
return
def addNoteKey(self, tHandle: str, style: T_NoteTypes, key: str) -> None:
"""Set notes key for a given item."""
if tHandle in self._items:
self._items[tHandle].addNoteKey(style, key)
return
def genNewNoteKey(self, tHandle: str, style: T_NoteTypes) -> str:
"""Set notes key for a given item."""
keys = set()
for item in self._items.values():
keys.update(item.noteKeys(style))
if style in NOTE_TYPES and (item := self._items.get(tHandle)):
for _ in range(1000):
key = style[:1] + "".join([KEY_SOURCE[randint(0, 31)] for _ in range(4)])
if key not in keys:
item.addNoteKey(style, key)
return key
return "err"
##
# Pack/Unpack
##
@@ -1007,12 +1026,13 @@ class IndexItem:
must be reset each time the item is re-indexed.
"""
__slots__ = ("_handle", "_item", "_headings", "_count")
__slots__ = ("_handle", "_item", "_headings", "_count", "_notes")
def __init__(self, tHandle: str, nwItem: NWItem) -> None:
self._handle = tHandle
self._item = nwItem
self._headings: dict[str, IndexHeading] = {TT_NONE: IndexHeading(TT_NONE)}
self._notes: dict[str, set[str]] = {}
self._count = 0
return
@@ -1080,6 +1100,13 @@ class IndexItem:
self._headings[sTitle].addReference(tagKey, refType)
return
def addNoteKey(self, style: T_NoteTypes, key: str) -> None:
"""Add a note key to the index."""
if style not in self._notes:
self._notes[style] = set()
self._notes[style].add(key)
return
##
# Data Methods
##
@@ -1101,6 +1128,10 @@ class IndexItem:
self._count += 1
return f"T{self._count:04d}"
def noteKeys(self, style: T_NoteTypes) -> set[str]:
"""Return a set of all note keys."""
return self._notes.get(style, set())
##
# Pack/Unpack
##
@@ -1119,6 +1150,8 @@ class IndexItem:
data["headings"] = heads
if refs:
data["references"] = refs
if self._notes:
data["notes"] = {style: list(keys) for style, keys in self._notes.items()}
return data
@@ -1132,6 +1165,14 @@ class IndexItem:
tHeading.unpackData(hData)
tHeading.unpackReferences(references.get(sTitle, {}))
self.addHeading(tHeading)
for style, keys in data.get("notes", {}).items():
if style not in NOTE_TYPES:
raise ValueError("The notes style is invalid")
if not isListInstance(keys, str):
raise ValueError("The notes keys must be a list of strings")
self._notes[style] = set(keys)
return
# END Class IndexItem
@@ -1314,162 +1355,6 @@ class IndexHeading:
# END Class IndexHeading
# =============================================================================================== #
# The Text Index Object
# =============================================================================================== #
KEY_SOURCE = "0123456789bcdfghjklmnopqrstvwxyz"
class TextIndex:
"""Core: Text Index Wrapper Class
A wrapper class that holds various global text entries.
"""
__slots__ = ("_comments", "_footnotes")
def __init__(self) -> None:
self._comments = TextRegistry("c")
self._footnotes = TextRegistry("f")
return
@property
def comments(self) -> TextRegistry:
"""Return the comments text registry."""
return self._comments
@property
def footnotes(self) -> TextRegistry:
"""Return the footnotes text registry."""
return self._footnotes
##
# Methods
##
def clear(self) -> None:
"""Clear the index."""
self._comments.clear()
self._footnotes.clear()
return
def removeHandle(self, handle: str) -> None:
"""Remove all entries for a given handle."""
self._comments.removeHandle(handle)
self._footnotes.removeHandle(handle)
return
##
# Pack/Unpack
##
def packData(self) -> dict[str, dict]:
"""Pack all the text comments into a single dictionary."""
return {
"comments": self._comments.packData(),
"footnotes": self._footnotes.packData(),
}
def unpackData(self, data: dict) -> None:
"""Unpack the text comments index."""
self._comments.unpackData(data.get("comments", {}))
self._footnotes.unpackData(data.get("footnotes", {}))
return
# END Class TextIndex
class TextRegistry:
"""Core: Text Registry Index Wrapper Class
A wrapper class that holds a category of text entries.
"""
__slots__ = ("_map", "_text", "_prefix")
def __init__(self, prefix: str) -> None:
self._map: dict[str, str] = {}
self._text: dict[str, str] = {}
self._prefix = prefix
return
def __len__(self) -> int:
return len(self._text)
def __getitem__(self, key: str) -> str | None:
return self._text.get(key, (0, None))[1]
def __contains__(self, key: str) -> bool:
return key in self._text
##
# Methods
##
def clear(self) -> None:
"""Clear the index."""
self._map.clear()
self._text.clear()
return
def add(self, key: str, handle: str, text: str) -> None:
"""Add a new text entry."""
self._map[key] = handle
self._text[key] = text
return
def keysForHandle(self, handle: str) -> list[str]:
"""Return all keys for a given handle."""
return [k for k, v in self._map.items() if v == handle]
def removeHandle(self, handle: str) -> None:
"""Iterate through the data and remove entries for a handle."""
for key in [k for k, v in self._map.items() if v == handle]:
del self._text[key]
return
def newKey(self) -> str:
"""Generate a new key."""
key = self._prefix + "".join([KEY_SOURCE[randint(0, 31)] for _ in range(4)])
if key in self._text:
key = self.newKey()
return key
##
# Pack/Unpack
##
def packData(self) -> dict[str, dict[str, str]]:
"""Pack all the text entries into a dictionary."""
return {k: {"handle": self._map[k], "text": v} for k, v in self._text.items()}
def unpackData(self, data: dict) -> None:
"""Unpack text entries from a dictionary."""
self.clear()
if not isinstance(data, dict):
raise ValueError("textEntry is not a dict")
for key, entry in data.items():
if not isinstance(key, str):
raise ValueError("textEntry key must be a string")
if not isinstance(entry, dict):
raise ValueError("textEntry entry is not a dict")
handle = entry.get("handle")
text = entry.get("text")
if not isHandle(handle):
raise ValueError("textEntry handle must be a handle")
if not isinstance(text, str):
raise ValueError("textEntry text is not a string")
self.add(key, handle, text)
return
# END Class TextEntry
# =============================================================================================== #
# Text Processing Functions
# =============================================================================================== #
novelwriter/core/status.py
+2 -2
View File
@@ -29,7 +29,7 @@ import logging
import random
from collections.abc import Iterable
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Literal
from PyQt5.QtCore import QPointF, Qt
from PyQt5.QtGui import QIcon, QPainter, QPainterPath, QPixmap, QColor, QPolygonF
@@ -75,7 +75,7 @@ class NWStatus:
__slots__ = ("_store", "_default", "_prefix", "_height")
def __init__(self, prefix: str) -> None:
def __init__(self, prefix: Literal["s", "i"]) -> None:
self._store: dict[str, StatusEntry] = {}
self._default = None
self._prefix = prefix[:1]
novelwriter/gui/doceditor.py
+5 -6
View File
@@ -1855,8 +1855,9 @@ class GuiDocEditor(QPlainTextEdit):
def _insertCommentStructure(self, style: nwComment) -> None:
"""Insert a shortcut/comment combo."""
if style == nwComment.FOOTNOTE:
key = SHARED.project.index.newCommentKey(style)
if self._docHandle and style == nwComment.FOOTNOTE:
self.saveText() # Index must be up to date
key = SHARED.project.index.newCommentKey(self._docHandle, style)
code = nwShortcode.COMMENT_STYLES[nwComment.FOOTNOTE]
cursor = self.textCursor()
@@ -1868,14 +1869,12 @@ class GuiDocEditor(QPlainTextEdit):
cursor.beginEditBlock()
cursor.insertText(code.format(key))
cursor.setPosition(block.position() + block.length())
cursor.setPosition(block.position() + block.length() - 1)
cursor.insertBlock()
cursor.insertBlock()
cursor.insertText(f"%Footnote.{key}: ")
cursor.insertBlock()
cursor.endEditBlock()
cursor.setPosition(cursor.position() - 1)
self.setTextCursor(cursor)
return
novelwriter/gui/dochighlight.py
+19 -19
View File
@@ -93,19 +93,19 @@ class GuiDocHighlighter(QSyntaxHighlighter):
colBreak.setAlpha(64)
# Create Character Formats
self._addCharFormat("header1", SHARED.theme.colHead, "bold", 1.8)
self._addCharFormat("header2", SHARED.theme.colHead, "bold", 1.6)
self._addCharFormat("header3", SHARED.theme.colHead, "bold", 1.4)
self._addCharFormat("header4", SHARED.theme.colHead, "bold", 1.2)
self._addCharFormat("head1h", SHARED.theme.colHeadH, "bold", 1.8)
self._addCharFormat("head2h", SHARED.theme.colHeadH, "bold", 1.6)
self._addCharFormat("head3h", SHARED.theme.colHeadH, "bold", 1.4)
self._addCharFormat("head4h", SHARED.theme.colHeadH, "bold", 1.2)
self._addCharFormat("bold", colEmph, "bold")
self._addCharFormat("italic", colEmph, "italic")
self._addCharFormat("strike", SHARED.theme.colHidden, "strike")
self._addCharFormat("mspaces", SHARED.theme.colError, "errline")
self._addCharFormat("nobreak", colBreak, "background")
self._addCharFormat("header1", SHARED.theme.colHead, "b", 1.8)
self._addCharFormat("header2", SHARED.theme.colHead, "b", 1.6)
self._addCharFormat("header3", SHARED.theme.colHead, "b", 1.4)
self._addCharFormat("header4", SHARED.theme.colHead, "b", 1.2)
self._addCharFormat("head1h", SHARED.theme.colHeadH, "b", 1.8)
self._addCharFormat("head2h", SHARED.theme.colHeadH, "b", 1.6)
self._addCharFormat("head3h", SHARED.theme.colHeadH, "b", 1.4)
self._addCharFormat("head4h", SHARED.theme.colHeadH, "b", 1.2)
self._addCharFormat("bold", colEmph, "b")
self._addCharFormat("italic", colEmph, "i")
self._addCharFormat("strike", SHARED.theme.colHidden, "s")
self._addCharFormat("mspaces", SHARED.theme.colError, "err")
self._addCharFormat("nobreak", colBreak, "bg")
self._addCharFormat("dialog1", SHARED.theme.colDialN)
self._addCharFormat("dialog2", SHARED.theme.colDialD)
self._addCharFormat("dialog3", SHARED.theme.colDialS)
@@ -117,7 +117,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self._addCharFormat("modifier", SHARED.theme.colMod)
self._addCharFormat("value", SHARED.theme.colVal)
self._addCharFormat("optional", SHARED.theme.colOpt)
self._addCharFormat("invalid", None, "errline")
self._addCharFormat("invalid", None, "err")
# Cache Spell Error Format
self._spellErr = QTextCharFormat()
@@ -442,16 +442,16 @@ class GuiDocHighlighter(QSyntaxHighlighter):
if style:
styles = style.split(",")
if "bold" in styles:
if "b" in styles:
charFormat.setFontWeight(QFont.Weight.Bold)
if "italic" in styles:
if "i" in styles:
charFormat.setFontItalic(True)
if "strike" in styles:
if "s" in styles:
charFormat.setFontStrikeOut(True)
if "errline" in styles:
if "err" in styles:
charFormat.setUnderlineColor(SHARED.theme.colError)
charFormat.setUnderlineStyle(QTextCharFormat.UnderlineStyle.SpellCheckUnderline)
if "background" in styles and color is not None:
if "bg" in styles and color is not None:
charFormat.setBackground(QBrush(color, Qt.BrushStyle.SolidPattern))
if size:
sample/content/636b6aa9b697b.nwd
+4 -4
View File
@@ -1,8 +1,8 @@
%%~name: Making a Scene
%%~path: 6a2d6d5f4f401/636b6aa9b697b
%%~kind: NOVEL/DOCUMENT
%%~hash: 06b80d830f3f4d5c703eff82067d4335b8b98151
%%~date: Unknown/2024-04-14 23:28:43
%%~hash: c7e664867218b3a9aac5c12119ef0ec63da2e5cc
%%~date: Unknown/2024-04-18 17:56:30
### Making a Scene
@pov: Jane
@@ -21,9 +21,9 @@ If you have the need for it, you can also add text that can be automatically rep
The editor also supports non breaking spaces, and the spell checker accepts long dashes—like this—as valid word separators. Regular dashes are also supported – and can be automatically inserted when typing two hyphens.
Thin spaces and thin non-breaking spaces are also supported from the Insert menu, and can be used to separate numbers from their units, like: 25 kg.[footnote:fq2ms]
Thin spaces and thin non-breaking spaces are also supported from the Insert menu, and can be used to separate numbers from their units, like: 25 kg.[footnote:f4xr5]
%Footnote.fq2ms: This is a footnote about non-breaking spaces.
%Footnote.f4xr5: Using a non-breaking space is the correct way to separate a number from its unit. This ensures that line wrapping does not split the two.
#### Some Section Here
sample/nwProject.nwx
+6 -6
View File
@@ -1,6 +1,6 @@
<?xml version='1.0' encoding='utf-8'?>
<novelWriterXML appVersion="2.5a1" hexVersion="0x020500a1" fileVersion="1.5" fileRevision="4" timeStamp="2024-04-14 23:31:44">
<project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="1879" autoCount="272" editTime="86979">
<novelWriterXML appVersion="2.5a2" hexVersion="0x020500a1" fileVersion="1.5" fileRevision="4" timeStamp="2024-04-18 17:57:37">
<project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="1916" autoCount="274" editTime="87487">
<name>Sample Project</name>
<author>Jane Smith</author>
</project>
@@ -36,7 +36,7 @@
<entry key="i56be10" count="1" red="220" green="138" blue="221" shape="BLOCK_4">Main</entry>
</importance>
</settings>
<content items="31" novelWords="998" notesWords="416">
<content items="31" novelWords="996" notesWords="416">
<item handle="7031beac91f75" parent="None" root="7031beac91f75" order="0" type="ROOT" class="NOVEL">
<meta expanded="yes" />
<name status="sc24b8f" import="ia857f0">Novel</name>
@@ -46,7 +46,7 @@
<name status="sc24b8f" import="ia857f0" active="yes">Title Page</name>
</item>
<item handle="974e400180a99" parent="7031beac91f75" root="7031beac91f75" order="1" type="FILE" class="NOVEL" layout="DOCUMENT">
<meta expanded="no" heading="H0" charCount="250" wordCount="49" paraCount="2" cursorPos="275" />
<meta expanded="no" heading="H0" charCount="233" wordCount="47" paraCount="2" cursorPos="275" />
<name status="sf12341" import="ia857f0" active="yes">Page</name>
</item>
<item handle="edca4be2fcaf8" parent="7031beac91f75" root="7031beac91f75" order="2" type="FILE" class="NOVEL" layout="DOCUMENT">
@@ -58,11 +58,11 @@
<name status="sf24ce6" import="ia857f0" active="yes">Chapter One</name>
</item>
<item handle="636b6aa9b697b" parent="6a2d6d5f4f401" root="7031beac91f75" order="0" type="FILE" class="NOVEL" layout="DOCUMENT">
<meta expanded="no" heading="H3" charCount="2953" wordCount="520" paraCount="15" cursorPos="2049" />
<meta expanded="no" heading="H3" charCount="2953" wordCount="520" paraCount="15" cursorPos="2155" />
<name status="s90e6c9" import="ia857f0" active="yes">Making a Scene</name>
</item>
<item handle="bc0cbd2a407f3" parent="6a2d6d5f4f401" root="7031beac91f75" order="1" type="FILE" class="NOVEL" layout="DOCUMENT">
<meta expanded="no" heading="H3" charCount="548" wordCount="108" paraCount="3" cursorPos="531" />
<meta expanded="no" heading="H3" charCount="563" wordCount="108" paraCount="3" cursorPos="691" />
<name status="s90e6c9" import="ia857f0" active="yes">Another Scene</name>
</item>
<item handle="ba8a28a246524" parent="7031beac91f75" root="7031beac91f75" order="4" type="FILE" class="NOVEL" layout="DOCUMENT">