Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove old tagsIndex

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2022-05-28 20:16:27 +02:00
parent d584f57a3b
commit 01be8be44f
3 changed files with 102 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/index.py
+28 -65
View File
@@ -35,13 +35,14 @@ from novelwriter.error import logException
from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode
from novelwriter.core.document import NWDoc
from novelwriter.common import (
checkInt, isHandle, isTitleTag, isItemClass, isItemLayout, jsonEncode
checkInt, isHandle, isTitleTag, isItemLayout, jsonEncode
)
logger = logging.getLogger(__name__)
H_VALID = ("H0", "H1", "H2", "H3", "H4")
H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4}
H_NONE = "T000000"
class NWIndex():
@@ -54,7 +55,6 @@ class NWIndex():
self._indexBroken = False
# Indices
self._tagIndex = {}
self._refIndex = {}
self._fileIndex = {}
self._fileMeta = {}
@@ -80,7 +80,6 @@ class NWIndex():
def clearIndex(self):
"""Clear the index dictionaries and time stamps.
"""
self._tagIndex = {}
self._refIndex = {}
self._fileIndex = {}
self._fileMeta = {}
@@ -98,9 +97,9 @@ class NWIndex():
"""
logger.debug("Removing item '%s' from the index", tHandle)
delTags = list(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex))
delTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags))
for tTag in delTags:
self._tagIndex.pop(tTag, None)
self._tags.pop(tTag, None)
self._refIndex.pop(tHandle, None)
self._fileIndex.pop(tHandle, None)
@@ -161,7 +160,6 @@ class NWIndex():
self._indexBroken = True
return False
self._tagIndex = theData.get("tagIndex", {})
self._refIndex = theData.get("refIndex", {})
self._fileIndex = theData.get("fileIndex", {})
self._fileMeta = theData.get("fileMeta", {})
@@ -188,16 +186,14 @@ class NWIndex():
self._indexBroken = True
return False
for tHandle, tData in theData.items():
self._tags = theData.get("tagsIndex", {})
for tHandle, tData in theData.get("itemIndex", {}).items():
nwItem = self.theProject.tree[tHandle]
if nwItem is not None:
tItem = IndexItem(tHandle, nwItem)
tItem.unpackData(tData)
self._items[tHandle] = tItem
self._generateTagsIndex()
# print(json.dumps(self._tags, indent=2, default=str))
logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000)
self._checkIndex()
@@ -215,7 +211,6 @@ class NWIndex():
try:
with open(indexFile, mode="w+", encoding="utf-8") as outFile:
outFile.write("{\n")
outFile.write(f' "tagIndex": {jsonEncode(self._tagIndex, n=1, nmax=2)},\n')
outFile.write(f' "refIndex": {jsonEncode(self._refIndex, n=1, nmax=3)},\n')
outFile.write(f' "fileIndex": {jsonEncode(self._fileIndex, n=1, nmax=3)},\n')
outFile.write(f' "fileMeta": {jsonEncode(self._fileMeta, n=1, nmax=2)}\n')
@@ -233,7 +228,10 @@ class NWIndex():
itemsIndex = {handle: item.packData() for handle, item in self._items.items()}
with open(indexFile, mode="w+", encoding="utf-8") as outFile:
outFile.write(jsonEncode(itemsIndex, nmax=3))
outFile.write("{\n")
outFile.write(f' "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n')
outFile.write(f' "itemIndex": {jsonEncode(itemsIndex, n=1, nmax=4)}\n')
outFile.write("}\n")
logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000)
@@ -289,9 +287,9 @@ class NWIndex():
self._fileIndex[tHandle] = {}
# Also clear references to the file in the tags index
clearTags = list(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex))
clearTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags))
for aTag in clearTags:
self._tagIndex.pop(aTag)
self._tags.pop(aTag)
# Scan the text content
nTitle = 0
@@ -395,7 +393,7 @@ class NWIndex():
def _indexPage(self, tHandle, itemLayout):
"""Index a page with no title.
"""
self._fileIndex[tHandle]["T000000"] = {
self._fileIndex[tHandle][H_NONE] = {
"level": "H0",
"title": "",
"layout": itemLayout.name,
@@ -446,8 +444,11 @@ class NWIndex():
sTitle = f"T{nTitle:06d}"
if theBits[0] == nwKeyWords.TAG_KEY:
self._tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name, sTitle]
self._tags[theBits[1]] = [tHandle, itemClass.name, sTitle]
self._tags[theBits[1]] = {
"handle": tHandle,
"heading": sTitle,
"class": itemClass.name,
}
if tHandle in self._items:
self._items[tHandle].setHeadingTag(sTitle, theBits[1])
@@ -521,8 +522,8 @@ class NWIndex():
# For a tag, only the first value is accepted, the rest are ignored
if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1:
if theBits[1] in self._tagIndex:
isGood[1] = self._tagIndex[theBits[1]][1] == tItem.itemHandle
if theBits[1] in self._tags:
isGood[1] = self._tags[theBits[1]].get("handle") == tItem.itemHandle
else:
isGood[1] = True
return isGood
@@ -530,8 +531,8 @@ class NWIndex():
# If we're still here, we check that the references exist
theKey = nwKeyWords.KEY_CLASS[theBits[0]].name
for n in range(1, nBits):
if theBits[n] in self._tagIndex:
isGood[n] = theKey == self._tagIndex[theBits[n]][2]
if theBits[n] in self._tags:
isGood[n] = theKey == self._tags[theBits[n]].get("class")
return isGood
@@ -674,7 +675,7 @@ class NWIndex():
return {}
theRefs = {}
theTags = set(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex))
theTags = set(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags))
if theTags:
for tHandle in self._refIndex:
for sTitle in self._refIndex[tHandle]:
@@ -687,10 +688,8 @@ class NWIndex():
def getTagSource(self, theTag):
"""Return the source location of a given tag.
"""
theRef = self._tagIndex.get(theTag, [])
if len(theRef) == 4:
return theRef[1], theRef[0], theRef[3]
return None, 0, "T000000"
ref = self._tags.get(theTag, {})
return ref.get("handle"), ref.get("heading", H_NONE)
##
# Internal Functions
@@ -712,17 +711,6 @@ class NWIndex():
return theHandles
def _generateTagsIndex(self):
"""Generate the reverse tags index from the loaded index data.
The tags index must be updated during runtime with new changes.
"""
self._tags = {}
for tHandle, tItem in self._items.items():
for sTitle, tHead in tItem.items():
if tHead.tag:
self._tags[tHead.tag] = (tHandle, tItem.itemClass.name, sTitle)
return
##
# Index Checkers
##
@@ -737,7 +725,6 @@ class NWIndex():
tStart = time()
try:
self._checkTagIndex()
self._checkRefIndex()
self._checkFileIndex()
self._checkFileMeta()
@@ -763,28 +750,6 @@ class NWIndex():
return
def _checkTagIndex(self):
"""Scan the tag index for errors.
Warning: This function raises exceptions.
"""
for tTag in self._tagIndex:
if not isinstance(tTag, str):
raise KeyError("tagIndex key is not a string")
tEntry = self._tagIndex[tTag]
if len(tEntry) != 4:
raise IndexError("tagIndex[a] expected 4 values")
if not isinstance(tEntry[0], int):
raise ValueError("tagIndex[a][0] is not an integer")
if not isHandle(tEntry[1]):
raise ValueError("tagIndex[a][1] is not a handle")
if not isItemClass(tEntry[2]):
raise ValueError("tagIndex[a][2] is not an nwItemClass")
if not isTitleTag(tEntry[3]):
raise ValueError("tagIndex[a][3] is not a title tag")
return
def _checkRefIndex(self):
"""Scan the reference index for errors.
Warning: This function raises exceptions.
@@ -968,8 +933,6 @@ def countWords(theText):
class IndexItem:
DEF_HKEY = "T000000"
def __init__(self, tHandle, tItem):
self._handle = tHandle
self._item = tItem
@@ -979,7 +942,7 @@ class IndexItem:
self._index = 0
# Add a placeholder heading
self._headings[self.DEF_HKEY] = IndexHeading(self.DEF_HKEY)
self._headings[H_NONE] = IndexHeading(H_NONE)
return
@@ -1016,8 +979,8 @@ class IndexItem:
return
def addHeading(self, tHeading):
if "T000000" in self._headings:
self._headings.pop("T000000")
if H_NONE in self._headings:
self._headings.pop(H_NONE)
self._headings[tHeading.key] = tHeading
return
novelwriter/gui/docviewer.py
+1 -1
View File
@@ -245,7 +245,7 @@ class GuiDocViewer(QTextBrowser):
index being up to date.
"""
logger.debug("Loading document from tag '%s'", theTag)
tHandle, _, sTitle = self.theProject.index.getTagSource(theTag)
tHandle, sTitle = self.theProject.index.getTagSource(theTag)
if tHandle is None:
self.theParent.makeAlert(self.tr(
"Could not find the reference for tag '{0}'. It either doesn't "
tests/test_core/test_core_index.py
+73 -69
View File
@@ -68,25 +68,25 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir):
assert theIndex.saveIndex() is True
# Take a copy of the index
tagIndex = str(theIndex._tagIndex)
tagIndex = str(theIndex._tags)
refIndex = str(theIndex._refIndex)
fileIndex = str(theIndex._fileIndex)
textCounts = str(theIndex._fileMeta)
# Delete a handle
assert theIndex._tagIndex.get("Bod", None) is not None
assert theIndex._tags.get("Bod", None) is not None
assert theIndex._refIndex.get("4c4f28287af27", None) is not None
assert theIndex._fileIndex.get("4c4f28287af27", None) is not None
assert theIndex._fileMeta.get("4c4f28287af27", None) is not None
theIndex.deleteHandle("4c4f28287af27")
assert theIndex._tagIndex.get("Bod", None) is None
assert theIndex._tags.get("Bod", None) is None
assert theIndex._refIndex.get("4c4f28287af27", None) is None
assert theIndex._fileIndex.get("4c4f28287af27", None) is None
assert theIndex._fileMeta.get("4c4f28287af27", None) is None
# Clear the index
theIndex.clearIndex()
assert theIndex._tagIndex == {}
assert theIndex._tags == {}
assert theIndex._refIndex == {}
assert theIndex._fileIndex == {}
assert theIndex._fileMeta == {}
@@ -99,16 +99,16 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir):
# Make the load pass
assert theIndex.loadIndex() is True
assert str(theIndex._tagIndex) == tagIndex
assert str(theIndex._tags) == tagIndex
assert str(theIndex._refIndex) == refIndex
assert str(theIndex._fileIndex) == fileIndex
assert str(theIndex._fileMeta) == textCounts
# Break the index and check that we notice
assert theIndex.indexBroken is False
theIndex._tagIndex["Bod"].append("Stuff")
theIndex._checkIndex()
assert theIndex.indexBroken is True
# assert theIndex.indexBroken is False
# theIndex._tagIndex["Bod"].append("Stuff")
# theIndex._checkIndex()
# assert theIndex.indexBroken is True
# Finalise
assert theProject.closeProject() is True
@@ -198,7 +198,9 @@ def testCoreIndex_CheckThese(nwMinimal, mockGUI):
"@pov: Jane\n"
"@invalid: John\n" # Checks for issue #688
))
assert theIndex._tagIndex == {"Jane": [2, cHandle, "CHARACTER", "T000001"]}
assert theIndex._tags == {
"Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"}
}
assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!"
assert theIndex.getReferences(nHandle, "T000001") == {
"@char": [],
@@ -309,7 +311,9 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI):
"This is a story about Jane Smith.\n\n"
"Well, not really.\n"
))
assert theIndex._tagIndex == {"Jane": [2, cHandle, "CHARACTER", "T000001"]}
assert theIndex._tags == {
"Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"}
}
assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!"
# Title Indexing
@@ -551,8 +555,8 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI):
# getTagSource
# ============
assert theIndex.getTagSource("Jane") == (cHandle, 2, "T000001")
assert theIndex.getTagSource("John") == (None, 0, "T000000")
assert theIndex.getTagSource("Jane") == (cHandle, "T000001")
assert theIndex.getTagSource("John") == (None, "T000000")
# getCounts
# =========
@@ -696,69 +700,69 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI):
# END Test testCoreIndex_ExtractData
@pytest.mark.core
def testCoreIndex_CheckTagIndex(mockGUI):
"""Test the tag index checker.
"""
theProject = NWProject(mockGUI)
theIndex = NWIndex(theProject)
# @pytest.mark.core
# def testCoreIndex_CheckTagIndex(mockGUI):
# """Test the tag index checker.
# """
# theProject = NWProject(mockGUI)
# theIndex = NWIndex(theProject)
# Valid Index
theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001"],
}
assert theIndex._checkTagIndex() is None
# # Valid Index
# theIndex._tagIndex = {
# "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001"],
# }
# assert theIndex._checkTagIndex() is None
# Wrong Key Type
theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"],
123456: [3, "bb2c23b3c42cc", "CHARACTER", "T000001"],
}
with pytest.raises(KeyError):
theIndex._checkTagIndex()
# # Wrong Key Type
# theIndex._tagIndex = {
# "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
# 123456: [3, "bb2c23b3c42cc", "CHARACTER", "T000001"],
# }
# with pytest.raises(KeyError):
# theIndex._checkTagIndex()
# Wrong Length
theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001", "Stuff"],
}
with pytest.raises(IndexError):
theIndex._checkTagIndex()
# # Wrong Length
# theIndex._tagIndex = {
# "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001", "Stuff"],
# }
# with pytest.raises(IndexError):
# theIndex._checkTagIndex()
# Wrong Type of Entry 0
theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": ["3", "bb2c23b3c42cc", "CHARACTER", "T000001"],
}
with pytest.raises(ValueError):
theIndex._checkTagIndex()
# # Wrong Type of Entry 0
# theIndex._tagIndex = {
# "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
# "Jane": ["3", "bb2c23b3c42cc", "CHARACTER", "T000001"],
# }
# with pytest.raises(ValueError):
# theIndex._checkTagIndex()
# Wrong Type of Entry 1
theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, 0xbb2c23b3c42cc, "CHARACTER", "T000001"],
}
with pytest.raises(ValueError):
theIndex._checkTagIndex()
# # Wrong Type of Entry 1
# theIndex._tagIndex = {
# "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
# "Jane": [3, 0xbb2c23b3c42cc, "CHARACTER", "T000001"],
# }
# with pytest.raises(ValueError):
# theIndex._checkTagIndex()
# Wrong Type of Entry 2
theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "INVALID_CLASS", "T000001"],
}
with pytest.raises(ValueError):
theIndex._checkTagIndex()
# # Wrong Type of Entry 2
# theIndex._tagIndex = {
# "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
# "Jane": [3, "bb2c23b3c42cc", "INVALID_CLASS", "T000001"],
# }
# with pytest.raises(ValueError):
# theIndex._checkTagIndex()
# Wrong Type of Entry 3
theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "CHARACTER", "INVALID"],
}
with pytest.raises(ValueError):
theIndex._checkTagIndex()
# # Wrong Type of Entry 3
# theIndex._tagIndex = {
# "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "INVALID"],
# }
# with pytest.raises(ValueError):
# theIndex._checkTagIndex()
# END Test testCoreIndex_CheckTagIndex
# # END Test testCoreIndex_CheckTagIndex
@pytest.mark.core
@@ -1225,7 +1229,7 @@ def testCoreIndex_CheckFileMeta(mockGUI):
with pytest.raises(ValueError):
theIndex._checkFileMeta()
# END Test testCoreIndex_CheckTextCounts
# END Test testCoreIndex_CheckFileMeta
@pytest.mark.core