Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove old tagsIndex

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2022-05-28 20:16:27 +02:00
parent d584f57a3b
commit 01be8be44f
3 changed files with 102 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/index.py
+28 -65
View File
@@ -35,13 +35,14 @@ from novelwriter.error import logException
from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode
from novelwriter.core.document import NWDoc from novelwriter.core.document import NWDoc
from novelwriter.common import ( from novelwriter.common import (
checkInt, isHandle, isTitleTag, isItemClass, isItemLayout, jsonEncode checkInt, isHandle, isTitleTag, isItemLayout, jsonEncode
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
H_VALID = ("H0", "H1", "H2", "H3", "H4") H_VALID = ("H0", "H1", "H2", "H3", "H4")
H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4} H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4}
H_NONE = "T000000"
class NWIndex(): class NWIndex():
@@ -54,7 +55,6 @@ class NWIndex():
self._indexBroken = False self._indexBroken = False
# Indices # Indices
self._tagIndex = {}
self._refIndex = {} self._refIndex = {}
self._fileIndex = {} self._fileIndex = {}
self._fileMeta = {} self._fileMeta = {}
@@ -80,7 +80,6 @@ class NWIndex():
def clearIndex(self): def clearIndex(self):
"""Clear the index dictionaries and time stamps. """Clear the index dictionaries and time stamps.
""" """
self._tagIndex = {}
self._refIndex = {} self._refIndex = {}
self._fileIndex = {} self._fileIndex = {}
self._fileMeta = {} self._fileMeta = {}
@@ -98,9 +97,9 @@ class NWIndex():
""" """
logger.debug("Removing item '%s' from the index", tHandle) logger.debug("Removing item '%s' from the index", tHandle)
delTags = list(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex)) delTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags))
for tTag in delTags: for tTag in delTags:
self._tagIndex.pop(tTag, None) self._tags.pop(tTag, None)
self._refIndex.pop(tHandle, None) self._refIndex.pop(tHandle, None)
self._fileIndex.pop(tHandle, None) self._fileIndex.pop(tHandle, None)
@@ -161,7 +160,6 @@ class NWIndex():
self._indexBroken = True self._indexBroken = True
return False return False
self._tagIndex = theData.get("tagIndex", {})
self._refIndex = theData.get("refIndex", {}) self._refIndex = theData.get("refIndex", {})
self._fileIndex = theData.get("fileIndex", {}) self._fileIndex = theData.get("fileIndex", {})
self._fileMeta = theData.get("fileMeta", {}) self._fileMeta = theData.get("fileMeta", {})
@@ -188,16 +186,14 @@ class NWIndex():
self._indexBroken = True self._indexBroken = True
return False return False
for tHandle, tData in theData.items(): self._tags = theData.get("tagsIndex", {})
for tHandle, tData in theData.get("itemIndex", {}).items():
nwItem = self.theProject.tree[tHandle] nwItem = self.theProject.tree[tHandle]
if nwItem is not None: if nwItem is not None:
tItem = IndexItem(tHandle, nwItem) tItem = IndexItem(tHandle, nwItem)
tItem.unpackData(tData) tItem.unpackData(tData)
self._items[tHandle] = tItem self._items[tHandle] = tItem
self._generateTagsIndex()
# print(json.dumps(self._tags, indent=2, default=str))
logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000)
self._checkIndex() self._checkIndex()
@@ -215,7 +211,6 @@ class NWIndex():
try: try:
with open(indexFile, mode="w+", encoding="utf-8") as outFile: with open(indexFile, mode="w+", encoding="utf-8") as outFile:
outFile.write("{\n") outFile.write("{\n")
outFile.write(f' "tagIndex": {jsonEncode(self._tagIndex, n=1, nmax=2)},\n')
outFile.write(f' "refIndex": {jsonEncode(self._refIndex, n=1, nmax=3)},\n') outFile.write(f' "refIndex": {jsonEncode(self._refIndex, n=1, nmax=3)},\n')
outFile.write(f' "fileIndex": {jsonEncode(self._fileIndex, n=1, nmax=3)},\n') outFile.write(f' "fileIndex": {jsonEncode(self._fileIndex, n=1, nmax=3)},\n')
outFile.write(f' "fileMeta": {jsonEncode(self._fileMeta, n=1, nmax=2)}\n') outFile.write(f' "fileMeta": {jsonEncode(self._fileMeta, n=1, nmax=2)}\n')
@@ -233,7 +228,10 @@ class NWIndex():
itemsIndex = {handle: item.packData() for handle, item in self._items.items()} itemsIndex = {handle: item.packData() for handle, item in self._items.items()}
with open(indexFile, mode="w+", encoding="utf-8") as outFile: with open(indexFile, mode="w+", encoding="utf-8") as outFile:
outFile.write(jsonEncode(itemsIndex, nmax=3)) outFile.write("{\n")
outFile.write(f' "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n')
outFile.write(f' "itemIndex": {jsonEncode(itemsIndex, n=1, nmax=4)}\n')
outFile.write("}\n")
logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000) logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000)
@@ -289,9 +287,9 @@ class NWIndex():
self._fileIndex[tHandle] = {} self._fileIndex[tHandle] = {}
# Also clear references to the file in the tags index # Also clear references to the file in the tags index
clearTags = list(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex)) clearTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags))
for aTag in clearTags: for aTag in clearTags:
self._tagIndex.pop(aTag) self._tags.pop(aTag)
# Scan the text content # Scan the text content
nTitle = 0 nTitle = 0
@@ -395,7 +393,7 @@ class NWIndex():
def _indexPage(self, tHandle, itemLayout): def _indexPage(self, tHandle, itemLayout):
"""Index a page with no title. """Index a page with no title.
""" """
self._fileIndex[tHandle]["T000000"] = { self._fileIndex[tHandle][H_NONE] = {
"level": "H0", "level": "H0",
"title": "", "title": "",
"layout": itemLayout.name, "layout": itemLayout.name,
@@ -446,8 +444,11 @@ class NWIndex():
sTitle = f"T{nTitle:06d}" sTitle = f"T{nTitle:06d}"
if theBits[0] == nwKeyWords.TAG_KEY: if theBits[0] == nwKeyWords.TAG_KEY:
self._tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name, sTitle] self._tags[theBits[1]] = {
self._tags[theBits[1]] = [tHandle, itemClass.name, sTitle] "handle": tHandle,
"heading": sTitle,
"class": itemClass.name,
}
if tHandle in self._items: if tHandle in self._items:
self._items[tHandle].setHeadingTag(sTitle, theBits[1]) self._items[tHandle].setHeadingTag(sTitle, theBits[1])
@@ -521,8 +522,8 @@ class NWIndex():
# For a tag, only the first value is accepted, the rest are ignored # For a tag, only the first value is accepted, the rest are ignored
if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1: if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1:
if theBits[1] in self._tagIndex: if theBits[1] in self._tags:
isGood[1] = self._tagIndex[theBits[1]][1] == tItem.itemHandle isGood[1] = self._tags[theBits[1]].get("handle") == tItem.itemHandle
else: else:
isGood[1] = True isGood[1] = True
return isGood return isGood
@@ -530,8 +531,8 @@ class NWIndex():
# If we're still here, we check that the references exist # If we're still here, we check that the references exist
theKey = nwKeyWords.KEY_CLASS[theBits[0]].name theKey = nwKeyWords.KEY_CLASS[theBits[0]].name
for n in range(1, nBits): for n in range(1, nBits):
if theBits[n] in self._tagIndex: if theBits[n] in self._tags:
isGood[n] = theKey == self._tagIndex[theBits[n]][2] isGood[n] = theKey == self._tags[theBits[n]].get("class")
return isGood return isGood
@@ -674,7 +675,7 @@ class NWIndex():
return {} return {}
theRefs = {} theRefs = {}
theTags = set(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex)) theTags = set(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags))
if theTags: if theTags:
for tHandle in self._refIndex: for tHandle in self._refIndex:
for sTitle in self._refIndex[tHandle]: for sTitle in self._refIndex[tHandle]:
@@ -687,10 +688,8 @@ class NWIndex():
def getTagSource(self, theTag): def getTagSource(self, theTag):
"""Return the source location of a given tag. """Return the source location of a given tag.
""" """
theRef = self._tagIndex.get(theTag, []) ref = self._tags.get(theTag, {})
if len(theRef) == 4: return ref.get("handle"), ref.get("heading", H_NONE)
return theRef[1], theRef[0], theRef[3]
return None, 0, "T000000"
## ##
# Internal Functions # Internal Functions
@@ -712,17 +711,6 @@ class NWIndex():
return theHandles return theHandles
def _generateTagsIndex(self):
"""Generate the reverse tags index from the loaded index data.
The tags index must be updated during runtime with new changes.
"""
self._tags = {}
for tHandle, tItem in self._items.items():
for sTitle, tHead in tItem.items():
if tHead.tag:
self._tags[tHead.tag] = (tHandle, tItem.itemClass.name, sTitle)
return
## ##
# Index Checkers # Index Checkers
## ##
@@ -737,7 +725,6 @@ class NWIndex():
tStart = time() tStart = time()
try: try:
self._checkTagIndex()
self._checkRefIndex() self._checkRefIndex()
self._checkFileIndex() self._checkFileIndex()
self._checkFileMeta() self._checkFileMeta()
@@ -763,28 +750,6 @@ class NWIndex():
return return
def _checkTagIndex(self):
"""Scan the tag index for errors.
Warning: This function raises exceptions.
"""
for tTag in self._tagIndex:
if not isinstance(tTag, str):
raise KeyError("tagIndex key is not a string")
tEntry = self._tagIndex[tTag]
if len(tEntry) != 4:
raise IndexError("tagIndex[a] expected 4 values")
if not isinstance(tEntry[0], int):
raise ValueError("tagIndex[a][0] is not an integer")
if not isHandle(tEntry[1]):
raise ValueError("tagIndex[a][1] is not a handle")
if not isItemClass(tEntry[2]):
raise ValueError("tagIndex[a][2] is not an nwItemClass")
if not isTitleTag(tEntry[3]):
raise ValueError("tagIndex[a][3] is not a title tag")
return
def _checkRefIndex(self): def _checkRefIndex(self):
"""Scan the reference index for errors. """Scan the reference index for errors.
Warning: This function raises exceptions. Warning: This function raises exceptions.
@@ -968,8 +933,6 @@ def countWords(theText):
class IndexItem: class IndexItem:
DEF_HKEY = "T000000"
def __init__(self, tHandle, tItem): def __init__(self, tHandle, tItem):
self._handle = tHandle self._handle = tHandle
self._item = tItem self._item = tItem
@@ -979,7 +942,7 @@ class IndexItem:
self._index = 0 self._index = 0
# Add a placeholder heading # Add a placeholder heading
self._headings[self.DEF_HKEY] = IndexHeading(self.DEF_HKEY) self._headings[H_NONE] = IndexHeading(H_NONE)
return return
@@ -1016,8 +979,8 @@ class IndexItem:
return return
def addHeading(self, tHeading): def addHeading(self, tHeading):
if "T000000" in self._headings: if H_NONE in self._headings:
self._headings.pop("T000000") self._headings.pop(H_NONE)
self._headings[tHeading.key] = tHeading self._headings[tHeading.key] = tHeading
return return
novelwriter/gui/docviewer.py
+1 -1
View File
@@ -245,7 +245,7 @@ class GuiDocViewer(QTextBrowser):
index being up to date. index being up to date.
""" """
logger.debug("Loading document from tag '%s'", theTag) logger.debug("Loading document from tag '%s'", theTag)
tHandle, _, sTitle = self.theProject.index.getTagSource(theTag) tHandle, sTitle = self.theProject.index.getTagSource(theTag)
if tHandle is None: if tHandle is None:
self.theParent.makeAlert(self.tr( self.theParent.makeAlert(self.tr(
"Could not find the reference for tag '{0}'. It either doesn't " "Could not find the reference for tag '{0}'. It either doesn't "
tests/test_core/test_core_index.py
+73 -69
View File
@@ -68,25 +68,25 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir):
assert theIndex.saveIndex() is True assert theIndex.saveIndex() is True
# Take a copy of the index # Take a copy of the index
tagIndex = str(theIndex._tagIndex) tagIndex = str(theIndex._tags)
refIndex = str(theIndex._refIndex) refIndex = str(theIndex._refIndex)
fileIndex = str(theIndex._fileIndex) fileIndex = str(theIndex._fileIndex)
textCounts = str(theIndex._fileMeta) textCounts = str(theIndex._fileMeta)
# Delete a handle # Delete a handle
assert theIndex._tagIndex.get("Bod", None) is not None assert theIndex._tags.get("Bod", None) is not None
assert theIndex._refIndex.get("4c4f28287af27", None) is not None assert theIndex._refIndex.get("4c4f28287af27", None) is not None
assert theIndex._fileIndex.get("4c4f28287af27", None) is not None assert theIndex._fileIndex.get("4c4f28287af27", None) is not None
assert theIndex._fileMeta.get("4c4f28287af27", None) is not None assert theIndex._fileMeta.get("4c4f28287af27", None) is not None
theIndex.deleteHandle("4c4f28287af27") theIndex.deleteHandle("4c4f28287af27")
assert theIndex._tagIndex.get("Bod", None) is None assert theIndex._tags.get("Bod", None) is None
assert theIndex._refIndex.get("4c4f28287af27", None) is None assert theIndex._refIndex.get("4c4f28287af27", None) is None
assert theIndex._fileIndex.get("4c4f28287af27", None) is None assert theIndex._fileIndex.get("4c4f28287af27", None) is None
assert theIndex._fileMeta.get("4c4f28287af27", None) is None assert theIndex._fileMeta.get("4c4f28287af27", None) is None
# Clear the index # Clear the index
theIndex.clearIndex() theIndex.clearIndex()
assert theIndex._tagIndex == {} assert theIndex._tags == {}
assert theIndex._refIndex == {} assert theIndex._refIndex == {}
assert theIndex._fileIndex == {} assert theIndex._fileIndex == {}
assert theIndex._fileMeta == {} assert theIndex._fileMeta == {}
@@ -99,16 +99,16 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir):
# Make the load pass # Make the load pass
assert theIndex.loadIndex() is True assert theIndex.loadIndex() is True
assert str(theIndex._tagIndex) == tagIndex assert str(theIndex._tags) == tagIndex
assert str(theIndex._refIndex) == refIndex assert str(theIndex._refIndex) == refIndex
assert str(theIndex._fileIndex) == fileIndex assert str(theIndex._fileIndex) == fileIndex
assert str(theIndex._fileMeta) == textCounts assert str(theIndex._fileMeta) == textCounts
# Break the index and check that we notice # Break the index and check that we notice
assert theIndex.indexBroken is False # assert theIndex.indexBroken is False
theIndex._tagIndex["Bod"].append("Stuff") # theIndex._tagIndex["Bod"].append("Stuff")
theIndex._checkIndex() # theIndex._checkIndex()
assert theIndex.indexBroken is True # assert theIndex.indexBroken is True
# Finalise # Finalise
assert theProject.closeProject() is True assert theProject.closeProject() is True
@@ -198,7 +198,9 @@ def testCoreIndex_CheckThese(nwMinimal, mockGUI):
"@pov: Jane\n" "@pov: Jane\n"
"@invalid: John\n" # Checks for issue #688 "@invalid: John\n" # Checks for issue #688
)) ))
assert theIndex._tagIndex == {"Jane": [2, cHandle, "CHARACTER", "T000001"]} assert theIndex._tags == {
"Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"}
}
assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!" assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!"
assert theIndex.getReferences(nHandle, "T000001") == { assert theIndex.getReferences(nHandle, "T000001") == {
"@char": [], "@char": [],
@@ -309,7 +311,9 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI):
"This is a story about Jane Smith.\n\n" "This is a story about Jane Smith.\n\n"
"Well, not really.\n" "Well, not really.\n"
)) ))
assert theIndex._tagIndex == {"Jane": [2, cHandle, "CHARACTER", "T000001"]} assert theIndex._tags == {
"Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"}
}
assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!" assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!"
# Title Indexing # Title Indexing
@@ -551,8 +555,8 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI):
# getTagSource # getTagSource
# ============ # ============
assert theIndex.getTagSource("Jane") == (cHandle, 2, "T000001") assert theIndex.getTagSource("Jane") == (cHandle, "T000001")
assert theIndex.getTagSource("John") == (None, 0, "T000000") assert theIndex.getTagSource("John") == (None, "T000000")
# getCounts # getCounts
# ========= # =========
@@ -696,69 +700,69 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI):
# END Test testCoreIndex_ExtractData # END Test testCoreIndex_ExtractData
@pytest.mark.core # @pytest.mark.core
def testCoreIndex_CheckTagIndex(mockGUI): # def testCoreIndex_CheckTagIndex(mockGUI):
"""Test the tag index checker. # """Test the tag index checker.
""" # """
theProject = NWProject(mockGUI) # theProject = NWProject(mockGUI)
theIndex = NWIndex(theProject) # theIndex = NWIndex(theProject)
# Valid Index # # Valid Index
theIndex._tagIndex = { # theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"], # "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], # "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001"],
} # }
assert theIndex._checkTagIndex() is None # assert theIndex._checkTagIndex() is None
# Wrong Key Type # # Wrong Key Type
theIndex._tagIndex = { # theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"], # "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
123456: [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], # 123456: [3, "bb2c23b3c42cc", "CHARACTER", "T000001"],
} # }
with pytest.raises(KeyError): # with pytest.raises(KeyError):
theIndex._checkTagIndex() # theIndex._checkTagIndex()
# Wrong Length # # Wrong Length
theIndex._tagIndex = { # theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"], # "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001", "Stuff"], # "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001", "Stuff"],
} # }
with pytest.raises(IndexError): # with pytest.raises(IndexError):
theIndex._checkTagIndex() # theIndex._checkTagIndex()
# Wrong Type of Entry 0 # # Wrong Type of Entry 0
theIndex._tagIndex = { # theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"], # "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": ["3", "bb2c23b3c42cc", "CHARACTER", "T000001"], # "Jane": ["3", "bb2c23b3c42cc", "CHARACTER", "T000001"],
} # }
with pytest.raises(ValueError): # with pytest.raises(ValueError):
theIndex._checkTagIndex() # theIndex._checkTagIndex()
# Wrong Type of Entry 1 # # Wrong Type of Entry 1
theIndex._tagIndex = { # theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"], # "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, 0xbb2c23b3c42cc, "CHARACTER", "T000001"], # "Jane": [3, 0xbb2c23b3c42cc, "CHARACTER", "T000001"],
} # }
with pytest.raises(ValueError): # with pytest.raises(ValueError):
theIndex._checkTagIndex() # theIndex._checkTagIndex()
# Wrong Type of Entry 2 # # Wrong Type of Entry 2
theIndex._tagIndex = { # theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"], # "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "INVALID_CLASS", "T000001"], # "Jane": [3, "bb2c23b3c42cc", "INVALID_CLASS", "T000001"],
} # }
with pytest.raises(ValueError): # with pytest.raises(ValueError):
theIndex._checkTagIndex() # theIndex._checkTagIndex()
# Wrong Type of Entry 3 # # Wrong Type of Entry 3
theIndex._tagIndex = { # theIndex._tagIndex = {
"John": [3, "14298de4d9524", "CHARACTER", "T000001"], # "John": [3, "14298de4d9524", "CHARACTER", "T000001"],
"Jane": [3, "bb2c23b3c42cc", "CHARACTER", "INVALID"], # "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "INVALID"],
} # }
with pytest.raises(ValueError): # with pytest.raises(ValueError):
theIndex._checkTagIndex() # theIndex._checkTagIndex()
# END Test testCoreIndex_CheckTagIndex # # END Test testCoreIndex_CheckTagIndex
@pytest.mark.core @pytest.mark.core
@@ -1225,7 +1229,7 @@ def testCoreIndex_CheckFileMeta(mockGUI):
with pytest.raises(ValueError): with pytest.raises(ValueError):
theIndex._checkFileMeta() theIndex._checkFileMeta()
# END Test testCoreIndex_CheckTextCounts # END Test testCoreIndex_CheckFileMeta
@pytest.mark.core @pytest.mark.core