Clean up the index code a bit and add index validation

novelwriter/core/index.py

@@ -34,7 +34,9 @@ from novelwriter.enum import nwItemType, nwItemLayout
 from novelwriter.error import logException
 from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode
 from novelwriter.core.document import NWDoc
-from novelwriter.common import checkInt, jsonEncode
+from novelwriter.common import (
+    checkInt, isHandle, isItemClass, isTitleTag, jsonEncode
+)
 
 logger = logging.getLogger(__name__)
 
@@ -74,13 +76,11 @@ class NWIndex():
     def clearIndex(self):
         """Clear the index dictionaries and time stamps.
         """
+        self._tags = {}
+        self._items = {}
         self._timeNovel = 0
         self._timeNotes = 0
         self._timeIndex = 0
-
-        self._tags = {}
-        self._items = {}
-
         return
 
     def deleteHandle(self, tHandle):
@@ -90,10 +90,8 @@ class NWIndex():
             return
 
         logger.debug("Removing item '%s' from the index", tHandle)
-
         for tTag in self._items[tHandle].allTags():
             self._tags.pop(tTag, None)
-
         self._items.pop(tHandle, None)
 
         return
@@ -144,30 +142,36 @@ class NWIndex():
             try:
                 with open(indexFile, mode="r", encoding="utf-8") as inFile:
                     theData = json.load(inFile)
-
             except Exception:
                 logger.error("Failed to load index file")
                 logException()
                 self._indexBroken = True
                 return False
 
-            self._tags = theData.get("tagsIndex", {})
+            try:
-            for tHandle, tData in theData.get("itemIndex", {}).items():
+                self._validateTagsIndex(theData["tagsIndex"])
-                nwItem = self.theProject.tree[tHandle]
+                self._validateItemIndex(theData["itemIndex"])
-                if nwItem is not None:
+            except Exception:
-                    tItem = IndexItem(tHandle, nwItem)
+                logger.error("The index content is invalid")
-                    tItem.unpackData(tData)
+                logException()
-                    self._items[tHandle] = tItem
+                self._indexBroken = True
+                return False
 
-            nowTime = round(time())
+        logger.debug("Checking index")
-            self._timeNovel = nowTime
+
-            self._timeNotes = nowTime
+        # Check that all files are indexed
-            self._timeIndex = nowTime
+        for fHandle in self.theProject.projFiles:
+            if fHandle not in self._items:
+                logger.warning("Item '%s' is not in the index", fHandle)
+                self.reIndexHandle(fHandle)
+
+        nowTime = round(time())
+        self._timeNovel = nowTime
+        self._timeNotes = nowTime
+        self._timeIndex = nowTime
 
         logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000)
 
-        self._checkIndex()
-
         return True
 
     def saveIndex(self):
@@ -214,11 +218,11 @@ class NWIndex():
             logger.info("Not indexing non-file item '%s'", tHandle)
             return False
 
+        # Delete the old entry and create a new
         self.deleteHandle(tHandle)
-
-        # Run word counter for the whole text
         self._items[tHandle] = IndexItem(tHandle, theItem)
 
+        # Run word counter for the whole text
         cC, wC, pC = countWords(theText)
         theItem.setCharCount(cC)
         theItem.setWordCount(wC)
@@ -249,7 +253,7 @@ class NWIndex():
                 continue
 
             if aLine.startswith("#"):
-                isTitle = self._indexTitle(tHandle, aLine, nLine, itemLayout)
+                isTitle = self._indexTitle(tHandle, aLine, nLine)
                 if isTitle and nLine > 0:
                     if nTitle > 0:
                         lastText = "\n".join(theLines[nTitle-1:nLine-1])
@@ -257,7 +261,7 @@ class NWIndex():
                     nTitle = nLine
 
             elif aLine.startswith("@"):
-                self._indexKeyword(tHandle, aLine, nLine, nTitle, itemClass)
+                self._indexKeyword(tHandle, aLine, nTitle, itemClass)
 
             elif aLine.startswith("%"):
                 if nTitle > 0:
@@ -274,7 +278,7 @@ class NWIndex():
             lastText = "\n".join(theLines[nTitle-1:])
             self._indexWordCounts(tHandle, lastText, nTitle)
 
-        # Index page with no titles and references
+        # Also count words on a page with no titles
         if nTitle == 0:
             self._indexWordCounts(tHandle, theText, nTitle)
 
@@ -292,7 +296,7 @@ class NWIndex():
     #  Internal Indexers
     ##
 
-    def _indexTitle(self, tHandle, aLine, nLine, itemLayout):
+    def _indexTitle(self, tHandle, aLine, nLine):
         """Save information about the title and its location in the
         file to the index.
         """
@@ -341,7 +345,7 @@ class NWIndex():
             self._items[tHandle].setHeadingSynopsis(sTitle, theText)
         return
 
-    def _indexKeyword(self, tHandle, aLine, nLine, nTitle, itemClass):
+    def _indexKeyword(self, tHandle, aLine, nTitle, itemClass):
         """Validate and save the information about a reference to a tag
         in another file.
         """
@@ -361,11 +365,9 @@ class NWIndex():
                 "heading": sTitle,
                 "class": itemClass.name,
             }
-            if tHandle in self._items:
+            self._items[tHandle].setHeadingTag(sTitle, theBits[1])
-                self._items[tHandle].setHeadingTag(sTitle, theBits[1])
         else:
-            if tHandle in self._items:
+            self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0])
-                self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0])
 
         return
 
@@ -627,26 +629,51 @@ class NWIndex():
 
         return theHandles
 
-    ##
+    def _validateTagsIndex(self, tagsIndex):
-    #  Index Checkers
+        """Iterate through the tagsIndex loaded from cache and check
-    ##
+        that it's valid.
-
-    def _checkIndex(self):
-        """Check that the entries in the index are valid and contain the
-        elements it should. Also check that each file present in the
-        contents folder when the project was loaded are also present in
-        the fileMeta index.
         """
-        logger.debug("Checking index")
+        self._tags = {}
-        tStart = time()
+        if not isinstance(tagsIndex, dict):
+            raise ValueError("tagsIndex is not a dict")
 
-        # If the index was ok, we check that project files are indexed
+        for tagKey, tagData in tagsIndex.items():
-        for fHandle in self.theProject.projFiles:
+            if not isinstance(tagKey, str):
-            if fHandle not in self._items:
+                raise ValueError("tagsIndex keys must be a strings")
-                logger.warning("Item '%s' is not in the index", fHandle)
+            if "handle" not in tagData:
-                self.reIndexHandle(fHandle)
+                raise KeyError("A tagIndex item is missing a handle entry")
+            if "heading" not in tagData:
+                raise KeyError("A tagIndex item is missing a heading entry")
+            if "class" not in tagData:
+                raise KeyError("A tagIndex item is missing a class entry")
+            if not isHandle(tagData["handle"]):
+                raise ValueError("tagsIndex handle must be a handle")
+            if not isTitleTag(tagData["heading"]):
+                raise ValueError("tagsIndex heading must be a title tag")
+            if not isItemClass(tagData["class"]):
+                raise ValueError("tagsIndex handle must be an nwItemClass")
 
-        logger.verbose("Index check completed in %.3f ms", (time() - tStart)*1000)
+        self._tags = tagsIndex
+
+        return
+
+    def _validateItemIndex(self, itemIndex):
+        """Iterate through the itemIndex loaded from cache and check
+        that it's valid.
+        """
+        self._items = {}
+        if not isinstance(itemIndex, dict):
+            raise ValueError("itemIndex is not a dict")
+
+        for tHandle, tData in itemIndex.items():
+            if not isHandle(tHandle):
+                raise ValueError("itemIndex keys must be handles")
+
+            nwItem = self.theProject.tree[tHandle]
+            if nwItem is not None:
+                tItem = IndexItem(tHandle, nwItem)
+                tItem.unpackData(tData)
+                self._items[tHandle] = tItem
 
         return
 
@@ -734,6 +761,10 @@ def countWords(theText):
     return charCount, wordCount, paraCount
 
 
+# =============================================================================================== #
+#  Indexer Objects
+# =============================================================================================== #
+
 class IndexItem:
 
     def __init__(self, tHandle, tItem):
@@ -773,20 +804,11 @@ class IndexItem:
     #  Setters
     ##
 
-    def setLevel(self, level):
-        if level in H_VALID:
-            self._level = level
-        else:
-            self._level = "H0"
-        return
-
     def updateLevel(self, level):
         """Set the level only if it is H0.
         """
-        if level in H_VALID and self._level == "H0":
+        if self._level == "H0":
             self._level = level
-        else:
-            self._level = "H0"
         return
 
     def addHeading(self, tHeading):
@@ -867,6 +889,8 @@ class IndexItem:
         self._level = data.get("level", "H0")
         references = data.get("references", {})
         for sTitle, hData in data.get("headings", {}).items():
+            if not isTitleTag(sTitle):
+                raise ValueError("The itemIndex contains an invalid title key")
             tHeading = IndexHeading(sTitle)
             tHeading.unpackData(hData)
             tHeading.unpackReferences(references.get(sTitle, {}))
@@ -940,8 +964,6 @@ class IndexHeading:
     def setLevel(self, level):
         if level in H_VALID:
             self._level = level
-        else:
-            self._level = "H0"
         return
 
     def setCounts(self, charCount, wordCount, paraCount):
@@ -1007,7 +1029,15 @@ class IndexHeading:
         """Unpack a set of references from a dictionary.
         """
         for tagKey, refTypes in data.items():
-            self._refs[tagKey] = set(refTypes)
+            if not isinstance(tagKey, str):
+                raise ValueError("itemIndex reference key must be a string")
+            if not isinstance(refTypes, list):
+                raise ValueError("itemIndex reference types must be a list")
+            for refType in refTypes:
+                if refType in nwKeyWords.VALID_KEYS:
+                    self.addReference(tagKey, refType)
+                else:
+                    raise ValueError("The itemIndex contains an invalid reference type")
         return
 
 # END Class IndexHeading

novelwriter/core/project.py

@@ -123,15 +123,15 @@ class NWProject():
     ##
 
     @property
-    def index(self) -> NWIndex:
+    def index(self):
         return self._projIndex
 
     @property
-    def tree(self) -> NWTree:
+    def tree(self):
         return self._projTree
 
     @property
-    def options(self) -> OptionState:
+    def options(self):
         return self._optState
 
     ##

tests/reference/coreIndex_LoadSave_tagsIndex.json

@@ -39,7 +39,7 @@
       }
     },
     "88243afbe5ed8": {
-      "level": "H0",
+      "level": "H3",
       "headings": {
         "T000001": {"level": "H3", "title": "Scene One", "tag": "", "cCount": 1197, "wCount": 174, "pCount": 2, "synopsis": "Aenean ut placerat velit. Etiam laoreet ullamcorper risus, eget lobortis enim scelerisque non. Suspendisse id maximus nunc, et mollis sapien. Curabitur vel semper sapien, non pulvinar dolor. Etiam finibus nisi vel mi molestie consectetur."},
         "T000013": {"level": "H4", "title": "Scene One, Section Two", "tag": "", "cCount": 1561, "wCount": 230, "pCount": 2, "synopsis": ""}
@@ -49,7 +49,7 @@
       }
     },
     "f96ec11c6a3da": {
-      "level": "H0",
+      "level": "H3",
       "headings": {
         "T000001": {"level": "H3", "title": "Scene Two", "tag": "", "cCount": 2034, "wCount": 299, "pCount": 3, "synopsis": "Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Integer sapien nulla, dictum at lacus a, dignissim consectetur dolor. Nunc vel eleifend lacus, eu dapibus orci."},
         "T000015": {"level": "H4", "title": "Scene Two, Section Two", "tag": "", "cCount": 2009, "wCount": 301, "pCount": 3, "synopsis": ""}