Move the item index into a wrapper class and combine the access functions

novelwriter/core/index.py

@@ -4,8 +4,9 @@ novelWriter – Project Index
 Data class for the project index of tags, headers and references
 
 File History:
-Created: 2019-04-22 [0.0.1] countWords
+Created: 2019-04-22 [0.0.1]  countWords
-Created: 2019-05-27 [0.1.4] NWIndex
+Created: 2019-05-27 [0.1.4]  NWIndex
+Created: 2022-05-28 [1.7rc1] IndexItem, IndexHeading
 
 This file is a part of novelWriter
 Copyright 2018–2022, Veronica Berglyd Olsen
@@ -56,7 +57,7 @@ class NWIndex():
 
         # Indices
         self._tags = {}
-        self._items = {}
+        self._itemIndex = ItemIndex(theProject)
 
         # TimeStamps
         self._timeNovel = 0
@@ -65,6 +66,10 @@ class NWIndex():
 
         return
 
+    ##
+    #  Properties
+    ##
+
     @property
     def indexBroken(self):
         return self._indexBroken
@@ -77,7 +82,7 @@ class NWIndex():
         """Clear the index dictionaries and time stamps.
         """
         self._tags = {}
-        self._items = {}
+        self._itemIndex.clear()
         self._timeNovel = 0
         self._timeNotes = 0
         self._timeIndex = 0
@@ -86,13 +91,11 @@ class NWIndex():
     def deleteHandle(self, tHandle):
         """Delete all entries of a given document handle.
         """
-        if tHandle not in self._items:
-            return
-
         logger.debug("Removing item '%s' from the index", tHandle)
-        for tTag in self._items[tHandle].allTags():
+        for tTag in self._itemIndex.allItemTags(tHandle):
             self._tags.pop(tTag, None)
-        self._items.pop(tHandle, None)
+
+        del self._itemIndex[tHandle]
 
         return
 
@@ -150,7 +153,7 @@ class NWIndex():
 
             try:
                 self._validateTagsIndex(theData["tagsIndex"])
-                self._validateItemIndex(theData["itemIndex"])
+                self._itemIndex.unpackData(theData["itemIndex"])
             except Exception:
                 logger.error("The index content is invalid")
                 logException()
@@ -161,7 +164,7 @@ class NWIndex():
 
         # Check that all files are indexed
         for fHandle in self.theProject.projFiles:
-            if fHandle not in self._items:
+            if fHandle not in self._itemIndex:
                 logger.warning("Item '%s' is not in the index", fHandle)
                 self.reIndexHandle(fHandle)
 
@@ -183,11 +186,11 @@ class NWIndex():
         tStart = time()
 
         try:
-            itemsIndex = {handle: item.packData() for handle, item in self._items.items()}
+            itemIndex = self._itemIndex.packData()
             with open(indexFile, mode="w+", encoding="utf-8") as outFile:
                 outFile.write("{\n")
                 outFile.write(f'  "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n')
-                outFile.write(f'  "itemIndex": {jsonEncode(itemsIndex, n=1, nmax=4)}\n')
+                outFile.write(f'  "itemIndex": {jsonEncode(itemIndex, n=1, nmax=4)}\n')
                 outFile.write("}\n")
 
         except Exception:
@@ -220,7 +223,7 @@ class NWIndex():
 
         # Delete the old entry and create a new
         self.deleteHandle(tHandle)
-        self._items[tHandle] = IndexItem(tHandle, theItem)
+        self._itemIndex.add(tHandle, theItem)
 
         # Run word counter for the whole text
         cC, wC, pC = countWords(theText)
@@ -240,9 +243,6 @@ class NWIndex():
             logger.debug("Not indexing inactive item '%s'", tHandle)
             return False
 
-        itemClass  = theItem.itemClass
-        itemLayout = theItem.itemLayout
-
         logger.debug("Indexing item with handle '%s'", tHandle)
 
         # Scan the text content
@@ -261,7 +261,7 @@ class NWIndex():
                     nTitle = nLine
 
             elif aLine.startswith("@"):
-                self._indexKeyword(tHandle, aLine, nTitle, itemClass)
+                self._indexKeyword(tHandle, aLine, nTitle, theItem.itemClass)
 
             elif aLine.startswith("%"):
                 if nTitle > 0:
@@ -285,7 +285,7 @@ class NWIndex():
         # Update timestamps for index changes
         nowTime = round(time())
         self._timeIndex = nowTime
-        if itemLayout == nwItemLayout.NOTE:
+        if theItem.itemLayout == nwItemLayout.NOTE:
             self._timeNotes = nowTime
         else:
             self._timeNovel = nowTime
@@ -296,7 +296,7 @@ class NWIndex():
     #  Internal Indexers
     ##
 
-    def _indexTitle(self, tHandle, aLine, nLine):
+    def _indexTitle(self, tHandle, aLine, nTitle):
         """Save information about the title and its location in the
         file to the index.
         """
@@ -321,28 +321,24 @@ class NWIndex():
         else:
             return False
 
-        sTitle = f"T{nLine:06d}"
+        sTitle = f"T{nTitle:06d}"
-        tItem = self._items[tHandle]
+        self._itemIndex.addItemHeading(tHandle, sTitle, hDepth, hText)
-        tItem.updateLevel(hDepth)
-        tItem.addHeading(IndexHeading(sTitle, hDepth, hText))
 
         return True
 
     def _indexWordCounts(self, tHandle, theText, nTitle):
         """Count text stats and save the counts to the index.
         """
-        cC, wC, pC = countWords(theText)
         sTitle = f"T{nTitle:06d}"
-        if tHandle in self._items:
+        cC, wC, pC = countWords(theText)
-            self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC)
+        self._itemIndex.setHeadingCounts(tHandle, sTitle,  cC, wC, pC)
         return
 
     def _indexSynopsis(self, tHandle, theText, nTitle):
         """Save the synopsis to the index.
         """
         sTitle = f"T{nTitle:06d}"
-        if tHandle in self._items:
+        self._itemIndex.setHeadingSynopsis(tHandle, sTitle, theText)
-            self._items[tHandle].setHeadingSynopsis(sTitle, theText)
         return
 
     def _indexKeyword(self, tHandle, aLine, nTitle, itemClass):
@@ -365,9 +361,9 @@ class NWIndex():
                 "heading": sTitle,
                 "class": itemClass.name,
             }
-            self._items[tHandle].setHeadingTag(sTitle, theBits[1])
+            self._itemIndex.setHeadingTag(tHandle, sTitle, theBits[1])
         else:
-            self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0])
+            self._itemIndex.addHeadingReferences(tHandle, sTitle, theBits[1:], theBits[0])
 
         return
 
@@ -447,35 +443,31 @@ class NWIndex():
     #  Extract Data
     ##
 
-    def novelStructure(self, skipExcluded=True):
+    def novelStructure(self, skipExcl=True):
         """Iterate over all titles in the novel, in the correct order as
         they appear in the tree view and in the respective document
         files, but skipping all note files.
         """
-        for tHandle in self._listNovelHandles(skipExcluded):
+        for tHandle, sTitle, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl):
-            for sTitle in self._items[tHandle].headings:
+            tKey = f"{tHandle}:{sTitle}"
-                tKey = f"{tHandle}:{sTitle}"
+            yield tKey, tHandle, sTitle, hItem
-                yield tKey, tHandle, sTitle, self._items[tHandle][sTitle]
+        return
 
-    def getNovelWordCount(self, skipExcluded=True):
+    def getNovelWordCount(self, skipExcl=True):
         """Count the number of words in the novel project.
         """
         wCount = 0
-        for tHandle in self._listNovelHandles(skipExcluded):
+        for _, _, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl):
-            for hItem in self._items[tHandle].entries:
+            wCount += hItem.wordCount
-                wCount += hItem.wordCount
-
         return wCount
 
-    def getNovelTitleCounts(self, skipExcluded=True):
+    def getNovelTitleCounts(self, skipExcl=True):
         """Count the number of titles in the novel project.
         """
         hCount = [0, 0, 0, 0, 0]
-        for tHandle in self._listNovelHandles(skipExcluded):
+        for _, _, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl):
-            for hItem in self._items[tHandle].entries:
+            iLevel = H_LEVEL.get(hItem.level, 0)
-                iLevel = H_LEVEL.get(hItem.level, 0)
+            hCount[iLevel] += 1
-                hCount[iLevel] += 1
-
         return hCount
 
     def getHandleWordCounts(self, tHandle):
@@ -483,7 +475,7 @@ class NWIndex():
         """
         return [
             (f"{tHandle}:{sTitle}", hItem.wordCount)
-            for sTitle, hItem in self._items.get(tHandle, {}).items()
+            for sTitle, hItem in self._itemIndex.iterItemHeaders(tHandle)
         ]
 
     def getHandleHeaders(self, tHandle):
@@ -491,39 +483,34 @@ class NWIndex():
         """
         return [
             (sTitle, hItem.level, hItem.title)
-            for sTitle, hItem in self._items.get(tHandle, {}).items()
+            for sTitle, hItem in self._itemIndex.iterItemHeaders(tHandle)
         ]
 
     def getHandleHeaderLevel(self, tHandle):
         """Get the header level of the first header of a handle.
         """
-        if tHandle in self._items:
+        return self._itemIndex.mainItemHeader(tHandle)
-            return self._items[tHandle].level
-        else:
-            return "H0"
 
-    def getTableOfContents(self, maxDepth, skipExcluded=True):
+    def getTableOfContents(self, maxDepth, skipExcl=True):
         """Generate a table of contents up to a maximum depth.
         """
         tOrder = []
         tData = {}
         pKey = None
-        for tHandle in self._listNovelHandles(skipExcluded):
+        for tHandle, sTitle, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl):
-            for sTitle in self._items[tHandle].headings:
+            tKey = f"{tHandle}:{sTitle}"
-                tKey = f"{tHandle}:{sTitle}"
+            iLevel = H_LEVEL.get(hItem.level, 0)
-                hItem = self._items[tHandle][sTitle]
+            if iLevel > maxDepth:
-                iLevel = H_LEVEL.get(hItem.level, 0)
+                if pKey in tData:
-                if iLevel > maxDepth:
+                    tData[pKey]["words"] += hItem.wordCount
-                    if pKey in tData:
+            else:
-                        tData[pKey]["words"] += hItem.wordCount
+                pKey = tKey
-                else:
+                tOrder.append(tKey)
-                    pKey = tKey
+                tData[tKey] = {
-                    tOrder.append(tKey)
+                    "level": iLevel,
-                    tData[tKey] = {
+                    "title": hItem.title,
-                        "level": iLevel,
+                    "words": hItem.wordCount,
-                        "title": hItem.title,
+                }
-                        "words": hItem.wordCount,
-                    }
 
         theToC = [(
             tKey,
@@ -538,35 +525,26 @@ class NWIndex():
         """Return the counts for a file, or a section of a file,
         starting at title sTitle if it is provided.
         """
-        cC = 0
+        tItem = self._itemIndex[tHandle]
-        wC = 0
+        if tItem is None:
-        pC = 0
+            return 0, 0, 0
 
         if sTitle is None:
-            if tHandle in self._items:
+            cItem = tItem.item
-                tItem = self._items[tHandle].item
-                cC = tItem.charCount
-                wC = tItem.wordCount
-                pC = tItem.paraCount
         else:
-            if tHandle in self._items:
+            cItem = tItem[sTitle]
-                if sTitle in self._items[tHandle]:
-                    hItem = self._items[tHandle][sTitle]
-                    cC = hItem.charCount
-                    wC = hItem.wordCount
-                    pC = hItem.paraCount
 
-        return cC, wC, pC
+        if cItem is not None:
+            return cItem.charCount, cItem.wordCount, cItem.paraCount
+
+        return 0, 0, 0
 
     def getReferences(self, tHandle, sTitle=None):
         """Extract all references made in a file, and optionally title
         section.
         """
         theRefs = {x: [] for x in nwKeyWords.KEY_CLASS}
-        if tHandle not in self._items:
+        for rTitle, hItem in self._itemIndex.iterItemHeaders(tHandle):
-            return theRefs
-
-        for rTitle, hItem in self._items[tHandle].items():
             if sTitle is None or sTitle == rTitle:
                 for aTag, refTypes in hItem.references.items():
                     for refType in refTypes:
@@ -578,28 +556,26 @@ class NWIndex():
     def getNovelData(self, tHandle, sTitle):
         """Return the novel data of a given handle and title.
         """
-        if tHandle in self._items:
+        if tHandle in self._itemIndex:
-            if sTitle in self._items[tHandle]:
+            return self._itemIndex[tHandle][sTitle]
-                return self._items[tHandle][sTitle]
         return None
 
     def getBackReferenceList(self, tHandle):
         """Build a list of files referring back to our file, specified
         by tHandle.
         """
-        if tHandle is None or tHandle not in self._items:
+        if tHandle is None or tHandle not in self._itemIndex:
             return {}
 
         theRefs = {}
-        theTags = self._items[tHandle].allTags()
+        theTags = self._itemIndex.allItemTags(tHandle)
         if not theTags:
             return theRefs
 
-        for aHandle, tItem in self._items.items():
+        for aHandle, sTitle, hItem in self._itemIndex.iterAllHeaders():
-            for sTitle, hItem in tItem.items():
+            for aTag in hItem.references:
-                for aTag in hItem.references:
+                if aTag in theTags and aHandle not in theRefs:
-                    if aTag in theTags and aHandle not in theRefs:
+                    theRefs[aHandle] = sTitle
-                        theRefs[aHandle] = sTitle
 
         return theRefs
 
@@ -613,22 +589,6 @@ class NWIndex():
     #  Internal Functions
     ##
 
-    def _listNovelHandles(self, skipExcluded):
-        """Return a list of all handles that exist in the novel index.
-        """
-        theHandles = []
-        for tItem in self.theProject.tree:
-            if tItem is None:
-                continue
-            if not tItem.isExported and skipExcluded:
-                continue
-            if tItem.itemLayout == nwItemLayout.NOTE:
-                continue
-            if tItem.itemHandle in self._items:
-                theHandles.append(tItem.itemHandle)
-
-        return theHandles
-
     def _validateTagsIndex(self, tagsIndex):
         """Iterate through the tagsIndex loaded from cache and check
         that it's valid.
@@ -657,15 +617,172 @@ class NWIndex():
 
         return
 
-    def _validateItemIndex(self, itemIndex):
+# END Class NWIndex
-        """Iterate through the itemIndex loaded from cache and check
+
-        that it's valid.
+
+# =============================================================================================== #
+#  Indexer Objects
+# =============================================================================================== #
+
+class ItemIndex:
+    """A wrapper object holding the indexed items.
+    """
+
+    def __init__(self, theProject):
+        self.theProject = theProject
+        self._items = {}
+        return
+
+    ##
+    #  Methods
+    ##
+
+    def clear(self):
+        """Clear the index.
         """
         self._items = {}
-        if not isinstance(itemIndex, dict):
+        return
+
+    def __contains__(self, tHandle):
+        """Check if an item exists in the index,
+        """
+        return tHandle in self._items
+
+    def __delitem__(self, tHandle):
+        """Delete an entry in the index.
+        """
+        self._items.pop(tHandle, None)
+        return
+
+    def __getitem__(self, tHandle):
+        """Return an item, or return None if it isn't found.
+        """
+        return self._items.get(tHandle, None)
+
+    def add(self, tHandle, tItem):
+        """Add a new item to the index. This will overwrite the item if
+        it already exists.
+        """
+        self._items[tHandle] = IndexItem(tHandle, tItem)
+        return
+
+    def mainItemHeader(self, tHandle):
+        """Return the primary item header for an item.
+        """
+        if tHandle in self._items:
+            return self._items[tHandle].level
+        return "H0"
+
+    def allItemTags(self, tHandle):
+        """Get all tags set for headings of an item.
+        """
+        if tHandle in self._items:
+            return self._items[tHandle].allTags()
+        return []
+
+    def iterItemHeaders(self, tHandle):
+        """Iterate over all item headers of an item.
+        """
+        if tHandle in self._items:
+            for sTitle, hItem in self._items[tHandle].items():
+                yield sTitle, hItem
+        return
+
+    def iterAllHeaders(self):
+        """Iterate through all items and headings in the index.
+        """
+        for tHandle, tItem in self._items.items():
+            for sTitle, hItem in tItem.items():
+                yield tHandle, sTitle, hItem
+        return
+
+    def iterNovelStructure(self, rootHandle=None, skipExcl=False):
+        """Iterate over all items and headers in the novel structure for
+        a given root handle, or for all if root handle is None.
+        """
+        for tItem in self.theProject.tree:
+            if tItem is None:
+                continue
+            if tItem.itemLayout == nwItemLayout.NOTE:
+                continue
+            if skipExcl and not tItem.isExported:
+                continue
+
+            tHandle = tItem.itemHandle
+            if tHandle not in self._items:
+                continue
+
+            if rootHandle is None:
+                for sTitle, hItem in self._items[tHandle].items():
+                    yield tHandle, sTitle, hItem
+            elif tItem.rootHandle == rootHandle:
+                for sTitle, hItem in self._items[tHandle].items():
+                    yield tHandle, sTitle, hItem
+            else:
+                continue
+
+        return
+
+    ##
+    #  Setters
+    ##
+
+    def addItemHeading(self, tHandle, sTitle, hDepth, hText):
+        """Set the main heading level of an item.
+        """
+        if tHandle in self._items:
+            tItem = self._items[tHandle]
+            tItem.updateLevel(hDepth)
+            tItem.addHeading(IndexHeading(sTitle, hDepth, hText))
+        return
+
+    def setHeadingCounts(self, tHandle, sTitle, cC, wC, pC):
+        """Set the character, word and paragraph counts of a heading
+        on a given item.
+        """
+        if tHandle in self._items:
+            self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC)
+        return
+
+    def setHeadingSynopsis(self, tHandle, sTitle, sText):
+        """Set the synopsis text for a heading on a given item.
+        """
+        if tHandle in self._items:
+            self._items[tHandle].setHeadingSynopsis(sTitle, sText)
+        return
+
+    def setHeadingTag(self, tHandle, sTitle, tagKey):
+        """Set the main tag for a heading on a given item.
+        """
+        if tHandle in self._items:
+            self._items[tHandle].setHeadingTag(sTitle, tagKey)
+        return
+
+    def addHeadingReferences(self, tHandle, sTitle, tagKeys, refType):
+        """Set the reference tags for a heading on a given item.
+        """
+        if tHandle in self._items:
+            self._items[tHandle].addHeadingReferences(sTitle, tagKeys, refType)
+        return
+
+    ##
+    #  Pack/Unpack
+    ##
+
+    def packData(self):
+        """Pack all the data of the index into a single dictionary.
+        """
+        return {handle: item.packData() for handle, item in self._items.items()}
+
+    def unpackData(self, data):
+        """Iterate through the itemIndex loaded from cache and check
+        that it's valid. This will raise errors if there is a problem.
+        """
+        self._items = {}
+        if not isinstance(data, dict):
             raise ValueError("itemIndex is not a dict")
 
-        for tHandle, tData in itemIndex.items():
+        for tHandle, tData in data.items():
             if not isHandle(tHandle):
                 raise ValueError("itemIndex keys must be handles")
 
@@ -677,100 +794,14 @@ class NWIndex():
 
         return
 
-# END Class NWIndex
+# END Class ItemIndex
 
 
-# =============================================================================================== #
-#  Simple Word Counter
-# =============================================================================================== #
-
-def countWords(theText):
-    """Count words in a piece of text, skipping special syntax and
-    comments.
-    """
-    charCount = 0
-    wordCount = 0
-    paraCount = 0
-    prevEmpty = True
-
-    if not isinstance(theText, str):
-        return charCount, wordCount, paraCount
-
-    # We need to treat dashes as word separators for counting words.
-    # The check+replace approach is much faster than direct replace for
-    # large texts, and a bit slower for small texts, but in the latter
-    # case it doesn't really matter.
-    if nwUnicode.U_ENDASH in theText:
-        theText = theText.replace(nwUnicode.U_ENDASH, " ")
-    if nwUnicode.U_EMDASH in theText:
-        theText = theText.replace(nwUnicode.U_EMDASH, " ")
-
-    for aLine in theText.splitlines():
-
-        countPara = True
-
-        if not aLine:
-            prevEmpty = True
-            continue
-        if aLine[0] == "@" or aLine[0] == "%":
-            continue
-
-        if aLine[0] == "[":
-            if aLine.startswith(("[NEWPAGE]", "[NEW PAGE]", "[VSPACE]")):
-                continue
-            elif aLine.startswith("[VSPACE:") and aLine.endswith("]"):
-                continue
-
-        elif aLine[0] == "#":
-            if aLine[:5] == "#### ":
-                aLine = aLine[5:]
-                countPara = False
-            elif aLine[:4] == "### ":
-                aLine = aLine[4:]
-                countPara = False
-            elif aLine[:3] == "## ":
-                aLine = aLine[3:]
-                countPara = False
-            elif aLine[:2] == "# ":
-                aLine = aLine[2:]
-                countPara = False
-            elif aLine[:3] == "#! ":
-                aLine = aLine[3:]
-                countPara = False
-            elif aLine[:4] == "##! ":
-                aLine = aLine[4:]
-                countPara = False
-
-        elif aLine[0] == ">" or aLine[-1] == "<":
-            if aLine[:2] == ">>":
-                aLine = aLine[2:].lstrip(" ")
-            elif aLine[:1] == ">":
-                aLine = aLine[1:].lstrip(" ")
-            if aLine[-2:] == "<<":
-                aLine = aLine[:-2].rstrip(" ")
-            elif aLine[-1:] == "<":
-                aLine = aLine[:-1].rstrip(" ")
-
-        wordCount += len(aLine.split())
-        charCount += len(aLine)
-        if countPara and prevEmpty:
-            paraCount += 1
-
-        prevEmpty = not countPara
-
-    return charCount, wordCount, paraCount
-
-
-# =============================================================================================== #
-#  Indexer Objects
-# =============================================================================================== #
-
 class IndexItem:
 
     def __init__(self, tHandle, tItem):
         self._handle = tHandle
         self._item = tItem
-
         self._level = "H0"
         self._headings = {}
         self._index = 0
@@ -780,6 +811,9 @@ class IndexItem:
 
         return
 
+    def __repr__(self):
+        return f"<IndexItem handle={self._handle}>"
+
     ##
     # Properties
     ##
@@ -792,47 +826,50 @@ class IndexItem:
     def level(self):
         return self._level
 
-    @property
-    def headings(self):
-        return sorted(self._headings.keys())
-
-    @property
-    def entries(self):
-        return self._headings.values()
-
     ##
     #  Setters
     ##
 
     def updateLevel(self, level):
-        """Set the level only if it is H0.
+        """Set the level only if it has not already been set.
         """
         if self._level == "H0":
             self._level = level
         return
 
     def addHeading(self, tHeading):
+        """Add a heading to the item. Also remove the placeholder entry
+        if it exists.
+        """
         if H_NONE in self._headings:
             self._headings.pop(H_NONE)
         self._headings[tHeading.key] = tHeading
         return
 
     def setHeadingCounts(self, sTitle, charCount, wordCount, paraCount):
+        """Set the character, word and paragraph count of a heading.
+        """
         if sTitle in self._headings:
             self._headings[sTitle].setCounts(charCount, wordCount, paraCount)
         return
 
     def setHeadingSynopsis(self, sTitle, synopText):
+        """Set the synopsis text of a heading.
+        """
         if sTitle in self._headings:
             self._headings[sTitle].setSynopsis(synopText)
         return
 
     def setHeadingTag(self, sTitle, tagKey):
+        """Set the tag of a heading.
+        """
         if sTitle in self._headings:
             self._headings[sTitle].setTag(tagKey)
         return
 
     def addHeadingReferences(self, sTitle, tagKeys, refType):
+        """Add a reference key and all its types to a heading.
+        """
         if sTitle in self._headings:
             for tagKey in tagKeys:
                 self._headings[sTitle].addReference(tagKey, refType)
@@ -917,6 +954,9 @@ class IndexHeading:
 
         return
 
+    def __repr__(self):
+        return f"<IndexHeading key={self._key}>"
+
     ##
     #  Properties
     ##
@@ -962,21 +1002,30 @@ class IndexHeading:
     ##
 
     def setLevel(self, level):
+        """Set the level of the header if it's a valid value.
+        """
         if level in H_VALID:
             self._level = level
         return
 
     def setCounts(self, charCount, wordCount, paraCount):
+        """Set the character, word and paragraph count. Make sure the
+        value is an integer and is not smaller than 0.
+        """
         self._charCount = max(0, checkInt(charCount, 0))
         self._wordCount = max(0, checkInt(wordCount, 0))
         self._paraCount = max(0, checkInt(paraCount, 0))
         return
 
     def setSynopsis(self, synopText):
+        """Set the synopsis text and make sure it is a string.
+        """
         self._synopsis = str(synopText)
         return
 
     def setTag(self, tagKey):
+        """Set the tag for references, and make sure it is a string.
+        """
         self._tag = str(tagKey)
         return
 
@@ -1041,3 +1090,84 @@ class IndexHeading:
         return
 
 # END Class IndexHeading
+
+
+# =============================================================================================== #
+#  Simple Word Counter
+# =============================================================================================== #
+
+def countWords(theText):
+    """Count words in a piece of text, skipping special syntax and
+    comments.
+    """
+    charCount = 0
+    wordCount = 0
+    paraCount = 0
+    prevEmpty = True
+
+    if not isinstance(theText, str):
+        return charCount, wordCount, paraCount
+
+    # We need to treat dashes as word separators for counting words.
+    # The check+replace approach is much faster than direct replace for
+    # large texts, and a bit slower for small texts, but in the latter
+    # case it doesn't really matter.
+    if nwUnicode.U_ENDASH in theText:
+        theText = theText.replace(nwUnicode.U_ENDASH, " ")
+    if nwUnicode.U_EMDASH in theText:
+        theText = theText.replace(nwUnicode.U_EMDASH, " ")
+
+    for aLine in theText.splitlines():
+
+        countPara = True
+
+        if not aLine:
+            prevEmpty = True
+            continue
+        if aLine[0] == "@" or aLine[0] == "%":
+            continue
+
+        if aLine[0] == "[":
+            if aLine.startswith(("[NEWPAGE]", "[NEW PAGE]", "[VSPACE]")):
+                continue
+            elif aLine.startswith("[VSPACE:") and aLine.endswith("]"):
+                continue
+
+        elif aLine[0] == "#":
+            if aLine[:5] == "#### ":
+                aLine = aLine[5:]
+                countPara = False
+            elif aLine[:4] == "### ":
+                aLine = aLine[4:]
+                countPara = False
+            elif aLine[:3] == "## ":
+                aLine = aLine[3:]
+                countPara = False
+            elif aLine[:2] == "# ":
+                aLine = aLine[2:]
+                countPara = False
+            elif aLine[:3] == "#! ":
+                aLine = aLine[3:]
+                countPara = False
+            elif aLine[:4] == "##! ":
+                aLine = aLine[4:]
+                countPara = False
+
+        elif aLine[0] == ">" or aLine[-1] == "<":
+            if aLine[:2] == ">>":
+                aLine = aLine[2:].lstrip(" ")
+            elif aLine[:1] == ">":
+                aLine = aLine[1:].lstrip(" ")
+            if aLine[-2:] == "<<":
+                aLine = aLine[:-2].rstrip(" ")
+            elif aLine[-1:] == "<":
+                aLine = aLine[:-1].rstrip(" ")
+
+        wordCount += len(aLine.split())
+        charCount += len(aLine)
+        if countPara and prevEmpty:
+            paraCount += 1
+
+        prevEmpty = not countPara
+
+    return charCount, wordCount, paraCount

novelwriter/gui/noveltree.py

@@ -251,9 +251,7 @@ class GuiNovelTree(QTreeWidget):
         currChapter = None
         currScene = None
 
-        for tKey, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(
+        for tKey, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(skipExcl=True):
-            skipExcluded=True
-        ):
 
             tItem = self._createTreeItem(tHandle, sTitle, tKey, novIdx)
             self._treeMap[tKey] = tItem

novelwriter/gui/outline.py

@@ -389,7 +389,7 @@ class GuiOutline(QTreeWidget):
         currChapter = None
         currScene = None
 
-        for _, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(skipExcluded=True):
+        for _, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(skipExcl=True):
 
             tItem = self._createTreeItem(tHandle, sTitle, novIdx)
 

tests/test_core/test_core_index.py

@@ -69,19 +69,19 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir):
 
     # Take a copy of the index
     tagIndex = str(theIndex._tags)
-    itemsIndex = str({handle: item.packData() for handle, item in theIndex._items.items()})
+    itemsIndex = str(theIndex._itemIndex.packData())
 
     # Delete a handle
     assert theIndex._tags.get("Bod", None) is not None
-    assert theIndex._items.get("4c4f28287af27", None) is not None
+    assert theIndex._itemIndex["4c4f28287af27"] is not None
     theIndex.deleteHandle("4c4f28287af27")
     assert theIndex._tags.get("Bod", None) is None
-    assert theIndex._items.get("4c4f28287af27", None) is None
+    assert theIndex._itemIndex["4c4f28287af27"] is None
 
     # Clear the index
     theIndex.clearIndex()
     assert theIndex._tags == {}
-    assert theIndex._items == {}
+    assert theIndex._itemIndex._items == {}
 
     # Make the load fail
     with monkeypatch.context() as mp:
@@ -92,9 +92,7 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir):
     assert theIndex.loadIndex() is True
 
     assert str(theIndex._tags) == tagIndex
-    assert str(
+    assert str(theIndex._itemIndex.packData()) == itemsIndex
-        {handle: item.packData() for handle, item in theIndex._items.items()}
-    ) == itemsIndex
 
     # Break the index and check that we notice
     # assert theIndex.indexBroken is False
@@ -328,40 +326,40 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI):
         "##### Title Five\n\n"  # Not interpreted as a title, the hashes are counted as a word
         "Paragraph Five.\n\n"
     ))
-    assert theIndex._items[nHandle]["T000001"].references == {}
+    assert theIndex._itemIndex[nHandle]["T000001"].references == {}
-    assert theIndex._items[nHandle]["T000007"].references == {}
+    assert theIndex._itemIndex[nHandle]["T000007"].references == {}
-    assert theIndex._items[nHandle]["T000013"].references == {}
+    assert theIndex._itemIndex[nHandle]["T000013"].references == {}
-    assert theIndex._items[nHandle]["T000019"].references == {}
+    assert theIndex._itemIndex[nHandle]["T000019"].references == {}
 
-    assert theIndex._items[nHandle]["T000001"].level == "H1"
+    assert theIndex._itemIndex[nHandle]["T000001"].level == "H1"
-    assert theIndex._items[nHandle]["T000007"].level == "H2"
+    assert theIndex._itemIndex[nHandle]["T000007"].level == "H2"
-    assert theIndex._items[nHandle]["T000013"].level == "H3"
+    assert theIndex._itemIndex[nHandle]["T000013"].level == "H3"
-    assert theIndex._items[nHandle]["T000019"].level == "H4"
+    assert theIndex._itemIndex[nHandle]["T000019"].level == "H4"
 
-    assert theIndex._items[nHandle]["T000001"].title == "Title One"
+    assert theIndex._itemIndex[nHandle]["T000001"].title == "Title One"
-    assert theIndex._items[nHandle]["T000007"].title == "Title Two"
+    assert theIndex._itemIndex[nHandle]["T000007"].title == "Title Two"
-    assert theIndex._items[nHandle]["T000013"].title == "Title Three"
+    assert theIndex._itemIndex[nHandle]["T000013"].title == "Title Three"
-    assert theIndex._items[nHandle]["T000019"].title == "Title Four"
+    assert theIndex._itemIndex[nHandle]["T000019"].title == "Title Four"
 
-    assert theIndex._items[nHandle]["T000001"].charCount == 23
+    assert theIndex._itemIndex[nHandle]["T000001"].charCount == 23
-    assert theIndex._items[nHandle]["T000007"].charCount == 23
+    assert theIndex._itemIndex[nHandle]["T000007"].charCount == 23
-    assert theIndex._items[nHandle]["T000013"].charCount == 27
+    assert theIndex._itemIndex[nHandle]["T000013"].charCount == 27
-    assert theIndex._items[nHandle]["T000019"].charCount == 56
+    assert theIndex._itemIndex[nHandle]["T000019"].charCount == 56
 
-    assert theIndex._items[nHandle]["T000001"].wordCount == 4
+    assert theIndex._itemIndex[nHandle]["T000001"].wordCount == 4
-    assert theIndex._items[nHandle]["T000007"].wordCount == 4
+    assert theIndex._itemIndex[nHandle]["T000007"].wordCount == 4
-    assert theIndex._items[nHandle]["T000013"].wordCount == 4
+    assert theIndex._itemIndex[nHandle]["T000013"].wordCount == 4
-    assert theIndex._items[nHandle]["T000019"].wordCount == 9
+    assert theIndex._itemIndex[nHandle]["T000019"].wordCount == 9
 
-    assert theIndex._items[nHandle]["T000001"].paraCount == 1
+    assert theIndex._itemIndex[nHandle]["T000001"].paraCount == 1
-    assert theIndex._items[nHandle]["T000007"].paraCount == 1
+    assert theIndex._itemIndex[nHandle]["T000007"].paraCount == 1
-    assert theIndex._items[nHandle]["T000013"].paraCount == 1
+    assert theIndex._itemIndex[nHandle]["T000013"].paraCount == 1
-    assert theIndex._items[nHandle]["T000019"].paraCount == 3
+    assert theIndex._itemIndex[nHandle]["T000019"].paraCount == 3
 
-    assert theIndex._items[nHandle]["T000001"].synopsis == "Synopsis One."
+    assert theIndex._itemIndex[nHandle]["T000001"].synopsis == "Synopsis One."
-    assert theIndex._items[nHandle]["T000007"].synopsis == "Synopsis Two."
+    assert theIndex._itemIndex[nHandle]["T000007"].synopsis == "Synopsis Two."
-    assert theIndex._items[nHandle]["T000013"].synopsis == "Synopsis Three."
+    assert theIndex._itemIndex[nHandle]["T000013"].synopsis == "Synopsis Three."
-    assert theIndex._items[nHandle]["T000019"].synopsis == "Synopsis Four."
+    assert theIndex._itemIndex[nHandle]["T000019"].synopsis == "Synopsis Four."
 
     # Note File
     assert theIndex.scanText(cHandle, (
@@ -370,13 +368,13 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI):
         "% synopsis: Synopsis One.\n\n"
         "Paragraph One.\n\n"
     ))
-    assert theIndex._items[cHandle]["T000001"].references == {}
+    assert theIndex._itemIndex[cHandle]["T000001"].references == {}
-    assert theIndex._items[cHandle]["T000001"].level == "H1"
+    assert theIndex._itemIndex[cHandle]["T000001"].level == "H1"
-    assert theIndex._items[cHandle]["T000001"].title == "Title One"
+    assert theIndex._itemIndex[cHandle]["T000001"].title == "Title One"
-    assert theIndex._items[cHandle]["T000001"].charCount == 23
+    assert theIndex._itemIndex[cHandle]["T000001"].charCount == 23
-    assert theIndex._items[cHandle]["T000001"].wordCount == 4
+    assert theIndex._itemIndex[cHandle]["T000001"].wordCount == 4
-    assert theIndex._items[cHandle]["T000001"].paraCount == 1
+    assert theIndex._itemIndex[cHandle]["T000001"].paraCount == 1
-    assert theIndex._items[cHandle]["T000001"].synopsis == "Synopsis One."
+    assert theIndex._itemIndex[cHandle]["T000001"].synopsis == "Synopsis One."
 
     # Valid and Invalid References
     assert theIndex.scanText(sHandle, (
@@ -387,7 +385,7 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI):
         "% synopsis: Synopsis One.\n\n"
         "Paragraph One.\n\n"
     ))
-    assert theIndex._items[sHandle]["T000001"].references == {
+    assert theIndex._itemIndex[sHandle]["T000001"].references == {
         "One": {"@pov"}, "Two": {"@char"}
     }
 
@@ -398,25 +396,25 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI):
         "#! My Project\n\n"
         ">> By Jane Doe <<\n\n"
     ))
-    assert theIndex._items[cHandle]["T000001"].references == {}
+    assert theIndex._itemIndex[cHandle]["T000001"].references == {}
-    assert theIndex._items[tHandle]["T000001"].level == "H1"
+    assert theIndex._itemIndex[tHandle]["T000001"].level == "H1"
-    assert theIndex._items[tHandle]["T000001"].title == "My Project"
+    assert theIndex._itemIndex[tHandle]["T000001"].title == "My Project"
-    assert theIndex._items[tHandle]["T000001"].charCount == 21
+    assert theIndex._itemIndex[tHandle]["T000001"].charCount == 21
-    assert theIndex._items[tHandle]["T000001"].wordCount == 5
+    assert theIndex._itemIndex[tHandle]["T000001"].wordCount == 5
-    assert theIndex._items[tHandle]["T000001"].paraCount == 1
+    assert theIndex._itemIndex[tHandle]["T000001"].paraCount == 1
-    assert theIndex._items[tHandle]["T000001"].synopsis == ""
+    assert theIndex._itemIndex[tHandle]["T000001"].synopsis == ""
 
     assert theIndex.scanText(tHandle, (
         "##! Prologue\n\n"
         "In the beginning there was time ...\n\n"
     ))
-    assert theIndex._items[cHandle]["T000001"].references == {}
+    assert theIndex._itemIndex[cHandle]["T000001"].references == {}
-    assert theIndex._items[tHandle]["T000001"].level == "H2"
+    assert theIndex._itemIndex[tHandle]["T000001"].level == "H2"
-    assert theIndex._items[tHandle]["T000001"].title == "Prologue"
+    assert theIndex._itemIndex[tHandle]["T000001"].title == "Prologue"
-    assert theIndex._items[tHandle]["T000001"].charCount == 43
+    assert theIndex._itemIndex[tHandle]["T000001"].charCount == 43
-    assert theIndex._items[tHandle]["T000001"].wordCount == 8
+    assert theIndex._itemIndex[tHandle]["T000001"].wordCount == 8
-    assert theIndex._items[tHandle]["T000001"].paraCount == 1
+    assert theIndex._itemIndex[tHandle]["T000001"].paraCount == 1
-    assert theIndex._items[tHandle]["T000001"].synopsis == ""
+    assert theIndex._itemIndex[tHandle]["T000001"].synopsis == ""
 
     # Page wo/Title
     # =============
@@ -425,25 +423,25 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI):
     assert theIndex.scanText(pHandle, (
         "This is a page with some text on it.\n\n"
     ))
-    assert theIndex._items[pHandle]["T000000"].references == {}
+    assert theIndex._itemIndex[pHandle]["T000000"].references == {}
-    assert theIndex._items[pHandle]["T000000"].level == "H0"
+    assert theIndex._itemIndex[pHandle]["T000000"].level == "H0"
-    assert theIndex._items[pHandle]["T000000"].title == ""
+    assert theIndex._itemIndex[pHandle]["T000000"].title == ""
-    assert theIndex._items[pHandle]["T000000"].charCount == 36
+    assert theIndex._itemIndex[pHandle]["T000000"].charCount == 36
-    assert theIndex._items[pHandle]["T000000"].wordCount == 9
+    assert theIndex._itemIndex[pHandle]["T000000"].wordCount == 9
-    assert theIndex._items[pHandle]["T000000"].paraCount == 1
+    assert theIndex._itemIndex[pHandle]["T000000"].paraCount == 1
-    assert theIndex._items[pHandle]["T000000"].synopsis == ""
+    assert theIndex._itemIndex[pHandle]["T000000"].synopsis == ""
 
     theProject.tree[pHandle]._layout = nwItemLayout.NOTE
     assert theIndex.scanText(pHandle, (
         "This is a page with some text on it.\n\n"
     ))
-    assert theIndex._items[pHandle]["T000000"].references == {}
+    assert theIndex._itemIndex[pHandle]["T000000"].references == {}
-    assert theIndex._items[pHandle]["T000000"].level == "H0"
+    assert theIndex._itemIndex[pHandle]["T000000"].level == "H0"
-    assert theIndex._items[pHandle]["T000000"].title == ""
+    assert theIndex._itemIndex[pHandle]["T000000"].title == ""
-    assert theIndex._items[pHandle]["T000000"].charCount == 36
+    assert theIndex._itemIndex[pHandle]["T000000"].charCount == 36
-    assert theIndex._items[pHandle]["T000000"].wordCount == 9
+    assert theIndex._itemIndex[pHandle]["T000000"].wordCount == 9
-    assert theIndex._items[pHandle]["T000000"].paraCount == 1
+    assert theIndex._itemIndex[pHandle]["T000000"].paraCount == 1
-    assert theIndex._items[pHandle]["T000000"].synopsis == ""
+    assert theIndex._itemIndex[pHandle]["T000000"].synopsis == ""
 
     assert theProject.closeProject() is True
 
@@ -488,13 +486,13 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI):
     theProject.tree[nHandle].setExported(False)
 
     theKeys = []
-    for aKey, _, _, _ in theIndex.novelStructure(skipExcluded=False):
+    for aKey, _, _, _ in theIndex.novelStructure(skipExcl=False):
         theKeys.append(aKey)
 
     assert theKeys == ["%s:T000001" % nHandle]
 
     theKeys = []
-    for aKey, _, _, _ in theIndex.novelStructure(skipExcluded=True):
+    for aKey, _, _, _ in theIndex.novelStructure(skipExcl=True):
         theKeys.append(aKey)
 
     assert theKeys == []
@@ -625,12 +623,29 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI):
     assert theIndex.scanText(sHandle, "### Scene One\n\n")
     assert theIndex.scanText(tHandle, "### Scene Two\n\n")
 
-    assert theIndex._listNovelHandles(False) == [nHandle, hHandle, sHandle, tHandle]
+    assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=False)] == [
-    assert theIndex._listNovelHandles(True) == [hHandle, sHandle, tHandle]
+        (nHandle, "T000001"),
+        (nHandle, "T000011"),
+        (hHandle, "T000001"),
+        (sHandle, "T000001"),
+        (tHandle, "T000001"),
+    ]
+
+    assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=True)] == [
+        (hHandle, "T000001"),
+        (sHandle, "T000001"),
+        (tHandle, "T000001"),
+    ]
 
     # Add a fake handle to the tree and check that it's ignored
     theProject.tree._treeOrder.append("0000000000000")
-    assert theIndex._listNovelHandles(False) == [nHandle, hHandle, sHandle, tHandle]
+    assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=False)] == [
+        (nHandle, "T000001"),
+        (nHandle, "T000011"),
+        (hHandle, "T000001"),
+        (sHandle, "T000001"),
+        (tHandle, "T000001"),
+    ]
     theProject.tree._treeOrder.remove("0000000000000")
 
     # Extract stats

tests/test_gui/test_gui_docviewer.py

@@ -48,7 +48,7 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum):
     # Rebuild the index
     nwGUI.mainMenu.aRebuildIndex.activate(QAction.Trigger)
     assert nwGUI.theProject.index._tags != {}
-    assert nwGUI.theProject.index._items != {}
+    assert nwGUI.theProject.index._itemIndex._items != {}
 
     # Select a document in the project tree
     nwGUI.treeView.setSelectedHandle("88243afbe5ed8")