Merge branch 'main' into release_2.0rc2

novelwriter/core/index.py

@@ -58,23 +58,23 @@ class NWIndex:
     The index data is cached in a JSON file between writing sessions.
     """
 
-    def __init__(self, theProject):
+    def __init__(self, project):
 
-        self.theProject = theProject
+        self._project = project
 
         # Storage and State
         self._tagsIndex = TagsIndex()
-        self._itemIndex = ItemIndex(theProject)
+        self._itemIndex = ItemIndex(project)
         self._indexBroken = False
 
         # TimeStamps
-        self._indexChange = 0
+        self._indexChange = 0.0
         self._rootChange = {}
 
         return
 
     def __repr__(self):
-        return f"<NWIndex project='{self.theProject.data.name}'>"
+        return f"<NWIndex project='{self._project.data.name}'>"
 
     ##
     #  Properties
@@ -93,10 +93,22 @@ class NWIndex:
         """
         self._tagsIndex.clear()
         self._itemIndex.clear()
-        self._indexChange = 0
+        self._indexChange = 0.0
         self._rootChange = {}
         return
 
+    def rebuildIndex(self):
+        """Rebuild the entire index from scratch.
+        """
+        self.clearIndex()
+        for nwItem in self._project.tree:
+            if nwItem is not None and nwItem.isFileType():
+                tHandle = nwItem.itemHandle
+                theDoc = self._project.storage.getDocument(tHandle)
+                self.scanText(tHandle, theDoc.readDocument() or "")
+        self._indexBroken = False
+        return
+
     def deleteHandle(self, tHandle):
         """Delete all entries of a given document handle.
         """
@@ -113,11 +125,11 @@ class NWIndex:
         moved from the archive or trash folders back into the active
         project.
         """
-        if not self.theProject.tree.checkType(tHandle, nwItemType.FILE):
+        if not self._project.tree.checkType(tHandle, nwItemType.FILE):
             return False
 
         logger.debug("Re-indexing item '%s'", tHandle)
-        theDoc = self.theProject.storage.getDocument(tHandle)
+        theDoc = self._project.storage.getDocument(tHandle)
         self.scanText(tHandle, theDoc.readDocument() or "")
 
         return True
@@ -125,13 +137,13 @@ class NWIndex:
     def indexChangedSince(self, checkTime):
         """Check if the index has changed since a given time.
         """
-        return self._indexChange > checkTime
+        return self._indexChange > float(checkTime)
 
     def rootChangedSince(self, rootHandle, checkTime):
         """Check if the index has changed since a given time for a
         given root item.
         """
-        return self._rootChange.get(rootHandle, self._indexChange) > checkTime
+        return self._rootChange.get(rootHandle, self._indexChange) > float(checkTime)
 
     ##
     #  Load and Save Index to/from File
@@ -140,7 +152,7 @@ class NWIndex:
     def loadIndex(self):
         """Load index from last session from the project meta folder.
         """
-        indexFile = self.theProject.storage.getMetaFile(nwFiles.INDEX_FILE)
+        indexFile = self._project.storage.getMetaFile(nwFiles.INDEX_FILE)
         if not isinstance(indexFile, Path):
             return False
 
@@ -171,12 +183,12 @@ class NWIndex:
         logger.debug("Checking index")
 
         # Check that all files are indexed
-        for fHandle in self.theProject.projFiles:
+        for fHandle in self._project.projFiles:
             if fHandle not in self._itemIndex:
                 logger.warning("Item '%s' is not in the index", fHandle)
                 self.reIndexHandle(fHandle)
 
-        self._indexChange = round(time())
+        self._indexChange = time()
 
         logger.debug("Index loaded in %.3f ms", (time() - tStart)*1000)
 
@@ -186,7 +198,7 @@ class NWIndex:
         """Save the current index as a json file in the project meta
         data folder.
         """
-        indexFile = self.theProject.storage.getMetaFile(nwFiles.INDEX_FILE)
+        indexFile = self._project.storage.getMetaFile(nwFiles.INDEX_FILE)
         if not isinstance(indexFile, Path):
             return False
 
@@ -222,7 +234,7 @@ class NWIndex:
         files before we save them, in which case we already have the
         text.
         """
-        theItem = self.theProject.tree[tHandle]
+        theItem = self._project.tree[tHandle]
         if theItem is None:
             logger.info("Not indexing unknown item '%s'", tHandle)
             return False
@@ -256,7 +268,7 @@ class NWIndex:
             self._scanActive(tHandle, theItem, theText, itemTags)
 
         # Update timestamps for index changes
-        nowTime = round(time())
+        nowTime = time()
         self._indexChange = nowTime
         self._rootChange[theItem.itemRoot] = nowTime
 
@@ -737,8 +749,8 @@ class ItemIndex:
     IndexHeading object for each header of the text.
     """
 
-    def __init__(self, theProject):
+    def __init__(self, project):
-        self.theProject = theProject
+        self._project = project
         self._items = {}
         return
 
@@ -802,7 +814,7 @@ class ItemIndex:
         """Iterate over all items and headers in the novel structure for
         a given root handle, or for all if root handle is None.
         """
-        for tItem in self.theProject.tree:
+        for tItem in self._project.tree:
             if tItem is None:
                 continue
             if tItem.isNoteLayout():
@@ -885,7 +897,7 @@ class ItemIndex:
             if not isHandle(tHandle):
                 raise ValueError("itemIndex keys must be handles")
 
-            nwItem = self.theProject.tree[tHandle]
+            nwItem = self._project.tree[tHandle]
             if nwItem is not None:
                 tItem = IndexItem(tHandle, nwItem)
                 tItem.unpackData(tData)

novelwriter/core/project.py

@@ -369,8 +369,11 @@ class NWProject(QObject):
 
         self._scanProjectFolder()
         self._index.loadIndex()
-        self.updateWordCounts()
+        if xmlReader.state == XMLReadState.WAS_LEGACY:
+            # Often, the index needs to be rebuilt when updating format
+            self._index.rebuildIndex()
 
+        self.updateWordCounts()
         self._projOpened = time()
         self._projAltered = False
 

novelwriter/core/tree.py

@@ -331,20 +331,15 @@ class NWTree:
     def setOrder(self, newOrder):
         """Reorders the tree based on a list of items.
         """
-        tmpOrder = []
+        tmpOrder = [tHandle for tHandle in newOrder if tHandle in self._projTree]
-
+        if not (len(tmpOrder) == len(newOrder) == len(self._treeOrder)):
-        # Add all known elements to a new temp list
+            # Something is wrong, so let's debug it
-        for tHandle in newOrder:
+            for tHandle in newOrder:
-            if tHandle in self._projTree:
+                if tHandle not in self._projTree:
-                tmpOrder.append(tHandle)
+                    logger.error("Handle '%s' in new tree order is not in old order", tHandle)
-            else:
+            for tHandle in self._treeOrder:
-                logger.error("Handle '%s' in new tree order is not in project tree", tHandle)
+                if tHandle not in tmpOrder:
-
+                    logger.warning("Handle '%s' in old tree order is not in new order", tHandle)
-        # Do a reverse lookup to check for items that will be lost
-        # This is mainly for debugging purposes
-        for tHandle in self._treeOrder:
-            if tHandle not in tmpOrder:
-                logger.warning("Handle '%s' in old tree order is not in new tree order", tHandle)
 
         # Save the temp list
         self._treeOrder = tmpOrder

novelwriter/guimain.py

@@ -833,17 +833,8 @@ class GuiMain(QMainWindow):
         tStart = time()
 
         self.projView.saveProjectTasks()
-        self.theProject.index.clearIndex()
+        self.theProject.index.rebuildIndex()
-
+        self.projView.populateTree()
-        for tItem in self.theProject.tree:
-            if tItem is None:  # pragma: no cover
-                continue  # This is a bug trap
-
-            logger.debug("Indexing '%s'", tItem.itemName)
-            if self.theProject.index.reIndexHandle(tItem.itemHandle):
-                # Update Word Counts
-                self.projView.propagateCount(tItem.itemHandle, tItem.wordCount, countChildren=True)
-                self.projView.setTreeItemValues(tItem.itemHandle)
 
         tEnd = time()
         self.setStatus(

tests/test_core/test_core_index.py

@@ -90,7 +90,7 @@ def testCoreIndex_LoadSave(monkeypatch, prjLipsum, mockGUI, tstPaths):
     assert theIndex._tagsIndex._tags == {}
     assert theIndex._itemIndex._items == {}
 
-    # No folder for sloading
+    # No folder for loading
     with monkeypatch.context() as mp:
         mp.setattr("novelwriter.core.storage.NWStorage.getMetaFile", lambda *a: None)
         assert theIndex.loadIndex() is False
@@ -108,6 +108,13 @@ def testCoreIndex_LoadSave(monkeypatch, prjLipsum, mockGUI, tstPaths):
     assert str(theIndex._tagsIndex.packData()) == tagIndex
     assert str(theIndex._itemIndex.packData()) == itemsIndex
 
+    # Rebuild index
+    theIndex.clearIndex()
+    theIndex.rebuildIndex()
+
+    assert str(theIndex._tagsIndex.packData()) == tagIndex
+    assert str(theIndex._itemIndex.packData()) == itemsIndex
+
     # Check File
     copyfile(projFile, testFile)
     assert cmpFiles(testFile, compFile)

tests/test_core/test_core_project.py

@@ -230,7 +230,7 @@ def testCoreProject_Open(monkeypatch, caplog, mockGUI, fncPath, mockRnd):
         assert "The file format of your project is about to be" in mockGUI.lastQuestion[1]
         mockGUI.askResponse = True
 
-    # Won't convert legacy file
+    # Won't open project from newer version
     with monkeypatch.context() as mp:
         mp.setattr(ProjectXMLReader, "hexVersion", property(lambda *a: 0x99999999))
         mockGUI.askResponse = False
@@ -245,6 +245,18 @@ def testCoreProject_Open(monkeypatch, caplog, mockGUI, fncPath, mockRnd):
 
     assert theProject.closeProject()
 
+    # Trigger an index rebuild
+    with monkeypatch.context() as mp:
+        mp.setattr(ProjectXMLReader, "state", property(lambda *a: XMLReadState.WAS_LEGACY))
+        mp.setattr("novelwriter.core.index.NWIndex.loadIndex", lambda *a: True)
+        mockGUI.askResponse = True
+        theProject.index._indexBroken = True
+        assert theProject.openProject(fncPath) is True
+        assert "The file format of your project is about to be" in mockGUI.lastQuestion[1]
+        assert theProject.index._indexBroken is False
+
+    assert theProject.closeProject()
+
 # END Test testCoreProject_Open
 
 

tests/test_core/test_core_tree.py

@@ -24,6 +24,7 @@ import random
 
 from pathlib import Path
 
+from mock import causeOSError
 from tools import readFile
 
 from novelwriter.enum import nwItemClass, nwItemType, nwItemLayout
@@ -230,6 +231,36 @@ def testCoreTree_BuildTree(mockGUI, mockItems):
 # END Test testCoreTree_BuildTree
 
 
+@pytest.mark.core
+def testCoreTree_PackUnpack(mockGUI, mockItems):
+    """Test packing and unpacking data.
+    """
+    theProject = NWProject(mockGUI)
+    theTree = NWTree(theProject)
+
+    aHandles = []
+    for tHandle, pHandle, nwItem in mockItems:
+        aHandles.append(tHandle)
+        theTree.append(tHandle, pHandle, nwItem)
+        theTree.updateItemData(tHandle)
+
+    assert len(theTree) == len(mockItems)
+
+    # Pack
+    tree = theTree.pack()
+    for i, (tHandle, pHandle, nwItem) in enumerate(mockItems):
+        assert tree[i]["itemAttr"]["handle"] == tHandle
+
+    # Unpack
+    theTree.clear()
+    assert len(theTree) == 0
+    assert theTree.handles() == []
+    assert theTree.unpack(tree) is True
+    assert theTree.handles() == aHandles
+
+# END Test testCoreTree_PackUnpack
+
+
 @pytest.mark.core
 def testCoreTree_Methods(mockGUI, mockItems):
     """Test various class methods.
@@ -272,6 +303,13 @@ def testCoreTree_Methods(mockGUI, mockItems):
     assert theTree.findRoot(nwItemClass.NOVEL) == "a000000000001"
     assert theTree.findRoot(nwItemClass.CHARACTER) == "a000000000004"
 
+    # Iter roots
+    roots = list(theTree.iterRoots(None))
+    assert roots[0][0] == "a000000000001"
+    assert roots[1][0] == "a000000000002"
+    assert roots[2][0] == "a000000000003"
+    assert roots[3][0] == "a000000000004"
+
     # Add a fake item to root and check that it can handle it
     theTree._treeRoots["0000000000000"] = NWItem(theProject)
     assert theTree.findRoot(nwItemClass.WORLD) is None
@@ -363,7 +401,7 @@ def testCoreTree_Stats(mockGUI, mockItems):
 
 
 @pytest.mark.core
-def testCoreTree_Reorder(mockGUI, mockItems):
+def testCoreTree_Reorder(caplog, mockGUI, mockItems):
     """Test changing tree order.
     """
     theProject = NWProject(mockGUI)
@@ -384,12 +422,16 @@ def testCoreTree_Reorder(mockGUI, mockItems):
     theTree.setOrder(bHandle)
     assert theTree.handles() == bHandle
 
+    caplog.clear()
     theTree.setOrder(bHandle + ["stuff"])
     assert theTree.handles() == bHandle
+    assert "Handle 'stuff' in new tree order is not in old order" in caplog.text
 
+    caplog.clear()
     theTree._treeOrder.append("stuff")
     theTree.setOrder(bHandle)
     assert theTree.handles() == bHandle
+    assert "Handle 'stuff' in old tree order is not in new order" in caplog.text
 
 # END Test testCoreTree_Reorder
 
@@ -421,6 +463,11 @@ def testCoreTree_ToCFile(monkeypatch, mockGUI, mockItems, tmpPath):
     theProject._storage._runtimePath = None
     assert theTree.writeToCFile() is False
 
+    theProject._storage._runtimePath = tmpPath
+    with monkeypatch.context() as mp:
+        mp.setattr("builtins.open", causeOSError)
+        assert theTree.writeToCFile() is False
+
     theProject._storage._runtimePath = tmpPath
     assert theTree.writeToCFile() is True