diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 249fd19d..bdd49925 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -58,23 +58,23 @@ class NWIndex: The index data is cached in a JSON file between writing sessions. """ - def __init__(self, theProject): + def __init__(self, project): - self.theProject = theProject + self._project = project # Storage and State self._tagsIndex = TagsIndex() - self._itemIndex = ItemIndex(theProject) + self._itemIndex = ItemIndex(project) self._indexBroken = False # TimeStamps - self._indexChange = 0 + self._indexChange = 0.0 self._rootChange = {} return def __repr__(self): - return f"" + return f"" ## # Properties @@ -93,10 +93,22 @@ class NWIndex: """ self._tagsIndex.clear() self._itemIndex.clear() - self._indexChange = 0 + self._indexChange = 0.0 self._rootChange = {} return + def rebuildIndex(self): + """Rebuild the entire index from scratch. + """ + self.clearIndex() + for nwItem in self._project.tree: + if nwItem is not None and nwItem.isFileType(): + tHandle = nwItem.itemHandle + theDoc = self._project.storage.getDocument(tHandle) + self.scanText(tHandle, theDoc.readDocument() or "") + self._indexBroken = False + return + def deleteHandle(self, tHandle): """Delete all entries of a given document handle. """ @@ -113,11 +125,11 @@ class NWIndex: moved from the archive or trash folders back into the active project. """ - if not self.theProject.tree.checkType(tHandle, nwItemType.FILE): + if not self._project.tree.checkType(tHandle, nwItemType.FILE): return False logger.debug("Re-indexing item '%s'", tHandle) - theDoc = self.theProject.storage.getDocument(tHandle) + theDoc = self._project.storage.getDocument(tHandle) self.scanText(tHandle, theDoc.readDocument() or "") return True @@ -125,13 +137,13 @@ class NWIndex: def indexChangedSince(self, checkTime): """Check if the index has changed since a given time. """ - return self._indexChange > checkTime + return self._indexChange > float(checkTime) def rootChangedSince(self, rootHandle, checkTime): """Check if the index has changed since a given time for a given root item. """ - return self._rootChange.get(rootHandle, self._indexChange) > checkTime + return self._rootChange.get(rootHandle, self._indexChange) > float(checkTime) ## # Load and Save Index to/from File @@ -140,7 +152,7 @@ class NWIndex: def loadIndex(self): """Load index from last session from the project meta folder. """ - indexFile = self.theProject.storage.getMetaFile(nwFiles.INDEX_FILE) + indexFile = self._project.storage.getMetaFile(nwFiles.INDEX_FILE) if not isinstance(indexFile, Path): return False @@ -171,12 +183,12 @@ class NWIndex: logger.debug("Checking index") # Check that all files are indexed - for fHandle in self.theProject.projFiles: + for fHandle in self._project.projFiles: if fHandle not in self._itemIndex: logger.warning("Item '%s' is not in the index", fHandle) self.reIndexHandle(fHandle) - self._indexChange = round(time()) + self._indexChange = time() logger.debug("Index loaded in %.3f ms", (time() - tStart)*1000) @@ -186,7 +198,7 @@ class NWIndex: """Save the current index as a json file in the project meta data folder. """ - indexFile = self.theProject.storage.getMetaFile(nwFiles.INDEX_FILE) + indexFile = self._project.storage.getMetaFile(nwFiles.INDEX_FILE) if not isinstance(indexFile, Path): return False @@ -222,7 +234,7 @@ class NWIndex: files before we save them, in which case we already have the text. """ - theItem = self.theProject.tree[tHandle] + theItem = self._project.tree[tHandle] if theItem is None: logger.info("Not indexing unknown item '%s'", tHandle) return False @@ -256,7 +268,7 @@ class NWIndex: self._scanActive(tHandle, theItem, theText, itemTags) # Update timestamps for index changes - nowTime = round(time()) + nowTime = time() self._indexChange = nowTime self._rootChange[theItem.itemRoot] = nowTime @@ -737,8 +749,8 @@ class ItemIndex: IndexHeading object for each header of the text. """ - def __init__(self, theProject): - self.theProject = theProject + def __init__(self, project): + self._project = project self._items = {} return @@ -802,7 +814,7 @@ class ItemIndex: """Iterate over all items and headers in the novel structure for a given root handle, or for all if root handle is None. """ - for tItem in self.theProject.tree: + for tItem in self._project.tree: if tItem is None: continue if tItem.isNoteLayout(): @@ -885,7 +897,7 @@ class ItemIndex: if not isHandle(tHandle): raise ValueError("itemIndex keys must be handles") - nwItem = self.theProject.tree[tHandle] + nwItem = self._project.tree[tHandle] if nwItem is not None: tItem = IndexItem(tHandle, nwItem) tItem.unpackData(tData) diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py index cdcb83d0..bb26ff5e 100644 --- a/novelwriter/core/project.py +++ b/novelwriter/core/project.py @@ -369,8 +369,11 @@ class NWProject(QObject): self._scanProjectFolder() self._index.loadIndex() - self.updateWordCounts() + if xmlReader.state == XMLReadState.WAS_LEGACY: + # Often, the index needs to be rebuilt when updating format + self._index.rebuildIndex() + self.updateWordCounts() self._projOpened = time() self._projAltered = False diff --git a/novelwriter/core/tree.py b/novelwriter/core/tree.py index 9d499aaf..1f9960de 100644 --- a/novelwriter/core/tree.py +++ b/novelwriter/core/tree.py @@ -331,20 +331,15 @@ class NWTree: def setOrder(self, newOrder): """Reorders the tree based on a list of items. """ - tmpOrder = [] - - # Add all known elements to a new temp list - for tHandle in newOrder: - if tHandle in self._projTree: - tmpOrder.append(tHandle) - else: - logger.error("Handle '%s' in new tree order is not in project tree", tHandle) - - # Do a reverse lookup to check for items that will be lost - # This is mainly for debugging purposes - for tHandle in self._treeOrder: - if tHandle not in tmpOrder: - logger.warning("Handle '%s' in old tree order is not in new tree order", tHandle) + tmpOrder = [tHandle for tHandle in newOrder if tHandle in self._projTree] + if not (len(tmpOrder) == len(newOrder) == len(self._treeOrder)): + # Something is wrong, so let's debug it + for tHandle in newOrder: + if tHandle not in self._projTree: + logger.error("Handle '%s' in new tree order is not in old order", tHandle) + for tHandle in self._treeOrder: + if tHandle not in tmpOrder: + logger.warning("Handle '%s' in old tree order is not in new order", tHandle) # Save the temp list self._treeOrder = tmpOrder diff --git a/novelwriter/guimain.py b/novelwriter/guimain.py index 20f7ce2d..56ba564f 100644 --- a/novelwriter/guimain.py +++ b/novelwriter/guimain.py @@ -833,17 +833,8 @@ class GuiMain(QMainWindow): tStart = time() self.projView.saveProjectTasks() - self.theProject.index.clearIndex() - - for tItem in self.theProject.tree: - if tItem is None: # pragma: no cover - continue # This is a bug trap - - logger.debug("Indexing '%s'", tItem.itemName) - if self.theProject.index.reIndexHandle(tItem.itemHandle): - # Update Word Counts - self.projView.propagateCount(tItem.itemHandle, tItem.wordCount, countChildren=True) - self.projView.setTreeItemValues(tItem.itemHandle) + self.theProject.index.rebuildIndex() + self.projView.populateTree() tEnd = time() self.setStatus( diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index 82bf53c8..6d85dfb0 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -90,7 +90,7 @@ def testCoreIndex_LoadSave(monkeypatch, prjLipsum, mockGUI, tstPaths): assert theIndex._tagsIndex._tags == {} assert theIndex._itemIndex._items == {} - # No folder for sloading + # No folder for loading with monkeypatch.context() as mp: mp.setattr("novelwriter.core.storage.NWStorage.getMetaFile", lambda *a: None) assert theIndex.loadIndex() is False @@ -108,6 +108,13 @@ def testCoreIndex_LoadSave(monkeypatch, prjLipsum, mockGUI, tstPaths): assert str(theIndex._tagsIndex.packData()) == tagIndex assert str(theIndex._itemIndex.packData()) == itemsIndex + # Rebuild index + theIndex.clearIndex() + theIndex.rebuildIndex() + + assert str(theIndex._tagsIndex.packData()) == tagIndex + assert str(theIndex._itemIndex.packData()) == itemsIndex + # Check File copyfile(projFile, testFile) assert cmpFiles(testFile, compFile) diff --git a/tests/test_core/test_core_project.py b/tests/test_core/test_core_project.py index d2931fd8..1f3dc73e 100644 --- a/tests/test_core/test_core_project.py +++ b/tests/test_core/test_core_project.py @@ -230,7 +230,7 @@ def testCoreProject_Open(monkeypatch, caplog, mockGUI, fncPath, mockRnd): assert "The file format of your project is about to be" in mockGUI.lastQuestion[1] mockGUI.askResponse = True - # Won't convert legacy file + # Won't open project from newer version with monkeypatch.context() as mp: mp.setattr(ProjectXMLReader, "hexVersion", property(lambda *a: 0x99999999)) mockGUI.askResponse = False @@ -245,6 +245,18 @@ def testCoreProject_Open(monkeypatch, caplog, mockGUI, fncPath, mockRnd): assert theProject.closeProject() + # Trigger an index rebuild + with monkeypatch.context() as mp: + mp.setattr(ProjectXMLReader, "state", property(lambda *a: XMLReadState.WAS_LEGACY)) + mp.setattr("novelwriter.core.index.NWIndex.loadIndex", lambda *a: True) + mockGUI.askResponse = True + theProject.index._indexBroken = True + assert theProject.openProject(fncPath) is True + assert "The file format of your project is about to be" in mockGUI.lastQuestion[1] + assert theProject.index._indexBroken is False + + assert theProject.closeProject() + # END Test testCoreProject_Open diff --git a/tests/test_core/test_core_tree.py b/tests/test_core/test_core_tree.py index 9c3d0961..731e11aa 100644 --- a/tests/test_core/test_core_tree.py +++ b/tests/test_core/test_core_tree.py @@ -24,6 +24,7 @@ import random from pathlib import Path +from mock import causeOSError from tools import readFile from novelwriter.enum import nwItemClass, nwItemType, nwItemLayout @@ -230,6 +231,36 @@ def testCoreTree_BuildTree(mockGUI, mockItems): # END Test testCoreTree_BuildTree +@pytest.mark.core +def testCoreTree_PackUnpack(mockGUI, mockItems): + """Test packing and unpacking data. + """ + theProject = NWProject(mockGUI) + theTree = NWTree(theProject) + + aHandles = [] + for tHandle, pHandle, nwItem in mockItems: + aHandles.append(tHandle) + theTree.append(tHandle, pHandle, nwItem) + theTree.updateItemData(tHandle) + + assert len(theTree) == len(mockItems) + + # Pack + tree = theTree.pack() + for i, (tHandle, pHandle, nwItem) in enumerate(mockItems): + assert tree[i]["itemAttr"]["handle"] == tHandle + + # Unpack + theTree.clear() + assert len(theTree) == 0 + assert theTree.handles() == [] + assert theTree.unpack(tree) is True + assert theTree.handles() == aHandles + +# END Test testCoreTree_PackUnpack + + @pytest.mark.core def testCoreTree_Methods(mockGUI, mockItems): """Test various class methods. @@ -272,6 +303,13 @@ def testCoreTree_Methods(mockGUI, mockItems): assert theTree.findRoot(nwItemClass.NOVEL) == "a000000000001" assert theTree.findRoot(nwItemClass.CHARACTER) == "a000000000004" + # Iter roots + roots = list(theTree.iterRoots(None)) + assert roots[0][0] == "a000000000001" + assert roots[1][0] == "a000000000002" + assert roots[2][0] == "a000000000003" + assert roots[3][0] == "a000000000004" + # Add a fake item to root and check that it can handle it theTree._treeRoots["0000000000000"] = NWItem(theProject) assert theTree.findRoot(nwItemClass.WORLD) is None @@ -363,7 +401,7 @@ def testCoreTree_Stats(mockGUI, mockItems): @pytest.mark.core -def testCoreTree_Reorder(mockGUI, mockItems): +def testCoreTree_Reorder(caplog, mockGUI, mockItems): """Test changing tree order. """ theProject = NWProject(mockGUI) @@ -384,12 +422,16 @@ def testCoreTree_Reorder(mockGUI, mockItems): theTree.setOrder(bHandle) assert theTree.handles() == bHandle + caplog.clear() theTree.setOrder(bHandle + ["stuff"]) assert theTree.handles() == bHandle + assert "Handle 'stuff' in new tree order is not in old order" in caplog.text + caplog.clear() theTree._treeOrder.append("stuff") theTree.setOrder(bHandle) assert theTree.handles() == bHandle + assert "Handle 'stuff' in old tree order is not in new order" in caplog.text # END Test testCoreTree_Reorder @@ -421,6 +463,11 @@ def testCoreTree_ToCFile(monkeypatch, mockGUI, mockItems, tmpPath): theProject._storage._runtimePath = None assert theTree.writeToCFile() is False + theProject._storage._runtimePath = tmpPath + with monkeypatch.context() as mp: + mp.setattr("builtins.open", causeOSError) + assert theTree.writeToCFile() is False + theProject._storage._runtimePath = tmpPath assert theTree.writeToCFile() is True