From 85ea9f5b1912c1d10f9fc07b7fb5bbb0ae8ec16f Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Tue, 25 Jan 2022 17:41:34 +0100 Subject: [PATCH] Add automatic index rebuild (#975) * Rebuild the index automatically if not all files are in the index on open * Update tests --- novelwriter/core/index.py | 71 +++++++++++++++------------- novelwriter/core/project.py | 6 +++ novelwriter/guimain.py | 3 +- tests/test_core/test_core_index.py | 2 +- tests/test_gui/test_gui_docviewer.py | 4 +- tests/test_gui/test_gui_noveltree.py | 4 -- 6 files changed, 48 insertions(+), 42 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 1aa34acd..fa4f01dd 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -167,7 +167,7 @@ class NWIndex(): logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) - self.checkIndex() + self._checkIndex() return True @@ -182,10 +182,10 @@ class NWIndex(): try: with open(indexFile, mode="w+", encoding="utf-8") as outFile: outFile.write("{\n") - outFile.write(f'"tagIndex": {jsonEncode(self._tagIndex, nmax=1)},\n') - outFile.write(f'"refIndex": {jsonEncode(self._refIndex, nmax=2)},\n') - outFile.write(f'"fileIndex": {jsonEncode(self._fileIndex, nmax=2)},\n') - outFile.write(f'"fileMeta": {jsonEncode(self._fileMeta, nmax=1)}\n') + outFile.write(f' "tagIndex": {jsonEncode(self._tagIndex, n=1, nmax=2)},\n') + outFile.write(f' "refIndex": {jsonEncode(self._refIndex, n=1, nmax=3)},\n') + outFile.write(f' "fileIndex": {jsonEncode(self._fileIndex, n=1, nmax=3)},\n') + outFile.write(f' "fileMeta": {jsonEncode(self._fileMeta, n=1, nmax=2)}\n') outFile.write("}\n") except Exception: @@ -197,33 +197,6 @@ class NWIndex(): return True - def checkIndex(self): - """Check that the entries in the index are valid and contain the - elements it should. - """ - logger.debug("Checking index") - tStart = time() - - try: - self._checkTagIndex() - self._checkRefIndex() - self._checkFileIndex() - self._checkFileMeta() - self._indexBroken = False - - except Exception: - logger.error("Error while checking index") - logException() - self._indexBroken = True - - logger.verbose("Index check took %.3f ms", (time() - tStart)*1000) - logger.debug("Index check complete") - - if self._indexBroken: - self.clearIndex() - - return - ## # Index Building ## @@ -687,6 +660,40 @@ class NWIndex(): # Index Checkers ## + def _checkIndex(self): + """Check that the entries in the index are valid and contain the + elements it should. Also check that each file present in the + contents folder when the project was loaded are also present in + the fileMeta index. + """ + logger.debug("Checking index") + tStart = time() + + try: + self._checkTagIndex() + self._checkRefIndex() + self._checkFileIndex() + self._checkFileMeta() + self._indexBroken = False + + except Exception: + logger.error("Error while checking index") + logException() + self._indexBroken = True + + # Check that project files are indexed + for fHandle in self.theProject.projFiles: + if fHandle not in self._fileMeta: + self._indexBroken = True + break + + logger.verbose("Index check completed in %.3f ms", (time() - tStart)*1000) + + if self._indexBroken: + self.clearIndex() + + return + def _checkTagIndex(self): """Scan the tag index for errors. Warning: This function raises exceptions. diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py index 8e5a8f6e..26d2c098 100644 --- a/novelwriter/core/project.py +++ b/novelwriter/core/project.py @@ -84,6 +84,7 @@ class NWProject(): self.projSpell = None # The spell check language, if different than default self.projLang = None # The project language, used for builds self.projFile = None # The file name of the project main XML file + self.projFiles = [] # A list of all files in the content folder on load # Project Meta self.projName = "" # Project name (working title) @@ -203,6 +204,7 @@ class NWProject(): self.projSpell = None self.projLang = None self.projFile = nwFiles.PROJ_FILE + self.projFiles = [] self.projName = "" self.bookTitle = "" self.bookAuthors = [] @@ -1352,6 +1354,7 @@ class NWProject(): # Then check the files in the data folder logger.debug("Checking files in project content folder") orphanFiles = [] + self.projFiles = [] for fileItem in os.listdir(self.projContent): if not fileItem.endswith(".nwd"): logger.warning("Skipping file: %s", fileItem) @@ -1359,11 +1362,14 @@ class NWProject(): if len(fileItem) != 17: logger.warning("Skipping file: %s", fileItem) continue + fHandle = fileItem[:13] if not isHandle(fHandle): logger.warning("Skipping file: %s", fileItem) continue + if fHandle in self.projTree: + self.projFiles.append(fHandle) logger.debug("Checking file %s, handle '%s': OK", fileItem, fHandle) else: logger.warning("Checking file %s, handle '%s': Orphaned", fileItem, fHandle) diff --git a/novelwriter/guimain.py b/novelwriter/guimain.py index c1f4e9d3..8674dda1 100644 --- a/novelwriter/guimain.py +++ b/novelwriter/guimain.py @@ -550,8 +550,7 @@ class GuiMain(QMainWindow): ), nwAlert.WARN) self.rebuildIndex() - # Make sure the changed status is set to false on all that was - # just opened + # Make sure the changed status is set to false on things opened qApp.processEvents() self.docEditor.setDocumentChanged(False) self.theProject.setProjectChanged(False) diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index a059b653..9a7f7b88 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -108,7 +108,7 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): # Break the index and check that we notice assert theIndex.indexBroken is False theIndex._tagIndex["Bod"].append("Stuff") - theIndex.checkIndex() + theIndex._checkIndex() assert theIndex.indexBroken is True # Finalise diff --git a/tests/test_gui/test_gui_docviewer.py b/tests/test_gui/test_gui_docviewer.py index e971c743..4d22b62d 100644 --- a/tests/test_gui/test_gui_docviewer.py +++ b/tests/test_gui/test_gui_docviewer.py @@ -46,9 +46,7 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum): nwGUI.theProject.projTree.setSeed(42) assert nwGUI.openProject(nwLipsum) - # Rebuild the index as it isn't automatically copied - assert nwGUI.theIndex._tagIndex == {} - assert nwGUI.theIndex._refIndex == {} + # Rebuild the index nwGUI.mainMenu.aRebuildIndex.activate(QAction.Trigger) assert nwGUI.theIndex._tagIndex != {} assert nwGUI.theIndex._refIndex != {} diff --git a/tests/test_gui/test_gui_noveltree.py b/tests/test_gui/test_gui_noveltree.py index fab2e0d2..57e6acec 100644 --- a/tests/test_gui/test_gui_noveltree.py +++ b/tests/test_gui/test_gui_noveltree.py @@ -61,10 +61,6 @@ def testGuiNovelTree_TreeItems(qtbot, monkeypatch, nwGUI, nwMinimal): ## nwGUI.projTabs.setCurrentIndex(nwGUI.idxNovelView) - - # The tree should be empty as there is no index - assert nwTree.topLevelItemCount() == 0 - nwGUI.rebuildIndex() nwTree._populateTree() assert nwTree.topLevelItemCount() == 1