Add automatic index rebuild (#975)

* Rebuild the index automatically if not all files are in the index on open
* Update tests

novelwriter/core/index.py

@@ -167,7 +167,7 @@ class NWIndex():
 
         logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000)
 
-        self.checkIndex()
+        self._checkIndex()
 
         return True
 
@@ -182,10 +182,10 @@ class NWIndex():
         try:
             with open(indexFile, mode="w+", encoding="utf-8") as outFile:
                 outFile.write("{\n")
-                outFile.write(f'"tagIndex": {jsonEncode(self._tagIndex, nmax=1)},\n')
+                outFile.write(f'  "tagIndex": {jsonEncode(self._tagIndex, n=1, nmax=2)},\n')
-                outFile.write(f'"refIndex": {jsonEncode(self._refIndex, nmax=2)},\n')
+                outFile.write(f'  "refIndex": {jsonEncode(self._refIndex, n=1, nmax=3)},\n')
-                outFile.write(f'"fileIndex": {jsonEncode(self._fileIndex, nmax=2)},\n')
+                outFile.write(f'  "fileIndex": {jsonEncode(self._fileIndex, n=1, nmax=3)},\n')
-                outFile.write(f'"fileMeta": {jsonEncode(self._fileMeta, nmax=1)}\n')
+                outFile.write(f'  "fileMeta": {jsonEncode(self._fileMeta, n=1, nmax=2)}\n')
                 outFile.write("}\n")
 
         except Exception:
@@ -197,33 +197,6 @@ class NWIndex():
 
         return True
 
-    def checkIndex(self):
-        """Check that the entries in the index are valid and contain the
-        elements it should.
-        """
-        logger.debug("Checking index")
-        tStart = time()
-
-        try:
-            self._checkTagIndex()
-            self._checkRefIndex()
-            self._checkFileIndex()
-            self._checkFileMeta()
-            self._indexBroken = False
-
-        except Exception:
-            logger.error("Error while checking index")
-            logException()
-            self._indexBroken = True
-
-        logger.verbose("Index check took %.3f ms", (time() - tStart)*1000)
-        logger.debug("Index check complete")
-
-        if self._indexBroken:
-            self.clearIndex()
-
-        return
-
     ##
     #  Index Building
     ##
@@ -687,6 +660,40 @@ class NWIndex():
     #  Index Checkers
     ##
 
+    def _checkIndex(self):
+        """Check that the entries in the index are valid and contain the
+        elements it should. Also check that each file present in the
+        contents folder when the project was loaded are also present in
+        the fileMeta index.
+        """
+        logger.debug("Checking index")
+        tStart = time()
+
+        try:
+            self._checkTagIndex()
+            self._checkRefIndex()
+            self._checkFileIndex()
+            self._checkFileMeta()
+            self._indexBroken = False
+
+        except Exception:
+            logger.error("Error while checking index")
+            logException()
+            self._indexBroken = True
+
+        # Check that project files are indexed
+        for fHandle in self.theProject.projFiles:
+            if fHandle not in self._fileMeta:
+                self._indexBroken = True
+                break
+
+        logger.verbose("Index check completed in %.3f ms", (time() - tStart)*1000)
+
+        if self._indexBroken:
+            self.clearIndex()
+
+        return
+
     def _checkTagIndex(self):
         """Scan the tag index for errors.
         Warning: This function raises exceptions.

novelwriter/core/project.py

@@ -84,6 +84,7 @@ class NWProject():
         self.projSpell   = None  # The spell check language, if different than default
         self.projLang    = None  # The project language, used for builds
         self.projFile    = None  # The file name of the project main XML file
+        self.projFiles   = []    # A list of all files in the content folder on load
 
         # Project Meta
         self.projName    = ""  # Project name (working title)
@@ -203,6 +204,7 @@ class NWProject():
         self.projSpell   = None
         self.projLang    = None
         self.projFile    = nwFiles.PROJ_FILE
+        self.projFiles   = []
         self.projName    = ""
         self.bookTitle   = ""
         self.bookAuthors = []
@@ -1352,6 +1354,7 @@ class NWProject():
         # Then check the files in the data folder
         logger.debug("Checking files in project content folder")
         orphanFiles = []
+        self.projFiles = []
         for fileItem in os.listdir(self.projContent):
             if not fileItem.endswith(".nwd"):
                 logger.warning("Skipping file: %s", fileItem)
@@ -1359,11 +1362,14 @@ class NWProject():
             if len(fileItem) != 17:
                 logger.warning("Skipping file: %s", fileItem)
                 continue
+
             fHandle = fileItem[:13]
             if not isHandle(fHandle):
                 logger.warning("Skipping file: %s", fileItem)
                 continue
+
             if fHandle in self.projTree:
+                self.projFiles.append(fHandle)
                 logger.debug("Checking file %s, handle '%s': OK", fileItem, fHandle)
             else:
                 logger.warning("Checking file %s, handle '%s': Orphaned", fileItem, fHandle)

novelwriter/guimain.py

@@ -550,8 +550,7 @@ class GuiMain(QMainWindow):
             ), nwAlert.WARN)
             self.rebuildIndex()
 
-        # Make sure the changed status is set to false on all that was
+        # Make sure the changed status is set to false on things opened
-        # just opened
         qApp.processEvents()
         self.docEditor.setDocumentChanged(False)
         self.theProject.setProjectChanged(False)

tests/test_core/test_core_index.py

@@ -108,7 +108,7 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir):
     # Break the index and check that we notice
     assert theIndex.indexBroken is False
     theIndex._tagIndex["Bod"].append("Stuff")
-    theIndex.checkIndex()
+    theIndex._checkIndex()
     assert theIndex.indexBroken is True
 
     # Finalise

tests/test_gui/test_gui_docviewer.py

@@ -46,9 +46,7 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum):
     nwGUI.theProject.projTree.setSeed(42)
     assert nwGUI.openProject(nwLipsum)
 
-    # Rebuild the index as it isn't automatically copied
+    # Rebuild the index
-    assert nwGUI.theIndex._tagIndex == {}
-    assert nwGUI.theIndex._refIndex == {}
     nwGUI.mainMenu.aRebuildIndex.activate(QAction.Trigger)
     assert nwGUI.theIndex._tagIndex != {}
     assert nwGUI.theIndex._refIndex != {}

tests/test_gui/test_gui_noveltree.py

@@ -61,10 +61,6 @@ def testGuiNovelTree_TreeItems(qtbot, monkeypatch, nwGUI, nwMinimal):
     ##
 
     nwGUI.projTabs.setCurrentIndex(nwGUI.idxNovelView)
-
-    # The tree should be empty as there is no index
-    assert nwTree.topLevelItemCount() == 0
-
     nwGUI.rebuildIndex()
     nwTree._populateTree()
     assert nwTree.topLevelItemCount() == 1