Merge pull request #114 from vkbo/index_changes

Index and Project Changes

nw/constants.py

@@ -22,6 +22,7 @@ class nwFiles():
 
     APP_ICON   = "novelWriter.svg"
     PROJ_FILE  = "nwProject.nwx"
+    PROJ_COUNT = "projCount.txt"
     PROJ_DICT  = "wordlist.txt"
     SESS_INFO  = "sessionInfo.log"
     INDEX_FILE = "tagsIndex.json"

nw/gui/elements/doctree.py

@@ -220,7 +220,7 @@ class GuiDocTree(QTreeWidget):
     def deleteItem(self, tHandle=None):
         """Delete items from the tree. Note that this does not delete the item from the item tree in
         the project object. However, since this is only meta data, there isn't really a need to do
-        that to save memory. As items not in the tree are not saved to the project file, a loaded
+        that to save memory. Items not in the tree are not saved to the project file, so a loaded
         project will be clean anyway.
         """
 
@@ -247,6 +247,7 @@ class GuiDocTree(QTreeWidget):
             self.clearSelection()
             trItemP.setSelected(True)
             self.theProject.setProjectChanged(True)
+            self.theParent.theIndex.deleteHandle(tHandle)
 
         elif nwItemS.itemType == nwItemType.FOLDER:
             logger.debug("User requested folder %s deleted" % tHandle)

nw/gui/winmain.py

@@ -307,6 +307,10 @@ class GuiMain(QMainWindow):
         if self.theProject.lastViewed is not None:
             self.viewDocument(self.theProject.lastViewed)
 
+        # Check if we need to rebuild the index
+        if self.theIndex.indexBroken:
+            self.rebuildIndex()
+
         return True
 
     def saveProject(self):
@@ -607,7 +611,7 @@ class GuiMain(QMainWindow):
         """
 
         if isinstance(theMessage, list):
-            popMsg = "<br>".join(theMessage)
+            popMsg = " ".join(theMessage)
             logMsg = theMessage
         else:
             popMsg = theMessage

nw/project/index.py

@@ -17,8 +17,9 @@ import nw
 from os                  import path
 
 from nw.project.document import NWDoc
-from nw.enum             import nwItemType, nwItemClass
+from nw.enum             import nwItemType, nwItemClass, nwItemLayout
 from nw.constants        import nwFiles
+from nw.enum             import nwAlert
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +36,7 @@ class NWIndex():
 
     NOTE_KEYS  = [TAG_KEY]
     NOVEL_KEYS = [PLOT_KEY, POV_KEY, CHAR_KEY, WORLD_KEY, TIME_KEY, OBJECT_KEY, CUSTOM_KEY]
+    VALID_KEYS = [TAG_KEY, PLOT_KEY, POV_KEY, CHAR_KEY, WORLD_KEY, TIME_KEY, OBJECT_KEY, CUSTOM_KEY]
     TAG_CLASS  = {
         CHAR_KEY   : [nwItemClass.CHARACTER, 1],
         POV_KEY    : [nwItemClass.CHARACTER, 2],
@@ -48,24 +50,43 @@ class NWIndex():
     def __init__(self, theProject, theParent):
 
         # Internal
-        self.theProject = theProject
+        self.theProject  = theProject
-        self.theParent  = theParent
+        self.theParent   = theParent
-        self.mainConf   = self.theParent.mainConf
+        self.mainConf    = self.theParent.mainConf
+        self.indexBroken = False
 
         # Indices
         self.tagIndex   = {}
         self.refIndex   = {}
         self.novelIndex = {}
+        self.noteIndex  = {}
 
         # Lists
         self.novelList  = []
 
         return
 
+    ##
+    #  Public Methods
+    ##
+
     def clearIndex(self):
         self.tagIndex   = {}
         self.refIndex   = {}
         self.novelIndex = {}
+        self.noteIndex  = {}
+        return
+
+    def deleteHandle(self, tHandle):
+
+        for tTag in self.tagIndex:
+            if self.tagIndex[tTag][1] == tHandle:
+                self.tagIndex.pop(tTag, None)
+
+        self.refIndex.pop(tHandle, None)
+        self.novelIndex.pop(tHandle, None)
+        self.noteIndex.pop(tHandle, None)
+
         return
 
     ##
@@ -95,6 +116,10 @@ class NWIndex():
                 self.refIndex = theData["refIndex"]
             if "novelIndex" in theData.keys():
                 self.novelIndex = theData["novelIndex"]
+            if "noteIndex" in theData.keys():
+                self.noteIndex = theData["noteIndex"]
+
+            self.checkIndex()
 
             return True
 
@@ -116,6 +141,7 @@ class NWIndex():
                     "tagIndex"   : self.tagIndex,
                     "refIndex"   : self.refIndex,
                     "novelIndex" : self.novelIndex,
+                    "noteIndex"  : self.noteIndex,
                 }, indent=nIndent))
         except Exception as e:
             logger.error("Failed to save index file")
@@ -124,6 +150,40 @@ class NWIndex():
 
         return True
 
+    def checkIndex(self):
+        """Check that the entries in the index are valid and contain the elements it should.
+        """
+
+        self.indexBroken = False
+
+        for tTag in self.tagIndex:
+            if len(self.tagIndex[tTag]) != 3:
+                self.indexBroken = True
+
+        for tHandle in self.refIndex:
+            for tEntry in self.refIndex[tHandle]:
+                if len(tEntry) != 4:
+                    self.indexBroken = True
+
+        for tHandle in self.novelIndex:
+            for tEntry in self.novelIndex[tHandle]:
+                if len(tEntry) != 4:
+                    self.indexBroken = True
+
+        for tHandle in self.noteIndex:
+            for tEntry in self.noteIndex[tHandle]:
+                if len(tEntry) != 4:
+                    self.indexBroken = True
+
+        if self.indexBroken:
+            self.clearIndex()
+            self.theParent.makeAlert(
+                "The project index loaded from cache contains errors. Triggering Rebuild Index.",
+                nwAlert.WARN
+            )
+
+        return
+
     ##
     #  Index Building
     ##
@@ -137,6 +197,7 @@ class NWIndex():
         theItem = self.theProject.getItem(tHandle)
         if theItem is None: return False
         if theItem.itemType != nwItemType.FILE: return False
+        if theItem.parHandle == self.theProject.trashRoot: return False
         itemClass  = theItem.itemClass
         itemLayout = theItem.itemLayout
 
@@ -148,6 +209,8 @@ class NWIndex():
             self.refIndex[tHandle]   = []
             isNovel = True
         else:
+            self.noteIndex[tHandle] = []
+            self.refIndex[tHandle]  = []
             isNovel = False
 
         # Also clear references to file in tag index
@@ -166,19 +229,16 @@ class NWIndex():
             nChar  = len(aLine)
             if nChar == 0: continue
             if aLine[0] == "#":
-                if isNovel:
+                isTitle = self.indexTitle(tHandle, isNovel, aLine, nLine, itemLayout)
-                    isTitle = self.indexTitle(tHandle, aLine, nLine, itemLayout)
+                if isTitle:
-                    if isTitle:
+                    nTitle = nLine
-                        nTitle = nLine
             elif aLine[0] == "@":
-                if isNovel:
+                self.indexNoteRef(tHandle, aLine, nLine, nTitle)
-                    self.indexNoteRef(tHandle, aLine, nLine, nTitle)
+                self.indexTag(tHandle, aLine, nLine, itemClass)
-                else:
-                    self.indexTag(tHandle, aLine, nLine, itemClass)
 
         return True
 
-    def indexTitle(self, tHandle, aLine, nLine, itemLayout):
+    def indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout):
         """Save information about the title and its location in the file.
         """
 
@@ -198,7 +258,12 @@ class NWIndex():
             return False
 
         if hText != "":
-            self.novelIndex[tHandle].append([nLine, hDepth, hText, itemLayout.name])
+            if isNovel:
+                if tHandle in self.novelIndex:
+                    self.novelIndex[tHandle].append([nLine, hDepth, hText, itemLayout.name])
+            else:
+                if tHandle in self.noteIndex:
+                    self.noteIndex[tHandle].append([nLine, hDepth, hText, itemLayout.name])
 
         return True
 
@@ -284,6 +349,11 @@ class NWIndex():
         if nBits == 0:
             return []
 
+        # Check that the key is valid
+        isGood[0] = theBits[0] in self.VALID_KEYS
+        if not isGood[0] or nBits == 1:
+            return isGood
+
         # If we have a tag, only the first value is accepted, the rest is ignored
         if theBits[0] == self.TAG_KEY and nBits > 1:
             isGood[0] = True
@@ -297,13 +367,6 @@ class NWIndex():
             return isGood
 
         # If we're still here, we better check that the references exist
-        if tItem.itemClass == nwItemClass.NOVEL:
-            isGood[0] = theBits[0] in self.NOVEL_KEYS
-        else:
-            isGood[0] = theBits[0] in self.NOTE_KEYS
-        if not isGood[0] or nBits == 1:
-            return isGood
-
         for n in range(1,nBits):
             if theBits[n] in self.tagIndex:
                 isGood[n] = self.TAG_CLASS[theBits[0]][0].name == self.tagIndex[theBits[n]][2]
@@ -329,19 +392,19 @@ class NWIndex():
 
         return True
 
-    def buildReferenceList(self, theHandle):
+    def buildReferenceList(self, tHandle):
-        """Build a list of files referring back to our file, specified by theHandle.
+        """Build a list of files referring back to our file, specified by tHandle.
         """
 
         theRefs = {}
 
-        tItem = self.theProject.getItem(theHandle)
+        tItem = self.theProject.getItem(tHandle)
-        if theHandle is None:
+        if tHandle is None:
             return theRefs
 
         theTag = None
         for tTag in self.tagIndex:
-            if theHandle == self.tagIndex[tTag][1]:
+            if tHandle == self.tagIndex[tTag][1]:
                 theTag = tTag
                 break
 

nw/project/project.py

@@ -14,6 +14,7 @@ import logging
 import nw
 
 from os       import path, mkdir, listdir
+from shutil   import copyfile
 from lxml     import etree
 from hashlib  import sha256
 from datetime import datetime
@@ -196,10 +197,16 @@ class NWProject():
         if not self._checkFolder(self.projMeta):  return
         if not self._checkFolder(self.projCache): return
 
-        nwXML = etree.parse(fileName)
+        try:
-        xRoot = nwXML.getroot()
+            nwXML = etree.parse(fileName)
+        except Exception as e:
+            self.makeAlert(["Failed to parse project xml.",str(e)], nwAlert.ERROR)
+            self.clearProject()
+            return False
+
+        xRoot   = nwXML.getroot()
+        nwxRoot = xRoot.tag
 
-        nwxRoot     = xRoot.tag
         appVersion  = xRoot.attrib["appVersion"]
         fileVersion = xRoot.attrib["fileVersion"]
 
@@ -288,6 +295,9 @@ class NWProject():
 
         logger.debug("Saving project: %s" % self.projPath)
 
+        # Save a copy of the current file, just in case
+        self._maintainPrevious()
+
         # Root element and project details
         logger.debug("Writing project meta")
         nwXML = etree.Element("novelWriterXML",attrib={
@@ -699,4 +709,41 @@ class NWProject():
             itemHandle = self._makeHandle(addSeed+"!")
         return itemHandle
 
+    def _maintainPrevious(self):
+        """This function will take the current project file and copy it into the project cache
+        folder with an incremental file extension added. These serve as a backup in case the xml
+        file gets corrupted.
+        """
+
+        countFile = path.join(self.projCache, nwFiles.PROJ_COUNT)
+        projCount = 0
+
+        if path.isfile(countFile):
+            try:
+                with open(countFile, mode="r") as inFile:
+                    projCount = int(inFile.read())+1
+            except:
+                projCount = 0
+
+        if projCount > 9:
+            projCount = 0
+
+        projBackup = "%s.%d" % (nwFiles.PROJ_FILE, projCount)
+
+        try:
+            copyfile(
+                path.join(self.projPath,self.projFile),
+                path.join(self.projCache,projBackup)
+            )
+        except:
+            logger.error("Failed to write to file %s" % projBackup)
+
+        try:
+            with open(countFile, mode="w") as outFile:
+                outFile.write(str(projCount))
+        except:
+            logger.error("Failed to write to file %s" % countFile)
+
+        return
+
 # END Class NWProject

tests/reference/gui/1_tagsIndex.json

@@ -1 +1 @@
-{"tagIndex": {"Jane": [3, "2fca346db6561", "CHARACTER"], "MainPlot": [3, "02d20bbd7e394", "PLOT"], "Home": [3, "7688b6ef52555", "WORLD"]}, "refIndex": {"31489056e0916": [[5, "@pov", "Jane", 3], [6, "@plot", "MainPlot", 3], [11, "@pov", "Jane", 8], [12, "@plot", "MainPlot", 8], [13, "@location", "Home", 8], [17, "@char", "Jane", 15]]}, "novelIndex": {"31489056e0916": [[1, 1, "Novel", "SCENE"], [3, 2, "Chapter", "SCENE"], [8, 3, "Scene", "SCENE"], [15, 4, "Some Section", "SCENE"]]}}
+{"tagIndex": {"Jane": [3, "2fca346db6561", "CHARACTER"], "MainPlot": [3, "02d20bbd7e394", "PLOT"], "Home": [3, "7688b6ef52555", "WORLD"]}, "refIndex": {"31489056e0916": [[5, "@pov", "Jane", 3], [6, "@plot", "MainPlot", 3], [11, "@pov", "Jane", 8], [12, "@plot", "MainPlot", 8], [13, "@location", "Home", 8], [17, "@char", "Jane", 15]], "2fca346db6561": [], "02d20bbd7e394": [], "7688b6ef52555": []}, "novelIndex": {"31489056e0916": [[1, 1, "Novel", "SCENE"], [3, 2, "Chapter", "SCENE"], [8, 3, "Scene", "SCENE"], [15, 4, "Some Section", "SCENE"]]}, "noteIndex": {"2fca346db6561": [[1, 1, "Jane Doe", "NOTE"]], "02d20bbd7e394": [[1, 1, "Main Plot", "NOTE"]], "7688b6ef52555": [[1, 1, "Main Location", "NOTE"]]}}