From afb1304e06cc2442c3cc06c42f936e5579bcb58e Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 11:12:41 +0100 Subject: [PATCH 1/8] Remove some of the restrictions on how files can be linked together --- nw/project/index.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/nw/project/index.py b/nw/project/index.py index 45e5753a..9097c462 100644 --- a/nw/project/index.py +++ b/nw/project/index.py @@ -17,7 +17,7 @@ import nw from os import path from nw.project.document import NWDoc -from nw.enum import nwItemType, nwItemClass +from nw.enum import nwItemType, nwItemClass, nwItemLayout from nw.constants import nwFiles logger = logging.getLogger(__name__) @@ -35,6 +35,7 @@ class NWIndex(): NOTE_KEYS = [TAG_KEY] NOVEL_KEYS = [PLOT_KEY, POV_KEY, CHAR_KEY, WORLD_KEY, TIME_KEY, OBJECT_KEY, CUSTOM_KEY] + VALID_KEYS = [TAG_KEY, PLOT_KEY, POV_KEY, CHAR_KEY, WORLD_KEY, TIME_KEY, OBJECT_KEY, CUSTOM_KEY] TAG_CLASS = { CHAR_KEY : [nwItemClass.CHARACTER, 1], POV_KEY : [nwItemClass.CHARACTER, 2], @@ -56,6 +57,7 @@ class NWIndex(): self.tagIndex = {} self.refIndex = {} self.novelIndex = {} + self.noteIndex = {} # Lists self.novelList = [] @@ -95,6 +97,8 @@ class NWIndex(): self.refIndex = theData["refIndex"] if "novelIndex" in theData.keys(): self.novelIndex = theData["novelIndex"] + if "noteIndex" in theData.keys(): + self.noteIndex = theData["noteIndex"] return True @@ -116,6 +120,7 @@ class NWIndex(): "tagIndex" : self.tagIndex, "refIndex" : self.refIndex, "novelIndex" : self.novelIndex, + "noteIndex" : self.noteIndex, }, indent=nIndent)) except Exception as e: logger.error("Failed to save index file") @@ -148,6 +153,8 @@ class NWIndex(): self.refIndex[tHandle] = [] isNovel = True else: + self.noteIndex[tHandle] = [] + self.refIndex[tHandle] = [] isNovel = False # Also clear references to file in tag index @@ -166,19 +173,16 @@ class NWIndex(): nChar = len(aLine) if nChar == 0: continue if aLine[0] == "#": - if isNovel: - isTitle = self.indexTitle(tHandle, aLine, nLine, itemLayout) - if isTitle: - nTitle = nLine + isTitle = self.indexTitle(tHandle, isNovel, aLine, nLine, itemLayout) + if isTitle: + nTitle = nLine elif aLine[0] == "@": - if isNovel: - self.indexNoteRef(tHandle, aLine, nLine, nTitle) - else: - self.indexTag(tHandle, aLine, nLine, itemClass) + self.indexNoteRef(tHandle, aLine, nLine, nTitle) + self.indexTag(tHandle, aLine, nLine, itemClass) return True - def indexTitle(self, tHandle, aLine, nLine, itemLayout): + def indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout): """Save information about the title and its location in the file. """ @@ -198,7 +202,12 @@ class NWIndex(): return False if hText != "": - self.novelIndex[tHandle].append([nLine, hDepth, hText, itemLayout.name]) + if isNovel: + if tHandle in self.novelIndex: + self.novelIndex[tHandle].append([nLine, hDepth, hText, itemLayout.name]) + else: + if tHandle in self.noteIndex: + self.noteIndex[tHandle].append([nLine, hDepth, hText, itemLayout.name]) return True @@ -284,6 +293,11 @@ class NWIndex(): if nBits == 0: return [] + # Check that the key is valid + isGood[0] = theBits[0] in self.VALID_KEYS + if not isGood[0] or nBits == 1: + return isGood + # If we have a tag, only the first value is accepted, the rest is ignored if theBits[0] == self.TAG_KEY and nBits > 1: isGood[0] = True @@ -297,13 +311,6 @@ class NWIndex(): return isGood # If we're still here, we better check that the references exist - if tItem.itemClass == nwItemClass.NOVEL: - isGood[0] = theBits[0] in self.NOVEL_KEYS - else: - isGood[0] = theBits[0] in self.NOTE_KEYS - if not isGood[0] or nBits == 1: - return isGood - for n in range(1,nBits): if theBits[n] in self.tagIndex: isGood[n] = self.TAG_CLASS[theBits[0]][0].name == self.tagIndex[theBits[n]][2] From 0a5d1e593c330525b72ac81d3638fb7486be583a Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 11:40:14 +0100 Subject: [PATCH 2/8] Make sure deleted items are also deleted from the index, so the index is up to date --- nw/gui/elements/doctree.py | 3 ++- nw/project/index.py | 28 +++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/nw/gui/elements/doctree.py b/nw/gui/elements/doctree.py index 75c745a6..d618e4ce 100644 --- a/nw/gui/elements/doctree.py +++ b/nw/gui/elements/doctree.py @@ -220,7 +220,7 @@ class GuiDocTree(QTreeWidget): def deleteItem(self, tHandle=None): """Delete items from the tree. Note that this does not delete the item from the item tree in the project object. However, since this is only meta data, there isn't really a need to do - that to save memory. As items not in the tree are not saved to the project file, a loaded + that to save memory. Items not in the tree are not saved to the project file, so a loaded project will be clean anyway. """ @@ -247,6 +247,7 @@ class GuiDocTree(QTreeWidget): self.clearSelection() trItemP.setSelected(True) self.theProject.setProjectChanged(True) + self.theParent.theIndex.deleteHandle(tHandle) elif nwItemS.itemType == nwItemType.FOLDER: logger.debug("User requested folder %s deleted" % tHandle) diff --git a/nw/project/index.py b/nw/project/index.py index 9097c462..93890664 100644 --- a/nw/project/index.py +++ b/nw/project/index.py @@ -64,10 +64,27 @@ class NWIndex(): return + ## + # Public Methods + ## + def clearIndex(self): self.tagIndex = {} self.refIndex = {} self.novelIndex = {} + self.noteIndex = {} + return + + def deleteHandle(self, tHandle): + + for tTag in self.tagIndex: + if self.tagIndex[tTag][1] == tHandle: + self.tagIndex.pop(tTag, None) + + self.refIndex.pop(tHandle, None) + self.novelIndex.pop(tHandle, None) + self.noteIndex.pop(tHandle, None) + return ## @@ -142,6 +159,7 @@ class NWIndex(): theItem = self.theProject.getItem(tHandle) if theItem is None: return False if theItem.itemType != nwItemType.FILE: return False + if theItem.parHandle == self.theProject.trashRoot: return False itemClass = theItem.itemClass itemLayout = theItem.itemLayout @@ -336,19 +354,19 @@ class NWIndex(): return True - def buildReferenceList(self, theHandle): - """Build a list of files referring back to our file, specified by theHandle. + def buildReferenceList(self, tHandle): + """Build a list of files referring back to our file, specified by tHandle. """ theRefs = {} - tItem = self.theProject.getItem(theHandle) - if theHandle is None: + tItem = self.theProject.getItem(tHandle) + if tHandle is None: return theRefs theTag = None for tTag in self.tagIndex: - if theHandle == self.tagIndex[tTag][1]: + if tHandle == self.tagIndex[tTag][1]: theTag = tTag break From 93f3e01365e833ee3490d23ffaed9b606b071dea Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 11:54:18 +0100 Subject: [PATCH 3/8] Added check index function --- nw/project/index.py | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/nw/project/index.py b/nw/project/index.py index 93890664..08b872dc 100644 --- a/nw/project/index.py +++ b/nw/project/index.py @@ -19,6 +19,7 @@ from os import path from nw.project.document import NWDoc from nw.enum import nwItemType, nwItemClass, nwItemLayout from nw.constants import nwFiles +from nw.enum import nwAlert logger = logging.getLogger(__name__) @@ -49,9 +50,10 @@ class NWIndex(): def __init__(self, theProject, theParent): # Internal - self.theProject = theProject - self.theParent = theParent - self.mainConf = self.theParent.mainConf + self.theProject = theProject + self.theParent = theParent + self.mainConf = self.theParent.mainConf + self.indexBroken = False # Indices self.tagIndex = {} @@ -117,6 +119,8 @@ class NWIndex(): if "noteIndex" in theData.keys(): self.noteIndex = theData["noteIndex"] + self.checkIndex() + return True return False @@ -146,6 +150,40 @@ class NWIndex(): return True + def checkIndex(self): + """Check that the entries in the index are valid and contain the elements it should. + """ + + self.indexBroken = False + + for tTag in self.tagIndex: + if len(self.tagIndex[tTag]) != 3: + self.indexBroken = True + + for tHandle in self.refIndex: + for tEntry in self.refIndex[tHandle]: + if len(tEntry) != 4: + self.indexBroken = True + + for tHandle in self.novelIndex: + for tEntry in self.novelIndex[tHandle]: + if len(tEntry) != 4: + self.indexBroken = True + + for tHandle in self.noteIndex: + for tEntry in self.noteIndex[tHandle]: + if len(tEntry) != 4: + self.indexBroken = True + + if self.indexBroken: + self.clearIndex() + self.theParent.makeAlert( + "The project index loaded from cache contains errors. Triggering Rebuild Index.", + nwAlert.WARN + ) + + return + ## # Index Building ## From 2af6caa52707cf690cb5cc36933a2b1a10a0808e Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 11:59:20 +0100 Subject: [PATCH 4/8] Set the proper location to rebuild a broken index --- nw/gui/winmain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nw/gui/winmain.py b/nw/gui/winmain.py index 680ae966..ec9635db 100644 --- a/nw/gui/winmain.py +++ b/nw/gui/winmain.py @@ -307,6 +307,10 @@ class GuiMain(QMainWindow): if self.theProject.lastViewed is not None: self.viewDocument(self.theProject.lastViewed) + # Check if we need to rebuild the index + if self.theIndex.indexBroken: + self.rebuildIndex() + return True def saveProject(self): From 4252df9682cd1e8425095a88426a7f8f533f3016 Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 12:25:37 +0100 Subject: [PATCH 5/8] Add backups of the project xml file as a rolling version in the cashe folder --- nw/project/project.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/nw/project/project.py b/nw/project/project.py index 06c3d593..23e46e9e 100644 --- a/nw/project/project.py +++ b/nw/project/project.py @@ -14,6 +14,7 @@ import logging import nw from os import path, mkdir, listdir +from shutil import copyfile from lxml import etree from hashlib import sha256 from datetime import datetime @@ -288,6 +289,9 @@ class NWProject(): logger.debug("Saving project: %s" % self.projPath) + # Save a copy of the current file, just in case + self._maintainPrevious() + # Root element and project details logger.debug("Writing project meta") nwXML = etree.Element("novelWriterXML",attrib={ @@ -699,4 +703,41 @@ class NWProject(): itemHandle = self._makeHandle(addSeed+"!") return itemHandle + def _maintainPrevious(self): + """This function will take the current project file and copy it into the project cache + folder with an incremental file extension added. These serve as a backup in case the xml + file gets corrupted. + """ + + countFile = path.join(self.projCache, "projCount.txt") + projCount = 0 + + if path.isfile(countFile): + try: + with open(countFile, mode="r") as inFile: + projCount = int(inFile.read())+1 + except: + projCount = 0 + + if projCount > 9: + projCount = 0 + + projBackup = "%s.%d" % (nwFiles.PROJ_FILE, projCount) + + try: + copyfile( + path.join(self.projPath,self.projFile), + path.join(self.projCache,projBackup) + ) + except: + logger.error("Failed to write to file %s" % projBackup) + + try: + with open(countFile, mode="w") as outFile: + outFile.write(str(projCount)) + except: + logger.error("Failed to write to file %s" % countFile) + + return + # END Class NWProject From dd97aefd606bbbe7e34ae7aa40803bcbf9d60441 Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 12:49:13 +0100 Subject: [PATCH 6/8] Added some checks for corrupt novel project files --- nw/constants.py | 1 + nw/gui/winmain.py | 2 +- nw/project/project.py | 15 ++++++++++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/nw/constants.py b/nw/constants.py index 92c693dc..51bde4e5 100644 --- a/nw/constants.py +++ b/nw/constants.py @@ -22,6 +22,7 @@ class nwFiles(): APP_ICON = "novelWriter.svg" PROJ_FILE = "nwProject.nwx" + PROJ_COUNT = "projCount.txt" PROJ_DICT = "wordlist.txt" SESS_INFO = "sessionInfo.log" INDEX_FILE = "tagsIndex.json" diff --git a/nw/gui/winmain.py b/nw/gui/winmain.py index ec9635db..872e9c66 100644 --- a/nw/gui/winmain.py +++ b/nw/gui/winmain.py @@ -611,7 +611,7 @@ class GuiMain(QMainWindow): """ if isinstance(theMessage, list): - popMsg = "
".join(theMessage) + popMsg = " ".join(theMessage) logMsg = theMessage else: popMsg = theMessage diff --git a/nw/project/project.py b/nw/project/project.py index 23e46e9e..00f31177 100644 --- a/nw/project/project.py +++ b/nw/project/project.py @@ -197,10 +197,15 @@ class NWProject(): if not self._checkFolder(self.projMeta): return if not self._checkFolder(self.projCache): return - nwXML = etree.parse(fileName) - xRoot = nwXML.getroot() + try: + nwXML = etree.parse(fileName) + except Exception as e: + self.makeAlert(["Failed to parse project xml.",str(e)], nwAlert.ERROR) + return False + + xRoot = nwXML.getroot() + nwxRoot = xRoot.tag - nwxRoot = xRoot.tag appVersion = xRoot.attrib["appVersion"] fileVersion = xRoot.attrib["fileVersion"] @@ -709,7 +714,7 @@ class NWProject(): file gets corrupted. """ - countFile = path.join(self.projCache, "projCount.txt") + countFile = path.join(self.projCache, nwFiles.PROJ_COUNT) projCount = 0 if path.isfile(countFile): @@ -718,7 +723,7 @@ class NWProject(): projCount = int(inFile.read())+1 except: projCount = 0 - + if projCount > 9: projCount = 0 From f60eeedce8171d1d6bc25f20ca05f38fe08aa218 Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 12:56:14 +0100 Subject: [PATCH 7/8] Clear the project if open fails --- nw/project/project.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nw/project/project.py b/nw/project/project.py index 00f31177..71891185 100644 --- a/nw/project/project.py +++ b/nw/project/project.py @@ -201,6 +201,7 @@ class NWProject(): nwXML = etree.parse(fileName) except Exception as e: self.makeAlert(["Failed to parse project xml.",str(e)], nwAlert.ERROR) + self.clearProject() return False xRoot = nwXML.getroot() From 22e5f1c081dedd136d72740cc8d810ccf88c2485 Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Sat, 2 Nov 2019 13:03:58 +0100 Subject: [PATCH 8/8] Fix test --- tests/reference/gui/1_tagsIndex.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/reference/gui/1_tagsIndex.json b/tests/reference/gui/1_tagsIndex.json index 57b2ed92..8c365f5c 100644 --- a/tests/reference/gui/1_tagsIndex.json +++ b/tests/reference/gui/1_tagsIndex.json @@ -1 +1 @@ -{"tagIndex": {"Jane": [3, "2fca346db6561", "CHARACTER"], "MainPlot": [3, "02d20bbd7e394", "PLOT"], "Home": [3, "7688b6ef52555", "WORLD"]}, "refIndex": {"31489056e0916": [[5, "@pov", "Jane", 3], [6, "@plot", "MainPlot", 3], [11, "@pov", "Jane", 8], [12, "@plot", "MainPlot", 8], [13, "@location", "Home", 8], [17, "@char", "Jane", 15]]}, "novelIndex": {"31489056e0916": [[1, 1, "Novel", "SCENE"], [3, 2, "Chapter", "SCENE"], [8, 3, "Scene", "SCENE"], [15, 4, "Some Section", "SCENE"]]}} \ No newline at end of file +{"tagIndex": {"Jane": [3, "2fca346db6561", "CHARACTER"], "MainPlot": [3, "02d20bbd7e394", "PLOT"], "Home": [3, "7688b6ef52555", "WORLD"]}, "refIndex": {"31489056e0916": [[5, "@pov", "Jane", 3], [6, "@plot", "MainPlot", 3], [11, "@pov", "Jane", 8], [12, "@plot", "MainPlot", 8], [13, "@location", "Home", 8], [17, "@char", "Jane", 15]], "2fca346db6561": [], "02d20bbd7e394": [], "7688b6ef52555": []}, "novelIndex": {"31489056e0916": [[1, 1, "Novel", "SCENE"], [3, 2, "Chapter", "SCENE"], [8, 3, "Scene", "SCENE"], [15, 4, "Some Section", "SCENE"]]}, "noteIndex": {"2fca346db6561": [[1, 1, "Jane Doe", "NOTE"]], "02d20bbd7e394": [[1, 1, "Main Plot", "NOTE"]], "7688b6ef52555": [[1, 1, "Main Location", "NOTE"]]}} \ No newline at end of file