Cleane dup project open and handling of legacy project content

nw/constants/constants.py

@@ -39,6 +39,8 @@ class nwFiles():
     PROJ_FILE   = "nwProject.nwx"
     PROJ_DICT   = "wordlist.txt"
     PROJ_LOCK   = "nwProject.lock"
+    TOC_TXT     = "ToC.txt"
+    TOC_JSON    = "ToC.json"
     SESS_INFO   = "sessionInfo.log"
     INDEX_FILE  = "tagsIndex.json"
     OPTS_FILE   = "guiOptions.json"

nw/core/__init__.py

@@ -9,7 +9,6 @@ from nw.core.spellcheck import NWSpellSimple
 from nw.core.tokenizer import Tokenizer
 from nw.core.tohtml import ToHtml
 from nw.core.tools import countWords
-from nw.core.tools import projectMaintenance
 from nw.core.tools import numberToWord
 
 __all__ = [
@@ -22,6 +21,5 @@ __all__ = [
     "Tokenizer",
     "ToHtml",
     "countWords",
-    "projectMaintenance",
     "numberToWord",
 ]

nw/core/project.py

@@ -42,7 +42,6 @@ from shutil import make_archive
 from PyQt5.QtWidgets import QMessageBox
 
 from nw.gui.tools import OptionState
-from nw.core.tools import projectMaintenance
 from nw.core.document import NWDoc
 from nw.common import checkString, checkBool, checkInt, formatTimeStamp
 from nw.constants import (
@@ -72,11 +71,12 @@ class NWProject():
         self.autoCount   = 0     # Meta data: number of automatic saves
 
         # Class Settings
-        self.projPath = None # The full path to where the currently open project is saved
+        self.projPath    = None # The full path to where the currently open project is saved
-        self.projMeta = None # The full path to the project's meta data folder
+        self.projMeta    = None # The full path to the project's meta data folder
-        self.projData = None # The full path to the project's data folder
+        self.projCache   = None # The full path to the project's cache folder
-        self.projDict = None # The spell check dictionary
+        self.projContent = None # The full path to the project's content folder
-        self.projFile = None # The file name of the project main XML file
+        self.projDict    = None # The spell check dictionary
+        self.projFile    = None # The file name of the project main XML file
 
         # Project Meta
         self.projName    = "" # Project name (working title)
@@ -196,7 +196,8 @@ class NWProject():
         # Project Settings
         self.projPath    = None
         self.projMeta    = None
-        self.projData    = None
+        self.projCache   = None
+        self.projContent = None
         self.projDict    = None
         self.projFile    = nwFiles.PROJ_FILE
         self.projName    = ""
@@ -248,14 +249,37 @@ class NWProject():
         self.projPath = path.abspath(path.dirname(fileName))
         logger.debug("Opening project: %s" % self.projPath)
 
-        self.projMeta = path.join(self.projPath,"meta")
+        # Standard Folders and Files
-        self.projData = path.join(self.projPath,"content")
+        # ==========================
-        self.projDict = path.join(self.projMeta, nwFiles.PROJ_DICT)
+
+        self.projMeta    = path.join(self.projPath, "meta")
+        self.projCache   = path.join(self.projPath, "cache")
+        self.projContent = path.join(self.projPath, "content")
+        self.projDict    = path.join(self.projMeta, nwFiles.PROJ_DICT)
 
         if not self._checkFolder(self.projMeta):
             return False
-        if not self._checkFolder(self.projData):
+        if not self._checkFolder(self.projCache):
             return False
+        if not self._checkFolder(self.projContent):
+            return False
+
+        # Check for Old Legacy Data
+        # =========================
+
+        errList = []
+        for projItem in listdir(self.projPath):
+            logger.verbose("Project contains: %s" % projItem)
+            if projItem.startswith("data_"):
+                self._legacyDataFolder(projItem)
+
+        if errList:
+            self.makeAlert(errList, nwAlert.ERROR)
+
+        self._deprecatedFiles()
+
+        # Project Lock
+        # ============
 
         if overrideLock:
             self._clearLockFile()
@@ -272,10 +296,8 @@ class NWProject():
         else:
             logger.verbose("Project is not locked")
 
-        try:
+        # Open The Project XML File
-            projectMaintenance(self)
+        # =========================
-        except Exception as E:
-            logger.error(str(E))
 
         try:
             nwXML = etree.parse(fileName)
@@ -321,6 +343,7 @@ class NWProject():
 
         # Check File Type
         # ===============
+
         if not nwxRoot == "novelWriterXML":
             self.makeAlert(
                 "Project file does not appear to be a novelWriterXML file.",
@@ -330,14 +353,17 @@ class NWProject():
 
         # Check Project Storage Version
         # =============================
+
         if fileVersion == "1.0":
             msgBox = QMessageBox()
             msgRes = msgBox.question(self.theParent, "Old Project Version", (
                 "The project file and data is created by a %s version lower than 0.7. "
                 "Do you want to upgrade the project to the most recent format?<br><br>"
                 "Note that after the upgrade, you cannot open the project with an older "
-                "version of novelWriter any more, so make sure you have a recent backup."
+                "version of %s any more, so make sure you have a recent backup."
-            ) % nw.__package__)
+            ) % (
+                nw.__package__, nw.__package__
+            ))
             if msgRes == QMessageBox.Yes:
                 self._updateStorage()
             else:
@@ -353,6 +379,7 @@ class NWProject():
 
         # Check novelWriter Version
         # =========================
+
         if int(hexVersion, 16) > int(nw.__hexversion__, 16) and self.mainConf.showGUI:
             msgBox = QMessageBox()
             msgRes = msgBox.question(self.theParent, "Version Conflict", (
@@ -367,6 +394,7 @@ class NWProject():
 
         # Start Parsing XML
         # =================
+
         for xChild in xRoot:
             if xChild.tag == "project":
                 logger.debug("Found project meta")
@@ -384,6 +412,7 @@ class NWProject():
                         self.bookAuthors.append(xItem.text)
                     elif xItem.tag == "backup":
                         self.doBackup = checkBool(xItem.text, False)
+
             elif xChild.tag == "settings":
                 logger.debug("Found project settings")
                 for xItem in xChild:
@@ -411,6 +440,7 @@ class NWProject():
                         for xEntry in xItem:
                             titleFormat[xEntry.tag] = checkString(xEntry.text, "", False)
                         self.setTitleFormat(titleFormat)
+
             elif xChild.tag == "content":
                 logger.debug("Found project content")
                 self.projTree.unpackXML(xChild)
@@ -444,14 +474,14 @@ class NWProject():
             return False
 
         self.projMeta = path.join(self.projPath, "meta")
-        self.projData = path.join(self.projPath, "content")
+        self.projContent = path.join(self.projPath, "content")
         saveTime = time()
 
         if not self._checkFolder(self.projPath):
             return False
         if not self._checkFolder(self.projMeta):
             return False
-        if not self._checkFolder(self.projData):
+        if not self._checkFolder(self.projContent):
             return False
 
         logger.debug("Saving project: %s" % self.projPath)
@@ -978,7 +1008,7 @@ class NWProject():
 
         # Then check the files in the data folder
         orphanFiles = []
-        for fileItem in listdir(self.projData):
+        for fileItem in listdir(self.projContent):
             if not fileItem.endswith(".nwd"):
                 logger.warning("Skipping file %s" % fileItem)
                 continue
@@ -1046,52 +1076,113 @@ class NWProject():
 
         return True
 
-    def _updateStorage(self):
+    ##
-        """Updates the project storage folder from 1.0 to 1.1.
+    #  Legacy Data Structure Handlers
+    ##
+
+    def _legacyDataFolder(self, theFolder):
+        """Clean up legacy data folders.
         """
-        contDir = path.join(self.projPath, "content")
-        self._checkFolder(contDir)
         errList = []
+        theData = path.join(self.projPath, theFolder)
+        if not path.isdir(theData):
+            errList.append("Not a folder: %s" % theData)
+            return errList
 
-        for projItem in listdir(self.projPath):
+        logger.info("Old data folder %s found" % theFolder)
-            itemPath = path.join(self.projPath, projItem)
+
-            if not path.isdir(itemPath) or not projItem.startswith("data_"):
+        # Move Documents to Content
+        # =========================
+        for dataItem in listdir(theData):
+            theFile = path.join(theData, dataItem)
+            if not path.isfile(theFile):
+                theErr = self._moveUnknownItem(theData, dataItem)
+                if theErr:
+                    errList.append(theErr)
                 continue
-            for dataFile in listdir(itemPath):
-                dataPath = path.join(itemPath, dataFile)
-                if dataFile.endswith(".bak"):
-                    try:
-                        unlink(dataPath)
-                        logger.info("Deleted file: %s" % dataPath)
-                    except:
-                        errList.append("Failed to delete: %s" % dataPath)
 
-                elif dataFile.endswith(".nwd") and len(dataFile) == 21:
+            if len(dataItem) == 21 and dataItem.endswith("_main.nwd"):
-                    tHandle = projItem[-1]+dataFile[:12]
+                tHandle = theFolder[-1]+dataItem[:12]
-                    newPath = path.join(contDir, tHandle+".nwd")
+                newPath = path.join(self.projContent, tHandle+".nwd")
-                    try:
+                try:
-                        rename(dataPath, newPath)
+                    rename(theFile, newPath)
-                        logger.info("Moved file: %s" % dataPath)
+                    logger.info("Moved file: %s" % theFile)
-                        logger.info("New location: %s" % newPath)
+                    logger.info("New location: %s" % newPath)
-                    except:
+                except Exception as e:
-                        errList.append("Failed to move: %s" % dataPath)
+                    logger.error(str(e))
+                    errList.append("Could not move: %s" % theFile)
 
-                else:
+            elif len(dataItem) == 21 and dataItem.endswith("_main.bak"):
-                    newPath = path.join(self.projPath, "unknown_"+dataFile)
+                try:
-                    try:
+                    unlink(theFile)
-                        rename(dataPath, newPath)
+                    logger.info("Deleted file: %s" % theFile)
-                        logger.info("Moved file: %s" % dataPath)
+                except Exception as e:
-                        logger.info("New location: %s" % newPath)
+                    logger.error(str(e))
-                    except:
+                    errList.append("Could not delete: %s" % theFile)
-                        errList.append("Failed to move: %s" % dataPath)
-            try:
-                rmdir(itemPath)
-                logger.info("Removed folder: %s" % itemPath)
-            except:
-                errList.append("Failed to delete: %s" % itemPath)
 
-        if errList:
+            else:
-            self.makeAlert(errList, nwAlert.ERROR)
+                theErr = self._moveUnknownItem(theData, dataItem)
+                if theErr:
+                    errList.append(theErr)
+
+        # Remove Data Folder
+        # ==================
+        try:
+            rmdir(theData)
+            logger.info("Removed folder: %s" % theFolder)
+        except:
+            errList.append("Failed to remove: %s" % theFolder)
+
+        return errList
+
+    def _moveUnknownItem(self, theDir, theItem):
+        """Move an item that doesn't belong in the project folder to
+        a junk folder.
+        """
+        theJunk = path.join(self.projPath, "junk")
+        if not self._checkFolder(theJunk):
+            return "Could not make folder: %s" % theJunk
+
+        theSrc = path.join(theDir, theItem)
+        theDst = path.join(theJunk, theItem)
+
+        try:
+            rename(theSrc, theDst)
+            logger.info("Moved to junk: %s" % theSrc)
+        except Exception as e:
+            logger.error(str(e))
+            return "Could not move item %s to junk." % theSrc
+
+        return ""
+
+    def _deprecatedFiles(self):
+        """Delete files that are no longer used by novelWriter.
+        """
+        rmList = []
+        rmList.append(path.join(self.projCache, "nwProject.nwx.0"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.1"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.2"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.3"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.4"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.5"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.6"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.7"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.8"))
+        rmList.append(path.join(self.projCache, "nwProject.nwx.9"))
+        rmList.append(path.join(self.projMeta,  "mainOptions.json"))
+        rmList.append(path.join(self.projMeta,  "exportOptions.json"))
+        rmList.append(path.join(self.projMeta,  "outlineOptions.json"))
+        rmList.append(path.join(self.projMeta,  "timelineOptions.json"))
+        rmList.append(path.join(self.projMeta,  "docMergeOptions.json"))
+        rmList.append(path.join(self.projMeta,  "sessionLogOptions.json"))
+
+        for rmFile in rmList:
+            if path.isfile(rmFile):
+                logger.info("Deleting: %s" % rmFile)
+                try:
+                    unlink(rmFile)
+                except Exception as e:
+                    logger.error(str(e))
 
         return
 

nw/core/tools.py

@@ -8,7 +8,6 @@
  File History:
  Created: 2019-04-22 [0.0.1] countWords
  Created: 2019-10-13 [0.2.3] numberToWord, _numberToWordEN
- Created: 2020-02-13 [0.4.3] projectMaintenance
  Merged:  2020-05-08 [0.4.5] All of the above into this file
 
  This file is a part of novelWriter
@@ -81,45 +80,6 @@ def countWords(theText):
 
     return charCount, wordCount, paraCount
 
-def projectMaintenance(theProject):
-    """Wrapper class for handling various tasks related to managing old
-    projects with content from older versions of novelWriter.
-    """
-    # Remove no longer used project cache folder
-    if path.isdir(theProject.projPath):
-        cacheDir = path.join(theProject.projPath, "cache")
-        if path.isdir(cacheDir):
-            logger.info("Deprecated cache folder content found")
-            rmList = []
-            for i in range(10):
-                rmList.append(path.join(cacheDir, "nwProject.nwx.%d" % i))
-            rmList.append(path.join(cacheDir, "projCount.txt"))
-            for rmFile in rmList:
-                if path.isfile(rmFile):
-                    logger.info("Deleting: %s" % rmFile)
-                    try:
-                        unlink(rmFile)
-                    except Exception as e:
-                        logger.error(str(e))
-
-    # Remove no longer used meta files
-    rmList = []
-    rmList.append(path.join(theProject.projMeta, "mainOptions.json"))
-    rmList.append(path.join(theProject.projMeta, "exportOptions.json"))
-    rmList.append(path.join(theProject.projMeta, "outlineOptions.json"))
-    rmList.append(path.join(theProject.projMeta, "timelineOptions.json"))
-    rmList.append(path.join(theProject.projMeta, "docMergeOptions.json"))
-    rmList.append(path.join(theProject.projMeta, "sessionLogOptions.json"))
-    for rmFile in rmList:
-        if path.isfile(rmFile):
-            logger.info("Deleting: %s" % rmFile)
-            try:
-                unlink(rmFile)
-            except Exception as e:
-                logger.error(str(e))
-
-    return
-
 def numberToWord(numVal, theLanguage):
     """Wrapper for converting numbers to words for chapter headings.
     """

sample/content/ae7339df26ded.nwd

@@ -2,6 +2,6 @@
 ### We Found John!
 
 @pov: John
-@location: Mars, OuterSpace
+@location: Mars
 
 Jane has been searching for a while, and she finally found John on Mars. He was indeed in space! What was he doing on Mars anyway? Well, it turns out, he was farming potatoes.

sample/content/b8136a5a774a0.nwd

@@ -1,4 +1,4 @@
-%%~ b8136a5a774a0:7031beac91f75:Delete Me!
+%%~ b8136a5a774a0:98acd8c76c93a:Delete Me!
 ### Delete Me!
 
 This scene is trash.

sample/content/edca4be2fcaf8.nwd

@@ -1,4 +1,4 @@
-%%~ edca4be2fcaf8:7031beac91f75:Part One
+%%~ edca4be2fcaf8:7031beac91f75:Part 1
 # Part One
 
 The first part.

sample/content/f1471bef9f2ae.nwd

@@ -5,10 +5,4 @@
 
 Space … it’s an awful lot of nothing, with bits in it here and there. Some of which, people like to call home.
 
-## Outer Space
 
-@tag: OuterSpace
-
-Now even further into space!
-
-You can have more than one tag in a file, as long as there is only one tag per heading.

sample/nwProject.nwx

@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<novelWriterXML appVersion="0.6.2" hexVersion="0x000602f0" fileVersion="1.1" saveCount="189" autoCount="30" timeStamp="2020-05-28 20:19:28">
+<novelWriterXML appVersion="0.7.0rc1" hexVersion="0x000700c1" fileVersion="1.1" saveCount="193" autoCount="29" timeStamp="2020-05-29 22:52:06">
   <project>
     <name>Sample Project</name>
     <title>Sample Project</title>
@@ -11,8 +11,8 @@
     <spellCheck>True</spellCheck>
     <autoOutline>True</autoOutline>
     <lastEdited>636b6aa9b697b</lastEdited>
-    <lastViewed>bc0cbd2a407f3</lastViewed>
+    <lastViewed>ba8a28a246524</lastViewed>
-    <lastWordCount>941</lastWordCount>
+    <lastWordCount>914</lastWordCount>
     <autoReplace>
       <A>B</A>
       <B>E</B>
@@ -73,7 +73,7 @@
       <expanded>False</expanded>
       <exported>True</exported>
       <layout>PAGE</layout>
-      <charCount>210</charCount>
+      <charCount>208</charCount>
       <wordCount>40</wordCount>
       <paraCount>2</paraCount>
       <cursorPos>213</cursorPos>
@@ -89,7 +89,7 @@
       <charCount>23</charCount>
       <wordCount>5</wordCount>
       <paraCount>1</paraCount>
-      <cursorPos>27</cursorPos>
+      <cursorPos>0</cursorPos>
     </item>
     <item handle="e7ded148d6e4a" order="3" parent="7031beac91f75">
       <name>A Folder</name>
@@ -122,7 +122,7 @@
       <charCount>1199</charCount>
       <wordCount>216</wordCount>
       <paraCount>7</paraCount>
-      <cursorPos>1066</cursorPos>
+      <cursorPos>1266</cursorPos>
     </item>
     <item handle="bc0cbd2a407f3" order="2" parent="e7ded148d6e4a">
       <name>Another Scene</name>
@@ -135,7 +135,7 @@
       <charCount>476</charCount>
       <wordCount>93</wordCount>
       <paraCount>3</paraCount>
-      <cursorPos>428</cursorPos>
+      <cursorPos>551</cursorPos>
     </item>
     <item handle="ba8a28a246524" order="3" parent="e7ded148d6e4a">
       <name>Interlude</name>
@@ -148,7 +148,7 @@
       <charCount>633</charCount>
       <wordCount>101</wordCount>
       <paraCount>3</paraCount>
-      <cursorPos>752</cursorPos>
+      <cursorPos>1238</cursorPos>
     </item>
     <item handle="96b68994dfa3d" order="4" parent="e7ded148d6e4a">
       <name>A Note on Structure</name>
@@ -161,7 +161,7 @@
       <charCount>1692</charCount>
       <wordCount>313</wordCount>
       <paraCount>6</paraCount>
-      <cursorPos>551</cursorPos>
+      <cursorPos>1721</cursorPos>
     </item>
     <item handle="88706ddc78b1b" order="5" parent="e7ded148d6e4a">
       <name>Chapter Two</name>
@@ -174,7 +174,7 @@
       <charCount>139</charCount>
       <wordCount>28</wordCount>
       <paraCount>1</paraCount>
-      <cursorPos>242</cursorPos>
+      <cursorPos>343</cursorPos>
     </item>
     <item handle="ae7339df26ded" order="6" parent="e7ded148d6e4a">
       <name>We Found John!</name>
@@ -214,7 +214,7 @@
       <charCount>49</charCount>
       <wordCount>9</wordCount>
       <paraCount>1</paraCount>
-      <cursorPos>65</cursorPos>
+      <cursorPos>24</cursorPos>
     </item>
     <item handle="bb2c23b3c42cc" order="1" parent="f7e2d9f330615">
       <name>Jane Smith</name>
@@ -227,7 +227,7 @@
       <charCount>55</charCount>
       <wordCount>9</wordCount>
       <paraCount>1</paraCount>
-      <cursorPos>71</cursorPos>
+      <cursorPos>25</cursorPos>
     </item>
     <item handle="15c4492bd5107" order="2" parent="None">
       <name>Locations</name>
@@ -247,7 +247,7 @@
       <charCount>76</charCount>
       <wordCount>15</wordCount>
       <paraCount>1</paraCount>
-      <cursorPos>93</cursorPos>
+      <cursorPos>20</cursorPos>
     </item>
     <item handle="f1471bef9f2ae" order="1" parent="15c4492bd5107">
       <name>Space</name>
@@ -257,10 +257,10 @@
       <expanded>False</expanded>
       <exported>True</exported>
       <layout>NOTE</layout>
-      <charCount>241</charCount>
+      <charCount>115</charCount>
-      <wordCount>51</wordCount>
+      <wordCount>24</wordCount>
-      <paraCount>3</paraCount>
+      <paraCount>1</paraCount>
-      <cursorPos>135</cursorPos>
+      <cursorPos>133</cursorPos>
     </item>
     <item handle="5eaea4e8cdee8" order="2" parent="15c4492bd5107">
       <name>Mars</name>