Merge pull request #253 from vkbo/project_structure

Project File Structure

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Version 0.7 [2020-xx-xx]
 
+**Project Structure**
+
+* The project folder structure has been simplified and cleaned up. We also now pin the main entry values in the main XML file. the XML file is now given version 1.1, and locking it to only be opened by version 0.7 or later. The project is converted on first open, if the user approves. PR #253.
+
 **Other Changes**
 
 * Dropped the usage of .bak copies of document files. This was the old method to ensure the document data was written successfully, but it uses twice the storage space. Instead, writing via a temp file is the safe way to save files. PR #248.

docs/source/technical.rst

@@ -30,9 +30,9 @@ The project XML file is indent-formatted, suitable for diff tools and version co
 Project Documents
 -----------------
 
-The project documents are saved in folders starting with ``data_``.
+The project documents are saved in a folder in the main project folder named ``content``.
 Each document has a file handle taken from the first 13 characters of a SHA256 hash of the system time when the file was first created.
-The documents are saved with a folder and filename derived from this hash.
+The documents are saved with a filename assembled from this hash and the file extension ``.nwd``.
 If you wish to find the physical location of a file in the project, you can either look it up in the project XML file, or select :menuselection:`Document --> Show File Details` in the menu when having the document open.
 
 The reason for this cryptic file naming is to avoid issues with file naming conventions and restrictions on different operating systems, and also to have a file name that does not depend on what the user names the files, or changes it to.

nw/core/document.py

@@ -37,8 +37,6 @@ logger = logging.getLogger(__name__)
 
 class NWDoc():
 
-    FILE_MN = "main.nwd"
-
     def __init__(self, theProject, theParent):
 
         self.mainConf    = nw.CONFIG
@@ -93,8 +91,9 @@ class NWDoc():
             if self.theItem.parHandle == self.theProject.projTree.trashRoot():
                 self.docEditable = False
 
-        docDir, docFile = self._assemblePath(self.docHandle, self.FILE_MN)
+        docDir = "content"
-        self.fileLoc = path.join(docDir,docFile)
+        docFile = self.docHandle+".nwd"
+        self.fileLoc = path.join(docDir, docFile)
         logger.debug("Opening document %s" % self.fileLoc)
         dataDir = path.join(self.theProject.projPath, docDir)
         docPath = path.join(dataDir, docFile)
@@ -139,8 +138,9 @@ class NWDoc():
         if self.docHandle is None or not self.docEditable:
             return False
 
-        docDir, docFile = self._assemblePath(self.docHandle, self.FILE_MN)
+        docDir = "content"
-        logger.debug("Saving document %s" % path.join(docDir,docFile))
+        docFile = self.docHandle+".nwd"
+        logger.debug("Saving document %s" % path.join(docDir, docFile))
         dataPath = path.join(self.theProject.projPath, docDir)
         docPath  = path.join(dataPath, docFile)
         if not path.isdir(dataPath):
@@ -159,12 +159,6 @@ class NWDoc():
             self.makeAlert(["Could not save document.",str(e)], nwAlert.ERROR)
             return False
 
-        # Remove bak files from old file save method, if one exists
-        # This part can eventually be removed
-        docBack = path.join(dataPath, docFile[:-3]+"bak")
-        if path.isfile(docBack):
-            unlink(docBack)
-
         # If we're here, the file was successfully saved, so we can
         # replace the temp file with the actual file
         if path.isfile(docPath):
@@ -179,7 +173,8 @@ class NWDoc():
         """Permanently delete a document source file and its backups
         from the project data folder.
         """
-        docDir, docFile = self._assemblePath(tHandle, self.FILE_MN)
+        docDir = "content"
+        docFile = self.docHandle+".nwd"
         dataPath = path.join(self.theProject.projPath, docDir)
         chkList = []
         chkList.append(path.join(dataPath, docFile))
@@ -226,18 +221,4 @@ class NWDoc():
 
         return theMeta, thePath
 
-    ##
-    #  Internal Functions
-    ##
-
-    @staticmethod
-    def _assemblePath(tHandle, docExt):
-        """Assemble the file path for a given handle.
-        """
-        if tHandle is None:
-            return None, None
-        docDir  = "data_"+tHandle[0]
-        docFile = tHandle[1:13]+"_"+docExt
-        return docDir, docFile
-
 # END Class NWDoc

nw/core/project.py

@@ -33,7 +33,7 @@
 import logging
 import nw
 
-from os import path, mkdir, listdir, unlink, rename
+from os import path, mkdir, listdir, unlink, rename, rmdir
 from lxml import etree
 from hashlib import sha256
 from time import time
@@ -74,6 +74,7 @@ class NWProject():
         # Class Settings
         self.projPath = None # The full path to where the currently open project is saved
         self.projMeta = None # The full path to the project's meta data folder
+        self.projData = None # The full path to the project's data folder
         self.projDict = None # The spell check dictionary
         self.projFile = None # The file name of the project main XML file
 
@@ -195,6 +196,7 @@ class NWProject():
         # Project Settings
         self.projPath    = None
         self.projMeta    = None
+        self.projData    = None
         self.projDict    = None
         self.projFile    = nwFiles.PROJ_FILE
         self.projName    = ""
@@ -247,10 +249,13 @@ class NWProject():
         logger.debug("Opening project: %s" % self.projPath)
 
         self.projMeta = path.join(self.projPath,"meta")
+        self.projData = path.join(self.projPath,"content")
         self.projDict = path.join(self.projMeta, nwFiles.PROJ_DICT)
 
         if not self._checkFolder(self.projMeta):
             return False
+        if not self._checkFolder(self.projData):
+            return False
 
         if overrideLock:
             self._clearLockFile()
@@ -291,7 +296,7 @@ class NWProject():
                 self.clearProject()
                 return False
 
-        xRoot   = nwXML.getroot()
+        xRoot = nwXML.getroot()
         nwxRoot = xRoot.tag
 
         appVersion = "Unknown"
@@ -314,13 +319,40 @@ class NWProject():
         logger.verbose("XML root is %s" % nwxRoot)
         logger.verbose("File version is %s" % fileVersion)
 
-        if not nwxRoot == "novelWriterXML" or not fileVersion == "1.0":
+        # Check File Type
+        # ===============
+        if not nwxRoot == "novelWriterXML":
             self.makeAlert(
-                "Project file does not appear to be a novelWriterXML file version 1.0",
+                "Project file does not appear to be a novelWriterXML file.",
                 nwAlert.ERROR
             )
             return False
 
+        # Check Project Storage Version
+        # =============================
+        if fileVersion == "1.0":
+            msgBox = QMessageBox()
+            msgRes = msgBox.question(self.theParent, "Old Project Version", (
+                "The project file and data is created by a %s version lower than 0.7. "
+                "Do you want to upgrade the project to the most recent format?<br><br>"
+                "Note that after the upgrade, you cannot open the project with an older "
+                "version of novelWriter any more, so make sure you have a recent backup."
+            ) % nw.__package__)
+            if msgRes == QMessageBox.Yes:
+                self._updateStorage()
+            else:
+                return False
+        elif fileVersion != "1.1":
+            self.makeAlert((
+                "Unknown or unsupported %s project format. "
+                "The project cannot be opened by this version of %s."
+            ) % (
+                nw.__package__, nw.__package__
+            ), nwAlert.ERROR)
+            return False
+
+        # Check novelWriter Version
+        # =========================
         if int(hexVersion, 16) > int(nw.__hexversion__, 16) and self.mainConf.showGUI:
             msgBox = QMessageBox()
             msgRes = msgBox.question(self.theParent, "Version Conflict", (
@@ -333,6 +365,8 @@ class NWProject():
             if msgRes != QMessageBox.Yes:
                 return False
 
+        # Start Parsing XML
+        # =================
         for xChild in xRoot:
             if xChild.tag == "project":
                 logger.debug("Found project meta")
@@ -404,16 +438,21 @@ class NWProject():
         file.
         """
         if self.projPath is None:
-            self.makeAlert("Project path not set, cannot save.", nwAlert.ERROR)
+            self.makeAlert(
+                "Project path not set, cannot save project.", nwAlert.ERROR
+            )
             return False
 
-        self.projMeta = path.join(self.projPath,"meta")
+        self.projMeta = path.join(self.projPath, "meta")
+        self.projData = path.join(self.projPath, "content")
         saveTime = time()
 
         if not self._checkFolder(self.projPath):
             return False
         if not self._checkFolder(self.projMeta):
             return False
+        if not self._checkFolder(self.projData):
+            return False
 
         logger.debug("Saving project: %s" % self.projPath)
 
@@ -427,7 +466,7 @@ class NWProject():
         nwXML = etree.Element("novelWriterXML",attrib={
             "appVersion"  : str(nw.__version__),
             "hexVersion"  : str(nw.__hexversion__),
-            "fileVersion" : "1.0",
+            "fileVersion" : "1.1",
             "saveCount"   : str(self.saveCount),
             "autoCount"   : str(self.autoCount),
             "timeStamp"   : formatTimeStamp(saveTime),
@@ -937,29 +976,20 @@ class NWProject():
         if self.projPath is None:
             return
 
-        # First, scan the project data folders
+        # Then check the files in the data folder
-        itemList = []
-        for subItem in listdir(self.projPath):
-            if subItem[:5] != "data_":
-                continue
-            dataDir = path.join(self.projPath,subItem)
-            for subFile in listdir(dataDir):
-                if subFile[-4:] == ".nwd":
-                    newItem = path.join(subItem,subFile)
-                    itemList.append(newItem)
-
-        # Then check the valid files
         orphanFiles = []
-        for fileItem in itemList:
+        for fileItem in listdir(self.projData):
-            if len(fileItem) != 28:
+            if not fileItem.endswith(".nwd"):
-                # Just to be safe, shouldn't happen
                 logger.warning("Skipping file %s" % fileItem)
                 continue
-            fHandle = fileItem[5]+fileItem[7:19]
+            if len(fileItem) != 17:
+                logger.warning("Skipping file %s" % fileItem)
+                continue
+            fHandle = fileItem[:13]
             if fHandle in self.projTree:
-                logger.debug("Checking file %s, handle %s: OK" % (fileItem,fHandle))
+                logger.debug("Checking file %s, handle %s: OK" % (fileItem, fHandle))
             else:
-                logger.debug("Checking file %s, handle %s: Orphaned" % (fileItem,fHandle))
+                logger.debug("Checking file %s, handle %s: Orphaned" % (fileItem, fHandle))
                 orphanFiles.append(fHandle)
 
         # Report status
@@ -1016,6 +1046,55 @@ class NWProject():
 
         return True
 
+    def _updateStorage(self):
+        """Updates the project storage folder from 1.0 to 1.1.
+        """
+        contDir = path.join(self.projPath, "content")
+        self._checkFolder(contDir)
+        errList = []
+
+        for projItem in listdir(self.projPath):
+            itemPath = path.join(self.projPath, projItem)
+            if not path.isdir(itemPath) or not projItem.startswith("data_"):
+                continue
+            for dataFile in listdir(itemPath):
+                dataPath = path.join(itemPath, dataFile)
+                if dataFile.endswith(".bak"):
+                    try:
+                        unlink(dataPath)
+                        logger.info("Deleted file: %s" % dataPath)
+                    except:
+                        errList.append("Failed to delete: %s" % dataPath)
+
+                elif dataFile.endswith(".nwd") and len(dataFile) == 21:
+                    tHandle = projItem[-1]+dataFile[:12]
+                    newPath = path.join(contDir, tHandle+".nwd")
+                    try:
+                        rename(dataPath, newPath)
+                        logger.info("Moved file: %s" % dataPath)
+                        logger.info("New location: %s" % newPath)
+                    except:
+                        errList.append("Failed to move: %s" % dataPath)
+
+                else:
+                    newPath = path.join(self.projPath, "unknown_"+dataFile)
+                    try:
+                        rename(dataPath, newPath)
+                        logger.info("Moved file: %s" % dataPath)
+                        logger.info("New location: %s" % newPath)
+                    except:
+                        errList.append("Failed to move: %s" % dataPath)
+            try:
+                rmdir(itemPath)
+                logger.info("Removed folder: %s" % itemPath)
+            except:
+                errList.append("Failed to delete: %s" % itemPath)
+
+        if errList:
+            self.makeAlert(errList, nwAlert.ERROR)
+
+        return
+
 # END Class NWProject
 
 # ================================================================================================ #

nw/core/tools.py

@@ -89,7 +89,7 @@ def projectMaintenance(theProject):
     if path.isdir(theProject.projPath):
         cacheDir = path.join(theProject.projPath, "cache")
         if path.isdir(cacheDir):
-            logger.info("Deprecated cache folder found")
+            logger.info("Deprecated cache folder content found")
             rmList = []
             for i in range(10):
                 rmList.append(path.join(cacheDir, "nwProject.nwx.%d" % i))
@@ -101,11 +101,6 @@ def projectMaintenance(theProject):
                         unlink(rmFile)
                     except Exception as e:
                         logger.error(str(e))
-            logger.info("Deleting: %s" % cacheDir)
-            try:
-                rmdir(cacheDir)
-            except Exception as e:
-                logger.error(str(e))
 
     # Remove no longer used meta files
     rmList = []

sample/nwProject.nwx

@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<novelWriterXML appVersion="0.6.1" hexVersion="0x000601f0" fileVersion="1.0" saveCount="186" autoCount="28" timeStamp="2020-05-28 09:59:15">
+<novelWriterXML appVersion="0.6.1" hexVersion="0x000601f0" fileVersion="1.1" saveCount="189" autoCount="28" timeStamp="2020-05-28 19:12:56">
   <project>
     <name>Sample Project</name>
     <title>Sample Project</title>
@@ -12,7 +12,7 @@
     <autoOutline>True</autoOutline>
     <lastEdited>636b6aa9b697b</lastEdited>
     <lastViewed>ba8a28a246524</lastViewed>
-    <lastWordCount>920</lastWordCount>
+    <lastWordCount>914</lastWordCount>
     <autoReplace>
       <A>B</A>
       <B>E</B>
@@ -290,9 +290,9 @@
       <expanded>False</expanded>
       <exported>True</exported>
       <layout>SCENE</layout>
-      <charCount>30</charCount>
+      <charCount>0</charCount>
-      <wordCount>6</wordCount>
+      <wordCount>0</wordCount>
-      <paraCount>1</paraCount>
+      <paraCount>0</paraCount>
       <cursorPos>36</cursorPos>
     </item>
   </content>

tests/test_gui.py

@@ -237,14 +237,14 @@ def testMainWindows(qtbot, nwTempGUI, nwRef, nwTemp):
     # Check the files
     refFile = path.join(nwTempGUI,"nwProject.nwx")
     assert cmpFiles(refFile, path.join(nwRef,"gui","1_nwProject.nwx"), [2])
-    refFile = path.join(nwTempGUI,"data_0","e17daca5f3e1_main.nwd")
+    refFile = path.join(nwTempGUI,"content","0e17daca5f3e1.nwd")
-    assert cmpFiles(refFile, path.join(nwRef,"gui","1_e17daca5f3e1_main.nwd"))
+    assert cmpFiles(refFile, path.join(nwRef,"gui","1_0e17daca5f3e1.nwd"))
-    refFile = path.join(nwTempGUI,"data_9","8010bd9270f9_main.nwd")
+    refFile = path.join(nwTempGUI,"content","98010bd9270f9.nwd")
-    assert cmpFiles(refFile, path.join(nwRef,"gui","1_8010bd9270f9_main.nwd"))
+    assert cmpFiles(refFile, path.join(nwRef,"gui","1_98010bd9270f9.nwd"))
-    refFile = path.join(nwTempGUI,"data_3","1489056e0916_main.nwd")
+    refFile = path.join(nwTempGUI,"content","31489056e0916.nwd")
-    assert cmpFiles(refFile, path.join(nwRef,"gui","1_1489056e0916_main.nwd"))
+    assert cmpFiles(refFile, path.join(nwRef,"gui","1_31489056e0916.nwd"))
-    refFile = path.join(nwTempGUI,"data_1","a6562590ef19_main.nwd")
+    refFile = path.join(nwTempGUI,"content","1a6562590ef19.nwd")
-    assert cmpFiles(refFile, path.join(nwRef,"gui","1_a6562590ef19_main.nwd"))
+    assert cmpFiles(refFile, path.join(nwRef,"gui","1_1a6562590ef19.nwd"))
 
     nwGUI.closeMain()
     # qtbot.stopForInteraction()