Merge pull request #253 from vkbo/project_structure
Project File Structure
This commit is contained in:
Veronica K. Berglyd Olsen
@@ -2,6 +2,10 @@
|
||||
|
||||
## Version 0.7 [2020-xx-xx]
|
||||
|
||||
**Project Structure**
|
||||
|
||||
* The project folder structure has been simplified and cleaned up. We also now pin the main entry values in the main XML file. the XML file is now given version 1.1, and locking it to only be opened by version 0.7 or later. The project is converted on first open, if the user approves. PR #253.
|
||||
|
||||
**Other Changes**
|
||||
|
||||
* Dropped the usage of .bak copies of document files. This was the old method to ensure the document data was written successfully, but it uses twice the storage space. Instead, writing via a temp file is the safe way to save files. PR #248.
|
||||
|
||||
@@ -30,9 +30,9 @@ The project XML file is indent-formatted, suitable for diff tools and version co
|
||||
Project Documents
|
||||
-----------------
|
||||
|
||||
The project documents are saved in folders starting with ``data_``.
|
||||
The project documents are saved in a folder in the main project folder named ``content``.
|
||||
Each document has a file handle taken from the first 13 characters of a SHA256 hash of the system time when the file was first created.
|
||||
The documents are saved with a folder and filename derived from this hash.
|
||||
The documents are saved with a filename assembled from this hash and the file extension ``.nwd``.
|
||||
If you wish to find the physical location of a file in the project, you can either look it up in the project XML file, or select :menuselection:`Document --> Show File Details` in the menu when having the document open.
|
||||
|
||||
The reason for this cryptic file naming is to avoid issues with file naming conventions and restrictions on different operating systems, and also to have a file name that does not depend on what the user names the files, or changes it to.
|
||||
|
||||
+8
-27
@@ -37,8 +37,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class NWDoc():
|
||||
|
||||
FILE_MN = "main.nwd"
|
||||
|
||||
def __init__(self, theProject, theParent):
|
||||
|
||||
self.mainConf = nw.CONFIG
|
||||
@@ -93,8 +91,9 @@ class NWDoc():
|
||||
if self.theItem.parHandle == self.theProject.projTree.trashRoot():
|
||||
self.docEditable = False
|
||||
|
||||
docDir, docFile = self._assemblePath(self.docHandle, self.FILE_MN)
|
||||
self.fileLoc = path.join(docDir,docFile)
|
||||
docDir = "content"
|
||||
docFile = self.docHandle+".nwd"
|
||||
self.fileLoc = path.join(docDir, docFile)
|
||||
logger.debug("Opening document %s" % self.fileLoc)
|
||||
dataDir = path.join(self.theProject.projPath, docDir)
|
||||
docPath = path.join(dataDir, docFile)
|
||||
@@ -139,8 +138,9 @@ class NWDoc():
|
||||
if self.docHandle is None or not self.docEditable:
|
||||
return False
|
||||
|
||||
docDir, docFile = self._assemblePath(self.docHandle, self.FILE_MN)
|
||||
logger.debug("Saving document %s" % path.join(docDir,docFile))
|
||||
docDir = "content"
|
||||
docFile = self.docHandle+".nwd"
|
||||
logger.debug("Saving document %s" % path.join(docDir, docFile))
|
||||
dataPath = path.join(self.theProject.projPath, docDir)
|
||||
docPath = path.join(dataPath, docFile)
|
||||
if not path.isdir(dataPath):
|
||||
@@ -159,12 +159,6 @@ class NWDoc():
|
||||
self.makeAlert(["Could not save document.",str(e)], nwAlert.ERROR)
|
||||
return False
|
||||
|
||||
# Remove bak files from old file save method, if one exists
|
||||
# This part can eventually be removed
|
||||
docBack = path.join(dataPath, docFile[:-3]+"bak")
|
||||
if path.isfile(docBack):
|
||||
unlink(docBack)
|
||||
|
||||
# If we're here, the file was successfully saved, so we can
|
||||
# replace the temp file with the actual file
|
||||
if path.isfile(docPath):
|
||||
@@ -179,7 +173,8 @@ class NWDoc():
|
||||
"""Permanently delete a document source file and its backups
|
||||
from the project data folder.
|
||||
"""
|
||||
docDir, docFile = self._assemblePath(tHandle, self.FILE_MN)
|
||||
docDir = "content"
|
||||
docFile = self.docHandle+".nwd"
|
||||
dataPath = path.join(self.theProject.projPath, docDir)
|
||||
chkList = []
|
||||
chkList.append(path.join(dataPath, docFile))
|
||||
@@ -226,18 +221,4 @@ class NWDoc():
|
||||
|
||||
return theMeta, thePath
|
||||
|
||||
##
|
||||
# Internal Functions
|
||||
##
|
||||
|
||||
@staticmethod
|
||||
def _assemblePath(tHandle, docExt):
|
||||
"""Assemble the file path for a given handle.
|
||||
"""
|
||||
if tHandle is None:
|
||||
return None, None
|
||||
docDir = "data_"+tHandle[0]
|
||||
docFile = tHandle[1:13]+"_"+docExt
|
||||
return docDir, docFile
|
||||
|
||||
# END Class NWDoc
|
||||
|
||||
+104
-25
@@ -33,7 +33,7 @@
|
||||
import logging
|
||||
import nw
|
||||
|
||||
from os import path, mkdir, listdir, unlink, rename
|
||||
from os import path, mkdir, listdir, unlink, rename, rmdir
|
||||
from lxml import etree
|
||||
from hashlib import sha256
|
||||
from time import time
|
||||
@@ -74,6 +74,7 @@ class NWProject():
|
||||
# Class Settings
|
||||
self.projPath = None # The full path to where the currently open project is saved
|
||||
self.projMeta = None # The full path to the project's meta data folder
|
||||
self.projData = None # The full path to the project's data folder
|
||||
self.projDict = None # The spell check dictionary
|
||||
self.projFile = None # The file name of the project main XML file
|
||||
|
||||
@@ -195,6 +196,7 @@ class NWProject():
|
||||
# Project Settings
|
||||
self.projPath = None
|
||||
self.projMeta = None
|
||||
self.projData = None
|
||||
self.projDict = None
|
||||
self.projFile = nwFiles.PROJ_FILE
|
||||
self.projName = ""
|
||||
@@ -247,10 +249,13 @@ class NWProject():
|
||||
logger.debug("Opening project: %s" % self.projPath)
|
||||
|
||||
self.projMeta = path.join(self.projPath,"meta")
|
||||
self.projData = path.join(self.projPath,"content")
|
||||
self.projDict = path.join(self.projMeta, nwFiles.PROJ_DICT)
|
||||
|
||||
if not self._checkFolder(self.projMeta):
|
||||
return False
|
||||
if not self._checkFolder(self.projData):
|
||||
return False
|
||||
|
||||
if overrideLock:
|
||||
self._clearLockFile()
|
||||
@@ -291,7 +296,7 @@ class NWProject():
|
||||
self.clearProject()
|
||||
return False
|
||||
|
||||
xRoot = nwXML.getroot()
|
||||
xRoot = nwXML.getroot()
|
||||
nwxRoot = xRoot.tag
|
||||
|
||||
appVersion = "Unknown"
|
||||
@@ -314,13 +319,40 @@ class NWProject():
|
||||
logger.verbose("XML root is %s" % nwxRoot)
|
||||
logger.verbose("File version is %s" % fileVersion)
|
||||
|
||||
if not nwxRoot == "novelWriterXML" or not fileVersion == "1.0":
|
||||
# Check File Type
|
||||
# ===============
|
||||
if not nwxRoot == "novelWriterXML":
|
||||
self.makeAlert(
|
||||
"Project file does not appear to be a novelWriterXML file version 1.0",
|
||||
"Project file does not appear to be a novelWriterXML file.",
|
||||
nwAlert.ERROR
|
||||
)
|
||||
return False
|
||||
|
||||
# Check Project Storage Version
|
||||
# =============================
|
||||
if fileVersion == "1.0":
|
||||
msgBox = QMessageBox()
|
||||
msgRes = msgBox.question(self.theParent, "Old Project Version", (
|
||||
"The project file and data is created by a %s version lower than 0.7. "
|
||||
"Do you want to upgrade the project to the most recent format?<br><br>"
|
||||
"Note that after the upgrade, you cannot open the project with an older "
|
||||
"version of novelWriter any more, so make sure you have a recent backup."
|
||||
) % nw.__package__)
|
||||
if msgRes == QMessageBox.Yes:
|
||||
self._updateStorage()
|
||||
else:
|
||||
return False
|
||||
elif fileVersion != "1.1":
|
||||
self.makeAlert((
|
||||
"Unknown or unsupported %s project format. "
|
||||
"The project cannot be opened by this version of %s."
|
||||
) % (
|
||||
nw.__package__, nw.__package__
|
||||
), nwAlert.ERROR)
|
||||
return False
|
||||
|
||||
# Check novelWriter Version
|
||||
# =========================
|
||||
if int(hexVersion, 16) > int(nw.__hexversion__, 16) and self.mainConf.showGUI:
|
||||
msgBox = QMessageBox()
|
||||
msgRes = msgBox.question(self.theParent, "Version Conflict", (
|
||||
@@ -333,6 +365,8 @@ class NWProject():
|
||||
if msgRes != QMessageBox.Yes:
|
||||
return False
|
||||
|
||||
# Start Parsing XML
|
||||
# =================
|
||||
for xChild in xRoot:
|
||||
if xChild.tag == "project":
|
||||
logger.debug("Found project meta")
|
||||
@@ -404,16 +438,21 @@ class NWProject():
|
||||
file.
|
||||
"""
|
||||
if self.projPath is None:
|
||||
self.makeAlert("Project path not set, cannot save.", nwAlert.ERROR)
|
||||
self.makeAlert(
|
||||
"Project path not set, cannot save project.", nwAlert.ERROR
|
||||
)
|
||||
return False
|
||||
|
||||
self.projMeta = path.join(self.projPath,"meta")
|
||||
self.projMeta = path.join(self.projPath, "meta")
|
||||
self.projData = path.join(self.projPath, "content")
|
||||
saveTime = time()
|
||||
|
||||
if not self._checkFolder(self.projPath):
|
||||
return False
|
||||
if not self._checkFolder(self.projMeta):
|
||||
return False
|
||||
if not self._checkFolder(self.projData):
|
||||
return False
|
||||
|
||||
logger.debug("Saving project: %s" % self.projPath)
|
||||
|
||||
@@ -427,7 +466,7 @@ class NWProject():
|
||||
nwXML = etree.Element("novelWriterXML",attrib={
|
||||
"appVersion" : str(nw.__version__),
|
||||
"hexVersion" : str(nw.__hexversion__),
|
||||
"fileVersion" : "1.0",
|
||||
"fileVersion" : "1.1",
|
||||
"saveCount" : str(self.saveCount),
|
||||
"autoCount" : str(self.autoCount),
|
||||
"timeStamp" : formatTimeStamp(saveTime),
|
||||
@@ -937,29 +976,20 @@ class NWProject():
|
||||
if self.projPath is None:
|
||||
return
|
||||
|
||||
# First, scan the project data folders
|
||||
itemList = []
|
||||
for subItem in listdir(self.projPath):
|
||||
if subItem[:5] != "data_":
|
||||
continue
|
||||
dataDir = path.join(self.projPath,subItem)
|
||||
for subFile in listdir(dataDir):
|
||||
if subFile[-4:] == ".nwd":
|
||||
newItem = path.join(subItem,subFile)
|
||||
itemList.append(newItem)
|
||||
|
||||
# Then check the valid files
|
||||
# Then check the files in the data folder
|
||||
orphanFiles = []
|
||||
for fileItem in itemList:
|
||||
if len(fileItem) != 28:
|
||||
# Just to be safe, shouldn't happen
|
||||
for fileItem in listdir(self.projData):
|
||||
if not fileItem.endswith(".nwd"):
|
||||
logger.warning("Skipping file %s" % fileItem)
|
||||
continue
|
||||
fHandle = fileItem[5]+fileItem[7:19]
|
||||
if len(fileItem) != 17:
|
||||
logger.warning("Skipping file %s" % fileItem)
|
||||
continue
|
||||
fHandle = fileItem[:13]
|
||||
if fHandle in self.projTree:
|
||||
logger.debug("Checking file %s, handle %s: OK" % (fileItem,fHandle))
|
||||
logger.debug("Checking file %s, handle %s: OK" % (fileItem, fHandle))
|
||||
else:
|
||||
logger.debug("Checking file %s, handle %s: Orphaned" % (fileItem,fHandle))
|
||||
logger.debug("Checking file %s, handle %s: Orphaned" % (fileItem, fHandle))
|
||||
orphanFiles.append(fHandle)
|
||||
|
||||
# Report status
|
||||
@@ -1016,6 +1046,55 @@ class NWProject():
|
||||
|
||||
return True
|
||||
|
||||
def _updateStorage(self):
|
||||
"""Updates the project storage folder from 1.0 to 1.1.
|
||||
"""
|
||||
contDir = path.join(self.projPath, "content")
|
||||
self._checkFolder(contDir)
|
||||
errList = []
|
||||
|
||||
for projItem in listdir(self.projPath):
|
||||
itemPath = path.join(self.projPath, projItem)
|
||||
if not path.isdir(itemPath) or not projItem.startswith("data_"):
|
||||
continue
|
||||
for dataFile in listdir(itemPath):
|
||||
dataPath = path.join(itemPath, dataFile)
|
||||
if dataFile.endswith(".bak"):
|
||||
try:
|
||||
unlink(dataPath)
|
||||
logger.info("Deleted file: %s" % dataPath)
|
||||
except:
|
||||
errList.append("Failed to delete: %s" % dataPath)
|
||||
|
||||
elif dataFile.endswith(".nwd") and len(dataFile) == 21:
|
||||
tHandle = projItem[-1]+dataFile[:12]
|
||||
newPath = path.join(contDir, tHandle+".nwd")
|
||||
try:
|
||||
rename(dataPath, newPath)
|
||||
logger.info("Moved file: %s" % dataPath)
|
||||
logger.info("New location: %s" % newPath)
|
||||
except:
|
||||
errList.append("Failed to move: %s" % dataPath)
|
||||
|
||||
else:
|
||||
newPath = path.join(self.projPath, "unknown_"+dataFile)
|
||||
try:
|
||||
rename(dataPath, newPath)
|
||||
logger.info("Moved file: %s" % dataPath)
|
||||
logger.info("New location: %s" % newPath)
|
||||
except:
|
||||
errList.append("Failed to move: %s" % dataPath)
|
||||
try:
|
||||
rmdir(itemPath)
|
||||
logger.info("Removed folder: %s" % itemPath)
|
||||
except:
|
||||
errList.append("Failed to delete: %s" % itemPath)
|
||||
|
||||
if errList:
|
||||
self.makeAlert(errList, nwAlert.ERROR)
|
||||
|
||||
return
|
||||
|
||||
# END Class NWProject
|
||||
|
||||
# ================================================================================================ #
|
||||
|
||||
+1
-6
@@ -89,7 +89,7 @@ def projectMaintenance(theProject):
|
||||
if path.isdir(theProject.projPath):
|
||||
cacheDir = path.join(theProject.projPath, "cache")
|
||||
if path.isdir(cacheDir):
|
||||
logger.info("Deprecated cache folder found")
|
||||
logger.info("Deprecated cache folder content found")
|
||||
rmList = []
|
||||
for i in range(10):
|
||||
rmList.append(path.join(cacheDir, "nwProject.nwx.%d" % i))
|
||||
@@ -101,11 +101,6 @@ def projectMaintenance(theProject):
|
||||
unlink(rmFile)
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
logger.info("Deleting: %s" % cacheDir)
|
||||
try:
|
||||
rmdir(cacheDir)
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
|
||||
# Remove no longer used meta files
|
||||
rmList = []
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<novelWriterXML appVersion="0.6.1" hexVersion="0x000601f0" fileVersion="1.0" saveCount="186" autoCount="28" timeStamp="2020-05-28 09:59:15">
|
||||
<novelWriterXML appVersion="0.6.1" hexVersion="0x000601f0" fileVersion="1.1" saveCount="189" autoCount="28" timeStamp="2020-05-28 19:12:56">
|
||||
<project>
|
||||
<name>Sample Project</name>
|
||||
<title>Sample Project</title>
|
||||
@@ -12,7 +12,7 @@
|
||||
<autoOutline>True</autoOutline>
|
||||
<lastEdited>636b6aa9b697b</lastEdited>
|
||||
<lastViewed>ba8a28a246524</lastViewed>
|
||||
<lastWordCount>920</lastWordCount>
|
||||
<lastWordCount>914</lastWordCount>
|
||||
<autoReplace>
|
||||
<A>B</A>
|
||||
<B>E</B>
|
||||
@@ -290,9 +290,9 @@
|
||||
<expanded>False</expanded>
|
||||
<exported>True</exported>
|
||||
<layout>SCENE</layout>
|
||||
<charCount>30</charCount>
|
||||
<wordCount>6</wordCount>
|
||||
<paraCount>1</paraCount>
|
||||
<charCount>0</charCount>
|
||||
<wordCount>0</wordCount>
|
||||
<paraCount>0</paraCount>
|
||||
<cursorPos>36</cursorPos>
|
||||
</item>
|
||||
</content>
|
||||
|
||||
+8
-8
@@ -237,14 +237,14 @@ def testMainWindows(qtbot, nwTempGUI, nwRef, nwTemp):
|
||||
# Check the files
|
||||
refFile = path.join(nwTempGUI,"nwProject.nwx")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_nwProject.nwx"), [2])
|
||||
refFile = path.join(nwTempGUI,"data_0","e17daca5f3e1_main.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_e17daca5f3e1_main.nwd"))
|
||||
refFile = path.join(nwTempGUI,"data_9","8010bd9270f9_main.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_8010bd9270f9_main.nwd"))
|
||||
refFile = path.join(nwTempGUI,"data_3","1489056e0916_main.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_1489056e0916_main.nwd"))
|
||||
refFile = path.join(nwTempGUI,"data_1","a6562590ef19_main.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_a6562590ef19_main.nwd"))
|
||||
refFile = path.join(nwTempGUI,"content","0e17daca5f3e1.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_0e17daca5f3e1.nwd"))
|
||||
refFile = path.join(nwTempGUI,"content","98010bd9270f9.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_98010bd9270f9.nwd"))
|
||||
refFile = path.join(nwTempGUI,"content","31489056e0916.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_31489056e0916.nwd"))
|
||||
refFile = path.join(nwTempGUI,"content","1a6562590ef19.nwd")
|
||||
assert cmpFiles(refFile, path.join(nwRef,"gui","1_1a6562590ef19.nwd"))
|
||||
|
||||
nwGUI.closeMain()
|
||||
# qtbot.stopForInteraction()
|
||||
|
||||
Reference in New Issue
Block a user