Rewrite project consistency chech as part of the tree class

novelwriter/core/document.py

@@ -1,7 +1,6 @@
 """
 novelWriter – Project Document
 ==============================
-Data class for a single novelWriter document
 
 File History:
 Created: 2018-09-29 [0.0.1]
@@ -41,8 +40,15 @@ logger = logging.getLogger(__name__)
 
 
 class NWDocument:
+    """Core: Document Class
 
-    def __init__(self, project: NWProject, tHandle: str) -> None:
+    A Class wrapping a single novelWriter document file. It represents
+    a project item of nwItemType FILE. The file is not guaranteed to
+    exist, even if the item does. In the case it doesn't exist, reading
+    it returns a None rather than an empty or non-empty string.
+    """
+
+    def __init__(self, project: NWProject, tHandle: str | None) -> None:
 
         self._project = project
 

novelwriter/core/index.py

@@ -76,7 +76,7 @@ class NWIndex:
     a rebuild of the index data.
     """
 
-    def __init__(self, project):
+    def __init__(self, project: NWProject):
 
         self._project = project
 
@@ -197,7 +197,7 @@ class NWIndex:
         logger.debug("Checking index")
 
         # Check that all files are indexed
-        for fHandle in self._project.projFiles:
+        for fHandle in self._project.storage.scanContent():
             if fHandle not in self._itemIndex:
                 logger.warning("Item '%s' is not in the index", fHandle)
                 self.reIndexHandle(fHandle)

novelwriter/core/project.py

@@ -45,7 +45,7 @@ from novelwriter.core.sessions import NWSessionLog
 from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter, XMLReadState
 from novelwriter.core.projectdata import NWProjectData
 from novelwriter.common import (
-    checkStringNone, formatTimeStamp, hexToInt, isHandle, makeFileNameSafe, minmax
+    checkStringNone, formatTimeStamp, hexToInt, makeFileNameSafe, minmax
 )
 
 logger = logging.getLogger(__name__)
@@ -75,7 +75,6 @@ class NWProject(QObject):
         # Project Status
         self._projChanged = False  # The project has unsaved changes
         self._lockedBy    = None   # Data on which computer has the project open
-        self._projFiles   = []     # A list of all files in the content folder on load
 
         # Internal Mapping
         self.tr = partial(QCoreApplication.translate, "NWProject")
@@ -121,10 +120,6 @@ class NWProject(QObject):
     def projChanged(self):
         return self._projChanged
 
-    @property
-    def projFiles(self):
-        return self._projFiles
-
     ##
     #  Item Methods
     ##
@@ -213,9 +208,6 @@ class NWProject(QObject):
         self._data = NWProjectData(self)
         self._session = NWSessionLog(self)
 
-        # Project Settings
-        self._projFiles = []
-
         return
 
     def openProject(self, projPath, overrideLock=False):
@@ -330,15 +322,13 @@ class NWProject(QObject):
         )
 
         # Check the project tree consistency
-        for tItem in self._tree:
+        # This also handles any orphaned files found
-            if tItem:
+        orphans, recovered = self._tree.checkConsistency(self.tr("Recovered"))
-                tHandle = tItem.itemHandle
+        if orphans > 0:
-                logger.debug("Checking item '%s'", tHandle)
+            self.mainGui.makeAlert(self.tr(
-                if not self._tree.updateItemData(tHandle):
+                "Found {0} orphaned file(s) in the project. {1} file(s) were recovered."
-                    logger.error("There was a problem the item, and it has been removed")
+            ).format(orphans, recovered), nwAlert.WARN)
-                    del self._tree[tHandle]  # The file will be re-added as orphaned
 
-        self._scanProjectFolder()
         self._index.loadIndex()
         if xmlReader.state == XMLReadState.WAS_LEGACY:
             # Often, the index needs to be rebuilt when updating format
@@ -646,9 +636,8 @@ class NWProject(QObject):
 
         return True
 
-    def _loadProjectLocalisation(self):
+    def _loadProjectLocalisation(self) -> bool:
-        """Load the language data for the current project language.
+        """Load the language data for the current project language."""
-        """
         if self._data.language is None or CONFIG._nwLangPath is None:
             self._langData = {}
             return False
@@ -661,7 +650,6 @@ class NWProject(QObject):
             with open(langFile, mode="r", encoding="utf-8") as inFile:
                 self._langData = json.load(inFile)
             logger.debug("Loaded project language file: %s", langFile.name)
-
         except Exception:
             logger.error("Failed to project language file")
             logException()
@@ -669,99 +657,4 @@ class NWProject(QObject):
 
         return True
 
-    def _scanProjectFolder(self):
-        """Scan the project folder and check that the files in it are
-        also in the project XML file. If they aren't, import them as
-        orphaned files so the user can either delete them, or put them
-        back into the project tree.
-        """
-        contentPath = self._storage.contentPath
-        if not isinstance(contentPath, Path):
-            return False
-
-        # Then check the files in the data folder
-        logger.debug("Checking files in project content folder")
-        orphanFiles = []
-        self._projFiles = []
-
-        for item in contentPath.iterdir():
-            itemName = item.name
-            if not itemName.endswith(".nwd"):
-                logger.warning("Skipping file: %s", itemName)
-                continue
-            if len(itemName) != 17:
-                logger.warning("Skipping file: %s", itemName)
-                continue
-
-            fHandle = itemName[:13]
-            if not isHandle(fHandle):
-                logger.warning("Skipping file: %s", itemName)
-                continue
-
-            if fHandle in self._tree:
-                self._projFiles.append(fHandle)
-                logger.debug("Checking file %s, handle '%s': OK", itemName, fHandle)
-            else:
-                logger.warning("Checking file %s, handle '%s': Orphaned", itemName, fHandle)
-                orphanFiles.append(fHandle)
-
-        # Report status
-        if len(orphanFiles) > 0:
-            self.mainGui.makeAlert(self.tr(
-                "Found {0} orphaned file(s) in project folder."
-            ).format(len(orphanFiles)), nwAlert.WARN)
-        else:
-            logger.debug("File check OK")
-            return
-
-        # Handle orphans
-        nOrph = 0
-        noWhere = False
-        oPrefix = self.tr("Recovered")
-        for oHandle in orphanFiles:
-
-            # Look for meta data
-            oName = ""
-            oParent = None
-            oClass = None
-            oLayout = None
-
-            aDoc = self._storage.getDocument(oHandle)
-            if aDoc.readDocument(isOrphan=True) is not None:
-                oName, oParent, oClass, oLayout = aDoc.getMeta()
-
-            if oName:
-                oName = self.tr("[{0}] {1}").format(
-                    oPrefix, oName.replace("[%s]" % oPrefix, "").strip()
-                )
-            else:
-                nOrph += 1
-                oName = self.tr("Recovered File {0}").format(nOrph)
-
-            # Recover file meta data
-            oClass = oClass or nwItemClass.NOVEL
-            oLayout = oLayout or nwItemLayout.NOTE
-
-            if oParent is None or oParent not in self._tree:
-                oParent = self._tree.findRoot(oClass)
-                if oParent is None:
-                    oParent = self._tree.findRoot(nwItemClass.NOVEL)
-
-            # If the file still has no parent item, skip it
-            if oParent is None:
-                noWhere = True
-                continue
-
-            nHandle = self._tree.create(oName, oParent, nwItemType.FILE, oClass, oLayout)
-            if nHandle is not None:
-                (contentPath / f"{oHandle}.nwd").rename(contentPath / f"{nHandle}.nwd")
-
-        if noWhere:
-            self.mainGui.makeAlert(self.tr(
-                "One or more orphaned files could not be added back into the project. "
-                "Make sure at least a Novel root folder exists."
-            ), nwAlert.WARN)
-
-        return True
-
 # END Class NWProject

novelwriter/core/storage.py

@@ -33,7 +33,7 @@ from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
 
 from novelwriter import CONFIG
 from novelwriter.error import logException
-from novelwriter.common import minmax
+from novelwriter.common import isHandle, minmax
 from novelwriter.constants import nwFiles
 from novelwriter.core.document import NWDocument
 from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
@@ -179,6 +179,16 @@ class NWStorage:
             return self._runtimePath / "meta" / fileName
         return None
 
+    def scanContent(self) -> list[str]:
+        """Scan the content folder and return the handle of all files
+        found in it. Files that do not match the pattern are ignored.
+        """
+        contentPath = self.contentPath
+        return [
+            item.stem for item in contentPath.iterdir()
+            if item.suffix == ".nwd" and isHandle(item.stem)
+        ] if contentPath else []
+
     def readLockFile(self) -> list:
         """Read the project lock file."""
         if self._lockFilePath is None:

novelwriter/core/tree.py

@@ -95,18 +95,15 @@ class NWTree:
 
     @overload
     def create(self, label: str, parent: None, itemType: nwItemType,
-               itemClass: nwItemClass = nwItemClass.NO_CLASS,
+               itemClass: nwItemClass = nwItemClass.NO_CLASS) -> str:
-               itemLayout: nwItemLayout = nwItemLayout.NO_LAYOUT) -> str:
         ...
 
     @overload
     def create(self, label: str, parent: str | None, itemType: nwItemType,
-               itemClass: nwItemClass = nwItemClass.NO_CLASS,
+               itemClass: nwItemClass = nwItemClass.NO_CLASS) -> str | None:
-               itemLayout: nwItemLayout = nwItemLayout.NO_LAYOUT) -> str | None:
         ...
 
-    def create(self, label, parent, itemType,
+    def create(self, label, parent, itemType, itemClass=nwItemClass.NO_CLASS):
-               itemClass=nwItemClass.NO_CLASS, itemLayout=nwItemLayout.NO_LAYOUT):
         """Create a new item in the project tree, and return its handle.
         If the item cannot be added to the project, None is returned.
         """
@@ -117,7 +114,6 @@ class NWTree:
             newItem.setParent(parent)
             newItem.setType(itemType)
             newItem.setClass(itemClass)
-            newItem.setLayout(itemLayout)
             self.append(newItem)
             self.updateItemData(tHandle)
             return tHandle
@@ -185,12 +181,64 @@ class NWTree:
         """
         self.clear()
         for item in data:
-            nwItem = NWItem(self._project, "NOTSET")  # Handle is set by unpack()
+            nwItem = NWItem(self._project, "")  # Handle is set by unpack()
             if nwItem.unpack(item):
                 self.append(nwItem)
                 nwItem.saveInitialCount()
         return
 
+    def checkConsistency(self, prefix: str) -> tuple[int, int]:
+        """Check the project tree consistency. Also check the content
+        folder and add back files that were discovered but were not
+        included in the tree. This function should only be called after
+        the project file has been processed, but before the loading of
+        the project returns. The functions requires a prefix string to
+        mark recovered files.
+        """
+        for tHandle in self._treeOrder:
+            if self.updateItemData(tHandle):
+                logger.debug("Checking item '%s' ... OK", tHandle)
+            else:
+                logger.error("Checking item '%s' ... ERROR", tHandle)
+                self.__delitem__(tHandle)  # The file will be re-added as orphaned
+
+        orphans = 0
+        recovered = 0
+        storage = self._project.storage
+        for cHandle in storage.scanContent():
+            if cHandle in self._treeOrder:
+                continue
+
+            orphans += 1
+            aDoc = storage.getDocument(cHandle)
+            aDoc.readDocument(isOrphan=True)
+            oName, oParent, oClass, oLayout = aDoc.getMeta()
+
+            oName = oName or cHandle
+            oParent = oParent if oParent in self._treeOrder else None
+            oClass = oClass or nwItemClass.NOVEL
+            oLayout = oLayout or nwItemLayout.NOTE
+
+            # If the parent doesn't exists, find a new home
+            if oParent is None:  # Add it to the first available class root
+                oParent = self.findRoot(oClass)
+            if oParent is None:  # Otherwise, add to the Novel root
+                oParent = self.findRoot(nwItemClass.NOVEL)
+            if oParent is None:  # If not, give up
+                continue
+
+            # Create a new item
+            newItem = NWItem(self._project, cHandle)
+            newItem.setName(f"[{prefix}] {oName}")
+            newItem.setParent(oParent)
+            newItem.setType(nwItemType.FILE)
+            newItem.setClass(oClass)
+            newItem.setLayout(oLayout)
+            if self.append(newItem):
+                recovered += 1
+
+        return orphans, recovered
+
     def writeToCFile(self) -> bool:
         """Write the convenience table of contents file in the root of
         the project directory.