From b024c0996d24eebf14341e5e81e7bf171a4d4aa6 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Tue, 8 Nov 2022 19:25:56 +0100 Subject: [PATCH] Move zipping of project to storage class --- novelwriter/core/project.py | 77 +++++++------------------ novelwriter/core/projectxml.py | 8 +-- novelwriter/core/storage.py | 61 ++++++++++++++++---- novelwriter/gui/mainmenu.py | 2 +- novelwriter/guimain.py | 2 +- tests/test_core/test_core_index.py | 3 +- tests/test_core/test_core_project.py | 26 ++++----- tests/test_core/test_core_projectxml.py | 12 ++-- 8 files changed, 96 insertions(+), 95 deletions(-) diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py index 12ffcfac..d117f109 100644 --- a/novelwriter/core/project.py +++ b/novelwriter/core/project.py @@ -27,7 +27,6 @@ from __future__ import annotations import os import json -import shutil import logging import novelwriter @@ -299,7 +298,6 @@ class NWProject(QObject): xmlParsed = xmlReader.read(self._data, projContent) appVersion = xmlReader.appVersion or self.tr("Unknown") - hexVersion = xmlReader.hexVersion or "0x0" if not xmlParsed: if xmlReader.state == XMLReadState.NOT_NWX_FILE: @@ -339,7 +337,7 @@ class NWProject(QObject): # Check novelWriter Version # ========================= - if hexToInt(hexVersion) > hexToInt(novelwriter.__hexversion__): + if xmlReader.hexVersion > hexToInt(novelwriter.__hexversion__): msgYes = self.mainGui.askQuestion( self.tr("Version Conflict"), self.tr( @@ -359,7 +357,7 @@ class NWProject(QObject): self._tree.unpack(projContent) self._options.loadSettings() - self._index.loadIndex() + self._loadProjectLocalisation() # Update recent projects self.mainConf.updateRecentCache( @@ -376,7 +374,7 @@ class NWProject(QObject): del self._tree[tHandle] # The file will be re-added as orphaned self._scanProjectFolder() - self._loadProjectLocalisation() + self._index.loadIndex() self.updateWordCounts() self._projOpened = time() @@ -466,21 +464,17 @@ class NWProject(QObject): self._data.itemImport.write(None, self.tr("Main"), (50, 200, 0)) return - ## - # Zip/Unzip Project - ## - - def zipIt(self, doNotify): + def backupProject(self, doNotify): """Create a zip file of the entire project. """ - if not self.mainGui.hasProject: + if not self._storage.isOpen(): logger.error("No project open") return False logger.info("Backing up project") self.mainGui.setStatus(self.tr("Backing up project ...")) - if not (self.mainConf.backupPath and os.path.isdir(self.mainConf.backupPath)): + if not self.mainConf.backupPath: self.mainGui.makeAlert(self.tr( "Cannot backup project because no valid backup path is set. " "Please set a valid backup location in Preferences." @@ -490,52 +484,37 @@ class NWProject(QObject): if not self._data.name: self.mainGui.makeAlert(self.tr( "Cannot backup project because no project name is set. " - "Please set a Working Title in Project Settings." + "Please set a Project Name in Project Settings." ), nwAlert.ERROR) return False cleanName = makeFileNameSafe(self._data.name) - baseDir = os.path.abspath(os.path.join(self.mainConf.backupPath, cleanName)) - if not os.path.isdir(baseDir): - try: - os.mkdir(baseDir) - logger.debug("Created folder: %s", baseDir) - except Exception as exc: - self.mainGui.makeAlert(self.tr( - "Could not create backup folder." - ), nwAlert.ERROR, exception=exc) - return False - - if baseDir and baseDir.startswith(str(self._storage.runtimePath)): + baseDir = Path(self.mainConf.backupPath) / cleanName + try: + baseDir.mkdir(exist_ok=True) + except Exception as exc: self.mainGui.makeAlert(self.tr( - "Cannot backup project because the backup path is within the " - "project folder to be backed up. Please choose a different " - "backup path in Preferences." - ), nwAlert.ERROR) + "Could not create backup folder." + ), nwAlert.ERROR, exception=exc) return False - archName = self.tr("Backup from {0}").format(formatTimeStamp(time(), fileSafe=True)) - baseName = os.path.join(baseDir, archName) - - try: - self._storage.clearLockFile() - shutil.make_archive(baseName, "zip", self._storage.runtimePath, ".") - self._storage.writeLockFile() - logger.info("Backup written to: %s", archName) + archName = baseDir / self.tr( + "Backup from {0}.zip" + ).format(formatTimeStamp(time(), fileSafe=True)) + if self._storage.zipIt(archName, compression=2): if doNotify: self.mainGui.makeAlert(self.tr( "Backup archive file written to: {0}" - ).format(f"{os.path.join(cleanName, archName)}.zip"), nwAlert.INFO) - - except Exception as exc: + ).format(str(archName), nwAlert.INFO)) + else: self.mainGui.makeAlert(self.tr( "Could not write backup archive." - ), nwAlert.ERROR, exception=exc) + ), nwAlert.ERROR) return False self.mainGui.setStatus(self.tr( "Project backed up to '{0}'" - ).format(f"{baseName}.zip")) + ).format(str(archName))) return True @@ -737,20 +716,6 @@ class NWProject(QObject): return True - def _checkFolder(self, thePath): - """Check if a folder exists, and if it doesn't, create it. - """ - if not os.path.isdir(thePath): - try: - os.mkdir(thePath) - logger.debug("Created folder: %s", thePath) - except Exception as exc: - self.mainGui.makeAlert(self.tr( - "Could not create folder." - ), nwAlert.ERROR, exception=exc) - return False - return True - def _scanProjectFolder(self): """Scan the project folder and check that the files in it are also in the project XML file. If they aren't, import them as diff --git a/novelwriter/core/projectxml.py b/novelwriter/core/projectxml.py index cf78a587..5b1b72f4 100644 --- a/novelwriter/core/projectxml.py +++ b/novelwriter/core/projectxml.py @@ -34,7 +34,7 @@ from pathlib import Path from novelwriter.common import ( checkBool, checkInt, checkString, checkStringNone, formatTimeStamp, - simplified, yesNo + hexToInt, simplified, yesNo ) from novelwriter.constants import nwFiles @@ -104,9 +104,9 @@ class ProjectXMLReader: self._state = XMLReadState.NO_ACTION self._root = "" - self._version = 0x0000 + self._version = 0x0 self._appVersion = "" - self._hexVersion = "" + self._hexVersion = 0x0 self._timeStamp = "" return @@ -200,7 +200,7 @@ class ProjectXMLReader: logger.debug("XML is '%s' version '%s'", self._root, fileVersion) self._appVersion = str(xRoot.attrib.get("appVersion", "")) - self._hexVersion = str(xRoot.attrib.get("hexVersion", "")) + self._hexVersion = hexToInt(xRoot.attrib.get("hexVersion", "")) self._timeStamp = str(xRoot.attrib.get("timeStamp", "")) for xSection in xRoot: diff --git a/novelwriter/core/storage.py b/novelwriter/core/storage.py index 6cfa9ffa..40751909 100644 --- a/novelwriter/core/storage.py +++ b/novelwriter/core/storage.py @@ -28,7 +28,9 @@ import novelwriter from time import time from pathlib import Path +from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile +from novelwriter.common import minmax from novelwriter.constants import nwFiles from novelwriter.core.document import NWDocument from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter @@ -232,18 +234,55 @@ class NWStorage: return True + def zipIt(self, target, compression=None): + """Zip the content of the project at its runtime location into a + zip file. This process will only grab files that are supposed to + be in the project. All non-project files will be left out. + """ + basePath = self._runtimePath + if not isinstance(basePath, Path): + logger.error("No path set") + return False + + baseMeta = basePath / "meta" + baseCont = basePath / "content" + files = [ + (basePath / nwFiles.PROJ_FILE, nwFiles.PROJ_FILE), + (baseMeta / nwFiles.OPTS_FILE, f"meta/{nwFiles.OPTS_FILE}"), + (baseMeta / nwFiles.SESS_STATS, f"meta/{nwFiles.SESS_STATS}"), + (baseMeta / nwFiles.INDEX_FILE, f"meta/{nwFiles.INDEX_FILE}"), + (baseMeta / nwFiles.PROJ_DICT, f"meta/{nwFiles.PROJ_DICT}"), + ] + for contItem in baseCont.iterdir(): + name = contItem.name + if contItem.is_file() and len(name) == 17 and name.endswith(".nwd"): + files.append((contItem, f"content/{name}")) + + comp = ZIP_STORED if compression is None else ZIP_DEFLATED + level = minmax(compression, 0, 9) if isinstance(compression, int) else None + try: + with ZipFile(target, mode="w", compression=comp, compresslevel=level) as zipObj: + logger.info("Creating archive: %s", target) + for srcPath, zipPath in files: + if srcPath.is_file(): + zipObj.write(srcPath, zipPath) + logger.debug("Added: %s", zipPath) + except Exception: + logger.error("Failed to create acrhive") + logException() + return False + + return True + ## # Internal Functions ## - def _zipIt(self, target): - pass - def _prepareStorage(self, checkLegacy=True, newProject=False): """Prepare the storage area for the project. """ path = self._runtimePath - if path is None: + if not isinstance(path, Path): logger.error("No path set") self.clear() return False @@ -336,13 +375,13 @@ class NWStorage: """Delete files that are no longer used by novelWriter. """ remove = [ - path / "meta" / "mainOptions.json", - path / "meta" / "exportOptions.json", - path / "meta" / "outlineOptions.json", - path / "meta" / "timelineOptions.json", - path / "meta" / "docMergeOptions.json", - path / "meta" / "sessionLogOptions.json", - path / "ToC.json", + path / "meta" / "mainOptions.json", # Replaced in 0.5 + path / "meta" / "exportOptions.json", # Replaced in 0.5 + path / "meta" / "outlineOptions.json", # Replaced in 0.5 + path / "meta" / "timelineOptions.json", # Replaced in 0.5 + path / "meta" / "docMergeOptions.json", # Replaced in 0.5 + path / "meta" / "sessionLogOptions.json", # Replaced in 0.5 + path / "ToC.json", # Dropped in 1.0 RC 1 ] for item in remove: if item.is_file(): diff --git a/novelwriter/gui/mainmenu.py b/novelwriter/gui/mainmenu.py index 51a74bae..a3b7eca0 100644 --- a/novelwriter/gui/mainmenu.py +++ b/novelwriter/gui/mainmenu.py @@ -824,7 +824,7 @@ class GuiMainMenu(QMenuBar): # Tools > Backup self.aBackupProject = QAction(self.tr("Backup Project"), self) - self.aBackupProject.triggered.connect(lambda: self.theProject.zipIt(True)) + self.aBackupProject.triggered.connect(lambda: self.theProject.backupProject(doNoify=True)) self.toolsMenu.addAction(self.aBackupProject) # Tools > Export Project diff --git a/novelwriter/guimain.py b/novelwriter/guimain.py index 6537e7a0..961a9458 100644 --- a/novelwriter/guimain.py +++ b/novelwriter/guimain.py @@ -409,7 +409,7 @@ class GuiMain(QMainWindow): if not msgYes: doBackup = False if doBackup: - self.theProject.zipIt(False) + self.theProject.backupProject(doNotify=False) else: saveOK = True diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index 624f6ec8..678f00e5 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -29,6 +29,7 @@ from mock import causeException from tools import C, buildTestProject, cmpFiles, writeFile from novelwriter.enum import nwItemClass, nwItemLayout +from novelwriter.constants import nwFiles from novelwriter.core.index import NWIndex, countWords, TagsIndex from novelwriter.core.project import NWProject @@ -38,7 +39,7 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): """Test core functionality of scaning, saving, loading and checking the index cache file. """ - projFile = os.path.join(nwLipsum, "meta", "tagsIndex.json") + projFile = os.path.join(nwLipsum, "meta", nwFiles.INDEX_FILE) testFile = os.path.join(outDir, "coreIndex_LoadSave_tagsIndex.json") compFile = os.path.join(refDir, "coreIndex_LoadSave_tagsIndex.json") diff --git a/tests/test_core/test_core_project.py b/tests/test_core/test_core_project.py index 368a7ddb..e2e6aad2 100644 --- a/tests/test_core/test_core_project.py +++ b/tests/test_core/test_core_project.py @@ -235,7 +235,7 @@ def testCoreProject_Open(monkeypatch, caplog, mockGUI, fncDir, mockRnd): # Won't convert legacy file with monkeypatch.context() as mp: - mp.setattr(ProjectXMLReader, "hexVersion", property(lambda *a: "0x99999999")) + mp.setattr(ProjectXMLReader, "hexVersion", property(lambda *a: 0x99999999)) mockGUI.askResponse = False assert theProject.openProject(fncDir) is False assert "This project was saved by a newer version" in mockGUI.lastQuestion[1] @@ -699,46 +699,42 @@ def testCoreProject_Backup(monkeypatch, mockGUI, fncDir, tmpDir): # No project mockGUI.hasProject = False - assert theProject.zipIt(doNotify=False) is False + assert theProject.backupProject(doNotify=False) is False mockGUI.hasProject = True # Invalid path theProject.mainConf.backupPath = None - assert theProject.zipIt(doNotify=False) is False + assert theProject.backupProject(doNotify=False) is False # Missing project name theProject.mainConf.backupPath = tmpDir theProject.data.setName("") - assert theProject.zipIt(doNotify=False) is False + assert theProject.backupProject(doNotify=False) is False # Non-existent folder theProject.mainConf.backupPath = os.path.join(tmpDir, "nonexistent") theProject.data.setName("Test Minimal") - assert theProject.zipIt(doNotify=False) is False - - # Same folder as project (causes infinite loop in zipping) - theProject.mainConf.backupPath = fncDir - assert theProject.zipIt(doNotify=False) is False + assert theProject.backupProject(doNotify=False) is False # Subfolder of project (causes infinite loop in zipping) theProject.mainConf.backupPath = os.path.join(fncDir, "subdir") - assert theProject.zipIt(doNotify=False) is False + assert theProject.backupProject(doNotify=False) is False # Set a valid folder theProject.mainConf.backupPath = tmpDir # Can't make folder with monkeypatch.context() as mp: - mp.setattr("os.mkdir", causeOSError) - assert theProject.zipIt(doNotify=False) is False + mp.setattr("pathlib.Path.mkdir", causeOSError) + assert theProject.backupProject(doNotify=False) is False # Can't write archive with monkeypatch.context() as mp: - mp.setattr("shutil.make_archive", causeOSError) - assert theProject.zipIt(doNotify=False) is False + mp.setattr("zipfile.ZipFile.write", causeOSError) + assert theProject.backupProject(doNotify=False) is False # Test correct settings - assert theProject.zipIt(doNotify=True) is True + assert theProject.backupProject(doNotify=True) is True theFiles = os.listdir(os.path.join(tmpDir, "Test Minimal")) assert len(theFiles) == 1 diff --git a/tests/test_core/test_core_projectxml.py b/tests/test_core/test_core_projectxml.py index 3bbc9e3a..b9dce90c 100644 --- a/tests/test_core/test_core_projectxml.py +++ b/tests/test_core/test_core_projectxml.py @@ -131,7 +131,7 @@ def testCoreProjectXML_ReadCurrent(monkeypatch, tstPaths, fncPath): assert xmlReader.xmlRoot == "novelWriterXML" assert xmlReader.xmlVersion == 0x0105 assert xmlReader.appVersion == "2.0-rc1" - assert xmlReader.hexVersion == "0x020000c1" + assert xmlReader.hexVersion == 0x020000c1 # Check loaded data assert data.name == "Sample Project" @@ -257,7 +257,7 @@ def testCoreProjectXML_ReadLegacy10(tstPaths, fncPath, mockRnd): assert xmlReader.xmlRoot == "novelWriterXML" assert xmlReader.xmlVersion == 0x0100 assert xmlReader.appVersion == "0.6.1" - assert xmlReader.hexVersion == "0x000601f0" + assert xmlReader.hexVersion == 0x000601f0 # Check loaded data assert data.name == "Sample Project" @@ -399,7 +399,7 @@ def testCoreProjectXML_ReadLegacy11(tstPaths, fncPath, mockRnd): assert xmlReader.xmlRoot == "novelWriterXML" assert xmlReader.xmlVersion == 0x0101 assert xmlReader.appVersion == "0.9.2" - assert xmlReader.hexVersion == "0x000902f0" + assert xmlReader.hexVersion == 0x000902f0 # Check loaded data assert data.name == "Sample Project" @@ -541,7 +541,7 @@ def testCoreProjectXML_ReadLegacy12(tstPaths, fncPath, mockRnd): assert xmlReader.xmlRoot == "novelWriterXML" assert xmlReader.xmlVersion == 0x0102 assert xmlReader.appVersion == "1.4.2" - assert xmlReader.hexVersion == "0x010402f0" + assert xmlReader.hexVersion == 0x010402f0 # Check loaded data assert data.name == "Sample Project" @@ -686,7 +686,7 @@ def testCoreProjectXML_ReadLegacy13(tstPaths, fncPath, mockRnd): assert xmlReader.xmlRoot == "novelWriterXML" assert xmlReader.xmlVersion == 0x0103 assert xmlReader.appVersion == "1.6.6" - assert xmlReader.hexVersion == "0x010606f0" + assert xmlReader.hexVersion == 0x010606f0 # Check loaded data assert data.name == "Sample Project" @@ -831,7 +831,7 @@ def testCoreProjectXML_ReadLegacy14(tstPaths, fncPath, mockRnd): assert xmlReader.xmlRoot == "novelWriterXML" assert xmlReader.xmlVersion == 0x0104 assert xmlReader.appVersion == "2.0-rc1" - assert xmlReader.hexVersion == "0x020000c1" + assert xmlReader.hexVersion == 0x020000c1 # Check loaded data assert data.name == "Sample Project"