From 738f022212c304f1b0eb2b51cd742f4570e5e062 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:45:29 +0100 Subject: [PATCH 1/3] Add a quick document read function in the storage class --- novelwriter/core/storage.py | 23 +++++++++++++++++++++-- tests/test_core/test_core_storage.py | 17 +++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/novelwriter/core/storage.py b/novelwriter/core/storage.py index 2740d19d..59ea24b3 100644 --- a/novelwriter/core/storage.py +++ b/novelwriter/core/storage.py @@ -27,18 +27,18 @@ import json import logging from enum import Enum +from pathlib import Path from time import time from typing import TYPE_CHECKING -from pathlib import Path from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile from novelwriter import CONFIG -from novelwriter.error import logException from novelwriter.common import isHandle, minmax from novelwriter.constants import nwFiles from novelwriter.core.document import NWDocument from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter from novelwriter.core.spellcheck import UserDictionary +from novelwriter.error import logException if TYPE_CHECKING: # pragma: no cover from novelwriter.core.project import NWProject @@ -287,6 +287,25 @@ class NWStorage: return self._runtimePath / "meta" / fileName return None + def getDocumentText(self, tHandle: str) -> str: + """Return the text of a document in a fast and efficient way.""" + if ( + isinstance(self._runtimePath, Path) + and (path := self._runtimePath / "content" / f"{tHandle}.nwd").is_file() + ): + try: + with open(path, mode="r", encoding="utf-8") as inFile: + line = "" + for _ in range(10): + if not (line := inFile.readline()).startswith(r"%%~"): + break + return line + inFile.read() + except Exception: + logger.error("Cannot read document with handle '%s'", tHandle) + logException() + return "" + return "" + def scanContent(self) -> list[str]: """Scan the content folder and return the handle of all files found in it. Files that do not match the pattern are ignored. diff --git a/tests/test_core/test_core_storage.py b/tests/test_core/test_core_storage.py index 213cf9b3..6ea0937d 100644 --- a/tests/test_core/test_core_storage.py +++ b/tests/test_core/test_core_storage.py @@ -26,15 +26,15 @@ import pytest from pathlib import Path from zipfile import ZipFile -from tools import C, buildTestProject from mocked import causeOSError +from tools import C, buildTestProject from novelwriter import CONFIG from novelwriter.constants import nwFiles -from novelwriter.core.project import NWProject -from novelwriter.core.storage import NWStorage, NWStorageOpen, NWStorageCreate, _LegacyStorage from novelwriter.core.document import NWDocument +from novelwriter.core.project import NWProject from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter +from novelwriter.core.storage import NWStorage, NWStorageOpen, NWStorageCreate, _LegacyStorage class MockProject: @@ -85,7 +85,7 @@ def testCoreStorage_CreateNewProject(mockGUI, fncPath): @pytest.mark.core -def testCoreStorage_InitProjectStorage(mockGUI, fncPath, mockRnd): +def testCoreStorage_InitProjectStorage(monkeypatch, mockGUI, fncPath, mockRnd): """Test initialising a project in a folder.""" project = NWProject() @@ -157,6 +157,15 @@ def testCoreStorage_InitProjectStorage(mockGUI, fncPath, mockRnd): assert isinstance(storage.getDocument(C.hSceneDoc), NWDocument) assert repr(storage.getDocument(C.hSceneDoc)) == f"" + # We can directly access the content of a document + assert storage.getDocumentText(C.hSceneDoc) == "### New Scene\n\n" + + # Check read text fallback + assert storage.getDocumentText(C.hInvalid) == "" + with monkeypatch.context() as mp: + mp.setattr("builtins.open", causeOSError) + assert storage.getDocumentText(C.hSceneDoc) == "" + project.closeProject() # END Test testCoreStorage_InitProjectStorage From 6a7262b817bd8b90158f6d0a97bf61a1e2407994 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:56:31 +0100 Subject: [PATCH 2/3] Use the quick read feature wherever we only need document text --- novelwriter/core/coretools.py | 31 +++++--------------------- novelwriter/core/index.py | 13 +++++------ novelwriter/core/project.py | 21 +++++++---------- novelwriter/core/tokenizer.py | 4 +--- novelwriter/dialogs/docsplit.py | 3 +-- tests/test_core/test_core_coretools.py | 3 --- tests/test_core/test_core_index.py | 5 ++--- 7 files changed, 22 insertions(+), 58 deletions(-) diff --git a/novelwriter/core/coretools.py b/novelwriter/core/coretools.py index 7be58fc0..7b26f71e 100644 --- a/novelwriter/core/coretools.py +++ b/novelwriter/core/coretools.py @@ -32,7 +32,6 @@ import shutil from collections.abc import Iterable from functools import partial from pathlib import Path -from time import time from zipfile import ZipFile, is_zipfile from PyQt5.QtCore import QCoreApplication, QRegularExpression @@ -102,9 +101,7 @@ class DocMerger: if srcItem is None: return False - inDoc = self._project.storage.getDocument(srcHandle) - docText = (inDoc.readDocument() or "").rstrip("\n") - + docText = self._project.storage.getDocumentText(srcHandle).rstrip("\n") if addComment: docInfo = srcItem.describeMe() docSt, _ = srcItem.getImportStatus(incIcon=False) @@ -123,9 +120,8 @@ class DocMerger: return False outDoc = self._project.storage.getDocument(self._targetDoc) - docText = (outDoc.readDocument() or "").rstrip("\n") - if docText: - self._targetText.insert(0, docText) + if text := (outDoc.readDocument() or "").rstrip("\n"): + self._targetText.insert(0, text) status = outDoc.writeDocument("\n\n".join(self._targetText) + "\n\n") if not status: @@ -293,11 +289,10 @@ class DocDuplicator: newItem.setParent(hMap[newItem.itemParent]) self._project.tree.updateItemData(newItem.itemHandle) if newItem.isFileType(): - oldDoc = self._project.storage.getDocument(tHandle) newDoc = self._project.storage.getDocument(newItem.itemHandle) if newDoc.fileExists(): return - newDoc.writeDocument(oldDoc.readDocument() or "") + newDoc.writeDocument(self._project.storage.getDocumentText(tHandle)) yield newItem.itemHandle, nHandle nHandle = None return @@ -308,17 +303,10 @@ class DocDuplicator: class DocSearch: def __init__(self) -> None: - # RegEx Object self._regEx = QRegularExpression() self.setCaseSensitive(False) self._words = False self._escape = True - - # Project Cache - self._uuid = "" - self._time = 0.0 - self._cache: dict[str, str] = {} - return ## @@ -347,11 +335,6 @@ class DocSearch: self, project: NWProject, search: str ) -> Iterable[tuple[NWItem, list[tuple[int, int, str]], bool]]: """Iteratively search through documents in a project.""" - if project.data.uuid != self._uuid or time() - self._time > 20.0: - self._cache = {} - - self._uuid = project.data.uuid - self._time = time() self._regEx.setPattern(self._buildPattern(search)) logger.debug("Searching with pattern '%s'", self._regEx.pattern()) @@ -359,11 +342,7 @@ class DocSearch: storage = project.storage for item in project.tree: if item.isFileType(): - tHandle = item.itemHandle - if (text := self._cache.get(tHandle)) is None: - text = storage.getDocument(tHandle).readDocument() or "" - self._cache[tHandle] = text - + text = storage.getDocumentText(item.itemHandle) rxItt = self._regEx.globalMatch(text) count = 0 capped = False diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 7160ea03..7ca7afe5 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -122,9 +122,8 @@ class NWIndex: self.clearIndex() for nwItem in self._project.tree: if nwItem.isFileType(): - tHandle = nwItem.itemHandle - doc = self._project.storage.getDocument(tHandle) - self.scanText(tHandle, doc.readDocument() or "", blockSignal=True) + text = self._project.storage.getDocumentText(nwItem.itemHandle) + self.scanText(nwItem.itemHandle, text, blockSignal=True) self._indexBroken = False SHARED.indexSignalProxy({"event": "buildIndex"}) return @@ -142,17 +141,15 @@ class NWIndex: }) return - def reIndexHandle(self, tHandle: str | None) -> bool: + def reIndexHandle(self, tHandle: str | None) -> None: """Put a file back into the index. This is used when files are moved from the archive or trash folders back into the active project. """ if tHandle and self._project.tree.checkType(tHandle, nwItemType.FILE): logger.debug("Re-indexing item '%s'", tHandle) - doc = self._project.storage.getDocument(tHandle) - self.scanText(tHandle, doc.readDocument() or "") - return True - return False + self.scanText(tHandle, self._project.storage.getDocumentText(tHandle)) + return def indexChangedSince(self, checkTime: int | float) -> bool: """Check if the index has changed since a given time.""" diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py index 80aee83f..4a399c24 100644 --- a/novelwriter/core/project.py +++ b/novelwriter/core/project.py @@ -175,12 +175,10 @@ class NWProject: """Write content to a new document after it is created. This will not run if the file exists and is not empty. """ - tItem = self._tree[tHandle] - if not (tItem and tItem.isFileType()): + if not ((tItem := self._tree[tHandle]) and tItem.isFileType()): return False - newDoc = self._storage.getDocument(tHandle) - if (newDoc.readDocument() or "").strip(): + if self._storage.getDocumentText(tHandle).strip(): return False indent = "#"*minmax(hLevel, 1, 4) @@ -191,7 +189,7 @@ class NWProject: else: tItem.setLayout(nwItemLayout.NOTE) - newDoc.writeDocument(text) + self._storage.getDocument(tHandle).writeDocument(text) self._index.scanText(tHandle, text) return True @@ -200,21 +198,18 @@ class NWProject: """Copy content to a new document after it is created. This will not run if the file exists and is not empty. """ - tItem = self._tree[tHandle] - if not (tItem and tItem.isFileType()): + if not ((tItem := self._tree[tHandle]) and tItem.isFileType()): return False - sItem = self._tree[sHandle] - if not (sItem and sItem.isFileType()): + if not ((sItem := self._tree[sHandle]) and sItem.isFileType()): return False - newDoc = self._storage.getDocument(tHandle) - if (newDoc.readDocument() or "").strip(): + if self._storage.getDocumentText(tHandle).strip(): return False logger.debug("Populating '%s' with text from '%s'", tHandle, sHandle) - text = self._storage.getDocument(sHandle).readDocument() or "" - newDoc.writeDocument(text) + text = self._storage.getDocumentText(sHandle) + self._storage.getDocument(tHandle).writeDocument(text) sItem.setLayout(tItem.itemLayout) self._index.scanText(tHandle, text) diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 9125ba01..d3f2b690 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -429,9 +429,7 @@ class Tokenizer(ABC): self._text = "" self._handle = None if nwItem := self._project.tree[tHandle]: - if text is None: - text = self._project.storage.getDocument(tHandle).readDocument() or "" - self._text = text + self._text = text or self._project.storage.getDocumentText(tHandle) self._handle = tHandle self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT return diff --git a/novelwriter/dialogs/docsplit.py b/novelwriter/dialogs/docsplit.py index b214d56b..f1c3d295 100644 --- a/novelwriter/dialogs/docsplit.py +++ b/novelwriter/dialogs/docsplit.py @@ -214,8 +214,7 @@ class GuiDocSplit(QDialog): spLevel = self.splitLevel.currentData() if not self._text: - inDoc = SHARED.project.storage.getDocument(sHandle) - self._text = (inDoc.readDocument() or "").splitlines() + self._text = SHARED.project.storage.getDocumentText(sHandle).splitlines() for lineNo, aLine in enumerate(self._text): diff --git a/tests/test_core/test_core_coretools.py b/tests/test_core/test_core_coretools.py index 1e205156..b3178ef6 100644 --- a/tests/test_core/test_core_coretools.py +++ b/tests/test_core/test_core_coretools.py @@ -423,9 +423,6 @@ def testCoreTools_DocSearch(monkeypatch, mockGUI, fncPath, mockRnd, ipsumText): assert result[1] == (C.hChapterDoc, [], False) assert result[2] == (C.hSceneDoc, [(8, 5, "Scene")], False) - # Cache - assert list(search._cache.keys()) == [C.hTitlePage, C.hChapterDoc, C.hSceneDoc] - # Patterns # ======== diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index f10ed45a..b04f8c44 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -60,9 +60,8 @@ def testCoreIndex_LoadSave(qtbot, monkeypatch, prjLipsum, mockGUI, tstPaths): "60bdf227455cc": False, # World ROOT } for tItem in project.tree: - assert index.reIndexHandle(tItem.itemHandle) is notIndexable.get(tItem.itemHandle, True) - - assert index.reIndexHandle(None) is False + index.reIndexHandle(tItem.itemHandle) + assert (tItem.itemHandle in index._itemIndex) is notIndexable.get(tItem.itemHandle, True) # No folder for saving with monkeypatch.context() as mp: From 147d6b95d7295ef255d6157a58f65e1441cab1e3 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Tue, 26 Mar 2024 12:14:07 +0100 Subject: [PATCH 3/3] Move the quick read function to the document class where it belongs --- novelwriter/core/document.py | 27 ++++++++++++++++++++++++--- novelwriter/core/storage.py | 17 ++--------------- tests/test_core/test_core_document.py | 20 +++++++++++++++++++- tests/test_core/test_core_storage.py | 7 +------ 4 files changed, 46 insertions(+), 25 deletions(-) diff --git a/novelwriter/core/document.py b/novelwriter/core/document.py index cf0e1f15..a93d8855 100644 --- a/novelwriter/core/document.py +++ b/novelwriter/core/document.py @@ -31,7 +31,7 @@ from typing import TYPE_CHECKING from pathlib import Path from novelwriter.enum import nwItemLayout, nwItemClass -from novelwriter.error import formatException +from novelwriter.error import formatException, logException from novelwriter.common import formatTimeStamp, isHandle from novelwriter.core.item import NWItem @@ -106,7 +106,28 @@ class NWDocument: return self._item ## - # Class Methods + # Static Methods + ## + + @staticmethod + def quickReadText(content: Path, tHandle: str) -> str: + """Return the text of a document in a fast and efficient way.""" + if (path := content / f"{tHandle}.nwd").is_file(): + try: + with open(path, mode="r", encoding="utf-8") as inFile: + line = "" + for _ in range(10): + if not (line := inFile.readline()).startswith(r"%%~"): + break + return line + inFile.read() + except Exception: + logger.error("Cannot read document with handle '%s'", tHandle) + logException() + return "" + return "" + + ## + # Methods ## def fileExists(self) -> bool: @@ -155,7 +176,7 @@ class NWDocument: try: with open(docPath, mode="r", encoding="utf-8") as inFile: # Check the first <= 10 lines for metadata - for i in range(10): + for _ in range(10): line = inFile.readline() if line.startswith(r"%%~"): self._parseMeta(line) diff --git a/novelwriter/core/storage.py b/novelwriter/core/storage.py index 59ea24b3..e8a2bb16 100644 --- a/novelwriter/core/storage.py +++ b/novelwriter/core/storage.py @@ -289,21 +289,8 @@ class NWStorage: def getDocumentText(self, tHandle: str) -> str: """Return the text of a document in a fast and efficient way.""" - if ( - isinstance(self._runtimePath, Path) - and (path := self._runtimePath / "content" / f"{tHandle}.nwd").is_file() - ): - try: - with open(path, mode="r", encoding="utf-8") as inFile: - line = "" - for _ in range(10): - if not (line := inFile.readline()).startswith(r"%%~"): - break - return line + inFile.read() - except Exception: - logger.error("Cannot read document with handle '%s'", tHandle) - logException() - return "" + if isinstance(self._runtimePath, Path): + return NWDocument.quickReadText(self._runtimePath / "content", tHandle) return "" def scanContent(self) -> list[str]: diff --git a/tests/test_core/test_core_document.py b/tests/test_core/test_core_document.py index 5a236afc..3dbdd454 100644 --- a/tests/test_core/test_core_document.py +++ b/tests/test_core/test_core_document.py @@ -47,6 +47,9 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd): assert bool(doc) is False assert doc.readDocument() is None assert doc.fileExists() is False + assert doc.hashError is False + assert doc.createdDate == "Unknown" + assert doc.updatedDate == "Unknown" # Non-existent handle doc = NWDocument(project, C.hInvalid) @@ -76,6 +79,7 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd): # Try to open a new (non-existent) file xHandle = project.newFile("New File", C.hNovelRoot) + assert xHandle is not None doc = NWDocument(project, xHandle) assert bool(doc) is True assert repr(doc) == f"" @@ -93,7 +97,6 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd): # Set handle and save text = "### Test File\n\nText ...\n\n" doc = NWDocument(project, xHandle) - assert doc.readDocument(xHandle) == "" # type: ignore assert doc.writeDocument(text) is True # Save again to ensure temp file and previous file is handled @@ -145,6 +148,21 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd): doc._handle = None assert doc.writeDocument(text) is False + # Quick Read + # ========== + + contPath = fncPath / "content" + assert NWDocument.quickReadText(contPath, xHandle) == ( + "### Test File\n\n" + "Text ...\n\n" + ) + + # Check read text fallback + assert NWDocument.quickReadText(contPath, "0000000000000") == "" + with monkeypatch.context() as mp: + mp.setattr("builtins.open", causeOSError) + assert NWDocument.quickReadText(contPath, xHandle) == "" + # Delete Document # =============== diff --git a/tests/test_core/test_core_storage.py b/tests/test_core/test_core_storage.py index 6ea0937d..76520078 100644 --- a/tests/test_core/test_core_storage.py +++ b/tests/test_core/test_core_storage.py @@ -64,6 +64,7 @@ def testCoreStorage_CreateNewProject(mockGUI, fncPath): assert bool(storage.getDocument(C.hSceneDoc)) is False assert storage.getMetaFile("file") is None assert storage.scanContent() == [] + assert storage.getDocumentText(C.hSceneDoc) == "" # Cannot prepare a non-empty folder (fncPath / "foobar.txt").touch() @@ -160,12 +161,6 @@ def testCoreStorage_InitProjectStorage(monkeypatch, mockGUI, fncPath, mockRnd): # We can directly access the content of a document assert storage.getDocumentText(C.hSceneDoc) == "### New Scene\n\n" - # Check read text fallback - assert storage.getDocumentText(C.hInvalid) == "" - with monkeypatch.context() as mp: - mp.setattr("builtins.open", causeOSError) - assert storage.getDocumentText(C.hSceneDoc) == "" - project.closeProject() # END Test testCoreStorage_InitProjectStorage