Add fast document read (#1777)

novelwriter/core/coretools.py

@@ -32,7 +32,6 @@ import shutil
 from collections.abc import Iterable
 from functools import partial
 from pathlib import Path
-from time import time
 from zipfile import ZipFile, is_zipfile
 
 from PyQt5.QtCore import QCoreApplication, QRegularExpression
@@ -102,9 +101,7 @@ class DocMerger:
         if srcItem is None:
             return False
 
-        inDoc = self._project.storage.getDocument(srcHandle)
+        docText = self._project.storage.getDocumentText(srcHandle).rstrip("\n")
-        docText = (inDoc.readDocument() or "").rstrip("\n")
-
         if addComment:
             docInfo = srcItem.describeMe()
             docSt, _ = srcItem.getImportStatus(incIcon=False)
@@ -123,9 +120,8 @@ class DocMerger:
             return False
 
         outDoc = self._project.storage.getDocument(self._targetDoc)
-        docText = (outDoc.readDocument() or "").rstrip("\n")
+        if text := (outDoc.readDocument() or "").rstrip("\n"):
-        if docText:
+            self._targetText.insert(0, text)
-            self._targetText.insert(0, docText)
 
         status = outDoc.writeDocument("\n\n".join(self._targetText) + "\n\n")
         if not status:
@@ -293,11 +289,10 @@ class DocDuplicator:
                     newItem.setParent(hMap[newItem.itemParent])
                     self._project.tree.updateItemData(newItem.itemHandle)
                 if newItem.isFileType():
-                    oldDoc = self._project.storage.getDocument(tHandle)
                     newDoc = self._project.storage.getDocument(newItem.itemHandle)
                     if newDoc.fileExists():
                         return
-                    newDoc.writeDocument(oldDoc.readDocument() or "")
+                    newDoc.writeDocument(self._project.storage.getDocumentText(tHandle))
                 yield newItem.itemHandle, nHandle
                 nHandle = None
         return
@@ -308,17 +303,10 @@ class DocDuplicator:
 class DocSearch:
 
     def __init__(self) -> None:
-        # RegEx Object
         self._regEx = QRegularExpression()
         self.setCaseSensitive(False)
         self._words = False
         self._escape = True
-
-        # Project Cache
-        self._uuid = ""
-        self._time = 0.0
-        self._cache: dict[str, str] = {}
-
         return
 
     ##
@@ -347,11 +335,6 @@ class DocSearch:
         self, project: NWProject, search: str
     ) -> Iterable[tuple[NWItem, list[tuple[int, int, str]], bool]]:
         """Iteratively search through documents in a project."""
-        if project.data.uuid != self._uuid or time() - self._time > 20.0:
-            self._cache = {}
-
-        self._uuid = project.data.uuid
-        self._time = time()
         self._regEx.setPattern(self._buildPattern(search))
         logger.debug("Searching with pattern '%s'", self._regEx.pattern())
 
@@ -359,11 +342,7 @@ class DocSearch:
         storage = project.storage
         for item in project.tree:
             if item.isFileType():
-                tHandle = item.itemHandle
+                text = storage.getDocumentText(item.itemHandle)
-                if (text := self._cache.get(tHandle)) is None:
-                    text = storage.getDocument(tHandle).readDocument() or ""
-                    self._cache[tHandle] = text
-
                 rxItt = self._regEx.globalMatch(text)
                 count = 0
                 capped = False

novelwriter/core/document.py

@@ -31,7 +31,7 @@ from typing import TYPE_CHECKING
 from pathlib import Path
 
 from novelwriter.enum import nwItemLayout, nwItemClass
-from novelwriter.error import formatException
+from novelwriter.error import formatException, logException
 from novelwriter.common import formatTimeStamp, isHandle
 from novelwriter.core.item import NWItem
 
@@ -106,7 +106,28 @@ class NWDocument:
         return self._item
 
     ##
-    #  Class Methods
+    #  Static Methods
+    ##
+
+    @staticmethod
+    def quickReadText(content: Path, tHandle: str) -> str:
+        """Return the text of a document in a fast and efficient way."""
+        if (path := content / f"{tHandle}.nwd").is_file():
+            try:
+                with open(path, mode="r", encoding="utf-8") as inFile:
+                    line = ""
+                    for _ in range(10):
+                        if not (line := inFile.readline()).startswith(r"%%~"):
+                            break
+                    return line + inFile.read()
+            except Exception:
+                logger.error("Cannot read document with handle '%s'", tHandle)
+                logException()
+                return ""
+        return ""
+
+    ##
+    #  Methods
     ##
 
     def fileExists(self) -> bool:
@@ -155,7 +176,7 @@ class NWDocument:
             try:
                 with open(docPath, mode="r", encoding="utf-8") as inFile:
                     # Check the first <= 10 lines for metadata
-                    for i in range(10):
+                    for _ in range(10):
                         line = inFile.readline()
                         if line.startswith(r"%%~"):
                             self._parseMeta(line)

novelwriter/core/index.py

@@ -122,9 +122,8 @@ class NWIndex:
         self.clearIndex()
         for nwItem in self._project.tree:
             if nwItem.isFileType():
-                tHandle = nwItem.itemHandle
+                text = self._project.storage.getDocumentText(nwItem.itemHandle)
-                doc = self._project.storage.getDocument(tHandle)
+                self.scanText(nwItem.itemHandle, text, blockSignal=True)
-                self.scanText(tHandle, doc.readDocument() or "", blockSignal=True)
         self._indexBroken = False
         SHARED.indexSignalProxy({"event": "buildIndex"})
         return
@@ -142,17 +141,15 @@ class NWIndex:
         })
         return
 
-    def reIndexHandle(self, tHandle: str | None) -> bool:
+    def reIndexHandle(self, tHandle: str | None) -> None:
         """Put a file back into the index. This is used when files are
         moved from the archive or trash folders back into the active
         project.
         """
         if tHandle and self._project.tree.checkType(tHandle, nwItemType.FILE):
             logger.debug("Re-indexing item '%s'", tHandle)
-            doc = self._project.storage.getDocument(tHandle)
+            self.scanText(tHandle, self._project.storage.getDocumentText(tHandle))
-            self.scanText(tHandle, doc.readDocument() or "")
+        return
-            return True
-        return False
 
     def indexChangedSince(self, checkTime: int | float) -> bool:
         """Check if the index has changed since a given time."""

novelwriter/core/project.py

@@ -175,12 +175,10 @@ class NWProject:
         """Write content to a new document after it is created. This
         will not run if the file exists and is not empty.
         """
-        tItem = self._tree[tHandle]
+        if not ((tItem := self._tree[tHandle]) and tItem.isFileType()):
-        if not (tItem and tItem.isFileType()):
             return False
 
-        newDoc = self._storage.getDocument(tHandle)
+        if self._storage.getDocumentText(tHandle).strip():
-        if (newDoc.readDocument() or "").strip():
             return False
 
         indent = "#"*minmax(hLevel, 1, 4)
@@ -191,7 +189,7 @@ class NWProject:
         else:
             tItem.setLayout(nwItemLayout.NOTE)
 
-        newDoc.writeDocument(text)
+        self._storage.getDocument(tHandle).writeDocument(text)
         self._index.scanText(tHandle, text)
 
         return True
@@ -200,21 +198,18 @@ class NWProject:
         """Copy content to a new document after it is created. This
         will not run if the file exists and is not empty.
         """
-        tItem = self._tree[tHandle]
+        if not ((tItem := self._tree[tHandle]) and tItem.isFileType()):
-        if not (tItem and tItem.isFileType()):
             return False
 
-        sItem = self._tree[sHandle]
+        if not ((sItem := self._tree[sHandle]) and sItem.isFileType()):
-        if not (sItem and sItem.isFileType()):
             return False
 
-        newDoc = self._storage.getDocument(tHandle)
+        if self._storage.getDocumentText(tHandle).strip():
-        if (newDoc.readDocument() or "").strip():
             return False
 
         logger.debug("Populating '%s' with text from '%s'", tHandle, sHandle)
-        text = self._storage.getDocument(sHandle).readDocument() or ""
+        text = self._storage.getDocumentText(sHandle)
-        newDoc.writeDocument(text)
+        self._storage.getDocument(tHandle).writeDocument(text)
         sItem.setLayout(tItem.itemLayout)
         self._index.scanText(tHandle, text)
 

novelwriter/core/storage.py

@@ -27,18 +27,18 @@ import json
 import logging
 
 from enum import Enum
+from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING
-from pathlib import Path
 from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
 
 from novelwriter import CONFIG
-from novelwriter.error import logException
 from novelwriter.common import isHandle, minmax
 from novelwriter.constants import nwFiles
 from novelwriter.core.document import NWDocument
 from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
 from novelwriter.core.spellcheck import UserDictionary
+from novelwriter.error import logException
 
 if TYPE_CHECKING:  # pragma: no cover
     from novelwriter.core.project import NWProject
@@ -287,6 +287,12 @@ class NWStorage:
             return self._runtimePath / "meta" / fileName
         return None
 
+    def getDocumentText(self, tHandle: str) -> str:
+        """Return the text of a document in a fast and efficient way."""
+        if isinstance(self._runtimePath, Path):
+            return NWDocument.quickReadText(self._runtimePath / "content", tHandle)
+        return ""
+
     def scanContent(self) -> list[str]:
         """Scan the content folder and return the handle of all files
         found in it. Files that do not match the pattern are ignored.

novelwriter/core/tokenizer.py

@@ -429,9 +429,7 @@ class Tokenizer(ABC):
         self._text = ""
         self._handle = None
         if nwItem := self._project.tree[tHandle]:
-            if text is None:
+            self._text = text or self._project.storage.getDocumentText(tHandle)
-                text = self._project.storage.getDocument(tHandle).readDocument() or ""
-            self._text = text
             self._handle = tHandle
             self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT
         return

novelwriter/dialogs/docsplit.py

@@ -214,8 +214,7 @@ class GuiDocSplit(QDialog):
 
         spLevel = self.splitLevel.currentData()
         if not self._text:
-            inDoc = SHARED.project.storage.getDocument(sHandle)
+            self._text = SHARED.project.storage.getDocumentText(sHandle).splitlines()
-            self._text = (inDoc.readDocument() or "").splitlines()
 
         for lineNo, aLine in enumerate(self._text):
 

tests/test_core/test_core_coretools.py

@@ -423,9 +423,6 @@ def testCoreTools_DocSearch(monkeypatch, mockGUI, fncPath, mockRnd, ipsumText):
     assert result[1] == (C.hChapterDoc, [], False)
     assert result[2] == (C.hSceneDoc, [(8, 5, "Scene")], False)
 
-    # Cache
-    assert list(search._cache.keys()) == [C.hTitlePage, C.hChapterDoc, C.hSceneDoc]
-
     # Patterns
     # ========
 

tests/test_core/test_core_document.py

@@ -47,6 +47,9 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
     assert bool(doc) is False
     assert doc.readDocument() is None
     assert doc.fileExists() is False
+    assert doc.hashError is False
+    assert doc.createdDate == "Unknown"
+    assert doc.updatedDate == "Unknown"
 
     # Non-existent handle
     doc = NWDocument(project, C.hInvalid)
@@ -76,6 +79,7 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
 
     # Try to open a new (non-existent) file
     xHandle = project.newFile("New File", C.hNovelRoot)
+    assert xHandle is not None
     doc = NWDocument(project, xHandle)
     assert bool(doc) is True
     assert repr(doc) == f"<NWDocument handle={xHandle}>"
@@ -93,7 +97,6 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
     # Set handle and save
     text = "### Test File\n\nText ...\n\n"
     doc = NWDocument(project, xHandle)
-    assert doc.readDocument(xHandle) == ""  # type: ignore
     assert doc.writeDocument(text) is True
 
     # Save again to ensure temp file and previous file is handled
@@ -145,6 +148,21 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
     doc._handle = None
     assert doc.writeDocument(text) is False
 
+    # Quick Read
+    # ==========
+
+    contPath = fncPath / "content"
+    assert NWDocument.quickReadText(contPath, xHandle) == (
+        "### Test File\n\n"
+        "Text ...\n\n"
+    )
+
+    # Check read text fallback
+    assert NWDocument.quickReadText(contPath, "0000000000000") == ""
+    with monkeypatch.context() as mp:
+        mp.setattr("builtins.open", causeOSError)
+        assert NWDocument.quickReadText(contPath, xHandle) == ""
+
     # Delete Document
     # ===============
 

tests/test_core/test_core_index.py

@@ -60,9 +60,8 @@ def testCoreIndex_LoadSave(qtbot, monkeypatch, prjLipsum, mockGUI, tstPaths):
         "60bdf227455cc": False,  # World ROOT
     }
     for tItem in project.tree:
-        assert index.reIndexHandle(tItem.itemHandle) is notIndexable.get(tItem.itemHandle, True)
+        index.reIndexHandle(tItem.itemHandle)
-
+        assert (tItem.itemHandle in index._itemIndex) is notIndexable.get(tItem.itemHandle, True)
-    assert index.reIndexHandle(None) is False
 
     # No folder for saving
     with monkeypatch.context() as mp:

tests/test_core/test_core_storage.py

@@ -26,15 +26,15 @@ import pytest
 from pathlib import Path
 from zipfile import ZipFile
 
-from tools import C, buildTestProject
 from mocked import causeOSError
+from tools import C, buildTestProject
 
 from novelwriter import CONFIG
 from novelwriter.constants import nwFiles
-from novelwriter.core.project import NWProject
-from novelwriter.core.storage import NWStorage, NWStorageOpen, NWStorageCreate, _LegacyStorage
 from novelwriter.core.document import NWDocument
+from novelwriter.core.project import NWProject
 from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
+from novelwriter.core.storage import NWStorage, NWStorageOpen, NWStorageCreate, _LegacyStorage
 
 
 class MockProject:
@@ -64,6 +64,7 @@ def testCoreStorage_CreateNewProject(mockGUI, fncPath):
     assert bool(storage.getDocument(C.hSceneDoc)) is False
     assert storage.getMetaFile("file") is None
     assert storage.scanContent() == []
+    assert storage.getDocumentText(C.hSceneDoc) == ""
 
     # Cannot prepare a non-empty folder
     (fncPath / "foobar.txt").touch()
@@ -85,7 +86,7 @@ def testCoreStorage_CreateNewProject(mockGUI, fncPath):
 
 
 @pytest.mark.core
-def testCoreStorage_InitProjectStorage(mockGUI, fncPath, mockRnd):
+def testCoreStorage_InitProjectStorage(monkeypatch, mockGUI, fncPath, mockRnd):
     """Test initialising a project in a folder."""
     project = NWProject()
 
@@ -157,6 +158,9 @@ def testCoreStorage_InitProjectStorage(mockGUI, fncPath, mockRnd):
     assert isinstance(storage.getDocument(C.hSceneDoc), NWDocument)
     assert repr(storage.getDocument(C.hSceneDoc)) == f"<NWDocument handle={C.hSceneDoc}>"
 
+    # We can directly access the content of a document
+    assert storage.getDocumentText(C.hSceneDoc) == "### New Scene\n\n"
+
     project.closeProject()
 
 # END Test testCoreStorage_InitProjectStorage