Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add fast document read (#1777)

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-03-26 12:19:50 +01:00
committed by GitHub
parent 9081380255 147d6b95d7
commit 8bb0588b2b
11 changed files with 81 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/coretools.py
+5 -26
View File
@@ -32,7 +32,6 @@ import shutil
from collections.abc import Iterable
from functools import partial
from pathlib import Path
from time import time
from zipfile import ZipFile, is_zipfile
from PyQt5.QtCore import QCoreApplication, QRegularExpression
@@ -102,9 +101,7 @@ class DocMerger:
if srcItem is None:
return False
inDoc = self._project.storage.getDocument(srcHandle)
docText = (inDoc.readDocument() or "").rstrip("\n")
docText = self._project.storage.getDocumentText(srcHandle).rstrip("\n")
if addComment:
docInfo = srcItem.describeMe()
docSt, _ = srcItem.getImportStatus(incIcon=False)
@@ -123,9 +120,8 @@ class DocMerger:
return False
outDoc = self._project.storage.getDocument(self._targetDoc)
docText = (outDoc.readDocument() or "").rstrip("\n")
if docText:
self._targetText.insert(0, docText)
if text := (outDoc.readDocument() or "").rstrip("\n"):
self._targetText.insert(0, text)
status = outDoc.writeDocument("\n\n".join(self._targetText) + "\n\n")
if not status:
@@ -293,11 +289,10 @@ class DocDuplicator:
newItem.setParent(hMap[newItem.itemParent])
self._project.tree.updateItemData(newItem.itemHandle)
if newItem.isFileType():
oldDoc = self._project.storage.getDocument(tHandle)
newDoc = self._project.storage.getDocument(newItem.itemHandle)
if newDoc.fileExists():
return
newDoc.writeDocument(oldDoc.readDocument() or "")
newDoc.writeDocument(self._project.storage.getDocumentText(tHandle))
yield newItem.itemHandle, nHandle
nHandle = None
return
@@ -308,17 +303,10 @@ class DocDuplicator:
class DocSearch:
def __init__(self) -> None:
# RegEx Object
self._regEx = QRegularExpression()
self.setCaseSensitive(False)
self._words = False
self._escape = True
# Project Cache
self._uuid = ""
self._time = 0.0
self._cache: dict[str, str] = {}
return
##
@@ -347,11 +335,6 @@ class DocSearch:
self, project: NWProject, search: str
) -> Iterable[tuple[NWItem, list[tuple[int, int, str]], bool]]:
"""Iteratively search through documents in a project."""
if project.data.uuid != self._uuid or time() - self._time > 20.0:
self._cache = {}
self._uuid = project.data.uuid
self._time = time()
self._regEx.setPattern(self._buildPattern(search))
logger.debug("Searching with pattern '%s'", self._regEx.pattern())
@@ -359,11 +342,7 @@ class DocSearch:
storage = project.storage
for item in project.tree:
if item.isFileType():
tHandle = item.itemHandle
if (text := self._cache.get(tHandle)) is None:
text = storage.getDocument(tHandle).readDocument() or ""
self._cache[tHandle] = text
text = storage.getDocumentText(item.itemHandle)
rxItt = self._regEx.globalMatch(text)
count = 0
capped = False
novelwriter/core/document.py
+24 -3
View File
@@ -31,7 +31,7 @@ from typing import TYPE_CHECKING
from pathlib import Path
from novelwriter.enum import nwItemLayout, nwItemClass
from novelwriter.error import formatException
from novelwriter.error import formatException, logException
from novelwriter.common import formatTimeStamp, isHandle
from novelwriter.core.item import NWItem
@@ -106,7 +106,28 @@ class NWDocument:
return self._item
##
# Class Methods
# Static Methods
##
@staticmethod
def quickReadText(content: Path, tHandle: str) -> str:
"""Return the text of a document in a fast and efficient way."""
if (path := content / f"{tHandle}.nwd").is_file():
try:
with open(path, mode="r", encoding="utf-8") as inFile:
line = ""
for _ in range(10):
if not (line := inFile.readline()).startswith(r"%%~"):
break
return line + inFile.read()
except Exception:
logger.error("Cannot read document with handle '%s'", tHandle)
logException()
return ""
return ""
##
# Methods
##
def fileExists(self) -> bool:
@@ -155,7 +176,7 @@ class NWDocument:
try:
with open(docPath, mode="r", encoding="utf-8") as inFile:
# Check the first <= 10 lines for metadata
for i in range(10):
for _ in range(10):
line = inFile.readline()
if line.startswith(r"%%~"):
self._parseMeta(line)
novelwriter/core/index.py
+5 -8
View File
@@ -122,9 +122,8 @@ class NWIndex:
self.clearIndex()
for nwItem in self._project.tree:
if nwItem.isFileType():
tHandle = nwItem.itemHandle
doc = self._project.storage.getDocument(tHandle)
self.scanText(tHandle, doc.readDocument() or "", blockSignal=True)
text = self._project.storage.getDocumentText(nwItem.itemHandle)
self.scanText(nwItem.itemHandle, text, blockSignal=True)
self._indexBroken = False
SHARED.indexSignalProxy({"event": "buildIndex"})
return
@@ -142,17 +141,15 @@ class NWIndex:
})
return
def reIndexHandle(self, tHandle: str | None) -> bool:
def reIndexHandle(self, tHandle: str | None) -> None:
"""Put a file back into the index. This is used when files are
moved from the archive or trash folders back into the active
project.
"""
if tHandle and self._project.tree.checkType(tHandle, nwItemType.FILE):
logger.debug("Re-indexing item '%s'", tHandle)
doc = self._project.storage.getDocument(tHandle)
self.scanText(tHandle, doc.readDocument() or "")
return True
return False
self.scanText(tHandle, self._project.storage.getDocumentText(tHandle))
return
def indexChangedSince(self, checkTime: int | float) -> bool:
"""Check if the index has changed since a given time."""
novelwriter/core/project.py
+8 -13
View File
@@ -175,12 +175,10 @@ class NWProject:
"""Write content to a new document after it is created. This
will not run if the file exists and is not empty.
"""
tItem = self._tree[tHandle]
if not (tItem and tItem.isFileType()):
if not ((tItem := self._tree[tHandle]) and tItem.isFileType()):
return False
newDoc = self._storage.getDocument(tHandle)
if (newDoc.readDocument() or "").strip():
if self._storage.getDocumentText(tHandle).strip():
return False
indent = "#"*minmax(hLevel, 1, 4)
@@ -191,7 +189,7 @@ class NWProject:
else:
tItem.setLayout(nwItemLayout.NOTE)
newDoc.writeDocument(text)
self._storage.getDocument(tHandle).writeDocument(text)
self._index.scanText(tHandle, text)
return True
@@ -200,21 +198,18 @@ class NWProject:
"""Copy content to a new document after it is created. This
will not run if the file exists and is not empty.
"""
tItem = self._tree[tHandle]
if not (tItem and tItem.isFileType()):
if not ((tItem := self._tree[tHandle]) and tItem.isFileType()):
return False
sItem = self._tree[sHandle]
if not (sItem and sItem.isFileType()):
if not ((sItem := self._tree[sHandle]) and sItem.isFileType()):
return False
newDoc = self._storage.getDocument(tHandle)
if (newDoc.readDocument() or "").strip():
if self._storage.getDocumentText(tHandle).strip():
return False
logger.debug("Populating '%s' with text from '%s'", tHandle, sHandle)
text = self._storage.getDocument(sHandle).readDocument() or ""
newDoc.writeDocument(text)
text = self._storage.getDocumentText(sHandle)
self._storage.getDocument(tHandle).writeDocument(text)
sItem.setLayout(tItem.itemLayout)
self._index.scanText(tHandle, text)
novelwriter/core/storage.py
+8 -2
View File
@@ -27,18 +27,18 @@ import json
import logging
from enum import Enum
from pathlib import Path
from time import time
from typing import TYPE_CHECKING
from pathlib import Path
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
from novelwriter import CONFIG
from novelwriter.error import logException
from novelwriter.common import isHandle, minmax
from novelwriter.constants import nwFiles
from novelwriter.core.document import NWDocument
from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
from novelwriter.core.spellcheck import UserDictionary
from novelwriter.error import logException
if TYPE_CHECKING: # pragma: no cover
from novelwriter.core.project import NWProject
@@ -287,6 +287,12 @@ class NWStorage:
return self._runtimePath / "meta" / fileName
return None
def getDocumentText(self, tHandle: str) -> str:
"""Return the text of a document in a fast and efficient way."""
if isinstance(self._runtimePath, Path):
return NWDocument.quickReadText(self._runtimePath / "content", tHandle)
return ""
def scanContent(self) -> list[str]:
"""Scan the content folder and return the handle of all files
found in it. Files that do not match the pattern are ignored.
novelwriter/core/tokenizer.py
+1 -3
View File
@@ -429,9 +429,7 @@ class Tokenizer(ABC):
self._text = ""
self._handle = None
if nwItem := self._project.tree[tHandle]:
if text is None:
text = self._project.storage.getDocument(tHandle).readDocument() or ""
self._text = text
self._text = text or self._project.storage.getDocumentText(tHandle)
self._handle = tHandle
self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT
return
novelwriter/dialogs/docsplit.py
+1 -2
View File
@@ -214,8 +214,7 @@ class GuiDocSplit(QDialog):
spLevel = self.splitLevel.currentData()
if not self._text:
inDoc = SHARED.project.storage.getDocument(sHandle)
self._text = (inDoc.readDocument() or "").splitlines()
self._text = SHARED.project.storage.getDocumentText(sHandle).splitlines()
for lineNo, aLine in enumerate(self._text):
tests/test_core/test_core_coretools.py
-3
View File
@@ -423,9 +423,6 @@ def testCoreTools_DocSearch(monkeypatch, mockGUI, fncPath, mockRnd, ipsumText):
assert result[1] == (C.hChapterDoc, [], False)
assert result[2] == (C.hSceneDoc, [(8, 5, "Scene")], False)
# Cache
assert list(search._cache.keys()) == [C.hTitlePage, C.hChapterDoc, C.hSceneDoc]
# Patterns
# ========
tests/test_core/test_core_document.py
+19 -1
View File
@@ -47,6 +47,9 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
assert bool(doc) is False
assert doc.readDocument() is None
assert doc.fileExists() is False
assert doc.hashError is False
assert doc.createdDate == "Unknown"
assert doc.updatedDate == "Unknown"
# Non-existent handle
doc = NWDocument(project, C.hInvalid)
@@ -76,6 +79,7 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
# Try to open a new (non-existent) file
xHandle = project.newFile("New File", C.hNovelRoot)
assert xHandle is not None
doc = NWDocument(project, xHandle)
assert bool(doc) is True
assert repr(doc) == f"<NWDocument handle={xHandle}>"
@@ -93,7 +97,6 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
# Set handle and save
text = "### Test File\n\nText ...\n\n"
doc = NWDocument(project, xHandle)
assert doc.readDocument(xHandle) == "" # type: ignore
assert doc.writeDocument(text) is True
# Save again to ensure temp file and previous file is handled
@@ -145,6 +148,21 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, fncPath, mockRnd):
doc._handle = None
assert doc.writeDocument(text) is False
# Quick Read
# ==========
contPath = fncPath / "content"
assert NWDocument.quickReadText(contPath, xHandle) == (
"### Test File\n\n"
"Text ...\n\n"
)
# Check read text fallback
assert NWDocument.quickReadText(contPath, "0000000000000") == ""
with monkeypatch.context() as mp:
mp.setattr("builtins.open", causeOSError)
assert NWDocument.quickReadText(contPath, xHandle) == ""
# Delete Document
# ===============
tests/test_core/test_core_index.py
+2 -3
View File
@@ -60,9 +60,8 @@ def testCoreIndex_LoadSave(qtbot, monkeypatch, prjLipsum, mockGUI, tstPaths):
"60bdf227455cc": False, # World ROOT
}
for tItem in project.tree:
assert index.reIndexHandle(tItem.itemHandle) is notIndexable.get(tItem.itemHandle, True)
assert index.reIndexHandle(None) is False
index.reIndexHandle(tItem.itemHandle)
assert (tItem.itemHandle in index._itemIndex) is notIndexable.get(tItem.itemHandle, True)
# No folder for saving
with monkeypatch.context() as mp:
tests/test_core/test_core_storage.py
+8 -4
View File
@@ -26,15 +26,15 @@ import pytest
from pathlib import Path
from zipfile import ZipFile
from tools import C, buildTestProject
from mocked import causeOSError
from tools import C, buildTestProject
from novelwriter import CONFIG
from novelwriter.constants import nwFiles
from novelwriter.core.project import NWProject
from novelwriter.core.storage import NWStorage, NWStorageOpen, NWStorageCreate, _LegacyStorage
from novelwriter.core.document import NWDocument
from novelwriter.core.project import NWProject
from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
from novelwriter.core.storage import NWStorage, NWStorageOpen, NWStorageCreate, _LegacyStorage
class MockProject:
@@ -64,6 +64,7 @@ def testCoreStorage_CreateNewProject(mockGUI, fncPath):
assert bool(storage.getDocument(C.hSceneDoc)) is False
assert storage.getMetaFile("file") is None
assert storage.scanContent() == []
assert storage.getDocumentText(C.hSceneDoc) == ""
# Cannot prepare a non-empty folder
(fncPath / "foobar.txt").touch()
@@ -85,7 +86,7 @@ def testCoreStorage_CreateNewProject(mockGUI, fncPath):
@pytest.mark.core
def testCoreStorage_InitProjectStorage(mockGUI, fncPath, mockRnd):
def testCoreStorage_InitProjectStorage(monkeypatch, mockGUI, fncPath, mockRnd):
"""Test initialising a project in a folder."""
project = NWProject()
@@ -157,6 +158,9 @@ def testCoreStorage_InitProjectStorage(mockGUI, fncPath, mockRnd):
assert isinstance(storage.getDocument(C.hSceneDoc), NWDocument)
assert repr(storage.getDocument(C.hSceneDoc)) == f"<NWDocument handle={C.hSceneDoc}>"
# We can directly access the content of a document
assert storage.getDocumentText(C.hSceneDoc) == "### New Scene\n\n"
project.closeProject()
# END Test testCoreStorage_InitProjectStorage