From 16954c3bbad0263939b80444e1cb4be75471bb2a Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:19:51 +0100 Subject: [PATCH] Add search caching --- novelwriter/constants.py | 1 + novelwriter/core/coretools.py | 80 ++++++++++++++++++++++++++++------- novelwriter/gui/doceditor.py | 25 +++++++---- novelwriter/gui/search.py | 49 +++++++++++++++++---- novelwriter/guimain.py | 2 + 5 files changed, 125 insertions(+), 32 deletions(-) diff --git a/novelwriter/constants.py b/novelwriter/constants.py index 96e3db99..a1e2b84a 100644 --- a/novelwriter/constants.py +++ b/novelwriter/constants.py @@ -53,6 +53,7 @@ class nwConst: # Gui Settings STATUS_MSG_TIMEOUT = 15000 # milliseconds + MAX_SEARCH_RESULT = 1000 # Dialogs DLG_FINISHED = 2 diff --git a/novelwriter/core/coretools.py b/novelwriter/core/coretools.py index d007a771..6492ed7d 100644 --- a/novelwriter/core/coretools.py +++ b/novelwriter/core/coretools.py @@ -306,25 +306,73 @@ class DocDuplicator: class DocSearch: - def __init__(self, project: NWProject, regEx: bool, doCase: bool, wholeWords: bool) -> None: - self._project = project - self._escape = not regEx - self._words = wholeWords - self._rxOpts = QRegularExpression.PatternOption.UseUnicodePropertiesOption - if not doCase: - self._rxOpts |= QRegularExpression.PatternOption.CaseInsensitiveOption + def __init__(self) -> None: + # RegEx Object + self._regEx = QRegularExpression() + self.setCaseSensitive(False) + self._words = False + self._escape = True + + # Project Cache + self._uuid = "" + self._cache: dict[str, str] = {} + return - def iterSearch(self, search: str) -> Iterable[tuple[NWItem, list[tuple[int, int, str]]]]: - """Iteratively search through documents in the project.""" + ## + # Methods + ## + + def setCaseSensitive(self, state: bool) -> None: + """Set the case sensitive search flag.""" + opts = QRegularExpression.PatternOption.UseUnicodePropertiesOption + if not state: + opts |= QRegularExpression.PatternOption.CaseInsensitiveOption + self._regEx.setPatternOptions(opts) + return + + def setWholeWords(self, state: bool) -> None: + """Set the whole words search flag.""" + self._words = state + return + + def setUserRegEx(self, state: bool) -> None: + """Set the escape flag to the opposite state.""" + self._escape = not state + return + + def clearTextCache(self, tHandle: str | None) -> None: + """Clear text cache for a given item, or all items if None.""" + if tHandle is None: + self._cache = {} + logger.debug("Search cache cleared") + elif tHandle in self._cache: + self._cache.pop(tHandle, None) + logger.debug("Search cache cleared for '%s'", tHandle) + return + + def iterSearch( + self, project: NWProject, search: str + ) -> Iterable[tuple[NWItem, list[tuple[int, int, str]]]]: + """Iteratively search through documents in a project.""" + if project.data.uuid != self._uuid: + self.clearTextCache(None) + + self._uuid = project.data.uuid + self._regEx.setPattern(self._buildPattern(search)) + logger.debug("Searching with pattern '%s'", self._regEx.pattern()) + num = len(search) - storage = self._project.storage - regEx = QRegularExpression(self._buildPattern(search), self._rxOpts) - logger.debug("Searching with pattern '%s'", regEx.pattern()) - for item in self._project.tree: + storage = project.storage + for item in project.tree: if item.isFileType(): - text = storage.getDocument(item.itemHandle).readDocument() or "" - rxItt = regEx.globalMatch(text) + tHandle = item.itemHandle + text = self._cache.get(tHandle) + if text is None: + text = storage.getDocument(tHandle).readDocument() or "" + self._cache[tHandle] = text + + rxItt = self._regEx.globalMatch(text) results = [] while rxItt.hasNext(): rxMatch = rxItt.next() @@ -332,7 +380,9 @@ class DocSearch: num = rxMatch.capturedLength() context = text[pos:pos+100].partition("\n")[0] results.append((pos, num, context)) + yield item, results + return ## diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index 6f6edfb6..46a0d52e 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -54,7 +54,7 @@ from PyQt5.QtWidgets import ( from novelwriter import CONFIG, SHARED from novelwriter.common import minmax, transferCase -from novelwriter.constants import nwKeyWords, nwShortcode, nwUnicode +from novelwriter.constants import nwConst, nwKeyWords, nwShortcode, nwUnicode from novelwriter.core.document import NWDocument from novelwriter.enum import nwDocAction, nwDocInsert, nwDocMode, nwItemClass, nwTrinary from novelwriter.extensions.eventfilters import WheelEventFilter @@ -91,6 +91,7 @@ class GuiDocEditor(QPlainTextEdit): # Custom Signals statusMessage = pyqtSignal(str) + docTextSaved = pyqtSignal(str) docCountsChanged = pyqtSignal(str, int, int, int) editedStatusChanged = pyqtSignal(bool) loadDocumentTagRequest = pyqtSignal(str, Enum) @@ -488,6 +489,7 @@ class GuiDocEditor(QPlainTextEdit): return False self.setDocumentChanged(False) + self.docTextSaved.emit(tHandle) oldHeader = self._nwItem.mainHeading oldCount = SHARED.project.index.getHandleHeaderCount(tHandle) @@ -603,13 +605,15 @@ class GuiDocEditor(QPlainTextEdit): # Setters ## - def setDocumentChanged(self, state: bool) -> bool: + def setDocumentChanged(self, state: bool) -> None: """Keep track of the document changed variable, and emit the document change signal. """ - self._docChanged = state - self.editedStatusChanged.emit(self._docChanged) - return self._docChanged + if self._docChanged != state: + logger.debug("Document changed status is '%s'", state) + self._docChanged = state + self.editedStatusChanged.emit(self._docChanged) + return def setCursorPosition(self, position: int) -> None: """Move the cursor to a given position in the document.""" @@ -1372,9 +1376,10 @@ class GuiDocEditor(QPlainTextEdit): cursor.setPosition(0) self.setTextCursor(cursor) - # Search up to a maximum of 1000, and make sure certain special - # searches like a regex search for .* don't loop infinitely - while self.find(searchFor, findOpt) and len(resE) <= 1000: + # Search up to a maximum of MAX_SEARCH_RESULT, and make sure + # certain special searches like a regex search for .* don't loop + # infinitely + while self.find(searchFor, findOpt) and len(resE) <= nwConst.MAX_SEARCH_RESULT: cursor = self.textCursor() if cursor.hasSelection(): resS.append(cursor.selectionStart()) @@ -2613,8 +2618,10 @@ class GuiDocEditSearch(QFrame): def setResultCount(self, currRes: int | None, resCount: int | None) -> None: """Set the count values for the current search.""" + lim = nwConst.MAX_SEARCH_RESULT + numCount = f"{lim:n}+" if (resCount or 0) > lim else f"{resCount:n}" sCurrRes = "?" if currRes is None else str(currRes) - sResCount = "?" if resCount is None else "1000+" if resCount > 1000 else str(resCount) + sResCount = "?" if resCount is None else numCount minWidth = SHARED.theme.getTextWidth(f"{sResCount}//{sResCount}", self.boxFont) self.resultLabel.setText(f"{sCurrRes}/{sResCount}") self.resultLabel.setMinimumWidth(minWidth) diff --git a/novelwriter/gui/search.py b/novelwriter/gui/search.py index d87d9ab4..600d28bd 100644 --- a/novelwriter/gui/search.py +++ b/novelwriter/gui/search.py @@ -29,11 +29,12 @@ from PyQt5.QtCore import QSize, Qt, pyqtSignal, pyqtSlot from PyQt5.QtGui import QKeyEvent, QPalette from PyQt5.QtWidgets import ( QHBoxLayout, QLabel, QLineEdit, QToolBar, QTreeWidget, QTreeWidgetItem, - QVBoxLayout, QWidget + QVBoxLayout, QWidget, qApp ) from novelwriter import CONFIG, SHARED from novelwriter.common import checkInt +from novelwriter.constants import nwConst from novelwriter.core.coretools import DocSearch from novelwriter.core.item import NWItem @@ -55,6 +56,8 @@ class GuiProjectSearch(QWidget): iPx = SHARED.theme.baseIconSize mPx = CONFIG.pxInt(2) + self._search = DocSearch() + # Header self.viewLabel = QLabel(self.tr("Project Search")) self.viewLabel.setFont(SHARED.theme.guiFontB) @@ -150,6 +153,12 @@ class GuiProjectSearch(QWidget): self.searchText.selectAll() return + def closeProjectTasks(self) -> None: + """Run close project tasks.""" + self.searchText.clear() + self.searchResult.clear() + return + ## # Events ## @@ -177,6 +186,16 @@ class GuiProjectSearch(QWidget): super().keyPressEvent(event) return + ## + # Public Slots + ## + + @pyqtSlot(str) + def clearSearchCache(self, tHandle: str) -> None: + """Process document content change.""" + self._search.clearTextCache(tHandle) + return + ## # Private Slots ## @@ -186,11 +205,10 @@ class GuiProjectSearch(QWidget): """Perform a search.""" self.searchResult.clear() if text := self.searchText.text(): - search = DocSearch( - SHARED.project, self.toggleRegEx.isChecked(), - self.toggleCase.isChecked(), self.toggleWord.isChecked() - ) - for item, results in search.iterSearch(text): + self._search.setUserRegEx(self.toggleRegEx.isChecked()) + self._search.setCaseSensitive(self.toggleCase.isChecked()) + self._search.setWholeWords(self.toggleWord.isChecked()) + for item, results in self._search.iterSearch(SHARED.project, text): self._appendResultSet(item, results) return @@ -234,6 +252,10 @@ class GuiProjectSearch(QWidget): # Internal Functions ## + def _initSearch(self) -> None: + """Initialise the search.""" + return + def _appendResultSet(self, nwItem: NWItem, results: list[tuple[int, int, str]]) -> None: """Populate the result tree.""" if results: @@ -242,20 +264,31 @@ class GuiProjectSearch(QWidget): nwItem.itemType, nwItem.itemClass, nwItem.itemLayout, nwItem.mainHeading ) + lim = nwConst.MAX_SEARCH_RESULT + count = len(results) + numResult = f"{count:n}" + if count > lim: + results = results[:lim] + numResult = f"{lim:n}+" tItem = QTreeWidgetItem() - tItem.setText(0, f"{nwItem.itemName} ({len(results)})") + tItem.setText(0, f"{nwItem.itemName} ({numResult})") tItem.setIcon(0, docIcon) tItem.setData(0, self.D_HANDLE, tHandle) + self.searchResult.addTopLevelItem(tItem) + rItems = [] for start, length, context in results: rItem = QTreeWidgetItem() rItem.setText(0, context) rItem.setData(0, self.D_RESULT, (tHandle, start, length)) rItems.append(rItem) + tItem.addChildren(rItems) tItem.setExpanded(True) - self.searchResult.addTopLevelItem(tItem) + + qApp.processEvents() + return # END Class GuiProjectSearch diff --git a/novelwriter/guimain.py b/novelwriter/guimain.py index 77293e0d..256fead4 100644 --- a/novelwriter/guimain.py +++ b/novelwriter/guimain.py @@ -260,6 +260,7 @@ class GuiMain(QMainWindow): self.docEditor.requestProjectItemSelected.connect(self.projView.setSelectedHandle) self.docEditor.requestProjectItemRenamed.connect(self.projView.renameTreeItem) self.docEditor.requestNewNoteCreation.connect(self.projView.createNewNote) + self.docEditor.docTextSaved.connect(self.projSearch.clearSearchCache) self.docViewer.documentLoaded.connect(self.docViewerPanel.updateHandle) self.docViewer.loadDocumentTagRequest.connect(self._followTag) @@ -394,6 +395,7 @@ class GuiMain(QMainWindow): self.outlineView.closeProjectTasks() self.novelView.closeProjectTasks() self.projView.closeProjectTasks() + self.projSearch.closeProjectTasks() self.itemDetails.clearDetails() self.mainStatus.clearStatus()