From 8bdd09400ad20290ac74ec3d273ff927c9db82b1 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 28 May 2022 14:11:58 +0200 Subject: [PATCH 01/13] Move the project index into the project class --- novelwriter/core/project.py | 16 +++++++++++++--- novelwriter/core/tohtml.py | 2 +- novelwriter/core/tomd.py | 2 +- novelwriter/core/toodt.py | 2 +- novelwriter/dialogs/projdetails.py | 9 ++++----- novelwriter/gui/custom.py | 10 +++++----- novelwriter/gui/doceditor.py | 13 ++++++------- novelwriter/gui/dochighlight.py | 7 ++++--- novelwriter/gui/docviewer.py | 4 ++-- novelwriter/gui/itemdetails.py | 2 +- novelwriter/gui/noveltree.py | 11 ++++++----- novelwriter/gui/outline.py | 9 ++++----- novelwriter/gui/outlinedetails.py | 6 +++--- novelwriter/gui/projtree.py | 17 +++++++++-------- novelwriter/guimain.py | 17 ++++++++--------- tests/mock.py | 1 - tests/test_gui/test_gui_docviewer.py | 4 ++-- 17 files changed, 70 insertions(+), 62 deletions(-) diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py index b791f545..24a281b6 100644 --- a/novelwriter/core/project.py +++ b/novelwriter/core/project.py @@ -37,6 +37,7 @@ from PyQt5.QtCore import QCoreApplication from novelwriter.core.tree import NWTree from novelwriter.core.item import NWItem +from novelwriter.core.index import NWIndex from novelwriter.core.status import NWStatus from novelwriter.core.options import OptionState from novelwriter.core.document import NWDoc @@ -62,9 +63,10 @@ class NWProject(): self.mainConf = novelwriter.CONFIG # Core Elements - self.optState = OptionState(self) # Project-specific GUI options - self.projTree = NWTree(self) # The project tree - self.langData = {} # Localisation data + self.optState = OptionState(self) # Project-specific GUI options + self.projTree = NWTree(self) # The project tree + self._projIndex = NWIndex(self) # The projecty index + self.langData = {} # Localisation data # Project Status self.projOpened = 0 # The time stamp of when the project file was opened @@ -116,6 +118,14 @@ class NWProject(): return + ## + # Properties + ## + + @property + def index(self): + return self._projIndex + ## # Item Methods ## diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py index baa5165d..86a21786 100644 --- a/novelwriter/core/tohtml.py +++ b/novelwriter/core/tohtml.py @@ -451,7 +451,7 @@ class ToHtml(Tokenizer): def _formatKeywords(self, tText): """Apply HTML formatting to keywords. """ - isValid, theBits, _ = self.theParent.theIndex.scanThis("@"+tText) + isValid, theBits, _ = self.theProject.index.scanThis("@"+tText) if not isValid or not theBits: return "" diff --git a/novelwriter/core/tomd.py b/novelwriter/core/tomd.py index bd468f55..48d23a35 100644 --- a/novelwriter/core/tomd.py +++ b/novelwriter/core/tomd.py @@ -193,7 +193,7 @@ class ToMarkdown(Tokenizer): def _formatKeywords(self, tText, tStyle): """Apply Markdown formatting to keywords. """ - isValid, theBits, _ = self.theParent.theIndex.scanThis("@"+tText) + isValid, theBits, _ = self.theProject.index.scanThis("@"+tText) if not isValid or not theBits: return "" diff --git a/novelwriter/core/toodt.py b/novelwriter/core/toodt.py index c0b1daee..59eaf30f 100644 --- a/novelwriter/core/toodt.py +++ b/novelwriter/core/toodt.py @@ -550,7 +550,7 @@ class ToOdt(Tokenizer): def _formatKeywords(self, tText): """Apply formatting to keywords. """ - isValid, theBits, _ = self.theParent.theIndex.scanThis("@"+tText) + isValid, theBits, _ = self.theProject.index.scanThis("@"+tText) if not isValid or not theBits: return "" diff --git a/novelwriter/dialogs/projdetails.py b/novelwriter/dialogs/projdetails.py index 54b6c995..491df296 100644 --- a/novelwriter/dialogs/projdetails.py +++ b/novelwriter/dialogs/projdetails.py @@ -145,7 +145,6 @@ class GuiProjectDetailsMain(QWidget): self.theParent = theParent self.theProject = theProject self.theTheme = theParent.theTheme - self.theIndex = theParent.theIndex fPx = self.theTheme.fontPixelSize fPt = self.theTheme.fontPointSize @@ -245,8 +244,9 @@ class GuiProjectDetailsMain(QWidget): def updateValues(self): """Set all the values. """ - hCounts = self.theIndex.getNovelTitleCounts() - nwCount = self.theIndex.getNovelWordCount() + pIndex = self.theProject.index + hCounts = pIndex.getNovelTitleCounts() + nwCount = pIndex.getNovelWordCount() edTime = self.theProject.getCurrentEditTime() self.wordCountVal.setText(f"{nwCount:n}") @@ -277,7 +277,6 @@ class GuiProjectDetailsContents(QWidget): self.theParent = theParent self.theProject = theProject self.theTheme = theParent.theTheme - self.theIndex = theParent.theIndex self.optState = theProject.optState # Internal @@ -424,7 +423,7 @@ class GuiProjectDetailsContents(QWidget): """Extract the data for the tree. """ self._theToC = [] - self._theToC = self.theIndex.getTableOfContents(2) + self._theToC = self.theProject.index.getTableOfContents(2) self._theToC.append(("", 0, self.tr("END"), 0)) return diff --git a/novelwriter/gui/custom.py b/novelwriter/gui/custom.py index 16d26b40..62dd3c91 100644 --- a/novelwriter/gui/custom.py +++ b/novelwriter/gui/custom.py @@ -409,10 +409,10 @@ class PagedDialog(QDialog): return - def addTab(self, tabWidget, tabLabel): + def addTab(self, widget, label): """Forwards the adding of tabs to the QTabWidget. """ - self._tabBox.addTab(tabWidget, tabLabel) + self._tabBox.addTab(widget, label) return def addControls(self, buttonBar): @@ -431,15 +431,15 @@ class VerticalTabBar(QTabBar): self._mW = novelwriter.CONFIG.pxInt(150) return - def tabSizeHint(self, theIndex): + def tabSizeHint(self, index): """Returns a transposed size hint for the rotated bar. """ - tSize = QTabBar.tabSizeHint(self, theIndex) + tSize = QTabBar.tabSizeHint(self, index) tSize.transpose() tSize.setWidth(min(tSize.width(), self._mW)) return tSize - def paintEvent(self, theEvent): + def paintEvent(self, event): """Custom implementation of the label painter that rotates the label 90 degrees. """ diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index 98648ab6..9cd99497 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -79,7 +79,6 @@ class GuiDocEditor(QTextEdit): self.mainConf = novelwriter.CONFIG self.theParent = theParent self.theTheme = theParent.theTheme - self.theIndex = theParent.theIndex self.theProject = theParent.theProject self._nwDocument = None @@ -401,7 +400,7 @@ class GuiDocEditor(QTextEdit): self.document().rootFrame().setFrameFormat(docFrame) self.docFooter.updateLineCount() - self._docHeaders = self.theIndex.getHandleHeaders(self._docHandle) + self._docHeaders = self.theProject.index.getHandleHeaders(self._docHandle) qApp.processEvents() self.document().clearUndoRedoStacks() @@ -506,9 +505,9 @@ class GuiDocEditor(QTextEdit): self.setDocumentChanged(False) - oldHeader = self.theIndex.getHandleHeaderLevel(tHandle) - self.theIndex.scanText(tHandle, docText) - newHeader = self.theIndex.getHandleHeaderLevel(tHandle) + oldHeader = self.theProject.index.getHandleHeaderLevel(tHandle) + self.theProject.index.scanText(tHandle, docText) + newHeader = self.theProject.index.getHandleHeaderLevel(tHandle) if self._updateHeaders(checkLevel=True): self.theParent.requestNovelTreeRefresh() @@ -2003,7 +2002,7 @@ class GuiDocEditor(QTextEdit): if self._docHandle is None: return False - newHeaders = self.theIndex.getHandleHeaders(self._docHandle) + newHeaders = self.theProject.index.getHandleHeaders(self._docHandle) if checkPos: newPos = [x[0] for x in newHeaders] oldPos = [x[0] for x in self._docHeaders] @@ -2943,7 +2942,7 @@ class GuiDocEditFooter(QWidget): else: theStatus, theIcon = self._theItem.getImportStatus() sIcon = theIcon.pixmap(self.sPx, self.sPx) - hLevel = self.theParent.theIndex.getHandleHeaderLevel(self._docHandle) + hLevel = self.theProject.index.getHandleHeaderLevel(self._docHandle) sText = f"{theStatus} / {self._theItem.describeMe(hLevel)}" self.statusIcon.setPixmap(sIcon) diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index eddc27c3..0133f91c 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -55,7 +55,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): self.spEnchant = spEnchant self.theParent = theParent self.theTheme = theParent.theTheme - self.theIndex = theParent.theIndex + self.theProject = theParent.theProject self.theHandle = None self.spellCheck = False self.spellRx = None @@ -287,9 +287,10 @@ class GuiDocHighlighter(QSyntaxHighlighter): if theText.startswith("@"): # Keywords and commands self.setCurrentBlockState(self.BLOCK_META) + pIndex = self.theProject.index tItem = self.theParent.theProject.projTree[self.theHandle] - isValid, theBits, thePos = self.theIndex.scanThis(theText) - isGood = self.theIndex.checkThese(theBits, tItem) + isValid, theBits, thePos = pIndex.scanThis(theText) + isGood = pIndex.checkThese(theBits, tItem) if isValid: for n, theBit in enumerate(theBits): xPos = thePos[n] diff --git a/novelwriter/gui/docviewer.py b/novelwriter/gui/docviewer.py index 4c293da5..615f121c 100644 --- a/novelwriter/gui/docviewer.py +++ b/novelwriter/gui/docviewer.py @@ -245,7 +245,7 @@ class GuiDocViewer(QTextBrowser): index being up to date. """ logger.debug("Loading document from tag '%s'", theTag) - tHandle, _, sTitle = self.theParent.theIndex.getTagSource(theTag) + tHandle, _, sTitle = self.theProject.index.getTagSource(theTag) if tHandle is None: self.theParent.makeAlert(self.tr( "Could not find the reference for tag '{0}'. It either doesn't " @@ -1199,7 +1199,7 @@ class GuiDocViewDetails(QScrollArea): if self.theParent.docViewer.stickyRef: return - theRefs = self.theParent.theIndex.getBackReferenceList(tHandle) + theRefs = self.theProject.index.getBackReferenceList(tHandle) theList = [] for tHandle in theRefs: tItem = self.theProject.projTree[tHandle] diff --git a/novelwriter/gui/itemdetails.py b/novelwriter/gui/itemdetails.py index 8b42b752..88419395 100644 --- a/novelwriter/gui/itemdetails.py +++ b/novelwriter/gui/itemdetails.py @@ -269,7 +269,7 @@ class GuiItemDetails(QWidget): # Layout # ====== - hLevel = self.theParent.theIndex.getHandleHeaderLevel(tHandle) + hLevel = self.theProject.index.getHandleHeaderLevel(tHandle) usageIcon = self.theTheme.getItemIcon( nwItem.itemType, nwItem.itemClass, nwItem.itemLayout, hLevel ) diff --git a/novelwriter/gui/noveltree.py b/novelwriter/gui/noveltree.py index d0732abe..91895c7a 100644 --- a/novelwriter/gui/noveltree.py +++ b/novelwriter/gui/noveltree.py @@ -54,7 +54,6 @@ class GuiNovelTree(QTreeWidget): self.theParent = theParent self.theTheme = theParent.theTheme self.theProject = theParent.theProject - self.theIndex = theParent.theIndex # Internal Variables self._treeMap = {} @@ -137,7 +136,7 @@ class GuiNovelTree(QTreeWidget): """ logger.verbose("Requesting refresh of the novel tree") treeChanged = self.theParent.treeView.changedSince(self._lastBuild) - indexChanged = self.theIndex.novelChangedSince(self._lastBuild) + indexChanged = self.theProject.index.novelChangedSince(self._lastBuild) if not (treeChanged or indexChanged or overRide): logger.verbose("No changes have been made to the novel index") return @@ -158,7 +157,7 @@ class GuiNovelTree(QTreeWidget): def updateWordCounts(self, tHandle): """Update the word count for a given handle. """ - tHeaders = self.theIndex.getHandleWordCounts(tHandle) + tHeaders = self.theProject.index.getHandleWordCounts(tHandle) for titleKey, wCount in tHeaders: if titleKey in self._treeMap: self._treeMap[titleKey].setText(self.C_WORDS, f"{wCount:n}") @@ -252,7 +251,9 @@ class GuiNovelTree(QTreeWidget): currChapter = None currScene = None - for tKey, tHandle, sTitle, novIdx in self.theIndex.novelStructure(skipExcluded=True): + for tKey, tHandle, sTitle, novIdx in self.theProject.index.novelStructure( + skipExcluded=True + ): tItem = self._createTreeItem(tHandle, sTitle, tKey, novIdx) self._treeMap[tKey] = tItem @@ -315,7 +316,7 @@ class GuiNovelTree(QTreeWidget): newItem.setText(self.C_WORDS, f"{wC:n}") newItem.setTextAlignment(self.C_WORDS, Qt.AlignRight) - theRefs = self.theIndex.getReferences(tHandle, sTitle) + theRefs = self.theProject.index.getReferences(tHandle, sTitle) newItem.setText(self.C_POV, ", ".join(theRefs[nwKeyWords.POV_KEY])) return newItem diff --git a/novelwriter/gui/outline.py b/novelwriter/gui/outline.py index 4028ccf6..2b7a654b 100644 --- a/novelwriter/gui/outline.py +++ b/novelwriter/gui/outline.py @@ -91,7 +91,6 @@ class GuiOutline(QTreeWidget): self.theParent = theParent self.theProject = theParent.theProject self.theTheme = theParent.theTheme - self.theIndex = theParent.theIndex self.optState = theParent.theProject.optState self.headerMenu = GuiOutlineHeaderMenu(self) @@ -182,7 +181,7 @@ class GuiOutline(QTreeWidget): # If the novel index or novel tree has changed since the tree # was last built, we rebuild the tree from the updated index. - indexChanged = self.theIndex.novelChangedSince(self._lastBuild) + indexChanged = self.theProject.index.novelChangedSince(self._lastBuild) doBuild = (novelChanged or indexChanged) and self.theProject.autoOutline if doBuild or overRide: logger.debug("Rebuilding Project Outline") @@ -388,7 +387,7 @@ class GuiOutline(QTreeWidget): currChapter = None currScene = None - for tKey, tHandle, sTitle, novIdx in self.theIndex.novelStructure(skipExcluded=True): + for _, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(skipExcluded=True): tItem = self._createTreeItem(tHandle, sTitle, novIdx) @@ -442,7 +441,7 @@ class GuiOutline(QTreeWidget): newItem = QTreeWidgetItem() hIcon = "doc_%s" % novIdx["level"].lower() - hLevel = self.theIndex.getHandleHeaderLevel(tHandle) + hLevel = self.theProject.index.getHandleHeaderLevel(tHandle) dIcon = self.theTheme.getItemIcon(nwItemType.FILE, None, nwItemLayout.DOCUMENT, hLevel) cC = int(novIdx["cCount"]) @@ -465,7 +464,7 @@ class GuiOutline(QTreeWidget): newItem.setTextAlignment(self._colIdx[nwOutline.WCOUNT], Qt.AlignRight) newItem.setTextAlignment(self._colIdx[nwOutline.PCOUNT], Qt.AlignRight) - theRefs = self.theIndex.getReferences(tHandle, sTitle) + theRefs = self.theProject.index.getReferences(tHandle, sTitle) newItem.setText(self._colIdx[nwOutline.POV], ", ".join(theRefs[nwKeyWords.POV_KEY])) newItem.setText(self._colIdx[nwOutline.FOCUS], ", ".join(theRefs[nwKeyWords.FOCUS_KEY])) newItem.setText(self._colIdx[nwOutline.CHAR], ", ".join(theRefs[nwKeyWords.CHAR_KEY])) diff --git a/novelwriter/gui/outlinedetails.py b/novelwriter/gui/outlinedetails.py index 00c20e44..92d41c3c 100644 --- a/novelwriter/gui/outlinedetails.py +++ b/novelwriter/gui/outlinedetails.py @@ -58,7 +58,6 @@ class GuiOutlineDetails(QScrollArea): self.theParent = theParent self.theProject = theParent.theProject self.theTheme = theParent.theTheme - self.theIndex = theParent.theIndex self.optState = theParent.theProject.optState # Sizes @@ -283,9 +282,10 @@ class GuiOutlineDetails(QScrollArea): """Update the content of the tree with the given handle and line number pointing to a header. """ + pIndex = self.theProject.index nwItem = self.theProject.projTree[tHandle] - novIdx = self.theIndex.getNovelData(tHandle, sTitle) - theRefs = self.theIndex.getReferences(tHandle, sTitle) + novIdx = pIndex.getNovelData(tHandle, sTitle) + theRefs = pIndex.getReferences(tHandle, sTitle) if nwItem is None or novIdx is None: return False diff --git a/novelwriter/gui/projtree.py b/novelwriter/gui/projtree.py index b134b901..fd9e21ce 100644 --- a/novelwriter/gui/projtree.py +++ b/novelwriter/gui/projtree.py @@ -61,7 +61,6 @@ class GuiProjectTree(QTreeWidget): self.theParent = theParent self.theTheme = theParent.theTheme self.theProject = theParent.theProject - self.theIndex = theParent.theIndex # Internal Variables self._treeMap = {} @@ -236,12 +235,14 @@ class GuiProjectTree(QTreeWidget): else: newText = f"# {nwItem.itemName}\n\n" + pIndex = self.theProject.index + # Save the text and index it newDoc.writeDocument(newText) - self.theIndex.scanText(tHandle, newText) + pIndex.scanText(tHandle, newText) # Get Word Counts - cC, wC, pC = self.theIndex.getCounts(tHandle) + cC, wC, pC = pIndex.getCounts(tHandle) nwItem.setCharCount(cC) nwItem.setWordCount(wC) nwItem.setParaCount(pC) @@ -542,7 +543,7 @@ class GuiProjectTree(QTreeWidget): expIcon = self.theTheme.getIcon("cross") itempStatus, statusIcon = nwItem.getImportStatus() - hLevel = self.theIndex.getHandleHeaderLevel(tHandle) + hLevel = self.theProject.index.getHandleHeaderLevel(tHandle) itemIcon = self.theTheme.getItemIcon( nwItem.itemType, nwItem.itemClass, nwItem.itemLayout, hLevel ) @@ -598,7 +599,7 @@ class GuiProjectTree(QTreeWidget): if self.theProject.projTree.checkType(pHandle, nwItemType.FILE): # A file has an internal word count we need to account # for, but a folder always has 0 words on its own. - pCount += self.theIndex.getCounts(pHandle)[1] + pCount += self.theProject.index.getCounts(pHandle)[1] self.propagateCount(pHandle, pCount, countChildren=False) @@ -817,9 +818,9 @@ class GuiProjectTree(QTreeWidget): # Update the index if nwItemS.isInactive(): - self.theIndex.deleteHandle(mHandle) + self.theProject.index.deleteHandle(mHandle) else: - self.theIndex.reIndexHandle(mHandle) + self.theProject.index.reIndexHandle(mHandle) self.setTreeItemValues(mHandle) @@ -854,7 +855,7 @@ class GuiProjectTree(QTreeWidget): ], nwAlert.ERROR) return False - self.theIndex.deleteHandle(tHandle) + self.theProject.index.deleteHandle(tHandle) del self.theProject.projTree[tHandle] self._treeMap.pop(tHandle, None) diff --git a/novelwriter/guimain.py b/novelwriter/guimain.py index a759972a..34f9abcd 100644 --- a/novelwriter/guimain.py +++ b/novelwriter/guimain.py @@ -50,7 +50,7 @@ from novelwriter.dialogs import ( from novelwriter.tools import ( GuiBuildNovel, GuiLipsum, GuiProjectWizard, GuiWritingStats ) -from novelwriter.core import NWProject, NWIndex +from novelwriter.core import NWProject from novelwriter.enum import ( nwItemType, nwItemClass, nwAlert, nwWidget, nwState, nwView ) @@ -86,7 +86,6 @@ class GuiMain(QMainWindow): # Core Classes and Settings self.theTheme = GuiTheme() self.theProject = NWProject(self) - self.theIndex = NWIndex(self.theProject) self.hasProject = False self.isFocusMode = False self.idleRefTime = time() @@ -420,7 +419,7 @@ class GuiMain(QMainWindow): self.idleRefTime = time() self.idleTime = 0.0 - self.theIndex.clearIndex() + self.theProject.index.clearIndex() self.clearGUI() self.hasProject = False self._changeView(nwView.PROJECT) @@ -497,7 +496,7 @@ class GuiMain(QMainWindow): self.idleTime = 0.0 # Load the tag index - self.theIndex.loadIndex() + self.theProject.index.loadIndex() # Update GUI self._updateWindowTitle(self.theProject.projName) @@ -516,7 +515,7 @@ class GuiMain(QMainWindow): self.viewDocument(self.theProject.lastViewed) # Check if we need to rebuild the index - if self.theIndex.indexBroken: + if self.theProject.index.indexBroken: self.makeAlert(self.tr( "The project index is outdated or broken. Rebuilding index." ), nwAlert.INFO) @@ -540,7 +539,7 @@ class GuiMain(QMainWindow): self.treeView.saveTreeOrder() if self.theProject.saveProject(autoSave=autoSave): - self.theIndex.saveIndex() + self.theProject.index.saveIndex() return True @@ -863,7 +862,7 @@ class GuiMain(QMainWindow): tStart = time() self.treeView.saveTreeOrder() - self.theIndex.clearIndex() + self.theProject.index.clearIndex() for tItem in self.theProject.projTree: @@ -874,10 +873,10 @@ class GuiMain(QMainWindow): if tItem is not None and tItem.itemType == nwItemType.FILE: logger.verbose("Scanning '%s'", tItem.itemName) - self.theIndex.reIndexHandle(tItem.itemHandle) + self.theProject.index.reIndexHandle(tItem.itemHandle) # Get Word Counts - cC, wC, pC = self.theIndex.getCounts(tItem.itemHandle) + cC, wC, pC = self.theProject.index.getCounts(tItem.itemHandle) tItem.setCharCount(cC) tItem.setWordCount(wC) tItem.setParaCount(pC) diff --git a/tests/mock.py b/tests/mock.py index 4b272a17..23938d41 100644 --- a/tests/mock.py +++ b/tests/mock.py @@ -29,7 +29,6 @@ class MockGuiMain(): def __init__(self): self.mainConf = None self.hasProject = True - self.theIndex = None self.theProject = None self.statusBar = MockStatusBar() diff --git a/tests/test_gui/test_gui_docviewer.py b/tests/test_gui/test_gui_docviewer.py index 0a11bc3d..48f61eff 100644 --- a/tests/test_gui/test_gui_docviewer.py +++ b/tests/test_gui/test_gui_docviewer.py @@ -47,8 +47,8 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum): # Rebuild the index nwGUI.mainMenu.aRebuildIndex.activate(QAction.Trigger) - assert nwGUI.theIndex._tagIndex != {} - assert nwGUI.theIndex._refIndex != {} + assert nwGUI.theProject.index._tagIndex != {} + assert nwGUI.theProject.index._refIndex != {} # Select a document in the project tree nwGUI.treeView.setSelectedHandle("88243afbe5ed8") From 44926a4e9a7039f5fe4ac871d1d89372c0cc723c Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 28 May 2022 14:33:25 +0200 Subject: [PATCH 02/13] Also make tree and options properties of the project --- novelwriter/core/document.py | 2 +- novelwriter/core/index.py | 6 +- novelwriter/core/project.py | 86 ++++++++++++---------- novelwriter/core/tokenizer.py | 6 +- novelwriter/dialogs/docmerge.py | 8 +- novelwriter/dialogs/docsplit.py | 13 ++-- novelwriter/dialogs/itemeditor.py | 2 +- novelwriter/dialogs/projdetails.py | 45 ++++++------ novelwriter/dialogs/projsettings.py | 23 +++--- novelwriter/dialogs/wordlist.py | 11 +-- novelwriter/gui/doceditor.py | 9 +-- novelwriter/gui/dochighlight.py | 2 +- novelwriter/gui/docviewer.py | 10 +-- novelwriter/gui/itemdetails.py | 2 +- novelwriter/gui/outline.py | 20 ++--- novelwriter/gui/outlinedetails.py | 3 +- novelwriter/gui/projtree.py | 44 +++++------ novelwriter/guimain.py | 12 +-- novelwriter/tools/build.py | 90 +++++++++++------------ novelwriter/tools/writingstats.py | 67 ++++++++--------- tests/test_core/test_core_document.py | 2 +- tests/test_core/test_core_index.py | 32 ++++---- tests/test_core/test_core_project.py | 50 ++++++------- tests/test_dialogs/test_dlg_itemeditor.py | 4 +- tests/test_gui/test_gui_doceditor.py | 18 ++--- tests/test_gui/test_gui_docviewer.py | 2 +- tests/test_gui/test_gui_guimain.py | 20 ++--- tests/test_gui/test_gui_projtree.py | 50 ++++++------- 28 files changed, 324 insertions(+), 315 deletions(-) diff --git a/novelwriter/core/document.py b/novelwriter/core/document.py index 2334c77c..5420d44e 100644 --- a/novelwriter/core/document.py +++ b/novelwriter/core/document.py @@ -52,7 +52,7 @@ class NWDoc(): self._docHandle = theHandle if self._docHandle is not None: - self._theItem = self.theProject.projTree[theHandle] + self._theItem = self.theProject.tree[theHandle] return diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 9b8dd47c..7647a9f6 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -107,7 +107,7 @@ class NWIndex(): project. """ logger.debug("Re-indexing item '%s'", tHandle) - if not self.theProject.projTree.checkType(tHandle, nwItemType.FILE): + if not self.theProject.tree.checkType(tHandle, nwItemType.FILE): return False theDoc = NWDoc(self.theProject, tHandle) @@ -207,7 +207,7 @@ class NWIndex(): files before we save them in which case we already have the text. """ - theItem = self.theProject.projTree[tHandle] + theItem = self.theProject.tree[tHandle] if theItem is None: logger.info("Not indexing unknown item '%s'", tHandle) return False @@ -639,7 +639,7 @@ class NWIndex(): """Return a list of all handles that exist in the novel index. """ theHandles = [] - for tItem in self.theProject.projTree: + for tItem in self.theProject.tree: if tItem is None: continue if not tItem.isExported and skipExcluded: diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py index 24a281b6..0a13cb16 100644 --- a/novelwriter/core/project.py +++ b/novelwriter/core/project.py @@ -63,10 +63,10 @@ class NWProject(): self.mainConf = novelwriter.CONFIG # Core Elements - self.optState = OptionState(self) # Project-specific GUI options - self.projTree = NWTree(self) # The project tree + self._optState = OptionState(self) # Project-specific GUI options + self._projTree = NWTree(self) # The project tree self._projIndex = NWIndex(self) # The projecty index - self.langData = {} # Localisation data + self._langData = {} # Localisation data # Project Status self.projOpened = 0 # The time stamp of when the project file was opened @@ -123,9 +123,17 @@ class NWProject(): ## @property - def index(self): + def index(self) -> NWIndex: return self._projIndex + @property + def tree(self) -> NWTree: + return self._projTree + + @property + def options(self) -> OptionState: + return self._optState + ## # Item Methods ## @@ -139,8 +147,8 @@ class NWProject(): newItem.setName(label) newItem.setType(nwItemType.ROOT) newItem.setClass(itemClass) - self.projTree.append(None, None, newItem) - self.projTree.updateItemData(newItem.itemHandle) + self._projTree.append(None, None, newItem) + self._projTree.updateItemData(newItem.itemHandle) return newItem.itemHandle def newFolder(self, label, pHandle): @@ -149,8 +157,8 @@ class NWProject(): newItem = NWItem(self) newItem.setName(label) newItem.setType(nwItemType.FOLDER) - self.projTree.append(None, pHandle, newItem) - self.projTree.updateItemData(newItem.itemHandle) + self._projTree.append(None, pHandle, newItem) + self._projTree.updateItemData(newItem.itemHandle) return newItem.itemHandle def newFile(self, label, pHandle): @@ -159,21 +167,21 @@ class NWProject(): newItem = NWItem(self) newItem.setName(label) newItem.setType(nwItemType.FILE) - self.projTree.append(None, pHandle, newItem) - self.projTree.updateItemData(newItem.itemHandle) + self._projTree.append(None, pHandle, newItem) + self._projTree.updateItemData(newItem.itemHandle) return newItem.itemHandle def trashFolder(self): """Add the special trash root folder to the project. """ - trashHandle = self.projTree.trashRoot() + trashHandle = self._projTree.trashRoot() if trashHandle is None: newItem = NWItem(self) newItem.setName(trConst(nwLabels.CLASS_NAME[nwItemClass.TRASH])) newItem.setType(nwItemType.ROOT) newItem.setClass(nwItemClass.TRASH) - self.projTree.append(None, None, newItem) - self.projTree.updateItemData(newItem.itemHandle) + self._projTree.append(None, None, newItem) + self._projTree.updateItemData(newItem.itemHandle) return newItem.itemHandle return trashHandle @@ -194,7 +202,7 @@ class NWProject(): self.autoCount = 0 # Project Tree - self.projTree.clear() + self._projTree.clear() # Project Settings self.projPath = None @@ -598,9 +606,9 @@ class NWProject(): elif xChild.tag == "content": logger.debug("Found project content") - self.projTree.unpackXML(xChild) + self._projTree.unpackXML(xChild) - self.optState.loadSettings() + self._optState.loadSettings() # Sort out old file locations if legacyList: @@ -618,12 +626,12 @@ class NWProject(): self.mainConf.saveRecentCache() # Check the project tree consistency - for tItem in self.projTree: + for tItem in self._projTree: tHandle = tItem.itemHandle logger.verbose("Checking item '%s'", tHandle) - if not self.projTree.updateItemData(tHandle): + if not self._projTree.updateItemData(tHandle): logger.error("There was a problem item '%s', and it has been removed", tHandle) - del self.projTree[tHandle] # The file will be re-added as orphaned + del self._projTree[tHandle] # The file will be re-added as orphaned self._scanProjectFolder() self._loadProjectLocalisation() @@ -710,7 +718,7 @@ class NWProject(): # Save Tree Content logger.debug("Writing project content") - self.projTree.packXML(nwXML) + self._projTree.packXML(nwXML) # Write the xml tree to file tempFile = os.path.join(self.projPath, self.projFile+"~") @@ -743,7 +751,7 @@ class NWProject(): return False # Save project GUI options - self.optState.saveSettings() + self._optState.saveSettings() # Update recent projects self.mainConf.updateRecentCache(self.projPath, self.projName, self.currWCount, saveTime) @@ -759,8 +767,8 @@ class NWProject(): """Close the current project and clear all meta data. """ logger.info("Closing project: %s", self.projPath) - self.optState.saveSettings() - self.projTree.writeToCFile() + self._optState.saveSettings() + self._projTree.writeToCFile() self._appendSessionStats(idleTime) self._clearLockFile() self.clearProject() @@ -1060,9 +1068,9 @@ class NWProject(): items in the GUI project tree. The user can rearrange the order by drag-and-drop. Forwarded to the NWTree class. """ - if len(self.projTree) != len(newOrder): + if len(self._projTree) != len(newOrder): logger.warning("Sizes of new and old tree order do not match") - self.projTree.setOrder(newOrder) + self._projTree.setOrder(newOrder) self.setProjectChanged(True) return True @@ -1156,16 +1164,16 @@ class NWProject(): capable of handling it. """ sentItems = [] - iterItems = self.projTree.handles() + iterItems = self._projTree.handles() n = 0 nMax = min(len(iterItems), 10000) while n < nMax: tHandle = iterItems[n] - tItem = self.projTree[tHandle] + tItem = self._projTree[tHandle] n += 1 if tItem is None: # Technically a bug since treeOrder is built from the - # same data as projTree + # same data as _projTree continue elif tItem.itemParent is None: # Item is a root, or already been identified as an @@ -1196,7 +1204,7 @@ class NWProject(): def updateWordCounts(self): """Update the total word count values. """ - wcNovel, wcNotes = self.projTree.sumWords() + wcNovel, wcNotes = self._projTree.sumWords() wcTotal = wcNovel + wcNotes if wcTotal != self.currWCount: self.currNovelWC = wcNovel @@ -1212,7 +1220,7 @@ class NWProject(): """ self.statusItems.resetCounts() self.importItems.resetCounts() - for nwItem in self.projTree: + for nwItem in self._projTree: if nwItem.isNovelLike(): self.statusItems.increment(nwItem.itemStatus) else: @@ -1224,7 +1232,7 @@ class NWProject(): return it. The variable is cast to a string before lookup. If the word does not exist, it returns itself. """ - return self.langData.get(str(theWord), str(theWord)) + return self._langData.get(str(theWord), str(theWord)) ## # Internal Functions @@ -1256,7 +1264,7 @@ class NWProject(): """Load the language data for the current project language. """ if self.projLang is None: - self.langData = {} + self._langData = {} return False langFile = os.path.join(self.mainConf.nwLangPath, "project_%s.json" % self.projLang) @@ -1265,7 +1273,7 @@ class NWProject(): try: with open(langFile, mode="r", encoding="utf-8") as inFile: - self.langData = json.load(inFile) + self._langData = json.load(inFile) logger.debug("Loaded project language file: %s", os.path.basename(langFile)) except Exception: @@ -1400,7 +1408,7 @@ class NWProject(): logger.warning("Skipping file: %s", fileItem) continue - if fHandle in self.projTree: + if fHandle in self._projTree: self.projFiles.append(fHandle) logger.debug("Checking file %s, handle '%s': OK", fileItem, fHandle) else: @@ -1447,10 +1455,10 @@ class NWProject(): if oLayout is None: oLayout = nwItemLayout.NOTE - if oParent is None or oParent not in self.projTree: - oParent = self.projTree.findRoot(oClass) + if oParent is None or oParent not in self._projTree: + oParent = self._projTree.findRoot(oClass) if oParent is None: - oParent = self.projTree.findRoot(nwItemClass.NOVEL) + oParent = self._projTree.findRoot(nwItemClass.NOVEL) # If the file still has no parent item, skip it if oParent is None: @@ -1462,8 +1470,8 @@ class NWProject(): orphItem.setType(nwItemType.FILE) orphItem.setClass(oClass) orphItem.setLayout(oLayout) - self.projTree.append(oHandle, oParent, orphItem) - self.projTree.updateItemData(orphItem.itemHandle) + self._projTree.append(oHandle, oParent, orphItem) + self._projTree.updateItemData(orphItem.itemHandle) if noWhere: self.theParent.makeAlert(self.tr( diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index 271ffd2c..bb5f3d8e 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -275,7 +275,7 @@ class Tokenizer(ABC): def addRootHeading(self, theHandle): """Add a heading at the start of a new root folder. """ - if not self.theProject.projTree.checkType(theHandle, nwItemType.ROOT): + if not self.theProject.tree.checkType(theHandle, nwItemType.ROOT): return False if self._isFirst: @@ -284,7 +284,7 @@ class Tokenizer(ABC): else: textAlign = self.A_PBB | self.A_CENTRE - theItem = self.theProject.projTree[theHandle] + theItem = self.theProject.tree[theHandle] locNotes = self._localLookup("Notes") theTitle = f"{locNotes}: {theItem.itemName}" self._theTokens = [] @@ -301,7 +301,7 @@ class Tokenizer(ABC): not set, load it from the file. """ self._theHandle = theHandle - self._theItem = self.theProject.projTree[theHandle] + self._theItem = self.theProject.tree[theHandle] if self._theItem is None: return False diff --git a/novelwriter/dialogs/docmerge.py b/novelwriter/dialogs/docmerge.py index c082dd3b..033f3698 100644 --- a/novelwriter/dialogs/docmerge.py +++ b/novelwriter/dialogs/docmerge.py @@ -125,13 +125,13 @@ class GuiDocMerge(QDialog): ), nwAlert.ERROR) return False - srcItem = self.theProject.projTree[self.sourceItem] + srcItem = self.theProject.tree[self.sourceItem] if srcItem is None: self.theParent.makeAlert(self.tr("Internal error."), nwAlert.ERROR) return False nHandle = self.theProject.newFile(srcItem.itemName, srcItem.itemParent) - newItem = self.theProject.projTree[nHandle] + newItem = self.theProject.tree[nHandle] newItem.setStatus(srcItem.itemStatus) newItem.setImport(srcItem.itemImport) @@ -170,7 +170,7 @@ class GuiDocMerge(QDialog): if tHandle is None: return False - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] if nwItem is None: return False @@ -182,7 +182,7 @@ class GuiDocMerge(QDialog): for sHandle in self.theParent.treeView.getTreeFromHandle(tHandle): newItem = QListWidgetItem() - nwItem = self.theProject.projTree[sHandle] + nwItem = self.theProject.tree[sHandle] if nwItem.itemType is not nwItemType.FILE: continue newItem.setText(nwItem.itemName) diff --git a/novelwriter/dialogs/docsplit.py b/novelwriter/dialogs/docsplit.py index ff2cb849..76e64d5f 100644 --- a/novelwriter/dialogs/docsplit.py +++ b/novelwriter/dialogs/docsplit.py @@ -50,7 +50,6 @@ class GuiDocSplit(QDialog): self.mainConf = novelwriter.CONFIG self.theParent = theParent self.theProject = theParent.theProject - self.optState = theParent.theProject.optState self.sourceItem = None self.sourceText = [] @@ -75,7 +74,7 @@ class GuiDocSplit(QDialog): self.splitLevel.addItem(self.tr("Split up to Header Level 3 (Scene)"), 3) self.splitLevel.addItem(self.tr("Split up to Header Level 4 (Section)"), 4) spIndex = self.splitLevel.findData( - self.optState.getInt("GuiDocSplit", "spLevel", 3) + self.theProject.options.getInt("GuiDocSplit", "spLevel", 3) ) if spIndex != -1: self.splitLevel.setCurrentIndex(spIndex) @@ -121,7 +120,7 @@ class GuiDocSplit(QDialog): ), nwAlert.ERROR) return False - srcItem = self.theProject.projTree[self.sourceItem] + srcItem = self.theProject.tree[self.sourceItem] if srcItem is None: self.theParent.makeAlert(self.tr( "Could not parse source document." @@ -184,7 +183,7 @@ class GuiDocSplit(QDialog): wTitle = wTitle.lstrip("#").strip() nHandle = self.theProject.newFile(wTitle, fHandle) - newItem = self.theProject.projTree[nHandle] + newItem = self.theProject.tree[nHandle] newItem.setStatus(srcItem.itemStatus) newItem.setImport(srcItem.itemImport) logger.verbose( @@ -211,7 +210,7 @@ class GuiDocSplit(QDialog): def _doClose(self): """Close the dialog window without doing anything. """ - self.optState.saveSettings() + self.theProject.options.saveSettings() self.close() return @@ -232,7 +231,7 @@ class GuiDocSplit(QDialog): if self.sourceItem is None: return False - nwItem = self.theProject.projTree[self.sourceItem] + nwItem = self.theProject.tree[self.sourceItem] if nwItem is None: return False @@ -249,7 +248,7 @@ class GuiDocSplit(QDialog): return False spLevel = self.splitLevel.currentData() - self.optState.setValue("GuiDocSplit", "spLevel", spLevel) + self.theProject.options.setValue("GuiDocSplit", "spLevel", spLevel) logger.debug( "Scanning document '%s' for headings level <= %d", self.sourceItem, spLevel diff --git a/novelwriter/dialogs/itemeditor.py b/novelwriter/dialogs/itemeditor.py index b5faec0d..acf07134 100644 --- a/novelwriter/dialogs/itemeditor.py +++ b/novelwriter/dialogs/itemeditor.py @@ -55,7 +55,7 @@ class GuiItemEditor(QDialog): # Build GUI ## - self.theItem = self.theProject.projTree[tHandle] + self.theItem = self.theProject.tree[tHandle] if self.theItem is None: self.close() return diff --git a/novelwriter/dialogs/projdetails.py b/novelwriter/dialogs/projdetails.py index 491df296..258b82e4 100644 --- a/novelwriter/dialogs/projdetails.py +++ b/novelwriter/dialogs/projdetails.py @@ -52,18 +52,18 @@ class GuiProjectDetails(PagedDialog): self.mainConf = novelwriter.CONFIG self.theParent = theParent self.theProject = theParent.theProject - self.optState = theParent.theProject.optState self.setWindowTitle(self.tr("Project Details")) wW = self.mainConf.pxInt(600) wH = self.mainConf.pxInt(400) + pOptions = self.theProject.options self.setMinimumWidth(wW) self.setMinimumHeight(wH) self.resize( - self.mainConf.pxInt(self.optState.getInt("GuiProjectDetails", "winWidth", wW)), - self.mainConf.pxInt(self.optState.getInt("GuiProjectDetails", "winHeight", wH)) + self.mainConf.pxInt(pOptions.getInt("GuiProjectDetails", "winWidth", wW)), + self.mainConf.pxInt(pOptions.getInt("GuiProjectDetails", "winHeight", wH)) ) self.tabMain = GuiProjectDetailsMain(self.theParent, self.theProject) @@ -120,16 +120,17 @@ class GuiProjectDetails(PagedDialog): countFrom = self.tabContents.poValue.value() clearDouble = self.tabContents.dblValue.isChecked() - self.optState.setValue("GuiProjectDetails", "winWidth", winWidth) - self.optState.setValue("GuiProjectDetails", "winHeight", winHeight) - self.optState.setValue("GuiProjectDetails", "widthCol0", widthCol0) - self.optState.setValue("GuiProjectDetails", "widthCol1", widthCol1) - self.optState.setValue("GuiProjectDetails", "widthCol2", widthCol2) - self.optState.setValue("GuiProjectDetails", "widthCol3", widthCol3) - self.optState.setValue("GuiProjectDetails", "widthCol4", widthCol4) - self.optState.setValue("GuiProjectDetails", "wordsPerPage", wordsPerPage) - self.optState.setValue("GuiProjectDetails", "countFrom", countFrom) - self.optState.setValue("GuiProjectDetails", "clearDouble", clearDouble) + pOptions = self.theProject.options + pOptions.setValue("GuiProjectDetails", "winWidth", winWidth) + pOptions.setValue("GuiProjectDetails", "winHeight", winHeight) + pOptions.setValue("GuiProjectDetails", "widthCol0", widthCol0) + pOptions.setValue("GuiProjectDetails", "widthCol1", widthCol1) + pOptions.setValue("GuiProjectDetails", "widthCol2", widthCol2) + pOptions.setValue("GuiProjectDetails", "widthCol3", widthCol3) + pOptions.setValue("GuiProjectDetails", "widthCol4", widthCol4) + pOptions.setValue("GuiProjectDetails", "wordsPerPage", wordsPerPage) + pOptions.setValue("GuiProjectDetails", "countFrom", countFrom) + pOptions.setValue("GuiProjectDetails", "clearDouble", clearDouble) return @@ -277,7 +278,6 @@ class GuiProjectDetailsContents(QWidget): self.theParent = theParent self.theProject = theProject self.theTheme = theParent.theTheme - self.optState = theProject.optState # Internal self._theToC = [] @@ -285,6 +285,7 @@ class GuiProjectDetailsContents(QWidget): iPx = self.theTheme.baseIconSize hPx = self.mainConf.pxInt(12) vPx = self.mainConf.pxInt(4) + pOptions = self.theProject.options # Contents Tree # ============= @@ -313,11 +314,11 @@ class GuiProjectDetailsContents(QWidget): treeHeader.setStretchLastSection(True) treeHeader.setMinimumSectionSize(hPx) - wCol0 = self.mainConf.pxInt(self.optState.getInt("GuiProjectDetails", "widthCol0", 200)) - wCol1 = self.mainConf.pxInt(self.optState.getInt("GuiProjectDetails", "widthCol1", 60)) - wCol2 = self.mainConf.pxInt(self.optState.getInt("GuiProjectDetails", "widthCol2", 60)) - wCol3 = self.mainConf.pxInt(self.optState.getInt("GuiProjectDetails", "widthCol3", 60)) - wCol4 = self.mainConf.pxInt(self.optState.getInt("GuiProjectDetails", "widthCol4", 90)) + wCol0 = self.mainConf.pxInt(pOptions.getInt("GuiProjectDetails", "widthCol0", 200)) + wCol1 = self.mainConf.pxInt(pOptions.getInt("GuiProjectDetails", "widthCol1", 60)) + wCol2 = self.mainConf.pxInt(pOptions.getInt("GuiProjectDetails", "widthCol2", 60)) + wCol3 = self.mainConf.pxInt(pOptions.getInt("GuiProjectDetails", "widthCol3", 60)) + wCol4 = self.mainConf.pxInt(pOptions.getInt("GuiProjectDetails", "widthCol4", 90)) self.tocTree.setColumnWidth(0, wCol0) self.tocTree.setColumnWidth(1, wCol1) @@ -329,9 +330,9 @@ class GuiProjectDetailsContents(QWidget): # Options # ======= - wordsPerPage = self.optState.getInt("GuiProjectDetails", "wordsPerPage", 350) - countFrom = self.optState.getInt("GuiProjectDetails", "countFrom", 1) - clearDouble = self.optState.getInt("GuiProjectDetails", "clearDouble", True) + wordsPerPage = pOptions.getInt("GuiProjectDetails", "wordsPerPage", 350) + countFrom = pOptions.getInt("GuiProjectDetails", "countFrom", 1) + clearDouble = pOptions.getInt("GuiProjectDetails", "clearDouble", True) wordsHelp = ( self.tr("Typical word count for a 5 by 8 inch book page with 11 pt font is 350.") diff --git a/novelwriter/dialogs/projsettings.py b/novelwriter/dialogs/projsettings.py index 927f4ddd..8bbdcee3 100644 --- a/novelwriter/dialogs/projsettings.py +++ b/novelwriter/dialogs/projsettings.py @@ -52,19 +52,19 @@ class GuiProjectSettings(PagedDialog): self.mainConf = novelwriter.CONFIG self.theParent = theParent self.theProject = theParent.theProject - self.optState = theParent.theProject.optState self.theProject.countStatus() self.setWindowTitle(self.tr("Project Settings")) wW = self.mainConf.pxInt(570) wH = self.mainConf.pxInt(375) + pOptions = self.theProject.options self.setMinimumWidth(wW) self.setMinimumHeight(wH) self.resize( - self.mainConf.pxInt(self.optState.getInt("GuiProjectSettings", "winWidth", wW)), - self.mainConf.pxInt(self.optState.getInt("GuiProjectSettings", "winHeight", wH)) + self.mainConf.pxInt(pOptions.getInt("GuiProjectSettings", "winWidth", wW)), + self.mainConf.pxInt(pOptions.getInt("GuiProjectSettings", "winHeight", wH)) ) self.tabMain = GuiProjectEditMain(self.theParent, self.theProject) @@ -152,11 +152,12 @@ class GuiProjectSettings(PagedDialog): statusColW = self.mainConf.rpxInt(self.tabStatus.listBox.columnWidth(0)) importColW = self.mainConf.rpxInt(self.tabImport.listBox.columnWidth(0)) - self.optState.setValue("GuiProjectSettings", "winWidth", winWidth) - self.optState.setValue("GuiProjectSettings", "winHeight", winHeight) - self.optState.setValue("GuiProjectSettings", "replaceColW", replaceColW) - self.optState.setValue("GuiProjectSettings", "statusColW", statusColW) - self.optState.setValue("GuiProjectSettings", "importColW", importColW) + pOptions = self.theProject.options + pOptions.setValue("GuiProjectSettings", "winWidth", winWidth) + pOptions.setValue("GuiProjectSettings", "winHeight", winHeight) + pOptions.setValue("GuiProjectSettings", "replaceColW", replaceColW) + pOptions.setValue("GuiProjectSettings", "statusColW", statusColW) + pOptions.setValue("GuiProjectSettings", "importColW", importColW) return @@ -261,7 +262,6 @@ class GuiProjectEditStatus(QWidget): self.mainConf = novelwriter.CONFIG self.theParent = theParent self.theProject = theProject - self.optState = theProject.optState self.theTheme = theParent.theTheme if isStatus: @@ -274,7 +274,7 @@ class GuiProjectEditStatus(QWidget): colSetting = "importColW" wCol0 = self.mainConf.pxInt( - self.optState.getInt("GuiProjectSettings", colSetting, 130) + self.theProject.options.getInt("GuiProjectSettings", colSetting, 130) ) self.colDeleted = [] @@ -534,11 +534,10 @@ class GuiProjectEditReplace(QWidget): self.theParent = theParent self.theTheme = theParent.theTheme self.theProject = theProject - self.optState = theProject.optState self.arChanged = False wCol0 = self.mainConf.pxInt( - self.optState.getInt("GuiProjectSettings", "replaceColW", 130) + self.theProject.options.getInt("GuiProjectSettings", "replaceColW", 130) ) pageLabel = self.tr("Text Replace List for Preview and Export") diff --git a/novelwriter/dialogs/wordlist.py b/novelwriter/dialogs/wordlist.py index 7dadf258..77a4fb5a 100644 --- a/novelwriter/dialogs/wordlist.py +++ b/novelwriter/dialogs/wordlist.py @@ -52,19 +52,19 @@ class GuiWordList(QDialog): self.theParent = theParent self.theTheme = theParent.theTheme self.theProject = theParent.theProject - self.optState = theParent.theProject.optState self.setWindowTitle(self.tr("Project Word List")) mS = self.mainConf.pxInt(250) wW = self.mainConf.pxInt(320) wH = self.mainConf.pxInt(340) + pOptions = self.theProject.options self.setMinimumWidth(mS) self.setMinimumHeight(mS) self.resize( - self.mainConf.pxInt(self.optState.getInt("GuiWordList", "winWidth", wW)), - self.mainConf.pxInt(self.optState.getInt("GuiWordList", "winHeight", wH)) + self.mainConf.pxInt(pOptions.getInt("GuiWordList", "winWidth", wW)), + self.mainConf.pxInt(pOptions.getInt("GuiWordList", "winHeight", wH)) ) # Main Widgets @@ -207,8 +207,9 @@ class GuiWordList(QDialog): winWidth = self.mainConf.rpxInt(self.width()) winHeight = self.mainConf.rpxInt(self.height()) - self.optState.setValue("GuiWordList", "winWidth", winWidth) - self.optState.setValue("GuiWordList", "winHeight", winHeight) + pOptions = self.theProject.options + pOptions.setValue("GuiWordList", "winWidth", winWidth) + pOptions.setValue("GuiWordList", "winHeight", winHeight) return diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index 9cd99497..64e72259 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -2701,15 +2701,15 @@ class GuiDocEditHeader(QWidget): if self.mainConf.showFullPath: tTitle = [] - tTree = self.theProject.projTree.getItemPath(tHandle) + tTree = self.theProject.tree.getItemPath(tHandle) for aHandle in reversed(tTree): - nwItem = self.theProject.projTree[aHandle] + nwItem = self.theProject.tree[aHandle] if nwItem is not None: tTitle.append(nwItem.itemName) sSep = " %s " % nwUnicode.U_RSAQUO self.theTitle.setText(sSep.join(tTitle)) else: - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] if nwItem is None: return False self.theTitle.setText(nwItem.itemName) @@ -2795,7 +2795,6 @@ class GuiDocEditFooter(QWidget): self.theParent = docEditor.theParent self.theProject = docEditor.theProject self.theTheme = docEditor.theTheme - self.optState = docEditor.theProject.optState self._theItem = None self._docHandle = None @@ -2918,7 +2917,7 @@ class GuiDocEditFooter(QWidget): logger.verbose("No handle set, so clearing the editor footer") self._theItem = None else: - self._theItem = self.theProject.projTree[self._docHandle] + self._theItem = self.theProject.tree[self._docHandle] self.setHasSelection(False) self.updateInfo() diff --git a/novelwriter/gui/dochighlight.py b/novelwriter/gui/dochighlight.py index 0133f91c..bd2d78eb 100644 --- a/novelwriter/gui/dochighlight.py +++ b/novelwriter/gui/dochighlight.py @@ -288,7 +288,7 @@ class GuiDocHighlighter(QSyntaxHighlighter): if theText.startswith("@"): # Keywords and commands self.setCurrentBlockState(self.BLOCK_META) pIndex = self.theProject.index - tItem = self.theParent.theProject.projTree[self.theHandle] + tItem = self.theParent.theProject.tree[self.theHandle] isValid, theBits, thePos = pIndex.scanThis(theText) isGood = pIndex.checkThese(theBits, tItem) if isValid: diff --git a/novelwriter/gui/docviewer.py b/novelwriter/gui/docviewer.py index 615f121c..2587f125 100644 --- a/novelwriter/gui/docviewer.py +++ b/novelwriter/gui/docviewer.py @@ -160,7 +160,7 @@ class GuiDocViewer(QTextBrowser): def loadText(self, tHandle, updateHistory=True): """Load text into the viewer from an item handle. """ - if not self.theProject.projTree.checkType(tHandle, nwItemType.FILE): + if not self.theProject.tree.checkType(tHandle, nwItemType.FILE): logger.warning("Item not found") return False @@ -863,15 +863,15 @@ class GuiDocViewHeader(QWidget): if self.mainConf.showFullPath: tTitle = [] - tTree = self.theProject.projTree.getItemPath(tHandle) + tTree = self.theProject.tree.getItemPath(tHandle) for aHandle in reversed(tTree): - nwItem = self.theProject.projTree[aHandle] + nwItem = self.theProject.tree[aHandle] if nwItem is not None: tTitle.append(nwItem.itemName) sSep = " %s " % nwUnicode.U_RSAQUO self.theTitle.setText(sSep.join(tTitle)) else: - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] if nwItem is None: return False self.theTitle.setText(nwItem.itemName) @@ -1202,7 +1202,7 @@ class GuiDocViewDetails(QScrollArea): theRefs = self.theProject.index.getBackReferenceList(tHandle) theList = [] for tHandle in theRefs: - tItem = self.theProject.projTree[tHandle] + tItem = self.theProject.tree[tHandle] if tItem is not None: theList.append("%s" % ( tHandle, theRefs[tHandle], self.linkStyle, tItem.itemName diff --git a/novelwriter/gui/itemdetails.py b/novelwriter/gui/itemdetails.py index 88419395..89a38b76 100644 --- a/novelwriter/gui/itemdetails.py +++ b/novelwriter/gui/itemdetails.py @@ -227,7 +227,7 @@ class GuiItemDetails(QWidget): self.clearDetails() return - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] if nwItem is None: self.clearDetails() return diff --git a/novelwriter/gui/outline.py b/novelwriter/gui/outline.py index 2b7a654b..26cb8029 100644 --- a/novelwriter/gui/outline.py +++ b/novelwriter/gui/outline.py @@ -91,7 +91,6 @@ class GuiOutline(QTreeWidget): self.theParent = theParent self.theProject = theParent.theProject self.theTheme = theParent.theTheme - self.optState = theParent.theProject.optState self.headerMenu = GuiOutlineHeaderMenu(self) self.setFrameStyle(QFrame.NoFrame) @@ -273,10 +272,12 @@ class GuiOutline(QTreeWidget): """Load the state of the main tree header, that is, column order and column width. """ + pOptions = self.theProject.options + # Load whatever we saved last time, regardless of wether it # contains the correct names or number of columns. The names # must be valid though. - tempOrder = self.optState.getValue("GuiOutline", "headerOrder", []) + tempOrder = pOptions.getValue("GuiOutline", "headerOrder", []) treeOrder = [] for hName in tempOrder: try: @@ -299,14 +300,14 @@ class GuiOutline(QTreeWidget): # We load whatever column widths and hidden states we find in # the file, and leave the rest in their default state. - tmpWidth = self.optState.getValue("GuiOutline", "columnWidth", {}) + tmpWidth = pOptions.getValue("GuiOutline", "columnWidth", {}) for hName in tmpWidth: try: self._colWidth[nwOutline[hName]] = self.mainConf.pxInt(tmpWidth[hName]) except Exception: logger.warning("Ignored unknown outline column '%s'", str(hName)) - tmpHidden = self.optState.getValue("GuiOutline", "columnHidden", {}) + tmpHidden = pOptions.getValue("GuiOutline", "columnHidden", {}) for hName in tmpHidden: try: self._colHidden[nwOutline[hName]] = tmpHidden[hName] @@ -347,10 +348,11 @@ class GuiOutline(QTreeWidget): if not logHidden and logWidth > 0: colWidth[hName] = logWidth - self.optState.setValue("GuiOutline", "headerOrder", treeOrder) - self.optState.setValue("GuiOutline", "columnWidth", colWidth) - self.optState.setValue("GuiOutline", "columnHidden", colHidden) - self.optState.saveSettings() + pOptions = self.theProject.options + pOptions.setValue("GuiOutline", "headerOrder", treeOrder) + pOptions.setValue("GuiOutline", "columnWidth", colWidth) + pOptions.setValue("GuiOutline", "columnHidden", colHidden) + pOptions.saveSettings() return @@ -437,7 +439,7 @@ class GuiOutline(QTreeWidget): def _createTreeItem(self, tHandle, sTitle, novIdx): """Populate a tree item with all the column values. """ - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] newItem = QTreeWidgetItem() hIcon = "doc_%s" % novIdx["level"].lower() diff --git a/novelwriter/gui/outlinedetails.py b/novelwriter/gui/outlinedetails.py index 92d41c3c..40a3d29e 100644 --- a/novelwriter/gui/outlinedetails.py +++ b/novelwriter/gui/outlinedetails.py @@ -58,7 +58,6 @@ class GuiOutlineDetails(QScrollArea): self.theParent = theParent self.theProject = theParent.theProject self.theTheme = theParent.theTheme - self.optState = theParent.theProject.optState # Sizes minTitle = 30*self.theTheme.textNWidth @@ -283,7 +282,7 @@ class GuiOutlineDetails(QScrollArea): number pointing to a header. """ pIndex = self.theProject.index - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] novIdx = pIndex.getNovelData(tHandle, sTitle) theRefs = pIndex.getReferences(tHandle, sTitle) if nwItem is None or novIdx is None: diff --git a/novelwriter/gui/projtree.py b/novelwriter/gui/projtree.py index fd9e21ce..e35662f1 100644 --- a/novelwriter/gui/projtree.py +++ b/novelwriter/gui/projtree.py @@ -180,14 +180,14 @@ class GuiProjectTree(QTreeWidget): elif itemType in (nwItemType.FILE, nwItemType.FOLDER): sHandle = self.getSelectedHandle() - if sHandle is None or sHandle not in self.theProject.projTree: + if sHandle is None or sHandle not in self.theProject.tree: self.theParent.makeAlert(self.tr( "Did not find anywhere to add the file or folder!" ), nwAlert.ERROR) return False # If the selected item is a file, the new item will be a sibling - pItem = self.theProject.projTree[sHandle] + pItem = self.theProject.tree[sHandle] if pItem.itemType == nwItemType.FILE: nHandle = sHandle sHandle = pItem.itemParent @@ -195,7 +195,7 @@ class GuiProjectTree(QTreeWidget): logger.error("Internal error") # Bug return False - if self.theProject.projTree.isTrash(sHandle): + if self.theProject.tree.isTrash(sHandle): self.theParent.makeAlert(self.tr( "Cannot add new files or folders to the Trash folder." ), nwAlert.ERROR) @@ -221,7 +221,7 @@ class GuiProjectTree(QTreeWidget): # Add the new item to the tree self.revealNewTreeItem(tHandle, nHandle) self.theParent.editItem(tHandle) - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] # If this is a folder, return here if nwItem.itemType != nwItemType.FILE: @@ -254,7 +254,7 @@ class GuiProjectTree(QTreeWidget): def revealNewTreeItem(self, tHandle, nHandle=None): """Reveal a newly added project item in the project tree. """ - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] if nwItem is None: return False @@ -375,7 +375,7 @@ class GuiProjectTree(QTreeWidget): logger.error("No project open") return False - trashHandle = self.theProject.projTree.trashRoot() + trashHandle = self.theProject.tree.trashRoot() logger.debug("Emptying Trash folder") if trashHandle is None: @@ -436,7 +436,7 @@ class GuiProjectTree(QTreeWidget): return False trItemS = self._getTreeItem(tHandle) - nwItemS = self.theProject.projTree[tHandle] + nwItemS = self.theProject.tree[tHandle] if trItemS is None or nwItemS is None: logger.error("Could not find tree item for deletion") @@ -477,7 +477,7 @@ class GuiProjectTree(QTreeWidget): logger.error("Could not delete item") return False - if self.theProject.projTree.isTrash(tHandle): + if self.theProject.tree.isTrash(tHandle): # If the file is in the trash folder already, as the # user if they want to permanently delete the file. doPermanent = False @@ -531,7 +531,7 @@ class GuiProjectTree(QTreeWidget): already coming from the project tree. """ trItem = self._getTreeItem(tHandle) - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] if trItem is None or nwItem is None: return @@ -596,7 +596,7 @@ class GuiProjectTree(QTreeWidget): pHandle = pItem.data(self.C_NAME, Qt.UserRole) if pHandle: - if self.theProject.projTree.checkType(pHandle, nwItemType.FILE): + if self.theProject.tree.checkType(pHandle, nwItemType.FILE): # A file has an internal word count we need to account # for, but a folder always has 0 words on its own. pCount += self.theProject.index.getCounts(pHandle)[1] @@ -711,7 +711,7 @@ class GuiProjectTree(QTreeWidget): if isinstance(selItem, QTreeWidgetItem): tHandle = selItem.data(self.C_NAME, Qt.UserRole) self.setSelectedHandle(tHandle) # Just to be safe - tItem = self.theProject.projTree[tHandle] + tItem = self.theProject.tree[tHandle] if tItem is not None: if self.ctxMenu.filterActions(tItem): # Only open menu if any actions remain after filter @@ -749,7 +749,7 @@ class GuiProjectTree(QTreeWidget): return tHandle = selItem.data(self.C_NAME, Qt.UserRole) - tItem = self.theProject.projTree[tHandle] + tItem = self.theProject.tree[tHandle] if tItem is None: return @@ -797,7 +797,7 @@ class GuiProjectTree(QTreeWidget): """Run various maintenance tasks for a moved item. """ trItemS = self._getTreeItem(tHandle) - nwItemS = self.theProject.projTree[tHandle] + nwItemS = self.theProject.tree[tHandle] trItemP = trItemS.parent() if trItemP is None: logger.error("Failed to find new parent item of '%s'", tHandle) @@ -814,7 +814,7 @@ class GuiProjectTree(QTreeWidget): logger.debug("A total of %d item(s) were moved", len(mHandles)) for mHandle in mHandles: logger.debug("Updating item '%s'", mHandle) - self.theProject.projTree.updateItemData(mHandle) + self.theProject.tree.updateItemData(mHandle) # Update the index if nwItemS.isInactive(): @@ -847,7 +847,7 @@ class GuiProjectTree(QTreeWidget): def _deleteTreeItem(self, tHandle): """Permanently delete a tree item from the project and the map. """ - if self.theProject.projTree.checkType(tHandle, nwItemType.FILE): + if self.theProject.tree.checkType(tHandle, nwItemType.FILE): delDoc = NWDoc(self.theProject, tHandle) if not delDoc.deleteDocument(): self.theParent.makeAlert([ @@ -856,7 +856,7 @@ class GuiProjectTree(QTreeWidget): return False self.theProject.index.deleteHandle(tHandle) - del self.theProject.projTree[tHandle] + del self.theProject.tree[tHandle] self._treeMap.pop(tHandle, None) return True @@ -869,7 +869,7 @@ class GuiProjectTree(QTreeWidget): cCount = tItem.childCount() # Update tree-related meta data - nwItem = self.theProject.projTree[tHandle] + nwItem = self.theProject.tree[tHandle] nwItem.setExpanded(tItem.isExpanded() and cCount > 0) nwItem.setOrder(tIndex) @@ -943,7 +943,7 @@ class GuiProjectTree(QTreeWidget): trItem = self._getTreeItem(trashHandle) if trItem is None: trItem = self._addTreeItem( - self.theProject.projTree[trashHandle] + self.theProject.tree[trashHandle] ) if trItem is not None: trItem.setExpanded(True) @@ -963,8 +963,8 @@ class GuiProjectTree(QTreeWidget): def _emitItemChange(self, tHandle): """Emit an item change signal for a given handle. """ - if self.theProject.projTree.checkType(tHandle, nwItemType.FILE): - nwItem = self.theProject.projTree[tHandle] + if self.theProject.tree.checkType(tHandle, nwItemType.FILE): + nwItem = self.theProject.tree[tHandle] if nwItem.isNovelLike(): self.novelItemChanged.emit() else: @@ -1047,9 +1047,9 @@ class GuiProjectTreeMenu(QMenu): logger.error("Failed to extract information to build tree context menu") return False - trashHandle = self.theTree.theProject.projTree.trashRoot() + trashHandle = self.theTree.theProject.tree.trashRoot() - inTrash = self.theTree.theProject.projTree.isTrash(theItem.itemHandle) + inTrash = self.theTree.theProject.tree.isTrash(theItem.itemHandle) isTrash = theItem.itemHandle == trashHandle and trashHandle is not None isFile = theItem.itemType == nwItemType.FILE diff --git a/novelwriter/guimain.py b/novelwriter/guimain.py index 34f9abcd..3a957577 100644 --- a/novelwriter/guimain.py +++ b/novelwriter/guimain.py @@ -572,7 +572,7 @@ class GuiMain(QMainWindow): logger.error("No project open") return False - if not self.theProject.projTree.checkType(tHandle, nwItemType.FILE): + if not self.theProject.tree.checkType(tHandle, nwItemType.FILE): logger.debug("Requested item '%s' is not a document", tHandle) return False @@ -600,8 +600,8 @@ class GuiMain(QMainWindow): nHandle = None # The next handle after tHandle fHandle = None # The first file handle we encounter foundIt = False # We've found tHandle, pick the next we see - for tItem in self.theProject.projTree: - if not self.theProject.projTree.checkType(tItem.itemHandle, nwItemType.FILE): + for tItem in self.theProject.tree: + if not self.theProject.tree.checkType(tItem.itemHandle, nwItemType.FILE): continue if fHandle is None: fHandle = tItem.itemHandle @@ -818,7 +818,7 @@ class GuiMain(QMainWindow): logger.warning("No item selected") return False - tItem = self.theProject.projTree[tHandle] + tItem = self.theProject.tree[tHandle] if tItem is None: return False if tItem.itemType == nwItemType.NO_TYPE: @@ -864,7 +864,7 @@ class GuiMain(QMainWindow): self.treeView.saveTreeOrder() self.theProject.index.clearIndex() - for tItem in self.theProject.projTree: + for tItem in self.theProject.tree: if tItem is not None: self.setStatus(self.tr("Indexing: '{0}'").format(tItem.itemName)) @@ -1560,7 +1560,7 @@ class GuiMain(QMainWindow): """ tHandle = self.treeView.getSelectedHandle() if tHandle is not None: - tItem = self.theProject.projTree[tHandle] + tItem = self.theProject.tree[tHandle] if tItem is None: return if tItem.itemType == nwItemType.FILE: diff --git a/novelwriter/tools/build.py b/novelwriter/tools/build.py index d84a0d4c..06849ee9 100644 --- a/novelwriter/tools/build.py +++ b/novelwriter/tools/build.py @@ -75,7 +75,6 @@ class GuiBuildNovel(QDialog): self.theParent = theParent self.theTheme = theParent.theTheme self.theProject = theParent.theProject - self.optState = theParent.theProject.optState self.htmlText = [] # List of html documents self.htmlStyle = [] # List of html styles @@ -86,9 +85,10 @@ class GuiBuildNovel(QDialog): self.setMinimumWidth(self.mainConf.pxInt(700)) self.setMinimumHeight(self.mainConf.pxInt(600)) + pOptions = self.theProject.options self.resize( - self.mainConf.pxInt(self.optState.getInt("GuiBuildNovel", "winWidth", 900)), - self.mainConf.pxInt(self.optState.getInt("GuiBuildNovel", "winHeight", 800)) + self.mainConf.pxInt(pOptions.getInt("GuiBuildNovel", "winWidth", 900)), + self.mainConf.pxInt(pOptions.getInt("GuiBuildNovel", "winHeight", 800)) ) self.docView = GuiBuildNovelDocView(self, self.theProject) @@ -174,12 +174,12 @@ class GuiBuildNovel(QDialog): self.hideScene = QSwitch(width=wS, height=hS) self.hideScene.setChecked( - self.optState.getBool("GuiBuildNovel", "hideScene", False) + pOptions.getBool("GuiBuildNovel", "hideScene", False) ) self.hideSection = QSwitch(width=wS, height=hS) self.hideSection.setChecked( - self.optState.getBool("GuiBuildNovel", "hideSection", True) + pOptions.getBool("GuiBuildNovel", "hideSection", True) ) # Wrapper boxes due to QGridView and QLineEdit expand bug @@ -235,7 +235,7 @@ class GuiBuildNovel(QDialog): self.textFont.setReadOnly(True) self.textFont.setMinimumWidth(xFmt) self.textFont.setText( - self.optState.getString("GuiBuildNovel", "textFont", self.mainConf.textFont) + pOptions.getString("GuiBuildNovel", "textFont", self.mainConf.textFont) ) self.fontButton = QPushButton("...") self.fontButton.setMaximumWidth(int(2.5*self.theTheme.getTextWidth("..."))) @@ -247,7 +247,7 @@ class GuiBuildNovel(QDialog): self.textSize.setMaximum(72) self.textSize.setSingleStep(1) self.textSize.setValue( - self.optState.getInt("GuiBuildNovel", "textSize", self.mainConf.textSize) + pOptions.getInt("GuiBuildNovel", "textSize", self.mainConf.textSize) ) self.lineHeight = QDoubleSpinBox(self) @@ -257,7 +257,7 @@ class GuiBuildNovel(QDialog): self.lineHeight.setSingleStep(0.05) self.lineHeight.setDecimals(2) self.lineHeight.setValue( - self.optState.getFloat("GuiBuildNovel", "lineHeight", 1.15) + pOptions.getFloat("GuiBuildNovel", "lineHeight", 1.15) ) # Wrapper box due to QGridView and QLineEdit expand bug @@ -291,12 +291,12 @@ class GuiBuildNovel(QDialog): self.justifyText = QSwitch(width=wS, height=hS) self.justifyText.setChecked( - self.optState.getBool("GuiBuildNovel", "justifyText", False) + pOptions.getBool("GuiBuildNovel", "justifyText", False) ) self.noStyling = QSwitch(width=wS, height=hS) self.noStyling.setChecked( - self.optState.getBool("GuiBuildNovel", "noStyling", False) + pOptions.getBool("GuiBuildNovel", "noStyling", False) ) self.styleForm.addWidget(justifyLabel, 1, 0, 1, 1, Qt.AlignLeft) @@ -316,22 +316,22 @@ class GuiBuildNovel(QDialog): self.includeSynopsis = QSwitch(width=wS, height=hS) self.includeSynopsis.setChecked( - self.optState.getBool("GuiBuildNovel", "incSynopsis", False) + pOptions.getBool("GuiBuildNovel", "incSynopsis", False) ) self.includeComments = QSwitch(width=wS, height=hS) self.includeComments.setChecked( - self.optState.getBool("GuiBuildNovel", "incComments", False) + pOptions.getBool("GuiBuildNovel", "incComments", False) ) self.includeKeywords = QSwitch(width=wS, height=hS) self.includeKeywords.setChecked( - self.optState.getBool("GuiBuildNovel", "incKeywords", False) + pOptions.getBool("GuiBuildNovel", "incKeywords", False) ) self.includeBody = QSwitch(width=wS, height=hS) self.includeBody.setChecked( - self.optState.getBool("GuiBuildNovel", "incBodyText", True) + pOptions.getBool("GuiBuildNovel", "incBodyText", True) ) synopsisLabel = QLabel(self.tr("Include synopsis")) @@ -360,17 +360,17 @@ class GuiBuildNovel(QDialog): self.novelFiles = QSwitch(width=wS, height=hS) self.novelFiles.setChecked( - self.optState.getBool("GuiBuildNovel", "addNovel", True) + pOptions.getBool("GuiBuildNovel", "addNovel", True) ) self.noteFiles = QSwitch(width=wS, height=hS) self.noteFiles.setChecked( - self.optState.getBool("GuiBuildNovel", "addNotes", False) + pOptions.getBool("GuiBuildNovel", "addNotes", False) ) self.ignoreFlag = QSwitch(width=wS, height=hS) self.ignoreFlag.setChecked( - self.optState.getBool("GuiBuildNovel", "ignoreFlag", False) + pOptions.getBool("GuiBuildNovel", "ignoreFlag", False) ) novelLabel = QLabel(self.tr("Include novel files")) @@ -396,12 +396,12 @@ class GuiBuildNovel(QDialog): self.replaceTabs = QSwitch(width=wS, height=hS) self.replaceTabs.setChecked( - self.optState.getBool("GuiBuildNovel", "replaceTabs", False) + pOptions.getBool("GuiBuildNovel", "replaceTabs", False) ) self.replaceUCode = QSwitch(width=wS, height=hS) self.replaceUCode.setChecked( - self.optState.getBool("GuiBuildNovel", "replaceUCode", False) + pOptions.getBool("GuiBuildNovel", "replaceUCode", False) ) tabsLabel = QLabel(self.tr("Replace tabs with spaces")) @@ -493,9 +493,9 @@ class GuiBuildNovel(QDialog): # Splitter Position boxWidth = self.mainConf.pxInt(350) - boxWidth = self.optState.getInt("GuiBuildNovel", "boxWidth", boxWidth) + boxWidth = pOptions.getInt("GuiBuildNovel", "boxWidth", boxWidth) docWidth = max(self.width() - boxWidth, 100) - docWidth = self.optState.getInt("GuiBuildNovel", "docWidth", docWidth) + docWidth = pOptions.getInt("GuiBuildNovel", "docWidth", docWidth) # The Tool Box self.toolsBox = QVBoxLayout() @@ -712,10 +712,10 @@ class GuiBuildNovel(QDialog): self.theParent.treeView.flushTreeOrder() self.theParent.saveDocument() - self.buildProgress.setMaximum(len(self.theProject.projTree)) + self.buildProgress.setMaximum(len(self.theProject.tree)) self.buildProgress.setValue(0) - for nItt, tItem in enumerate(self.theProject.projTree): + for nItt, tItem in enumerate(self.theProject.tree): noteRoot = noteFiles noteRoot &= tItem.itemType == nwItemType.ROOT @@ -1153,28 +1153,28 @@ class GuiBuildNovel(QDialog): self.theProject.setProjectLang(buildLang) # GUI Settings - self.optState.setValue("GuiBuildNovel", "hideScene", hideScene) - self.optState.setValue("GuiBuildNovel", "hideSection", hideSection) - self.optState.setValue("GuiBuildNovel", "winWidth", winWidth) - self.optState.setValue("GuiBuildNovel", "winHeight", winHeight) - self.optState.setValue("GuiBuildNovel", "boxWidth", boxWidth) - self.optState.setValue("GuiBuildNovel", "docWidth", docWidth) - self.optState.setValue("GuiBuildNovel", "justifyText", justifyText) - self.optState.setValue("GuiBuildNovel", "noStyling", noStyling) - self.optState.setValue("GuiBuildNovel", "textFont", textFont) - self.optState.setValue("GuiBuildNovel", "textSize", textSize) - self.optState.setValue("GuiBuildNovel", "lineHeight", lineHeight) - self.optState.setValue("GuiBuildNovel", "addNovel", novelFiles) - self.optState.setValue("GuiBuildNovel", "addNotes", noteFiles) - self.optState.setValue("GuiBuildNovel", "ignoreFlag", ignoreFlag) - self.optState.setValue("GuiBuildNovel", "incSynopsis", incSynopsis) - self.optState.setValue("GuiBuildNovel", "incComments", incComments) - self.optState.setValue("GuiBuildNovel", "incKeywords", incKeywords) - self.optState.setValue("GuiBuildNovel", "incBodyText", incBodyText) - self.optState.setValue("GuiBuildNovel", "replaceTabs", replaceTabs) - self.optState.setValue("GuiBuildNovel", "replaceUCode", replaceUCode) - - self.optState.saveSettings() + pOptions = self.theProject.options + pOptions.setValue("GuiBuildNovel", "hideScene", hideScene) + pOptions.setValue("GuiBuildNovel", "hideSection", hideSection) + pOptions.setValue("GuiBuildNovel", "winWidth", winWidth) + pOptions.setValue("GuiBuildNovel", "winHeight", winHeight) + pOptions.setValue("GuiBuildNovel", "boxWidth", boxWidth) + pOptions.setValue("GuiBuildNovel", "docWidth", docWidth) + pOptions.setValue("GuiBuildNovel", "justifyText", justifyText) + pOptions.setValue("GuiBuildNovel", "noStyling", noStyling) + pOptions.setValue("GuiBuildNovel", "textFont", textFont) + pOptions.setValue("GuiBuildNovel", "textSize", textSize) + pOptions.setValue("GuiBuildNovel", "lineHeight", lineHeight) + pOptions.setValue("GuiBuildNovel", "addNovel", novelFiles) + pOptions.setValue("GuiBuildNovel", "addNotes", noteFiles) + pOptions.setValue("GuiBuildNovel", "ignoreFlag", ignoreFlag) + pOptions.setValue("GuiBuildNovel", "incSynopsis", incSynopsis) + pOptions.setValue("GuiBuildNovel", "incComments", incComments) + pOptions.setValue("GuiBuildNovel", "incKeywords", incKeywords) + pOptions.setValue("GuiBuildNovel", "incBodyText", incBodyText) + pOptions.setValue("GuiBuildNovel", "replaceTabs", replaceTabs) + pOptions.setValue("GuiBuildNovel", "replaceUCode", replaceUCode) + pOptions.saveSettings() return diff --git a/novelwriter/tools/writingstats.py b/novelwriter/tools/writingstats.py index 4ce83b66..aff7bc3d 100644 --- a/novelwriter/tools/writingstats.py +++ b/novelwriter/tools/writingstats.py @@ -67,33 +67,34 @@ class GuiWritingStats(QDialog): self.theParent = theParent self.theTheme = theParent.theTheme self.theProject = theParent.theProject - self.optState = theParent.theProject.optState self.logData = [] self.filterData = [] self.timeFilter = 0.0 self.wordOffset = 0 + pOptions = self.theProject.options + self.setWindowTitle(self.tr("Writing Statistics")) self.setMinimumWidth(self.mainConf.pxInt(420)) self.setMinimumHeight(self.mainConf.pxInt(400)) self.resize( - self.mainConf.pxInt(self.optState.getInt("GuiWritingStats", "winWidth", 550)), - self.mainConf.pxInt(self.optState.getInt("GuiWritingStats", "winHeight", 500)) + self.mainConf.pxInt(pOptions.getInt("GuiWritingStats", "winWidth", 550)), + self.mainConf.pxInt(pOptions.getInt("GuiWritingStats", "winHeight", 500)) ) # List Box wCol0 = self.mainConf.pxInt( - self.optState.getInt("GuiWritingStats", "widthCol0", 180) + pOptions.getInt("GuiWritingStats", "widthCol0", 180) ) wCol1 = self.mainConf.pxInt( - self.optState.getInt("GuiWritingStats", "widthCol1", 80) + pOptions.getInt("GuiWritingStats", "widthCol1", 80) ) wCol2 = self.mainConf.pxInt( - self.optState.getInt("GuiWritingStats", "widthCol2", 80) + pOptions.getInt("GuiWritingStats", "widthCol2", 80) ) wCol3 = self.mainConf.pxInt( - self.optState.getInt("GuiWritingStats", "widthCol3", 80) + pOptions.getInt("GuiWritingStats", "widthCol3", 80) ) self.listBox = QTreeWidget() @@ -115,9 +116,9 @@ class GuiWritingStats(QDialog): hHeader.setTextAlignment(self.C_IDLE, Qt.AlignRight) hHeader.setTextAlignment(self.C_COUNT, Qt.AlignRight) - sortCol = checkIntRange(self.optState.getInt("GuiWritingStats", "sortCol", 0), 0, 2, 0) + sortCol = checkIntRange(pOptions.getInt("GuiWritingStats", "sortCol", 0), 0, 2, 0) sortOrder = checkIntTuple( - self.optState.getInt("GuiWritingStats", "sortOrder", Qt.DescendingOrder), + pOptions.getInt("GuiWritingStats", "sortOrder", Qt.DescendingOrder), (Qt.AscendingOrder, Qt.DescendingOrder), Qt.DescendingOrder ) self.listBox.sortByColumn(sortCol, sortOrder) @@ -190,37 +191,37 @@ class GuiWritingStats(QDialog): self.incNovel = QSwitch(width=2*sPx, height=sPx) self.incNovel.setChecked( - self.optState.getBool("GuiWritingStats", "incNovel", True) + pOptions.getBool("GuiWritingStats", "incNovel", True) ) self.incNovel.clicked.connect(self._updateListBox) self.incNotes = QSwitch(width=2*sPx, height=sPx) self.incNotes.setChecked( - self.optState.getBool("GuiWritingStats", "incNotes", True) + pOptions.getBool("GuiWritingStats", "incNotes", True) ) self.incNotes.clicked.connect(self._updateListBox) self.hideZeros = QSwitch(width=2*sPx, height=sPx) self.hideZeros.setChecked( - self.optState.getBool("GuiWritingStats", "hideZeros", True) + pOptions.getBool("GuiWritingStats", "hideZeros", True) ) self.hideZeros.clicked.connect(self._updateListBox) self.hideNegative = QSwitch(width=2*sPx, height=sPx) self.hideNegative.setChecked( - self.optState.getBool("GuiWritingStats", "hideNegative", False) + pOptions.getBool("GuiWritingStats", "hideNegative", False) ) self.hideNegative.clicked.connect(self._updateListBox) self.groupByDay = QSwitch(width=2*sPx, height=sPx) self.groupByDay.setChecked( - self.optState.getBool("GuiWritingStats", "groupByDay", False) + pOptions.getBool("GuiWritingStats", "groupByDay", False) ) self.groupByDay.clicked.connect(self._updateListBox) self.showIdleTime = QSwitch(width=2*sPx, height=sPx) self.showIdleTime.setChecked( - self.optState.getBool("GuiWritingStats", "showIdleTime", False) + pOptions.getBool("GuiWritingStats", "showIdleTime", False) ) self.showIdleTime.clicked.connect(self._updateListBox) @@ -244,7 +245,7 @@ class GuiWritingStats(QDialog): self.histMax.setMaximum(100000) self.histMax.setSingleStep(100) self.histMax.setValue( - self.optState.getInt("GuiWritingStats", "histMax", 2000) + pOptions.getInt("GuiWritingStats", "histMax", 2000) ) self.histMax.valueChanged.connect(self._updateListBox) @@ -323,23 +324,23 @@ class GuiWritingStats(QDialog): showIdleTime = self.showIdleTime.isChecked() histMax = self.histMax.value() - self.optState.setValue("GuiWritingStats", "winWidth", winWidth) - self.optState.setValue("GuiWritingStats", "winHeight", winHeight) - self.optState.setValue("GuiWritingStats", "widthCol0", widthCol0) - self.optState.setValue("GuiWritingStats", "widthCol1", widthCol1) - self.optState.setValue("GuiWritingStats", "widthCol2", widthCol2) - self.optState.setValue("GuiWritingStats", "widthCol3", widthCol3) - self.optState.setValue("GuiWritingStats", "sortCol", sortCol) - self.optState.setValue("GuiWritingStats", "sortOrder", sortOrder) - self.optState.setValue("GuiWritingStats", "incNovel", incNovel) - self.optState.setValue("GuiWritingStats", "incNotes", incNotes) - self.optState.setValue("GuiWritingStats", "hideZeros", hideZeros) - self.optState.setValue("GuiWritingStats", "hideNegative", hideNegative) - self.optState.setValue("GuiWritingStats", "groupByDay", groupByDay) - self.optState.setValue("GuiWritingStats", "showIdleTime", showIdleTime) - self.optState.setValue("GuiWritingStats", "histMax", histMax) - - self.optState.saveSettings() + pOptions = self.theProject.options + pOptions.setValue("GuiWritingStats", "winWidth", winWidth) + pOptions.setValue("GuiWritingStats", "winHeight", winHeight) + pOptions.setValue("GuiWritingStats", "widthCol0", widthCol0) + pOptions.setValue("GuiWritingStats", "widthCol1", widthCol1) + pOptions.setValue("GuiWritingStats", "widthCol2", widthCol2) + pOptions.setValue("GuiWritingStats", "widthCol3", widthCol3) + pOptions.setValue("GuiWritingStats", "sortCol", sortCol) + pOptions.setValue("GuiWritingStats", "sortOrder", sortOrder) + pOptions.setValue("GuiWritingStats", "incNovel", incNovel) + pOptions.setValue("GuiWritingStats", "incNotes", incNotes) + pOptions.setValue("GuiWritingStats", "hideZeros", hideZeros) + pOptions.setValue("GuiWritingStats", "hideNegative", hideNegative) + pOptions.setValue("GuiWritingStats", "groupByDay", groupByDay) + pOptions.setValue("GuiWritingStats", "showIdleTime", showIdleTime) + pOptions.setValue("GuiWritingStats", "histMax", histMax) + pOptions.saveSettings() self.close() return diff --git a/tests/test_core/test_core_document.py b/tests/test_core/test_core_document.py index 881290d6..2f80e45a 100644 --- a/tests/test_core/test_core_document.py +++ b/tests/test_core/test_core_document.py @@ -64,7 +64,7 @@ def testCoreDocument_LoadSave(monkeypatch, mockGUI, nwMinimal): assert theDoc.readDocument() == "### New Scene\n\n" # Try to open a new (non-existent) file - nHandle = theProject.projTree.findRoot(nwItemClass.NOVEL) + nHandle = theProject.tree.findRoot(nwItemClass.NOVEL) assert nHandle is not None xHandle = theProject.newFile("New File", nHandle) theDoc = NWDoc(theProject, xHandle) diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index 93ec35c6..0d070eba 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -54,7 +54,7 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): "6c6afb1247750": False, # Plot ROOT "60bdf227455cc": False, # World ROOT } - for tItem in theProject.projTree: + for tItem in theProject.tree: assert theIndex.reIndexHandle(tItem.itemHandle) is notIndexable.get(tItem.itemHandle, True) assert theIndex.reIndexHandle(None) is False @@ -180,8 +180,8 @@ def testCoreIndex_CheckThese(nwMinimal, mockGUI): theIndex = NWIndex(theProject) nHandle = theProject.newFile("Hello", "a508bb932959c") cHandle = theProject.newFile("Jane", "afb3043c7b2b3") - nItem = theProject.projTree[nHandle] - cItem = theProject.projTree[cHandle] + nItem = theProject.tree[nHandle] + cItem = theProject.tree[cHandle] assert theIndex.novelChangedSince(0) is False assert theIndex.notesChangedSince(0) is False @@ -258,7 +258,7 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): # Some items for fail to scan tests dHandle = theProject.newFolder("Folder", "a508bb932959c") xHandle = theProject.newFile("No Layout", "a508bb932959c") - xItem = theProject.projTree[xHandle] + xItem = theProject.tree[xHandle] xItem.setLayout(nwItemLayout.NO_LAYOUT) # Check invalid data @@ -272,18 +272,18 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): # Create the trash folder tHandle = theProject.trashFolder() - assert theProject.projTree[tHandle] is not None + assert theProject.tree[tHandle] is not None xItem.setParent(tHandle) - theProject.projTree.updateItemData(xItem.itemHandle) + theProject.tree.updateItemData(xItem.itemHandle) assert xItem.itemRoot == tHandle assert xItem.itemClass == nwItemClass.TRASH assert theIndex.scanText(xHandle, "Hello World!") is False # Create the archive root aHandle = theProject.newRoot(nwItemClass.ARCHIVE) - assert theProject.projTree[aHandle] is not None + assert theProject.tree[aHandle] is not None xItem.setParent(aHandle) - theProject.projTree.updateItemData(xItem.itemHandle) + theProject.tree.updateItemData(xItem.itemHandle) assert theIndex.scanText(xHandle, "Hello World!") is False # Make some usable items @@ -433,7 +433,7 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): # Page wo/Title # ============= - theProject.projTree[pHandle]._layout = nwItemLayout.DOCUMENT + theProject.tree[pHandle]._layout = nwItemLayout.DOCUMENT assert theIndex.scanText(pHandle, ( "This is a page with some text on it.\n\n" )) @@ -446,7 +446,7 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): assert theIndex._fileIndex[pHandle]["T000000"]["pCount"] == 1 assert theIndex._fileIndex[pHandle]["T000000"]["synopsis"] == "" - theProject.projTree[pHandle]._layout = nwItemLayout.NOTE + theProject.tree[pHandle]._layout = nwItemLayout.NOTE assert theIndex.scanText(pHandle, ( "This is a page with some text on it.\n\n" )) @@ -499,7 +499,7 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): assert theKeys == ["%s:T000001" % nHandle] # Check that excluded files can be skipped - theProject.projTree[nHandle].setExported(False) + theProject.tree[nHandle].setExported(False) theKeys = [] for aKey, _, _, _ in theIndex.novelStructure(skipExcluded=False): @@ -631,9 +631,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): sHandle = theProject.newFile("Scene One", "a508bb932959c") tHandle = theProject.newFile("Scene Two", "a508bb932959c") - theProject.projTree[hHandle].itemLayout == nwItemLayout.DOCUMENT - theProject.projTree[sHandle].itemLayout == nwItemLayout.DOCUMENT - theProject.projTree[tHandle].itemLayout == nwItemLayout.DOCUMENT + theProject.tree[hHandle].itemLayout == nwItemLayout.DOCUMENT + theProject.tree[sHandle].itemLayout == nwItemLayout.DOCUMENT + theProject.tree[tHandle].itemLayout == nwItemLayout.DOCUMENT assert theIndex.scanText(hHandle, "## Chapter One\n\n") assert theIndex.scanText(sHandle, "### Scene One\n\n") @@ -643,9 +643,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): assert theIndex._listNovelHandles(True) == [hHandle, sHandle, tHandle] # Add a fake handle to the tree and check that it's ignored - theProject.projTree._treeOrder.append("0000000000000") + theProject.tree._treeOrder.append("0000000000000") assert theIndex._listNovelHandles(False) == [nHandle, hHandle, sHandle, tHandle] - theProject.projTree._treeOrder.remove("0000000000000") + theProject.tree._treeOrder.remove("0000000000000") # Extract stats assert theIndex.getNovelWordCount(False) == 34 diff --git a/tests/test_core/test_core_project.py b/tests/test_core/test_core_project.py index e6e5092d..005277c6 100644 --- a/tests/test_core/test_core_project.py +++ b/tests/test_core/test_core_project.py @@ -634,17 +634,17 @@ def testCoreProject_AccessItems(nwMinimal, mockGUI): "afb3043c7b2b3", # ROOT: Characters "9d5247ab588e0", # ROOT: World ] - assert theProject.projTree.handles() == oldOrder + assert theProject.tree.handles() == oldOrder assert theProject.setTreeOrder(newOrder) - assert theProject.projTree.handles() == newOrder + assert theProject.tree.handles() == newOrder # Add a non-existing item - theProject.projTree._treeOrder.append("01234567789abc") + theProject.tree._treeOrder.append("01234567789abc") # Add an item with a non-existent parent nHandle = theProject.newFile("Test File", "a6d311a93600a") - theProject.projTree[nHandle].setParent("cba9876543210") - assert theProject.projTree[nHandle].itemParent == "cba9876543210" + theProject.tree[nHandle].setParent("cba9876543210") + assert theProject.tree[nHandle].itemParent == "cba9876543210" retOrder = [] for tItem in theProject.getProjectItems(): @@ -661,7 +661,7 @@ def testCoreProject_AccessItems(nwMinimal, mockGUI): "f5ab3e30151e1", # FILE: New Chapter "8c659a11cd429", # FILE: New Scene ] - assert theProject.projTree[nHandle].itemParent is None + assert theProject.tree[nHandle].itemParent is None # END Test testCoreProject_AccessItems @@ -679,15 +679,15 @@ def testCoreProject_StatusImport(mockGUI, fncDir, mockRnd): # Change Status # ============= - theProject.projTree["0000000000014"].setStatus("Finished") - theProject.projTree["0000000000015"].setStatus("Draft") - theProject.projTree["0000000000016"].setStatus("Note") - theProject.projTree["0000000000017"].setStatus("Finished") + theProject.tree["0000000000014"].setStatus("Finished") + theProject.tree["0000000000015"].setStatus("Draft") + theProject.tree["0000000000016"].setStatus("Note") + theProject.tree["0000000000017"].setStatus("Finished") - assert theProject.projTree["0000000000014"].itemStatus == statusKeys[3] - assert theProject.projTree["0000000000015"].itemStatus == statusKeys[2] - assert theProject.projTree["0000000000016"].itemStatus == statusKeys[1] - assert theProject.projTree["0000000000017"].itemStatus == statusKeys[3] + assert theProject.tree["0000000000014"].itemStatus == statusKeys[3] + assert theProject.tree["0000000000015"].itemStatus == statusKeys[2] + assert theProject.tree["0000000000016"].itemStatus == statusKeys[1] + assert theProject.tree["0000000000017"].itemStatus == statusKeys[3] newList = [ {"key": statusKeys[0], "name": "New", "cols": (1, 1, 1)}, @@ -723,9 +723,9 @@ def testCoreProject_StatusImport(mockGUI, fncDir, mockRnd): # ================= fHandle = theProject.newFile("Jane Doe", "8b9d2e465e150") - theProject.projTree[fHandle].setImport("Main") + theProject.tree[fHandle].setImport("Main") - assert theProject.projTree[fHandle].itemImport == importKeys[3] + assert theProject.tree[fHandle].itemImport == importKeys[3] newList = [ {"key": importKeys[0], "name": "New", "cols": (1, 1, 1)}, {"key": importKeys[1], "name": "Minor", "cols": (2, 2, 2)}, @@ -851,7 +851,7 @@ def testCoreProject_Methods(monkeypatch, mockGUI, tmpDir, fncDir, mockRnd): # Trash folder # Should create on first call, and just returned on later calls hTrash = "0000000000018" - assert theProject.projTree[hTrash] is None + assert theProject.tree[hTrash] is None assert theProject.trashFolder() == hTrash assert theProject.trashFolder() == hTrash @@ -929,11 +929,11 @@ def testCoreProject_Methods(monkeypatch, mockGUI, tmpDir, fncDir, mockRnd): "0000000000010", "0000000000011", "0000000000012", "0000000000016", "0000000000017", ] - assert theProject.projTree.handles() == oldOrder + assert theProject.tree.handles() == oldOrder assert theProject.setTreeOrder(newOrder) - assert theProject.projTree.handles() == newOrder + assert theProject.tree.handles() == newOrder assert theProject.setTreeOrder(oldOrder) - assert theProject.projTree.handles() == oldOrder + assert theProject.tree.handles() == oldOrder # Session stats theProject.currWCount = 200 @@ -1003,7 +1003,7 @@ def testCoreProject_OrphanedFiles(mockGUI, nwLipsum): theProject = NWProject(mockGUI) assert theProject.openProject(nwLipsum) is True - assert theProject.projTree["636b6aa9b697b"] is None + assert theProject.tree["636b6aa9b697b"] is None # Add a file with non-existent parent # This file will be renoved from the project on open @@ -1041,11 +1041,11 @@ def testCoreProject_OrphanedFiles(mockGUI, nwLipsum): assert theProject.openProject(nwLipsum) assert theProject.projPath is not None - assert theProject.projTree["636b6aa9b697bb"] is None - assert theProject.projTree["abcdefghijklm"] is None + assert theProject.tree["636b6aa9b697bb"] is None + assert theProject.tree["abcdefghijklm"] is None # First Item with Meta Data - oItem = theProject.projTree["636b6aa9b697b"] + oItem = theProject.tree["636b6aa9b697b"] assert oItem is not None assert oItem.itemName == "[Recovered] Mars" assert oItem.itemHandle == "636b6aa9b697b" @@ -1055,7 +1055,7 @@ def testCoreProject_OrphanedFiles(mockGUI, nwLipsum): assert oItem.itemLayout == nwItemLayout.NOTE # Second Item without Meta Data - oItem = theProject.projTree["736b6aa9b697b"] + oItem = theProject.tree["736b6aa9b697b"] assert oItem is not None assert oItem.itemName == "Recovered File 1" assert oItem.itemHandle == "736b6aa9b697b" diff --git a/tests/test_dialogs/test_dlg_itemeditor.py b/tests/test_dialogs/test_dlg_itemeditor.py index c221138e..95b1ee71 100644 --- a/tests/test_dialogs/test_dlg_itemeditor.py +++ b/tests/test_dialogs/test_dlg_itemeditor.py @@ -65,9 +65,9 @@ def testDlgItemEditor_Dialog(qtbot, monkeypatch, nwGUI, fncProj, mockRnd): assert nwGUI.editItem() is False # Invalid Type - nwGUI.theProject.projTree[tHandle]._type = nwItemType.NO_TYPE + nwGUI.theProject.tree[tHandle]._type = nwItemType.NO_TYPE assert nwGUI.editItem() is False - nwGUI.theProject.projTree[tHandle]._type = nwItemType.FILE + nwGUI.theProject.tree[tHandle]._type = nwItemType.FILE # Open Properly assert nwGUI.editItem() is True diff --git a/tests/test_gui/test_gui_doceditor.py b/tests/test_gui/test_gui_doceditor.py index 7f3aa30c..1727b69f 100644 --- a/tests/test_gui/test_gui_doceditor.py +++ b/tests/test_gui/test_gui_doceditor.py @@ -185,10 +185,10 @@ def testGuiEditor_SaveText(qtbot, monkeypatch, caplog, nwGUI, nwMinimal, ipsumTe assert "Could not save document." in caplog.text # Change header level - assert nwGUI.theProject.projTree[sHandle].itemLayout == nwItemLayout.DOCUMENT + assert nwGUI.theProject.tree[sHandle].itemLayout == nwItemLayout.DOCUMENT nwGUI.docEditor.replaceText(longText[1:]) assert nwGUI.docEditor.saveText() is True - assert nwGUI.theProject.projTree[sHandle].itemLayout == nwItemLayout.DOCUMENT + assert nwGUI.theProject.tree[sHandle].itemLayout == nwItemLayout.DOCUMENT # Regular save assert nwGUI.docEditor.saveText() is True @@ -236,9 +236,9 @@ def testGuiEditor_MetaData(qtbot, monkeypatch, nwGUI, nwMinimal): assert nwGUI.docEditor.setCursorPosition(None) is False assert nwGUI.docEditor.setCursorPosition(10) is True assert nwGUI.docEditor.getCursorPosition() == 10 - assert nwGUI.theProject.projTree[sHandle].cursorPos != 10 + assert nwGUI.theProject.tree[sHandle].cursorPos != 10 nwGUI.docEditor.saveCursorPosition() - assert nwGUI.theProject.projTree[sHandle].cursorPos == 10 + assert nwGUI.theProject.tree[sHandle].cursorPos == 10 assert nwGUI.docEditor.setCursorLine(None) is False assert nwGUI.docEditor.setCursorLine(2) is True @@ -1226,8 +1226,8 @@ def testGuiEditor_WordCounters(qtbot, monkeypatch, caplog, nwGUI, nwMinimal, ips # Open a document and populate it sHandle = "8c659a11cd429" - nwGUI.theProject.projTree[sHandle]._initCount = 0 # Clear item's count - nwGUI.theProject.projTree[sHandle]._wordCount = 0 # Clear item's count + nwGUI.theProject.tree[sHandle]._initCount = 0 # Clear item's count + nwGUI.theProject.tree[sHandle]._wordCount = 0 # Clear item's count assert nwGUI.openDocument(sHandle) is True qtbot.wait(stepDelay) @@ -1253,9 +1253,9 @@ def testGuiEditor_WordCounters(qtbot, monkeypatch, caplog, nwGUI, nwMinimal, ips nwGUI.docEditor.wCounterDoc.run() # nwGUI.docEditor._updateDocCounts(cC, wC, pC) qtbot.wait(stepDelay) - assert nwGUI.theProject.projTree[sHandle]._charCount == cC - assert nwGUI.theProject.projTree[sHandle]._wordCount == wC - assert nwGUI.theProject.projTree[sHandle]._paraCount == pC + assert nwGUI.theProject.tree[sHandle]._charCount == cC + assert nwGUI.theProject.tree[sHandle]._wordCount == wC + assert nwGUI.theProject.tree[sHandle]._paraCount == pC assert nwGUI.docEditor.docFooter.wordsText.text() == f"Words: {wC} (+{wC})" # Select all text diff --git a/tests/test_gui/test_gui_docviewer.py b/tests/test_gui/test_gui_docviewer.py index 48f61eff..d3f245d1 100644 --- a/tests/test_gui/test_gui_docviewer.py +++ b/tests/test_gui/test_gui_docviewer.py @@ -140,7 +140,7 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum): nwGUI.docViewer.reloadText() # Change document title - nwItem = nwGUI.theProject.projTree["4c4f28287af27"] + nwItem = nwGUI.theProject.tree["4c4f28287af27"] nwItem.setName("Test Title") assert nwItem.itemName == "Test Title" nwGUI.docViewer.updateDocInfo("4c4f28287af27") diff --git a/tests/test_gui/test_gui_guimain.py b/tests/test_gui/test_gui_guimain.py index e0d7e2ec..5e00815d 100644 --- a/tests/test_gui/test_gui_guimain.py +++ b/tests/test_gui/test_gui_guimain.py @@ -181,10 +181,10 @@ def testGuiMain_Editing(qtbot, monkeypatch, nwGUI, fncProj, refDir, outDir, mock assert nwGUI.saveProject() assert nwGUI.closeProject() - assert len(nwGUI.theProject.projTree) == 0 - assert len(nwGUI.theProject.projTree._treeOrder) == 0 - assert len(nwGUI.theProject.projTree._treeRoots) == 0 - assert nwGUI.theProject.projTree.trashRoot() is None + assert len(nwGUI.theProject.tree) == 0 + assert len(nwGUI.theProject.tree._treeOrder) == 0 + assert len(nwGUI.theProject.tree._treeRoots) == 0 + assert nwGUI.theProject.tree.trashRoot() is None assert nwGUI.theProject.projPath is None assert nwGUI.theProject.projMeta is None assert nwGUI.theProject.projFile == "nwProject.nwx" @@ -208,10 +208,10 @@ def testGuiMain_Editing(qtbot, monkeypatch, nwGUI, fncProj, refDir, outDir, mock qtbot.wait(stepDelay) # Check that we loaded the data - assert len(nwGUI.theProject.projTree) == 8 - assert len(nwGUI.theProject.projTree._treeOrder) == 8 - assert len(nwGUI.theProject.projTree._treeRoots) == 4 - assert nwGUI.theProject.projTree.trashRoot() is None + assert len(nwGUI.theProject.tree) == 8 + assert len(nwGUI.theProject.tree._treeOrder) == 8 + assert len(nwGUI.theProject.tree._treeRoots) == 4 + assert nwGUI.theProject.tree.trashRoot() is None assert nwGUI.theProject.projPath == fncProj assert nwGUI.theProject.projMeta == os.path.join(fncProj, "meta") assert nwGUI.theProject.projFile == "nwProject.nwx" @@ -464,11 +464,11 @@ def testGuiMain_Editing(qtbot, monkeypatch, nwGUI, fncProj, refDir, outDir, mock # Check a Quick Create and Delete assert nwGUI.treeView.newTreeItem(nwItemType.FILE, None) newHandle = nwGUI.treeView.getSelectedHandle() - assert nwGUI.theProject.projTree["0000000000020"] is not None + assert nwGUI.theProject.tree["0000000000020"] is not None assert nwGUI.treeView.deleteItem() assert nwGUI.treeView.setSelectedHandle(newHandle) assert nwGUI.treeView.deleteItem() - assert nwGUI.theProject.projTree["0000000000024"] is not None # Trash + assert nwGUI.theProject.tree["0000000000024"] is not None # Trash assert nwGUI.saveProject() # Check the files diff --git a/tests/test_gui/test_gui_projtree.py b/tests/test_gui/test_gui_projtree.py index 5d3354f9..39e0b4dc 100644 --- a/tests/test_gui/test_gui_projtree.py +++ b/tests/test_gui/test_gui_projtree.py @@ -63,7 +63,7 @@ def testGuiProjTree_NewItems(qtbot, caplog, monkeypatch, nwGUI, fncDir, mockRnd) # Create root item assert nwTree.newTreeItem(nwItemType.ROOT, nwItemClass.WORLD) is True - assert "0000000000010" in nwGUI.theProject.projTree + assert "0000000000010" in nwGUI.theProject.tree # File/Folder Items # ================= @@ -78,42 +78,42 @@ def testGuiProjTree_NewItems(qtbot, caplog, monkeypatch, nwGUI, fncDir, mockRnd) # Create new folder as child of Novel folder nwTree.setSelectedHandle("0000000000008") assert nwTree.newTreeItem(nwItemType.FOLDER) is True - assert nwGUI.theProject.projTree["0000000000011"].itemParent == "0000000000008" - assert nwGUI.theProject.projTree["0000000000011"].itemRoot == "0000000000008" - assert nwGUI.theProject.projTree["0000000000011"].itemClass == nwItemClass.NOVEL + assert nwGUI.theProject.tree["0000000000011"].itemParent == "0000000000008" + assert nwGUI.theProject.tree["0000000000011"].itemRoot == "0000000000008" + assert nwGUI.theProject.tree["0000000000011"].itemClass == nwItemClass.NOVEL # Add a new file in the new folder nwTree.setSelectedHandle("0000000000011") assert nwTree.newTreeItem(nwItemType.FILE) is True - assert nwGUI.theProject.projTree["0000000000012"].itemParent == "0000000000011" - assert nwGUI.theProject.projTree["0000000000012"].itemRoot == "0000000000008" - assert nwGUI.theProject.projTree["0000000000012"].itemClass == nwItemClass.NOVEL + assert nwGUI.theProject.tree["0000000000012"].itemParent == "0000000000011" + assert nwGUI.theProject.tree["0000000000012"].itemRoot == "0000000000008" + assert nwGUI.theProject.tree["0000000000012"].itemClass == nwItemClass.NOVEL # Add a new file next to the other new file nwTree.setSelectedHandle("0000000000012") assert nwTree.newTreeItem(nwItemType.FILE) is True - assert nwGUI.theProject.projTree["0000000000013"].itemParent == "0000000000011" - assert nwGUI.theProject.projTree["0000000000013"].itemRoot == "0000000000008" - assert nwGUI.theProject.projTree["0000000000013"].itemClass == nwItemClass.NOVEL + assert nwGUI.theProject.tree["0000000000013"].itemParent == "0000000000011" + assert nwGUI.theProject.tree["0000000000013"].itemRoot == "0000000000008" + assert nwGUI.theProject.tree["0000000000013"].itemClass == nwItemClass.NOVEL assert nwGUI.openDocument("0000000000013") assert nwGUI.docEditor.getText() == "### New Document\n\n" # Add a new file to the characters folder nwTree.setSelectedHandle("000000000000a") assert nwTree.newTreeItem(nwItemType.FILE) is True - assert nwGUI.theProject.projTree["0000000000014"].itemParent == "000000000000a" - assert nwGUI.theProject.projTree["0000000000014"].itemRoot == "000000000000a" - assert nwGUI.theProject.projTree["0000000000014"].itemClass == nwItemClass.CHARACTER + assert nwGUI.theProject.tree["0000000000014"].itemParent == "000000000000a" + assert nwGUI.theProject.tree["0000000000014"].itemRoot == "000000000000a" + assert nwGUI.theProject.tree["0000000000014"].itemClass == nwItemClass.CHARACTER assert nwGUI.openDocument("0000000000014") assert nwGUI.docEditor.getText() == "# New Note\n\n" # Make sure the sibling folder bug trap works nwTree.setSelectedHandle("0000000000013") - nwGUI.theProject.projTree["0000000000013"].setParent(None) # This should not happen + nwGUI.theProject.tree["0000000000013"].setParent(None) # This should not happen caplog.clear() assert nwTree.newTreeItem(nwItemType.FILE) is False assert "Internal error" in caplog.text - nwGUI.theProject.projTree["0000000000013"].setParent("0000000000011") + nwGUI.theProject.tree["0000000000013"].setParent("0000000000011") # Get the trash folder nwTree._addTrashRoot() @@ -242,22 +242,22 @@ def testGuiProjTree_MoveItems(qtbot, monkeypatch, nwGUI, fncDir, mockRnd): # =========== nwTree.setSelectedHandle("0000000000008") - assert nwGUI.theProject.projTree._treeOrder.index("0000000000008") == 0 + assert nwGUI.theProject.tree._treeOrder.index("0000000000008") == 0 # Move novel folder up assert nwTree.moveTreeItem(-1) is False nwTree.flushTreeOrder() - assert nwGUI.theProject.projTree._treeOrder.index("0000000000008") == 0 + assert nwGUI.theProject.tree._treeOrder.index("0000000000008") == 0 # Move novel folder down assert nwTree.moveTreeItem(1) is True nwTree.flushTreeOrder() - assert nwGUI.theProject.projTree._treeOrder.index("0000000000008") == 1 + assert nwGUI.theProject.tree._treeOrder.index("0000000000008") == 1 # Move novel folder up again assert nwTree.moveTreeItem(-1) is True nwTree.flushTreeOrder() - assert nwGUI.theProject.projTree._treeOrder.index("0000000000008") == 0 + assert nwGUI.theProject.tree._treeOrder.index("0000000000008") == 0 # Clean up # qtbot.stopForInteraction() @@ -341,7 +341,7 @@ def testGuiProjTree_DeleteItems(qtbot, caplog, monkeypatch, nwGUI, fncDir, mockR "000000000000d", "000000000000e", "000000000000f", "0000000000010" ] - trashHandle = nwGUI.theProject.projTree.trashRoot() + trashHandle = nwGUI.theProject.tree.trashRoot() assert nwTree.getTreeFromHandle(trashHandle) == [ trashHandle, "0000000000012", "0000000000011" ] @@ -349,30 +349,30 @@ def testGuiProjTree_DeleteItems(qtbot, caplog, monkeypatch, nwGUI, fncDir, mockR # Delete the first file again (permanent), and ask for permission # Also open the document in the editor, which should trigger a close assert os.path.isfile(os.path.join(prjDir, "content", "0000000000012.nwd")) - assert "0000000000012" in nwGUI.theProject.projTree + assert "0000000000012" in nwGUI.theProject.tree assert nwGUI.docEditor.docHandle() is None assert nwGUI.openDocument("0000000000012") is True assert nwGUI.docEditor.docHandle() == "0000000000012" assert nwTree.deleteItem("0000000000012") is True assert nwGUI.docEditor.docHandle() is None assert not os.path.isfile(os.path.join(prjDir, "content", "0000000000012.nwd")) - assert "0000000000012" not in nwGUI.theProject.projTree + assert "0000000000012" not in nwGUI.theProject.tree assert nwTree.getTreeFromHandle(trashHandle) == [ trashHandle, "0000000000011" ] # Delete the second file, and skip asking for permission assert os.path.isfile(os.path.join(prjDir, "content", "0000000000011.nwd")) - assert "0000000000011" in nwGUI.theProject.projTree + assert "0000000000011" in nwGUI.theProject.tree assert nwTree.deleteItem("0000000000011", alreadyAsked=True) is True assert not os.path.isfile(os.path.join(prjDir, "content", "0000000000011.nwd")) - assert "0000000000011" not in nwGUI.theProject.projTree + assert "0000000000011" not in nwGUI.theProject.tree assert nwTree.getTreeFromHandle(trashHandle) == [trashHandle] # Delete Folder # ============= - trashHandle = nwGUI.theProject.projTree.trashRoot() + trashHandle = nwGUI.theProject.tree.trashRoot() # Add a folder with two files nwTree.setSelectedHandle("0000000000009") From b5b00744117a74d2c797ae644e964588c457ec5c Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 28 May 2022 18:48:31 +0200 Subject: [PATCH 03/13] Rewritten index storage using classes instead --- novelwriter/core/index.py | 226 +++++++++++++++++++++++++++++++++++++- 1 file changed, 225 insertions(+), 1 deletion(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 7647a9f6..6745a637 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -35,7 +35,7 @@ from novelwriter.error import logException from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode from novelwriter.core.document import NWDoc from novelwriter.common import ( - isHandle, isTitleTag, isItemClass, isItemLayout, jsonEncode + checkInt, isHandle, isTitleTag, isItemClass, isItemLayout, jsonEncode ) logger = logging.getLogger(__name__) @@ -59,6 +59,9 @@ class NWIndex(): self._fileIndex = {} self._fileMeta = {} + self._tags = {} + self._items = {} + # TimeStamps self._timeNovel = 0 self._timeNotes = 0 @@ -84,6 +87,10 @@ class NWIndex(): self._timeNovel = 0 self._timeNotes = 0 self._timeIndex = 0 + + self._tags = {} + self._items = {} + return def deleteHandle(self, tHandle): @@ -194,6 +201,18 @@ class NWIndex(): logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000) + indexFile = os.path.join(self.theProject.projMeta, "tagsIndex2.json") + tStart = time() + + itemsIndex = {} + for item in self._items.values(): + item.packData(itemsIndex) + + with open(indexFile, mode="w+", encoding="utf-8") as outFile: + outFile.write(jsonEncode(itemsIndex, nmax=3)) + + logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000) + return True ## @@ -219,6 +238,11 @@ class NWIndex(): cC, wC, pC = countWords(theText) self._fileMeta[tHandle] = ["H0", cC, wC, pC] + self._items[tHandle] = IndexItem(tHandle, theItem) + theItem.setCharCount(cC) + theItem.setWordCount(wC) + theItem.setParaCount(pC) + # If the file's meta data is missing, or the file is out of the # main project, we don't index the content if theItem.itemLayout == nwItemLayout.NO_LAYOUT: @@ -338,6 +362,10 @@ class NWIndex(): # first header level is recorded in the file meta index self._fileMeta[tHandle][0] = hDepth + tItem = self._items[tHandle] + tItem.updateLevel(hDepth) + tItem.addHeading(IndexHeading(sTitle, hDepth, hText)) + return True def _indexPage(self, tHandle, itemLayout): @@ -364,6 +392,8 @@ class NWIndex(): self._fileIndex[tHandle][sTitle]["cCount"] = cC self._fileIndex[tHandle][sTitle]["wCount"] = wC self._fileIndex[tHandle][sTitle]["pCount"] = pC + if tHandle in self._items: + self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC) return def _indexSynopsis(self, tHandle, theText, nTitle): @@ -373,6 +403,8 @@ class NWIndex(): if tHandle in self._fileIndex: if sTitle in self._fileIndex[tHandle]: self._fileIndex[tHandle][sTitle]["synopsis"] = theText + if tHandle in self._items: + self._items[tHandle].setHeadingSynopsis(sTitle, theText) return def _indexKeyword(self, tHandle, aLine, nLine, nTitle, itemClass): @@ -391,6 +423,8 @@ class NWIndex(): sTitle = f"T{nTitle:06d}" if theBits[0] == nwKeyWords.TAG_KEY: self._tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name, sTitle] + if tHandle in self._items: + self._items[tHandle].setHeadingTag(sTitle, theBits[1]) else: if tHandle not in self._refIndex: @@ -399,6 +433,8 @@ class NWIndex(): self._refIndex[tHandle][sTitle] = [] for aVal in theBits[1:]: self._refIndex[tHandle][sTitle].append([nLine, theBits[0], aVal]) + if tHandle in self._items: + self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0]) return @@ -892,3 +928,191 @@ def countWords(theText): prevEmpty = not countPara return charCount, wordCount, paraCount + + +class IndexItem: + + DEF_HKEY = "T000000" + + def __init__(self, tHandle, tItem): + self._handle = tHandle + self._item = tItem + + self._level = "H0" + self._headings = {} + + # Add a placeholder heading + self._headings[self.DEF_HKEY] = IndexHeading(self.DEF_HKEY) + + return + + ## + # Properties + ## + + @property + def level(self): + return self._level + + ## + # Setters + ## + + def setLevel(self, level): + if level in H_VALID: + self._level = level + else: + self._level = "H0" + return + + def updateLevel(self, level): + """Set the level only if it is H0. + """ + if level in H_VALID and self._level == "H0": + self._level = level + else: + self._level = "H0" + return + + def addHeading(self, tHeading): + if "T000000" in self._headings: + self._headings.pop("T000000") + self._headings[tHeading.key] = tHeading + return + + def setHeadingCounts(self, sTitle, charCount, wordCount, paraCount): + if sTitle in self._headings: + self._headings[sTitle].setCounts(charCount, wordCount, paraCount) + return + + def setHeadingSynopsis(self, sTitle, synopText): + if sTitle in self._headings: + self._headings[sTitle].setSynopsis(synopText) + return + + def setHeadingTag(self, sTitle, tagKey): + if sTitle in self._headings: + self._headings[sTitle].setTag(tagKey) + return + + def addHeadingReferences(self, sTitle, tagKeys, refType): + if sTitle in self._headings: + for tagKey in tagKeys: + self._headings[sTitle].addReference(tagKey, refType) + return + + ## + # Data Methods + ## + + def packData(self, container): + """Pack the indexed item's data into an existing dictionary. + """ + container[self._handle] = { + "firstLevel": self._level, + } + container[self._handle]["headings"] = { + key: value.packData() for key, value in self._headings.items() + } + container[self._handle]["references"] = { + key: value.packReferences() for key, value in self._headings.items() + } + return + +# END Class IndexItem + + +class IndexHeading: + + def __init__(self, key, level="H0", title=""): + self._key = key + self._level = level + self._title = title + + self._charCount = 0 + self._wordCount = 0 + self._paraCount = 0 + self._synopsis = "" + + self._tag = "" + self._refs = {} + + return + + ## + # Properties + ## + + @property + def key(self): + return self._key + + ## + # Setters + ## + + def setLevel(self, level): + if level in H_VALID: + self._level = level + else: + self._level = "H0" + return + + def setCounts(self, charCount, wordCount, paraCount): + self._charCount = max(0, checkInt(charCount, 0)) + self._wordCount = max(0, checkInt(wordCount, 0)) + self._paraCount = max(0, checkInt(paraCount, 0)) + return + + def setSynopsis(self, synopText): + self._synopsis = str(synopText) + return + + def setTag(self, tagKey): + self._tag = str(tagKey) + return + + def addReference(self, tagKey, refType): + """Add a record of a reference tag, and what keyword types it is + associated with. + """ + if tagKey not in self._refs: + self._refs[tagKey] = set() + self._refs[tagKey].add(refType) + return + + ## + # Data Methods + ## + + def packData(self): + """Pack the values into a dictionary for saving to cache. + """ + return { + "level": self._level, + "title": self._title, + "tag": self._tag, + "cCount": self._charCount, + "wCount": self._wordCount, + "pCount": self._paraCount, + "synopsis": self._synopsis, + } + + def packReferences(self): + return {key: list(value) for key, value in self._refs.items()} + + def unpackData(self, data): + """Unpack a title entry + """ + self._setLevel(data.get("level", "H0")) + self._title = str(data.get("title", "")) + self._tag = str(data.get("tag", "")) + self.setCounts( + data.get("cCount", 0), + data.get("wCount", 0), + data.get("pCount", 0), + ) + self._synopsis = str(data.get("synopsis", "")) + return + +# END Class IndexHeading From d584f57a3b916a0a8f081e187b9d3a4fc2b0871f Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 28 May 2022 19:45:33 +0200 Subject: [PATCH 04/13] Loading of new index now works --- novelwriter/core/index.py | 114 ++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 17 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 6745a637..bd82df0c 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -173,6 +173,33 @@ class NWIndex(): logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) + indexFile = os.path.join(self.theProject.projMeta, "tagsIndex2.json") + tStart = time() + + if os.path.isfile(indexFile): + logger.debug("Loading index file") + try: + with open(indexFile, mode="r", encoding="utf-8") as inFile: + theData = json.load(inFile) + + except Exception: + logger.error("Failed to load index file") + logException() + self._indexBroken = True + return False + + for tHandle, tData in theData.items(): + nwItem = self.theProject.tree[tHandle] + if nwItem is not None: + tItem = IndexItem(tHandle, nwItem) + tItem.unpackData(tData) + self._items[tHandle] = tItem + + self._generateTagsIndex() + # print(json.dumps(self._tags, indent=2, default=str)) + + logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) + self._checkIndex() return True @@ -204,10 +231,7 @@ class NWIndex(): indexFile = os.path.join(self.theProject.projMeta, "tagsIndex2.json") tStart = time() - itemsIndex = {} - for item in self._items.values(): - item.packData(itemsIndex) - + itemsIndex = {handle: item.packData() for handle, item in self._items.items()} with open(indexFile, mode="w+", encoding="utf-8") as outFile: outFile.write(jsonEncode(itemsIndex, nmax=3)) @@ -423,6 +447,7 @@ class NWIndex(): sTitle = f"T{nTitle:06d}" if theBits[0] == nwKeyWords.TAG_KEY: self._tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name, sTitle] + self._tags[theBits[1]] = [tHandle, itemClass.name, sTitle] if tHandle in self._items: self._items[tHandle].setHeadingTag(sTitle, theBits[1]) @@ -687,6 +712,17 @@ class NWIndex(): return theHandles + def _generateTagsIndex(self): + """Generate the reverse tags index from the loaded index data. + The tags index must be updated during runtime with new changes. + """ + self._tags = {} + for tHandle, tItem in self._items.items(): + for sTitle, tHead in tItem.items(): + if tHead.tag: + self._tags[tHead.tag] = (tHandle, tItem.itemClass.name, sTitle) + return + ## # Index Checkers ## @@ -940,6 +976,7 @@ class IndexItem: self._level = "H0" self._headings = {} + self._index = 0 # Add a placeholder heading self._headings[self.DEF_HKEY] = IndexHeading(self.DEF_HKEY) @@ -954,6 +991,10 @@ class IndexItem: def level(self): return self._level + @property + def itemClass(self): + return self._item.itemClass + ## # Setters ## @@ -1005,18 +1046,44 @@ class IndexItem: # Data Methods ## - def packData(self, container): - """Pack the indexed item's data into an existing dictionary. + def __getitem__(self, sTitle): + return self._headings.get(sTitle, None) + + def items(self): + return self._headings.items() + + ## + # Pack/Unpack + ## + + def packData(self): + """Pack the indexed item's data into a dictionary. """ - container[self._handle] = { - "firstLevel": self._level, - } - container[self._handle]["headings"] = { - key: value.packData() for key, value in self._headings.items() - } - container[self._handle]["references"] = { - key: value.packReferences() for key, value in self._headings.items() - } + heads = {} + refs = {} + for sTitle, hItem in self._headings.items(): + heads[sTitle] = hItem.packData() + hRefs = hItem.packReferences() + if hRefs: + refs[sTitle] = hRefs + + data = {"level": self._level} + data["headings"] = heads + if refs: + data["references"] = refs + + return data + + def unpackData(self, data): + """Unpack an item entry from the data. + """ + self._level = data.get("level", "H0") + references = data.get("references", {}) + for sTitle, hData in data.get("headings", {}).items(): + tHeading = IndexHeading(sTitle) + tHeading.unpackData(hData) + tHeading.unpackReferences(references.get(sTitle, {})) + self.addHeading(tHeading) return # END Class IndexItem @@ -1047,6 +1114,10 @@ class IndexHeading: def key(self): return self._key + @property + def tag(self): + return self._tag + ## # Setters ## @@ -1099,12 +1170,14 @@ class IndexHeading: } def packReferences(self): + """Pack references into a dictionary for saving to cache. + """ return {key: list(value) for key, value in self._refs.items()} def unpackData(self, data): - """Unpack a title entry + """Unpack a heading entry from a dictionary. """ - self._setLevel(data.get("level", "H0")) + self.setLevel(data.get("level", "H0")) self._title = str(data.get("title", "")) self._tag = str(data.get("tag", "")) self.setCounts( @@ -1115,4 +1188,11 @@ class IndexHeading: self._synopsis = str(data.get("synopsis", "")) return + def unpackReferences(self, data): + """Unpack a set of references from a dictionary. + """ + for tagKey, refTypes in data.items(): + self._refs[tagKey] = set(refTypes) + return + # END Class IndexHeading From 01be8be44f036106dfc95c4761b416e2966bfa48 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 28 May 2022 20:16:27 +0200 Subject: [PATCH 05/13] Remove old tagsIndex --- novelwriter/core/index.py | 93 ++++++------------- novelwriter/gui/docviewer.py | 2 +- tests/test_core/test_core_index.py | 142 +++++++++++++++-------------- 3 files changed, 102 insertions(+), 135 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index bd82df0c..bcdc6f3c 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -35,13 +35,14 @@ from novelwriter.error import logException from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode from novelwriter.core.document import NWDoc from novelwriter.common import ( - checkInt, isHandle, isTitleTag, isItemClass, isItemLayout, jsonEncode + checkInt, isHandle, isTitleTag, isItemLayout, jsonEncode ) logger = logging.getLogger(__name__) H_VALID = ("H0", "H1", "H2", "H3", "H4") H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4} +H_NONE = "T000000" class NWIndex(): @@ -54,7 +55,6 @@ class NWIndex(): self._indexBroken = False # Indices - self._tagIndex = {} self._refIndex = {} self._fileIndex = {} self._fileMeta = {} @@ -80,7 +80,6 @@ class NWIndex(): def clearIndex(self): """Clear the index dictionaries and time stamps. """ - self._tagIndex = {} self._refIndex = {} self._fileIndex = {} self._fileMeta = {} @@ -98,9 +97,9 @@ class NWIndex(): """ logger.debug("Removing item '%s' from the index", tHandle) - delTags = list(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex)) + delTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags)) for tTag in delTags: - self._tagIndex.pop(tTag, None) + self._tags.pop(tTag, None) self._refIndex.pop(tHandle, None) self._fileIndex.pop(tHandle, None) @@ -161,7 +160,6 @@ class NWIndex(): self._indexBroken = True return False - self._tagIndex = theData.get("tagIndex", {}) self._refIndex = theData.get("refIndex", {}) self._fileIndex = theData.get("fileIndex", {}) self._fileMeta = theData.get("fileMeta", {}) @@ -188,16 +186,14 @@ class NWIndex(): self._indexBroken = True return False - for tHandle, tData in theData.items(): + self._tags = theData.get("tagsIndex", {}) + for tHandle, tData in theData.get("itemIndex", {}).items(): nwItem = self.theProject.tree[tHandle] if nwItem is not None: tItem = IndexItem(tHandle, nwItem) tItem.unpackData(tData) self._items[tHandle] = tItem - self._generateTagsIndex() - # print(json.dumps(self._tags, indent=2, default=str)) - logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) self._checkIndex() @@ -215,7 +211,6 @@ class NWIndex(): try: with open(indexFile, mode="w+", encoding="utf-8") as outFile: outFile.write("{\n") - outFile.write(f' "tagIndex": {jsonEncode(self._tagIndex, n=1, nmax=2)},\n') outFile.write(f' "refIndex": {jsonEncode(self._refIndex, n=1, nmax=3)},\n') outFile.write(f' "fileIndex": {jsonEncode(self._fileIndex, n=1, nmax=3)},\n') outFile.write(f' "fileMeta": {jsonEncode(self._fileMeta, n=1, nmax=2)}\n') @@ -233,7 +228,10 @@ class NWIndex(): itemsIndex = {handle: item.packData() for handle, item in self._items.items()} with open(indexFile, mode="w+", encoding="utf-8") as outFile: - outFile.write(jsonEncode(itemsIndex, nmax=3)) + outFile.write("{\n") + outFile.write(f' "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n') + outFile.write(f' "itemIndex": {jsonEncode(itemsIndex, n=1, nmax=4)}\n') + outFile.write("}\n") logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000) @@ -289,9 +287,9 @@ class NWIndex(): self._fileIndex[tHandle] = {} # Also clear references to the file in the tags index - clearTags = list(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex)) + clearTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags)) for aTag in clearTags: - self._tagIndex.pop(aTag) + self._tags.pop(aTag) # Scan the text content nTitle = 0 @@ -395,7 +393,7 @@ class NWIndex(): def _indexPage(self, tHandle, itemLayout): """Index a page with no title. """ - self._fileIndex[tHandle]["T000000"] = { + self._fileIndex[tHandle][H_NONE] = { "level": "H0", "title": "", "layout": itemLayout.name, @@ -446,8 +444,11 @@ class NWIndex(): sTitle = f"T{nTitle:06d}" if theBits[0] == nwKeyWords.TAG_KEY: - self._tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name, sTitle] - self._tags[theBits[1]] = [tHandle, itemClass.name, sTitle] + self._tags[theBits[1]] = { + "handle": tHandle, + "heading": sTitle, + "class": itemClass.name, + } if tHandle in self._items: self._items[tHandle].setHeadingTag(sTitle, theBits[1]) @@ -521,8 +522,8 @@ class NWIndex(): # For a tag, only the first value is accepted, the rest are ignored if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1: - if theBits[1] in self._tagIndex: - isGood[1] = self._tagIndex[theBits[1]][1] == tItem.itemHandle + if theBits[1] in self._tags: + isGood[1] = self._tags[theBits[1]].get("handle") == tItem.itemHandle else: isGood[1] = True return isGood @@ -530,8 +531,8 @@ class NWIndex(): # If we're still here, we check that the references exist theKey = nwKeyWords.KEY_CLASS[theBits[0]].name for n in range(1, nBits): - if theBits[n] in self._tagIndex: - isGood[n] = theKey == self._tagIndex[theBits[n]][2] + if theBits[n] in self._tags: + isGood[n] = theKey == self._tags[theBits[n]].get("class") return isGood @@ -674,7 +675,7 @@ class NWIndex(): return {} theRefs = {} - theTags = set(filter(lambda x: self._tagIndex[x][1] == tHandle, self._tagIndex)) + theTags = set(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags)) if theTags: for tHandle in self._refIndex: for sTitle in self._refIndex[tHandle]: @@ -687,10 +688,8 @@ class NWIndex(): def getTagSource(self, theTag): """Return the source location of a given tag. """ - theRef = self._tagIndex.get(theTag, []) - if len(theRef) == 4: - return theRef[1], theRef[0], theRef[3] - return None, 0, "T000000" + ref = self._tags.get(theTag, {}) + return ref.get("handle"), ref.get("heading", H_NONE) ## # Internal Functions @@ -712,17 +711,6 @@ class NWIndex(): return theHandles - def _generateTagsIndex(self): - """Generate the reverse tags index from the loaded index data. - The tags index must be updated during runtime with new changes. - """ - self._tags = {} - for tHandle, tItem in self._items.items(): - for sTitle, tHead in tItem.items(): - if tHead.tag: - self._tags[tHead.tag] = (tHandle, tItem.itemClass.name, sTitle) - return - ## # Index Checkers ## @@ -737,7 +725,6 @@ class NWIndex(): tStart = time() try: - self._checkTagIndex() self._checkRefIndex() self._checkFileIndex() self._checkFileMeta() @@ -763,28 +750,6 @@ class NWIndex(): return - def _checkTagIndex(self): - """Scan the tag index for errors. - Warning: This function raises exceptions. - """ - for tTag in self._tagIndex: - if not isinstance(tTag, str): - raise KeyError("tagIndex key is not a string") - - tEntry = self._tagIndex[tTag] - if len(tEntry) != 4: - raise IndexError("tagIndex[a] expected 4 values") - if not isinstance(tEntry[0], int): - raise ValueError("tagIndex[a][0] is not an integer") - if not isHandle(tEntry[1]): - raise ValueError("tagIndex[a][1] is not a handle") - if not isItemClass(tEntry[2]): - raise ValueError("tagIndex[a][2] is not an nwItemClass") - if not isTitleTag(tEntry[3]): - raise ValueError("tagIndex[a][3] is not a title tag") - - return - def _checkRefIndex(self): """Scan the reference index for errors. Warning: This function raises exceptions. @@ -968,8 +933,6 @@ def countWords(theText): class IndexItem: - DEF_HKEY = "T000000" - def __init__(self, tHandle, tItem): self._handle = tHandle self._item = tItem @@ -979,7 +942,7 @@ class IndexItem: self._index = 0 # Add a placeholder heading - self._headings[self.DEF_HKEY] = IndexHeading(self.DEF_HKEY) + self._headings[H_NONE] = IndexHeading(H_NONE) return @@ -1016,8 +979,8 @@ class IndexItem: return def addHeading(self, tHeading): - if "T000000" in self._headings: - self._headings.pop("T000000") + if H_NONE in self._headings: + self._headings.pop(H_NONE) self._headings[tHeading.key] = tHeading return diff --git a/novelwriter/gui/docviewer.py b/novelwriter/gui/docviewer.py index 2587f125..2b59cf8e 100644 --- a/novelwriter/gui/docviewer.py +++ b/novelwriter/gui/docviewer.py @@ -245,7 +245,7 @@ class GuiDocViewer(QTextBrowser): index being up to date. """ logger.debug("Loading document from tag '%s'", theTag) - tHandle, _, sTitle = self.theProject.index.getTagSource(theTag) + tHandle, sTitle = self.theProject.index.getTagSource(theTag) if tHandle is None: self.theParent.makeAlert(self.tr( "Could not find the reference for tag '{0}'. It either doesn't " diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index 0d070eba..d2e76643 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -68,25 +68,25 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): assert theIndex.saveIndex() is True # Take a copy of the index - tagIndex = str(theIndex._tagIndex) + tagIndex = str(theIndex._tags) refIndex = str(theIndex._refIndex) fileIndex = str(theIndex._fileIndex) textCounts = str(theIndex._fileMeta) # Delete a handle - assert theIndex._tagIndex.get("Bod", None) is not None + assert theIndex._tags.get("Bod", None) is not None assert theIndex._refIndex.get("4c4f28287af27", None) is not None assert theIndex._fileIndex.get("4c4f28287af27", None) is not None assert theIndex._fileMeta.get("4c4f28287af27", None) is not None theIndex.deleteHandle("4c4f28287af27") - assert theIndex._tagIndex.get("Bod", None) is None + assert theIndex._tags.get("Bod", None) is None assert theIndex._refIndex.get("4c4f28287af27", None) is None assert theIndex._fileIndex.get("4c4f28287af27", None) is None assert theIndex._fileMeta.get("4c4f28287af27", None) is None # Clear the index theIndex.clearIndex() - assert theIndex._tagIndex == {} + assert theIndex._tags == {} assert theIndex._refIndex == {} assert theIndex._fileIndex == {} assert theIndex._fileMeta == {} @@ -99,16 +99,16 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): # Make the load pass assert theIndex.loadIndex() is True - assert str(theIndex._tagIndex) == tagIndex + assert str(theIndex._tags) == tagIndex assert str(theIndex._refIndex) == refIndex assert str(theIndex._fileIndex) == fileIndex assert str(theIndex._fileMeta) == textCounts # Break the index and check that we notice - assert theIndex.indexBroken is False - theIndex._tagIndex["Bod"].append("Stuff") - theIndex._checkIndex() - assert theIndex.indexBroken is True + # assert theIndex.indexBroken is False + # theIndex._tagIndex["Bod"].append("Stuff") + # theIndex._checkIndex() + # assert theIndex.indexBroken is True # Finalise assert theProject.closeProject() is True @@ -198,7 +198,9 @@ def testCoreIndex_CheckThese(nwMinimal, mockGUI): "@pov: Jane\n" "@invalid: John\n" # Checks for issue #688 )) - assert theIndex._tagIndex == {"Jane": [2, cHandle, "CHARACTER", "T000001"]} + assert theIndex._tags == { + "Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"} + } assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!" assert theIndex.getReferences(nHandle, "T000001") == { "@char": [], @@ -309,7 +311,9 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "This is a story about Jane Smith.\n\n" "Well, not really.\n" )) - assert theIndex._tagIndex == {"Jane": [2, cHandle, "CHARACTER", "T000001"]} + assert theIndex._tags == { + "Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"} + } assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!" # Title Indexing @@ -551,8 +555,8 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # getTagSource # ============ - assert theIndex.getTagSource("Jane") == (cHandle, 2, "T000001") - assert theIndex.getTagSource("John") == (None, 0, "T000000") + assert theIndex.getTagSource("Jane") == (cHandle, "T000001") + assert theIndex.getTagSource("John") == (None, "T000000") # getCounts # ========= @@ -696,69 +700,69 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # END Test testCoreIndex_ExtractData -@pytest.mark.core -def testCoreIndex_CheckTagIndex(mockGUI): - """Test the tag index checker. - """ - theProject = NWProject(mockGUI) - theIndex = NWIndex(theProject) +# @pytest.mark.core +# def testCoreIndex_CheckTagIndex(mockGUI): +# """Test the tag index checker. +# """ +# theProject = NWProject(mockGUI) +# theIndex = NWIndex(theProject) - # Valid Index - theIndex._tagIndex = { - "John": [3, "14298de4d9524", "CHARACTER", "T000001"], - "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], - } - assert theIndex._checkTagIndex() is None +# # Valid Index +# theIndex._tagIndex = { +# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], +# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], +# } +# assert theIndex._checkTagIndex() is None - # Wrong Key Type - theIndex._tagIndex = { - "John": [3, "14298de4d9524", "CHARACTER", "T000001"], - 123456: [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], - } - with pytest.raises(KeyError): - theIndex._checkTagIndex() +# # Wrong Key Type +# theIndex._tagIndex = { +# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], +# 123456: [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], +# } +# with pytest.raises(KeyError): +# theIndex._checkTagIndex() - # Wrong Length - theIndex._tagIndex = { - "John": [3, "14298de4d9524", "CHARACTER", "T000001"], - "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001", "Stuff"], - } - with pytest.raises(IndexError): - theIndex._checkTagIndex() +# # Wrong Length +# theIndex._tagIndex = { +# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], +# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001", "Stuff"], +# } +# with pytest.raises(IndexError): +# theIndex._checkTagIndex() - # Wrong Type of Entry 0 - theIndex._tagIndex = { - "John": [3, "14298de4d9524", "CHARACTER", "T000001"], - "Jane": ["3", "bb2c23b3c42cc", "CHARACTER", "T000001"], - } - with pytest.raises(ValueError): - theIndex._checkTagIndex() +# # Wrong Type of Entry 0 +# theIndex._tagIndex = { +# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], +# "Jane": ["3", "bb2c23b3c42cc", "CHARACTER", "T000001"], +# } +# with pytest.raises(ValueError): +# theIndex._checkTagIndex() - # Wrong Type of Entry 1 - theIndex._tagIndex = { - "John": [3, "14298de4d9524", "CHARACTER", "T000001"], - "Jane": [3, 0xbb2c23b3c42cc, "CHARACTER", "T000001"], - } - with pytest.raises(ValueError): - theIndex._checkTagIndex() +# # Wrong Type of Entry 1 +# theIndex._tagIndex = { +# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], +# "Jane": [3, 0xbb2c23b3c42cc, "CHARACTER", "T000001"], +# } +# with pytest.raises(ValueError): +# theIndex._checkTagIndex() - # Wrong Type of Entry 2 - theIndex._tagIndex = { - "John": [3, "14298de4d9524", "CHARACTER", "T000001"], - "Jane": [3, "bb2c23b3c42cc", "INVALID_CLASS", "T000001"], - } - with pytest.raises(ValueError): - theIndex._checkTagIndex() +# # Wrong Type of Entry 2 +# theIndex._tagIndex = { +# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], +# "Jane": [3, "bb2c23b3c42cc", "INVALID_CLASS", "T000001"], +# } +# with pytest.raises(ValueError): +# theIndex._checkTagIndex() - # Wrong Type of Entry 3 - theIndex._tagIndex = { - "John": [3, "14298de4d9524", "CHARACTER", "T000001"], - "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "INVALID"], - } - with pytest.raises(ValueError): - theIndex._checkTagIndex() +# # Wrong Type of Entry 3 +# theIndex._tagIndex = { +# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], +# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "INVALID"], +# } +# with pytest.raises(ValueError): +# theIndex._checkTagIndex() -# END Test testCoreIndex_CheckTagIndex +# # END Test testCoreIndex_CheckTagIndex @pytest.mark.core @@ -1225,7 +1229,7 @@ def testCoreIndex_CheckFileMeta(mockGUI): with pytest.raises(ValueError): theIndex._checkFileMeta() -# END Test testCoreIndex_CheckTextCounts +# END Test testCoreIndex_CheckFileMeta @pytest.mark.core From 94ba7a2f94041dfaf20de2eeabdbc04a198c0fff Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sat, 28 May 2022 21:59:52 +0200 Subject: [PATCH 06/13] Use the new index instead of the old --- novelwriter/core/index.py | 400 ++++------ novelwriter/gui/noveltree.py | 8 +- novelwriter/gui/outline.py | 16 +- novelwriter/gui/outlinedetails.py | 14 +- .../coreIndex_LoadSave_tagsIndex.json | 216 +++--- tests/test_core/test_core_index.py | 700 ++---------------- tests/test_gui/test_gui_docviewer.py | 4 +- 7 files changed, 343 insertions(+), 1015 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index bcdc6f3c..95485bf9 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -34,9 +34,7 @@ from novelwriter.enum import nwItemType, nwItemLayout from novelwriter.error import logException from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode from novelwriter.core.document import NWDoc -from novelwriter.common import ( - checkInt, isHandle, isTitleTag, isItemLayout, jsonEncode -) +from novelwriter.common import checkInt, jsonEncode logger = logging.getLogger(__name__) @@ -55,10 +53,6 @@ class NWIndex(): self._indexBroken = False # Indices - self._refIndex = {} - self._fileIndex = {} - self._fileMeta = {} - self._tags = {} self._items = {} @@ -80,9 +74,6 @@ class NWIndex(): def clearIndex(self): """Clear the index dictionaries and time stamps. """ - self._refIndex = {} - self._fileIndex = {} - self._fileMeta = {} self._timeNovel = 0 self._timeNotes = 0 self._timeIndex = 0 @@ -95,15 +86,15 @@ class NWIndex(): def deleteHandle(self, tHandle): """Delete all entries of a given document handle. """ + if tHandle not in self._items: + return + logger.debug("Removing item '%s' from the index", tHandle) - delTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags)) - for tTag in delTags: + for tTag in self._items[tHandle].allTags(): self._tags.pop(tTag, None) - self._refIndex.pop(tHandle, None) - self._fileIndex.pop(tHandle, None) - self._fileMeta.pop(tHandle, None) + self._items.pop(tHandle, None) return @@ -148,32 +139,6 @@ class NWIndex(): indexFile = os.path.join(self.theProject.projMeta, nwFiles.INDEX_FILE) tStart = time() - if os.path.isfile(indexFile): - logger.debug("Loading index file") - try: - with open(indexFile, mode="r", encoding="utf-8") as inFile: - theData = json.load(inFile) - - except Exception: - logger.error("Failed to load index file") - logException() - self._indexBroken = True - return False - - self._refIndex = theData.get("refIndex", {}) - self._fileIndex = theData.get("fileIndex", {}) - self._fileMeta = theData.get("fileMeta", {}) - - nowTime = round(time()) - self._timeNovel = nowTime - self._timeNotes = nowTime - self._timeIndex = nowTime - - logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) - - indexFile = os.path.join(self.theProject.projMeta, "tagsIndex2.json") - tStart = time() - if os.path.isfile(indexFile): logger.debug("Loading index file") try: @@ -194,6 +159,11 @@ class NWIndex(): tItem.unpackData(tData) self._items[tHandle] = tItem + nowTime = round(time()) + self._timeNovel = nowTime + self._timeNotes = nowTime + self._timeIndex = nowTime + logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) self._checkIndex() @@ -209,11 +179,11 @@ class NWIndex(): tStart = time() try: + itemsIndex = {handle: item.packData() for handle, item in self._items.items()} with open(indexFile, mode="w+", encoding="utf-8") as outFile: outFile.write("{\n") - outFile.write(f' "refIndex": {jsonEncode(self._refIndex, n=1, nmax=3)},\n') - outFile.write(f' "fileIndex": {jsonEncode(self._fileIndex, n=1, nmax=3)},\n') - outFile.write(f' "fileMeta": {jsonEncode(self._fileMeta, n=1, nmax=2)}\n') + outFile.write(f' "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n') + outFile.write(f' "itemIndex": {jsonEncode(itemsIndex, n=1, nmax=4)}\n') outFile.write("}\n") except Exception: @@ -223,18 +193,6 @@ class NWIndex(): logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000) - indexFile = os.path.join(self.theProject.projMeta, "tagsIndex2.json") - tStart = time() - - itemsIndex = {handle: item.packData() for handle, item in self._items.items()} - with open(indexFile, mode="w+", encoding="utf-8") as outFile: - outFile.write("{\n") - outFile.write(f' "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n') - outFile.write(f' "itemIndex": {jsonEncode(itemsIndex, n=1, nmax=4)}\n') - outFile.write("}\n") - - logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000) - return True ## @@ -256,11 +214,12 @@ class NWIndex(): logger.info("Not indexing non-file item '%s'", tHandle) return False - # Run word counter for the whole text - cC, wC, pC = countWords(theText) - self._fileMeta[tHandle] = ["H0", cC, wC, pC] + self.deleteHandle(tHandle) + # Run word counter for the whole text self._items[tHandle] = IndexItem(tHandle, theItem) + + cC, wC, pC = countWords(theText) theItem.setCharCount(cC) theItem.setWordCount(wC) theItem.setParaCount(pC) @@ -282,15 +241,6 @@ class NWIndex(): logger.debug("Indexing item with handle '%s'", tHandle) - # Delete or reset old entries for the file - self._refIndex.pop(tHandle, None) - self._fileIndex[tHandle] = {} - - # Also clear references to the file in the tags index - clearTags = list(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags)) - for aTag in clearTags: - self._tags.pop(aTag) - # Scan the text content nTitle = 0 theLines = theText.splitlines() @@ -326,7 +276,6 @@ class NWIndex(): # Index page with no titles and references if nTitle == 0: - self._indexPage(tHandle, itemLayout) self._indexWordCounts(tHandle, theText, nTitle) # Update timestamps for index changes @@ -369,51 +318,17 @@ class NWIndex(): return False sTitle = f"T{nLine:06d}" - self._fileIndex[tHandle][sTitle] = { - "level": hDepth, - "title": hText, - "layout": itemLayout.name, - "cCount": 0, - "wCount": 0, - "pCount": 0, - "synopsis": "", - } - - if self._fileMeta[tHandle][0] == "H0": - # Since this initialises to H0, this ensures that only the - # first header level is recorded in the file meta index - self._fileMeta[tHandle][0] = hDepth - tItem = self._items[tHandle] tItem.updateLevel(hDepth) tItem.addHeading(IndexHeading(sTitle, hDepth, hText)) return True - def _indexPage(self, tHandle, itemLayout): - """Index a page with no title. - """ - self._fileIndex[tHandle][H_NONE] = { - "level": "H0", - "title": "", - "layout": itemLayout.name, - "cCount": 0, - "wCount": 0, - "pCount": 0, - "synopsis": "", - } - return - def _indexWordCounts(self, tHandle, theText, nTitle): """Count text stats and save the counts to the index. """ cC, wC, pC = countWords(theText) sTitle = f"T{nTitle:06d}" - if tHandle in self._fileIndex: - if sTitle in self._fileIndex[tHandle]: - self._fileIndex[tHandle][sTitle]["cCount"] = cC - self._fileIndex[tHandle][sTitle]["wCount"] = wC - self._fileIndex[tHandle][sTitle]["pCount"] = pC if tHandle in self._items: self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC) return @@ -422,9 +337,6 @@ class NWIndex(): """Save the synopsis to the index. """ sTitle = f"T{nTitle:06d}" - if tHandle in self._fileIndex: - if sTitle in self._fileIndex[tHandle]: - self._fileIndex[tHandle][sTitle]["synopsis"] = theText if tHandle in self._items: self._items[tHandle].setHeadingSynopsis(sTitle, theText) return @@ -451,14 +363,7 @@ class NWIndex(): } if tHandle in self._items: self._items[tHandle].setHeadingTag(sTitle, theBits[1]) - else: - if tHandle not in self._refIndex: - self._refIndex[tHandle] = {} - if sTitle not in self._refIndex[tHandle]: - self._refIndex[tHandle][sTitle] = [] - for aVal in theBits[1:]: - self._refIndex[tHandle][sTitle].append([nLine, theBits[0], aVal]) if tHandle in self._items: self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0]) @@ -546,17 +451,17 @@ class NWIndex(): files, but skipping all note files. """ for tHandle in self._listNovelHandles(skipExcluded): - for sTitle in sorted(self._fileIndex[tHandle]): + for sTitle in self._items[tHandle].headings: tKey = f"{tHandle}:{sTitle}" - yield tKey, tHandle, sTitle, self._fileIndex[tHandle][sTitle] + yield tKey, tHandle, sTitle, self._items[tHandle][sTitle] def getNovelWordCount(self, skipExcluded=True): """Count the number of words in the novel project. """ wCount = 0 for tHandle in self._listNovelHandles(skipExcluded): - for sTitle in self._fileIndex[tHandle]: - wCount += self._fileIndex[tHandle][sTitle]["wCount"] + for hItem in self._items[tHandle].entries: + wCount += hItem.wordCount return wCount @@ -565,8 +470,8 @@ class NWIndex(): """ hCount = [0, 0, 0, 0, 0] for tHandle in self._listNovelHandles(skipExcluded): - for sTitle in self._fileIndex[tHandle]: - iLevel = H_LEVEL.get(self._fileIndex[tHandle][sTitle]["level"], 0) + for hItem in self._items[tHandle].entries: + iLevel = H_LEVEL.get(hItem.level, 0) hCount[iLevel] += 1 return hCount @@ -574,19 +479,26 @@ class NWIndex(): def getHandleWordCounts(self, tHandle): """Get all header word counts for a specific handle. """ - hRecord = self._fileIndex.get(tHandle, {}) - return [(f"{tHandle}:{sTitle}", sData["wCount"]) for sTitle, sData in hRecord.items()] + return [ + (f"{tHandle}:{sTitle}", hItem.wordCount) + for sTitle, hItem in self._items.get(tHandle, {}).items() + ] def getHandleHeaders(self, tHandle): """Get all headers for a specific handle. """ - hRecord = self._fileIndex.get(tHandle, {}) - return [(sTitle, sData["level"], sData["title"]) for sTitle, sData in hRecord.items()] + return [ + (sTitle, hItem.level, hItem.title) + for sTitle, hItem in self._items.get(tHandle, {}).items() + ] def getHandleHeaderLevel(self, tHandle): """Get the header level of the first header of a handle. """ - return self._fileMeta.get(tHandle, ["H0"])[0] + if tHandle in self._items: + return self._items[tHandle].level + else: + return "H0" def getTableOfContents(self, maxDepth, skipExcluded=True): """Generate a table of contents up to a maximum depth. @@ -595,21 +507,20 @@ class NWIndex(): tData = {} pKey = None for tHandle in self._listNovelHandles(skipExcluded): - for sTitle in sorted(self._fileIndex[tHandle]): + for sTitle in self._items[tHandle].headings: tKey = f"{tHandle}:{sTitle}" - theData = self._fileIndex[tHandle][sTitle] - iLevel = H_LEVEL.get(theData["level"], 0) + hItem = self._items[tHandle][sTitle] + iLevel = H_LEVEL.get(hItem.level, 0) if iLevel > maxDepth: if pKey in tData: - theData["wCount"] - tData[pKey]["words"] += theData["wCount"] + tData[pKey]["words"] += hItem.wordCount else: pKey = tKey tOrder.append(tKey) tData[tKey] = { "level": iLevel, - "title": theData["title"], - "words": theData["wCount"], + "title": hItem.title, + "words": hItem.wordCount, } theToC = [( @@ -630,16 +541,18 @@ class NWIndex(): pC = 0 if sTitle is None: - if tHandle in self._fileMeta: - cC = self._fileMeta[tHandle][1] - wC = self._fileMeta[tHandle][2] - pC = self._fileMeta[tHandle][3] + if tHandle in self._items: + tItem = self._items[tHandle].item + cC = tItem.charCount + wC = tItem.wordCount + pC = tItem.paraCount else: - if tHandle in self._fileIndex: - if sTitle in self._fileIndex[tHandle]: - cC = self._fileIndex[tHandle][sTitle]["cCount"] - wC = self._fileIndex[tHandle][sTitle]["wCount"] - pC = self._fileIndex[tHandle][sTitle]["pCount"] + if tHandle in self._items: + if sTitle in self._items[tHandle]: + hItem = self._items[tHandle][sTitle] + cC = hItem.charCount + wC = hItem.wordCount + pC = hItem.paraCount return cC, wC, pC @@ -648,40 +561,43 @@ class NWIndex(): section. """ theRefs = {x: [] for x in nwKeyWords.KEY_CLASS} - if tHandle not in self._refIndex: + if tHandle not in self._items: return theRefs - for refTitle in self._refIndex[tHandle]: - for aTag in self._refIndex[tHandle][refTitle]: - if len(aTag) == 3 and (sTitle is None or sTitle == refTitle): - if aTag[1] in theRefs: - theRefs[aTag[1]].append(aTag[2]) + for rTitle, hItem in self._items[tHandle].items(): + if sTitle is None or sTitle == rTitle: + for aTag, refTypes in hItem.references.items(): + for refType in refTypes: + if refType in theRefs: + theRefs[refType].append(aTag) return theRefs def getNovelData(self, tHandle, sTitle): """Return the novel data of a given handle and title. """ - if tHandle in self._fileIndex: - if sTitle in self._fileIndex[tHandle]: - return self._fileIndex[tHandle][sTitle] + if tHandle in self._items: + if sTitle in self._items[tHandle]: + return self._items[tHandle][sTitle] return None def getBackReferenceList(self, tHandle): """Build a list of files referring back to our file, specified by tHandle. """ - if tHandle is None: + if tHandle is None or tHandle not in self._items: return {} theRefs = {} - theTags = set(filter(lambda x: self._tags[x].get("handle") == tHandle, self._tags)) - if theTags: - for tHandle in self._refIndex: - for sTitle in self._refIndex[tHandle]: - for _, _, tTag in self._refIndex[tHandle][sTitle]: - if tTag in theTags and tHandle not in theRefs: - theRefs[tHandle] = sTitle + theTags = self._items[tHandle].allTags() + if not theTags: + return theRefs + + for aHandle, tItem in self._items.items(): + for sTitle, hItem in tItem.items(): + for aTag in hItem.references: + if aTag in theTags and aHandle not in theRefs: + theRefs[aHandle] = sTitle return theRefs @@ -706,7 +622,7 @@ class NWIndex(): continue if tItem.itemLayout == nwItemLayout.NOTE: continue - if tItem.itemHandle in self._fileIndex: + if tItem.itemHandle in self._items: theHandles.append(tItem.itemHandle) return theHandles @@ -724,25 +640,9 @@ class NWIndex(): logger.debug("Checking index") tStart = time() - try: - self._checkRefIndex() - self._checkFileIndex() - self._checkFileMeta() - self._indexBroken = False - - except Exception: - logger.error("Error while checking index") - logException() - self._indexBroken = True - - if self._indexBroken: - self.clearIndex() - logger.verbose("Index check completed in %.3f ms", (time() - tStart)*1000) - return - # If the index was ok, we check that project files are indexed for fHandle in self.theProject.projFiles: - if fHandle not in self._fileMeta: + if fHandle not in self._items: logger.warning("Item '%s' is not in the index", fHandle) self.reIndexHandle(fHandle) @@ -750,103 +650,6 @@ class NWIndex(): return - def _checkRefIndex(self): - """Scan the reference index for errors. - Warning: This function raises exceptions. - """ - for tHandle in self._refIndex: - if not isHandle(tHandle): - raise KeyError("refIndex key is not a handle") - - hEntry = self._refIndex[tHandle] - for sTitle in hEntry: - if not isTitleTag(sTitle): - raise KeyError("refIndex[a] key is not a title tag") - - sEntry = hEntry[sTitle] - for tEntry in sEntry: - if len(tEntry) != 3: - raise IndexError("refIndex[a][b][i] expected 3 values") - if not isinstance(tEntry[0], int): - raise ValueError("refIndex[a][b][i][0] is not an integer") - if not tEntry[1] in nwKeyWords.VALID_KEYS: - raise ValueError("refIndex[a][b][i][1] is not a keyword") - if not isinstance(tEntry[2], str): - raise ValueError("refIndex[a][b][i][2] is not a string") - - return - - def _checkFileIndex(self): - """Scan the file index for errors. - Warning: This function raises exceptions. - """ - for tHandle in self._fileIndex: - if not isHandle(tHandle): - raise KeyError("fileIndex key is not a handle") - - hEntry = self._fileIndex[tHandle] - for sTitle in self._fileIndex[tHandle]: - if not isTitleTag(sTitle): - raise KeyError("fileIndex[a] key is not a title tag") - - sEntry = hEntry[sTitle] - if len(sEntry) != 7: - raise IndexError("fileIndex[a][b] expected 7 values") - - if "level" not in sEntry: - raise KeyError("fileIndex[a][b] has no 'level' key") - if "title" not in sEntry: - raise KeyError("fileIndex[a][b] has no 'title' key") - if "layout" not in sEntry: - raise KeyError("fileIndex[a][b] has no 'layout' key") - if "cCount" not in sEntry: - raise KeyError("fileIndex[a][b] has no 'cCount' key") - if "wCount" not in sEntry: - raise KeyError("fileIndex[a][b] has no 'wCount' key") - if "pCount" not in sEntry: - raise KeyError("fileIndex[a][b] has no 'pCount' key") - if "synopsis" not in sEntry: - raise KeyError("fileIndex[a][b] has no 'synopsis' key") - - if not sEntry["level"] in H_VALID: - raise ValueError("fileIndex[a][b][level] is not a header level") - if not isinstance(sEntry["title"], str): - raise ValueError("fileIndex[a][b][title] is not a string") - if not isItemLayout(sEntry["layout"]): - raise ValueError("fileIndex[a][b][layout] is not an nwItemLayout") - if not isinstance(sEntry["cCount"], int): - raise ValueError("fileIndex[a][b][cCount] is not an integer") - if not isinstance(sEntry["wCount"], int): - raise ValueError("fileIndex[a][b][wCount] is not an integer") - if not isinstance(sEntry["pCount"], int): - raise ValueError("fileIndex[a][b][pCount] is not an integer") - if not isinstance(sEntry["synopsis"], str): - raise ValueError("fileIndex[a][b][synopsis] is not a string") - - return - - def _checkFileMeta(self): - """Scan the text counts index for errors. - Warning: This function raises exceptions. - """ - for tHandle in self._fileMeta: - if not isHandle(tHandle): - raise KeyError("fileMeta key is not a handle") - - tEntry = self._fileMeta[tHandle] - if len(tEntry) != 4: - raise IndexError("fileMeta[a] expected 4 values") - if not tEntry[0] in H_VALID: - raise ValueError("fileMeta[a][0] is not a header level") - if not isinstance(tEntry[1], int): - raise ValueError("fileMeta[a][1] is not an integer") - if not isinstance(tEntry[2], int): - raise ValueError("fileMeta[a][2] is not an integer") - if not isinstance(tEntry[3], int): - raise ValueError("fileMeta[a][3] is not an integer") - - return - # END Class NWIndex @@ -950,13 +753,21 @@ class IndexItem: # Properties ## + @property + def item(self): + return self._item + @property def level(self): return self._level @property - def itemClass(self): - return self._item.itemClass + def headings(self): + return sorted(self._headings.keys()) + + @property + def entries(self): + return self._headings.values() ## # Setters @@ -1012,9 +823,22 @@ class IndexItem: def __getitem__(self, sTitle): return self._headings.get(sTitle, None) + def __contains__(self, sTitle): + return sTitle in self._headings + def items(self): return self._headings.items() + def allTags(self): + """Return a list of all tags in the current item. + """ + tags = [] + for hItem in self._headings.values(): + tag = hItem.tag + if tag: + tags.append(tag) + return tags + ## # Pack/Unpack ## @@ -1077,10 +901,38 @@ class IndexHeading: def key(self): return self._key + @property + def level(self): + return self._level + + @property + def title(self): + return self._title + + @property + def charCount(self): + return self._charCount + + @property + def wordCount(self): + return self._wordCount + + @property + def paraCount(self): + return self._paraCount + + @property + def synopsis(self): + return self._synopsis + @property def tag(self): return self._tag + @property + def references(self): + return self._refs + ## # Setters ## diff --git a/novelwriter/gui/noveltree.py b/novelwriter/gui/noveltree.py index 91895c7a..28edb2d9 100644 --- a/novelwriter/gui/noveltree.py +++ b/novelwriter/gui/noveltree.py @@ -258,7 +258,7 @@ class GuiNovelTree(QTreeWidget): tItem = self._createTreeItem(tHandle, sTitle, tKey, novIdx) self._treeMap[tKey] = tItem - tLevel = novIdx["level"] + tLevel = novIdx.level if tLevel == "H1": self.addTopLevelItem(tItem) currTitle = tItem @@ -305,12 +305,12 @@ class GuiNovelTree(QTreeWidget): """Populate a tree item with all the column values. """ newItem = QTreeWidgetItem() - hIcon = "doc_%s" % novIdx["level"].lower() + hIcon = "doc_%s" % novIdx.level.lower() theData = (tHandle, sTitle[1:].lstrip("0"), titleKey) - wC = int(novIdx["wCount"]) + wC = int(novIdx.wordCount) - newItem.setText(self.C_TITLE, novIdx["title"]) + newItem.setText(self.C_TITLE, novIdx.title) newItem.setData(self.C_TITLE, Qt.UserRole, theData) newItem.setIcon(self.C_TITLE, self.theTheme.getIcon(hIcon)) newItem.setText(self.C_WORDS, f"{wC:n}") diff --git a/novelwriter/gui/outline.py b/novelwriter/gui/outline.py index 26cb8029..e3886b87 100644 --- a/novelwriter/gui/outline.py +++ b/novelwriter/gui/outline.py @@ -393,7 +393,7 @@ class GuiOutline(QTreeWidget): tItem = self._createTreeItem(tHandle, sTitle, novIdx) - tLevel = novIdx["level"] + tLevel = novIdx.level if tLevel == "H1": self.addTopLevelItem(tItem) currTitle = tItem @@ -441,24 +441,24 @@ class GuiOutline(QTreeWidget): """ nwItem = self.theProject.tree[tHandle] newItem = QTreeWidgetItem() - hIcon = "doc_%s" % novIdx["level"].lower() + hIcon = "doc_%s" % novIdx.level.lower() hLevel = self.theProject.index.getHandleHeaderLevel(tHandle) dIcon = self.theTheme.getItemIcon(nwItemType.FILE, None, nwItemLayout.DOCUMENT, hLevel) - cC = int(novIdx["cCount"]) - wC = int(novIdx["wCount"]) - pC = int(novIdx["pCount"]) + cC = int(novIdx.charCount) + wC = int(novIdx.wordCount) + pC = int(novIdx.paraCount) - newItem.setText(self._colIdx[nwOutline.TITLE], novIdx["title"]) + newItem.setText(self._colIdx[nwOutline.TITLE], novIdx.title) newItem.setData(self._colIdx[nwOutline.TITLE], Qt.UserRole, tHandle) newItem.setIcon(self._colIdx[nwOutline.TITLE], self.theTheme.getIcon(hIcon)) - newItem.setText(self._colIdx[nwOutline.LEVEL], novIdx["level"]) + newItem.setText(self._colIdx[nwOutline.LEVEL], novIdx.level) newItem.setText(self._colIdx[nwOutline.LABEL], nwItem.itemName) newItem.setIcon(self._colIdx[nwOutline.LABEL], dIcon) newItem.setText(self._colIdx[nwOutline.LINE], sTitle[1:].lstrip("0")) newItem.setData(self._colIdx[nwOutline.LINE], Qt.UserRole, sTitle) - newItem.setText(self._colIdx[nwOutline.SYNOP], novIdx["synopsis"]) + newItem.setText(self._colIdx[nwOutline.SYNOP], novIdx.synopsis) newItem.setText(self._colIdx[nwOutline.CCOUNT], f"{cC:n}") newItem.setText(self._colIdx[nwOutline.WCOUNT], f"{wC:n}") newItem.setText(self._colIdx[nwOutline.PCOUNT], f"{pC:n}") diff --git a/novelwriter/gui/outlinedetails.py b/novelwriter/gui/outlinedetails.py index 40a3d29e..f6f86e67 100644 --- a/novelwriter/gui/outlinedetails.py +++ b/novelwriter/gui/outlinedetails.py @@ -288,26 +288,26 @@ class GuiOutlineDetails(QScrollArea): if nwItem is None or novIdx is None: return False - if novIdx["level"] in self.LVL_MAP: - self.titleLabel.setText("%s" % self.tr(self.LVL_MAP[novIdx["level"]])) + if novIdx.level in self.LVL_MAP: + self.titleLabel.setText("%s" % self.tr(self.LVL_MAP[novIdx.level])) else: self.titleLabel.setText("%s" % self.tr("Title")) - self.titleValue.setText(novIdx["title"]) + self.titleValue.setText(novIdx.title) itemStatus, _ = nwItem.getImportStatus() self.fileValue.setText(nwItem.itemName) self.itemValue.setText(itemStatus) - cC = checkInt(novIdx["cCount"], 0) - wC = checkInt(novIdx["wCount"], 0) - pC = checkInt(novIdx["pCount"], 0) + cC = checkInt(novIdx.charCount, 0) + wC = checkInt(novIdx.wordCount, 0) + pC = checkInt(novIdx.paraCount, 0) self.cCValue.setText(f"{cC:n}") self.wCValue.setText(f"{wC:n}") self.pCValue.setText(f"{pC:n}") - self.synopValue.setText(novIdx["synopsis"]) + self.synopValue.setText(novIdx.synopsis) self.povKeyValue.setText(self._formatTags(theRefs, nwKeyWords.POV_KEY)) self.focKeyValue.setText(self._formatTags(theRefs, nwKeyWords.FOCUS_KEY)) diff --git a/tests/reference/coreIndex_LoadSave_tagsIndex.json b/tests/reference/coreIndex_LoadSave_tagsIndex.json index fb4d9acd..44adc7ad 100644 --- a/tests/reference/coreIndex_LoadSave_tagsIndex.json +++ b/tests/reference/coreIndex_LoadSave_tagsIndex.json @@ -1,99 +1,125 @@ { -"tagIndex": { - "Bod": [3, "4c4f28287af27", "CHARACTER", "T000001"], - "Main": [3, "2426c6f0ca922", "PLOT", "T000001"], - "Europe": [3, "04468803b92e1", "WORLD", "T000001"] -}, -"refIndex": { - "fb609cd8319dc": { - "T000001": [[3, "@pov", "Bod"], [4, "@plot", "Main"], [5, "@location", "Europe"]] + "tagsIndex": { + "Bod": {"handle": "4c4f28287af27", "heading": "T000001", "class": "CHARACTER"}, + "Main": {"handle": "2426c6f0ca922", "heading": "T000001", "class": "PLOT"}, + "Europe": {"handle": "04468803b92e1", "heading": "T000001", "class": "WORLD"} }, - "88243afbe5ed8": { - "T000001": [[3, "@pov", "Bod"], [4, "@plot", "Main"], [5, "@location", "Europe"]] - }, - "f96ec11c6a3da": { - "T000001": [[3, "@pov", "Bod"], [4, "@plot", "Main"], [5, "@location", "Europe"]] - }, - "441420a886d82": { - "T000001": [[3, "@pov", "Bod"], [4, "@plot", "Main"], [5, "@location", "Europe"]] - }, - "eb103bc70c90c": { - "T000001": [[3, "@pov", "Bod"], [4, "@plot", "Main"], [5, "@location", "Europe"]] - }, - "f8c0562e50f1b": { - "T000001": [[3, "@pov", "Bod"], [4, "@plot", "Main"], [5, "@location", "Europe"]] - }, - "47666c91c7ccf": { - "T000001": [[3, "@pov", "Bod"], [4, "@plot", "Main"], [5, "@location", "Europe"]] - }, - "4c4f28287af27": { - "T000001": [[4, "@plot", "Main"]] + "itemIndex": { + "7a992350f3eb6": { + "level": "H1", + "headings": { + "T000001": {"level": "H1", "title": "Lorem Ipsum", "tag": "", "cCount": 230, "wCount": 40, "pCount": 3, "synopsis": ""} + } + }, + "8c58a65414c23": { + "level": "H0", + "headings": { + "T000000": {"level": "H0", "title": "", "tag": "", "cCount": 1058, "wCount": 176, "pCount": 2, "synopsis": ""} + } + }, + "88d59a277361b": { + "level": "H2", + "headings": { + "T000001": {"level": "H2", "title": "Prologue", "tag": "", "cCount": 584, "wCount": 92, "pCount": 1, "synopsis": "Explanation from the lipsum.com website."} + } + }, + "db7e733775d4d": { + "level": "H1", + "headings": { + "T000001": {"level": "H1", "title": "Act One", "tag": "", "cCount": 35, "wCount": 6, "pCount": 1, "synopsis": ""} + } + }, + "fb609cd8319dc": { + "level": "H2", + "headings": { + "T000001": {"level": "H2", "title": "Chapter One", "tag": "", "cCount": 419, "wCount": 67, "pCount": 1, "synopsis": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque at aliquam quam."} + }, + "references": { + "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + } + }, + "88243afbe5ed8": { + "level": "H0", + "headings": { + "T000001": {"level": "H3", "title": "Scene One", "tag": "", "cCount": 1197, "wCount": 174, "pCount": 2, "synopsis": "Aenean ut placerat velit. Etiam laoreet ullamcorper risus, eget lobortis enim scelerisque non. Suspendisse id maximus nunc, et mollis sapien. Curabitur vel semper sapien, non pulvinar dolor. Etiam finibus nisi vel mi molestie consectetur."}, + "T000013": {"level": "H4", "title": "Scene One, Section Two", "tag": "", "cCount": 1561, "wCount": 230, "pCount": 2, "synopsis": ""} + }, + "references": { + "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + } + }, + "f96ec11c6a3da": { + "level": "H0", + "headings": { + "T000001": {"level": "H3", "title": "Scene Two", "tag": "", "cCount": 2034, "wCount": 299, "pCount": 3, "synopsis": "Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Integer sapien nulla, dictum at lacus a, dignissim consectetur dolor. Nunc vel eleifend lacus, eu dapibus orci."}, + "T000015": {"level": "H4", "title": "Scene Two, Section Two", "tag": "", "cCount": 2009, "wCount": 301, "pCount": 3, "synopsis": ""} + }, + "references": { + "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + } + }, + "846352075de7d": { + "level": "H2", + "headings": { + "T000001": {"level": "H2", "title": "Why do we use it?", "tag": "", "cCount": 631, "wCount": 109, "pCount": 3, "synopsis": ""} + } + }, + "441420a886d82": { + "level": "H2", + "headings": { + "T000001": {"level": "H2", "title": "Chapter Two", "tag": "", "cCount": 477, "wCount": 70, "pCount": 1, "synopsis": "Curabitur a elit posuere, varius ex et, convallis neque. Phasellus sagittis pharetra sem vitae dapibus. Curabitur varius lorem non pulvinar congue."} + }, + "references": { + "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + } + }, + "eb103bc70c90c": { + "level": "H3", + "headings": { + "T000001": {"level": "H3", "title": "Scene Three", "tag": "", "cCount": 3006, "wCount": 439, "pCount": 4, "synopsis": "Aenean ut libero ut lectus porttitor rhoncus vel et massa. Nam pretium, nibh et varius vehicula, urna metus blandit eros, euismod pharetra diam diam et libero. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos."} + }, + "references": { + "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + } + }, + "f8c0562e50f1b": { + "level": "H3", + "headings": { + "T000001": {"level": "H3", "title": "Scene Four", "tag": "", "cCount": 3839, "wCount": 563, "pCount": 6, "synopsis": "Nam tempor blandit magna laoreet aliquet. Vestibulum auctor posuere leo, ac gravida nisi rhoncus varius. Aenean posuere dolor vitae condimentum volutpat. Donec egestas volutpat risus, quis luctus justo."} + }, + "references": { + "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + } + }, + "47666c91c7ccf": { + "level": "H3", + "headings": { + "T000001": {"level": "H3", "title": "Scene Five", "tag": "", "cCount": 3644, "wCount": 543, "pCount": 5, "synopsis": "Praesent eget est porta, dictum ante in, egestas risus. Mauris risus mauris, consequat aliquam mauris et, feugiat iaculis ipsum. Aliquam arcu ipsum, fermentum ut arcu sed, lobortis euismod sem. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus."} + }, + "references": { + "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + } + }, + "4c4f28287af27": { + "level": "H1", + "headings": { + "T000001": {"level": "H1", "title": "Nobody Owens", "tag": "Bod", "cCount": 1864, "wCount": 284, "pCount": 3, "synopsis": ""} + }, + "references": { + "T000001": {"Main": ["@plot"]} + } + }, + "2426c6f0ca922": { + "level": "H1", + "headings": { + "T000001": {"level": "H1", "title": "Main Plot", "tag": "Main", "cCount": 1369, "wCount": 195, "pCount": 2, "synopsis": ""} + } + }, + "04468803b92e1": { + "level": "H1", + "headings": { + "T000001": {"level": "H1", "title": "Ancient Europe", "tag": "Europe", "cCount": 1770, "wCount": 259, "pCount": 3, "synopsis": ""} + } + } } -}, -"fileIndex": { - "7a992350f3eb6": { - "T000001": {"level": "H1", "title": "Lorem Ipsum", "layout": "DOCUMENT", "cCount": 230, "wCount": 40, "pCount": 3, "synopsis": ""} - }, - "8c58a65414c23": { - "T000000": {"level": "H0", "title": "", "layout": "DOCUMENT", "cCount": 1058, "wCount": 176, "pCount": 2, "synopsis": ""} - }, - "88d59a277361b": { - "T000001": {"level": "H2", "title": "Prologue", "layout": "DOCUMENT", "cCount": 584, "wCount": 92, "pCount": 1, "synopsis": "Explanation from the lipsum.com website."} - }, - "db7e733775d4d": { - "T000001": {"level": "H1", "title": "Act One", "layout": "DOCUMENT", "cCount": 35, "wCount": 6, "pCount": 1, "synopsis": ""} - }, - "fb609cd8319dc": { - "T000001": {"level": "H2", "title": "Chapter One", "layout": "DOCUMENT", "cCount": 419, "wCount": 67, "pCount": 1, "synopsis": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque at aliquam quam."} - }, - "88243afbe5ed8": { - "T000001": {"level": "H3", "title": "Scene One", "layout": "DOCUMENT", "cCount": 1197, "wCount": 174, "pCount": 2, "synopsis": "Aenean ut placerat velit. Etiam laoreet ullamcorper risus, eget lobortis enim scelerisque non. Suspendisse id maximus nunc, et mollis sapien. Curabitur vel semper sapien, non pulvinar dolor. Etiam finibus nisi vel mi molestie consectetur."}, - "T000013": {"level": "H4", "title": "Scene One, Section Two", "layout": "DOCUMENT", "cCount": 1561, "wCount": 230, "pCount": 2, "synopsis": ""} - }, - "f96ec11c6a3da": { - "T000001": {"level": "H3", "title": "Scene Two", "layout": "DOCUMENT", "cCount": 2034, "wCount": 299, "pCount": 3, "synopsis": "Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Integer sapien nulla, dictum at lacus a, dignissim consectetur dolor. Nunc vel eleifend lacus, eu dapibus orci."}, - "T000015": {"level": "H4", "title": "Scene Two, Section Two", "layout": "DOCUMENT", "cCount": 2009, "wCount": 301, "pCount": 3, "synopsis": ""} - }, - "846352075de7d": { - "T000001": {"level": "H2", "title": "Why do we use it?", "layout": "DOCUMENT", "cCount": 631, "wCount": 109, "pCount": 3, "synopsis": ""} - }, - "441420a886d82": { - "T000001": {"level": "H2", "title": "Chapter Two", "layout": "DOCUMENT", "cCount": 477, "wCount": 70, "pCount": 1, "synopsis": "Curabitur a elit posuere, varius ex et, convallis neque. Phasellus sagittis pharetra sem vitae dapibus. Curabitur varius lorem non pulvinar congue."} - }, - "eb103bc70c90c": { - "T000001": {"level": "H3", "title": "Scene Three", "layout": "DOCUMENT", "cCount": 3006, "wCount": 439, "pCount": 4, "synopsis": "Aenean ut libero ut lectus porttitor rhoncus vel et massa. Nam pretium, nibh et varius vehicula, urna metus blandit eros, euismod pharetra diam diam et libero. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos."} - }, - "f8c0562e50f1b": { - "T000001": {"level": "H3", "title": "Scene Four", "layout": "DOCUMENT", "cCount": 3839, "wCount": 563, "pCount": 6, "synopsis": "Nam tempor blandit magna laoreet aliquet. Vestibulum auctor posuere leo, ac gravida nisi rhoncus varius. Aenean posuere dolor vitae condimentum volutpat. Donec egestas volutpat risus, quis luctus justo."} - }, - "47666c91c7ccf": { - "T000001": {"level": "H3", "title": "Scene Five", "layout": "DOCUMENT", "cCount": 3644, "wCount": 543, "pCount": 5, "synopsis": "Praesent eget est porta, dictum ante in, egestas risus. Mauris risus mauris, consequat aliquam mauris et, feugiat iaculis ipsum. Aliquam arcu ipsum, fermentum ut arcu sed, lobortis euismod sem. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus."} - }, - "4c4f28287af27": { - "T000001": {"level": "H1", "title": "Nobody Owens", "layout": "NOTE", "cCount": 1864, "wCount": 284, "pCount": 3, "synopsis": ""} - }, - "2426c6f0ca922": { - "T000001": {"level": "H1", "title": "Main Plot", "layout": "NOTE", "cCount": 1369, "wCount": 195, "pCount": 2, "synopsis": ""} - }, - "04468803b92e1": { - "T000001": {"level": "H1", "title": "Ancient Europe", "layout": "NOTE", "cCount": 1770, "wCount": 259, "pCount": 3, "synopsis": ""} - } -}, -"fileMeta": { - "7a992350f3eb6": ["H1", 230, 40, 3], - "8c58a65414c23": ["H0", 1058, 176, 2], - "88d59a277361b": ["H2", 584, 92, 1], - "db7e733775d4d": ["H1", 35, 6, 1], - "fb609cd8319dc": ["H2", 419, 67, 1], - "88243afbe5ed8": ["H3", 2758, 404, 4], - "f96ec11c6a3da": ["H3", 4043, 600, 6], - "846352075de7d": ["H2", 631, 109, 3], - "441420a886d82": ["H2", 477, 70, 1], - "eb103bc70c90c": ["H3", 3006, 439, 4], - "f8c0562e50f1b": ["H3", 3839, 563, 6], - "47666c91c7ccf": ["H3", 3644, 543, 5], - "4c4f28287af27": ["H1", 1864, 284, 3], - "2426c6f0ca922": ["H1", 1369, 195, 2], - "04468803b92e1": ["H1", 1770, 259, 3] -} } diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index d2e76643..eff85fec 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -69,27 +69,19 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): # Take a copy of the index tagIndex = str(theIndex._tags) - refIndex = str(theIndex._refIndex) - fileIndex = str(theIndex._fileIndex) - textCounts = str(theIndex._fileMeta) + itemsIndex = str({handle: item.packData() for handle, item in theIndex._items.items()}) # Delete a handle assert theIndex._tags.get("Bod", None) is not None - assert theIndex._refIndex.get("4c4f28287af27", None) is not None - assert theIndex._fileIndex.get("4c4f28287af27", None) is not None - assert theIndex._fileMeta.get("4c4f28287af27", None) is not None + assert theIndex._items.get("4c4f28287af27", None) is not None theIndex.deleteHandle("4c4f28287af27") assert theIndex._tags.get("Bod", None) is None - assert theIndex._refIndex.get("4c4f28287af27", None) is None - assert theIndex._fileIndex.get("4c4f28287af27", None) is None - assert theIndex._fileMeta.get("4c4f28287af27", None) is None + assert theIndex._items.get("4c4f28287af27", None) is None # Clear the index theIndex.clearIndex() assert theIndex._tags == {} - assert theIndex._refIndex == {} - assert theIndex._fileIndex == {} - assert theIndex._fileMeta == {} + assert theIndex._items == {} # Make the load fail with monkeypatch.context() as mp: @@ -100,9 +92,9 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): assert theIndex.loadIndex() is True assert str(theIndex._tags) == tagIndex - assert str(theIndex._refIndex) == refIndex - assert str(theIndex._fileIndex) == fileIndex - assert str(theIndex._fileMeta) == textCounts + assert str( + {handle: item.packData() for handle, item in theIndex._items.items()} + ) == itemsIndex # Break the index and check that we notice # assert theIndex.indexBroken is False @@ -201,7 +193,7 @@ def testCoreIndex_CheckThese(nwMinimal, mockGUI): assert theIndex._tags == { "Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"} } - assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!" + assert theIndex.getNovelData(nHandle, "T000001").title == "Hello World!" assert theIndex.getReferences(nHandle, "T000001") == { "@char": [], "@custom": [], @@ -314,7 +306,7 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): assert theIndex._tags == { "Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"} } - assert theIndex.getNovelData(nHandle, "T000001")["title"] == "Hello World!" + assert theIndex.getNovelData(nHandle, "T000001").title == "Hello World!" # Title Indexing # ============== @@ -336,42 +328,40 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "##### Title Five\n\n" # Not interpreted as a title, the hashes are counted as a word "Paragraph Five.\n\n" )) - assert nHandle not in theIndex._refIndex + assert theIndex._items[nHandle]["T000001"].references == {} + assert theIndex._items[nHandle]["T000007"].references == {} + assert theIndex._items[nHandle]["T000013"].references == {} + assert theIndex._items[nHandle]["T000019"].references == {} - assert theIndex._fileIndex[nHandle]["T000001"]["level"] == "H1" - assert theIndex._fileIndex[nHandle]["T000007"]["level"] == "H2" - assert theIndex._fileIndex[nHandle]["T000013"]["level"] == "H3" - assert theIndex._fileIndex[nHandle]["T000019"]["level"] == "H4" + assert theIndex._items[nHandle]["T000001"].level == "H1" + assert theIndex._items[nHandle]["T000007"].level == "H2" + assert theIndex._items[nHandle]["T000013"].level == "H3" + assert theIndex._items[nHandle]["T000019"].level == "H4" - assert theIndex._fileIndex[nHandle]["T000001"]["title"] == "Title One" - assert theIndex._fileIndex[nHandle]["T000007"]["title"] == "Title Two" - assert theIndex._fileIndex[nHandle]["T000013"]["title"] == "Title Three" - assert theIndex._fileIndex[nHandle]["T000019"]["title"] == "Title Four" + assert theIndex._items[nHandle]["T000001"].title == "Title One" + assert theIndex._items[nHandle]["T000007"].title == "Title Two" + assert theIndex._items[nHandle]["T000013"].title == "Title Three" + assert theIndex._items[nHandle]["T000019"].title == "Title Four" - assert theIndex._fileIndex[nHandle]["T000001"]["layout"] == "DOCUMENT" - assert theIndex._fileIndex[nHandle]["T000007"]["layout"] == "DOCUMENT" - assert theIndex._fileIndex[nHandle]["T000013"]["layout"] == "DOCUMENT" - assert theIndex._fileIndex[nHandle]["T000019"]["layout"] == "DOCUMENT" + assert theIndex._items[nHandle]["T000001"].charCount == 23 + assert theIndex._items[nHandle]["T000007"].charCount == 23 + assert theIndex._items[nHandle]["T000013"].charCount == 27 + assert theIndex._items[nHandle]["T000019"].charCount == 56 - assert theIndex._fileIndex[nHandle]["T000001"]["cCount"] == 23 - assert theIndex._fileIndex[nHandle]["T000007"]["cCount"] == 23 - assert theIndex._fileIndex[nHandle]["T000013"]["cCount"] == 27 - assert theIndex._fileIndex[nHandle]["T000019"]["cCount"] == 56 + assert theIndex._items[nHandle]["T000001"].wordCount == 4 + assert theIndex._items[nHandle]["T000007"].wordCount == 4 + assert theIndex._items[nHandle]["T000013"].wordCount == 4 + assert theIndex._items[nHandle]["T000019"].wordCount == 9 - assert theIndex._fileIndex[nHandle]["T000001"]["wCount"] == 4 - assert theIndex._fileIndex[nHandle]["T000007"]["wCount"] == 4 - assert theIndex._fileIndex[nHandle]["T000013"]["wCount"] == 4 - assert theIndex._fileIndex[nHandle]["T000019"]["wCount"] == 9 + assert theIndex._items[nHandle]["T000001"].paraCount == 1 + assert theIndex._items[nHandle]["T000007"].paraCount == 1 + assert theIndex._items[nHandle]["T000013"].paraCount == 1 + assert theIndex._items[nHandle]["T000019"].paraCount == 3 - assert theIndex._fileIndex[nHandle]["T000001"]["pCount"] == 1 - assert theIndex._fileIndex[nHandle]["T000007"]["pCount"] == 1 - assert theIndex._fileIndex[nHandle]["T000013"]["pCount"] == 1 - assert theIndex._fileIndex[nHandle]["T000019"]["pCount"] == 3 - - assert theIndex._fileIndex[nHandle]["T000001"]["synopsis"] == "Synopsis One." - assert theIndex._fileIndex[nHandle]["T000007"]["synopsis"] == "Synopsis Two." - assert theIndex._fileIndex[nHandle]["T000013"]["synopsis"] == "Synopsis Three." - assert theIndex._fileIndex[nHandle]["T000019"]["synopsis"] == "Synopsis Four." + assert theIndex._items[nHandle]["T000001"].synopsis == "Synopsis One." + assert theIndex._items[nHandle]["T000007"].synopsis == "Synopsis Two." + assert theIndex._items[nHandle]["T000013"].synopsis == "Synopsis Three." + assert theIndex._items[nHandle]["T000019"].synopsis == "Synopsis Four." # Note File assert theIndex.scanText(cHandle, ( @@ -380,15 +370,13 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "% synopsis: Synopsis One.\n\n" "Paragraph One.\n\n" )) - assert cHandle not in theIndex._refIndex - - assert theIndex._fileIndex[cHandle]["T000001"]["level"] == "H1" - assert theIndex._fileIndex[cHandle]["T000001"]["title"] == "Title One" - assert theIndex._fileIndex[cHandle]["T000001"]["layout"] == "NOTE" - assert theIndex._fileIndex[cHandle]["T000001"]["cCount"] == 23 - assert theIndex._fileIndex[cHandle]["T000001"]["wCount"] == 4 - assert theIndex._fileIndex[cHandle]["T000001"]["pCount"] == 1 - assert theIndex._fileIndex[cHandle]["T000001"]["synopsis"] == "Synopsis One." + assert theIndex._items[cHandle]["T000001"].references == {} + assert theIndex._items[cHandle]["T000001"].level == "H1" + assert theIndex._items[cHandle]["T000001"].title == "Title One" + assert theIndex._items[cHandle]["T000001"].charCount == 23 + assert theIndex._items[cHandle]["T000001"].wordCount == 4 + assert theIndex._items[cHandle]["T000001"].paraCount == 1 + assert theIndex._items[cHandle]["T000001"].synopsis == "Synopsis One." # Valid and Invalid References assert theIndex.scanText(sHandle, ( @@ -399,9 +387,9 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "% synopsis: Synopsis One.\n\n" "Paragraph One.\n\n" )) - assert theIndex._refIndex[sHandle]["T000001"] == ( - [[3, "@pov", "One"], [5, "@char", "Two"]] - ) + assert theIndex._items[sHandle]["T000001"].references == { + "One": {"@pov"}, "Two": {"@char"} + } # Special Titles # ============== @@ -410,29 +398,25 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "#! My Project\n\n" ">> By Jane Doe <<\n\n" )) - assert tHandle not in theIndex._refIndex - - assert theIndex._fileIndex[tHandle]["T000001"]["level"] == "H1" - assert theIndex._fileIndex[tHandle]["T000001"]["title"] == "My Project" - assert theIndex._fileIndex[tHandle]["T000001"]["layout"] == "DOCUMENT" - assert theIndex._fileIndex[tHandle]["T000001"]["cCount"] == 21 - assert theIndex._fileIndex[tHandle]["T000001"]["wCount"] == 5 - assert theIndex._fileIndex[tHandle]["T000001"]["pCount"] == 1 - assert theIndex._fileIndex[tHandle]["T000001"]["synopsis"] == "" + assert theIndex._items[cHandle]["T000001"].references == {} + assert theIndex._items[tHandle]["T000001"].level == "H1" + assert theIndex._items[tHandle]["T000001"].title == "My Project" + assert theIndex._items[tHandle]["T000001"].charCount == 21 + assert theIndex._items[tHandle]["T000001"].wordCount == 5 + assert theIndex._items[tHandle]["T000001"].paraCount == 1 + assert theIndex._items[tHandle]["T000001"].synopsis == "" assert theIndex.scanText(tHandle, ( "##! Prologue\n\n" "In the beginning there was time ...\n\n" )) - assert tHandle not in theIndex._refIndex - - assert theIndex._fileIndex[tHandle]["T000001"]["level"] == "H2" - assert theIndex._fileIndex[tHandle]["T000001"]["title"] == "Prologue" - assert theIndex._fileIndex[tHandle]["T000001"]["layout"] == "DOCUMENT" - assert theIndex._fileIndex[tHandle]["T000001"]["cCount"] == 43 - assert theIndex._fileIndex[tHandle]["T000001"]["wCount"] == 8 - assert theIndex._fileIndex[tHandle]["T000001"]["pCount"] == 1 - assert theIndex._fileIndex[tHandle]["T000001"]["synopsis"] == "" + assert theIndex._items[cHandle]["T000001"].references == {} + assert theIndex._items[tHandle]["T000001"].level == "H2" + assert theIndex._items[tHandle]["T000001"].title == "Prologue" + assert theIndex._items[tHandle]["T000001"].charCount == 43 + assert theIndex._items[tHandle]["T000001"].wordCount == 8 + assert theIndex._items[tHandle]["T000001"].paraCount == 1 + assert theIndex._items[tHandle]["T000001"].synopsis == "" # Page wo/Title # ============= @@ -441,27 +425,25 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): assert theIndex.scanText(pHandle, ( "This is a page with some text on it.\n\n" )) - assert pHandle in theIndex._fileIndex - assert theIndex._fileIndex[pHandle]["T000000"]["level"] == "H0" - assert theIndex._fileIndex[pHandle]["T000000"]["title"] == "" - assert theIndex._fileIndex[pHandle]["T000000"]["layout"] == "DOCUMENT" - assert theIndex._fileIndex[pHandle]["T000000"]["cCount"] == 36 - assert theIndex._fileIndex[pHandle]["T000000"]["wCount"] == 9 - assert theIndex._fileIndex[pHandle]["T000000"]["pCount"] == 1 - assert theIndex._fileIndex[pHandle]["T000000"]["synopsis"] == "" + assert theIndex._items[pHandle]["T000000"].references == {} + assert theIndex._items[pHandle]["T000000"].level == "H0" + assert theIndex._items[pHandle]["T000000"].title == "" + assert theIndex._items[pHandle]["T000000"].charCount == 36 + assert theIndex._items[pHandle]["T000000"].wordCount == 9 + assert theIndex._items[pHandle]["T000000"].paraCount == 1 + assert theIndex._items[pHandle]["T000000"].synopsis == "" theProject.tree[pHandle]._layout = nwItemLayout.NOTE assert theIndex.scanText(pHandle, ( "This is a page with some text on it.\n\n" )) - assert pHandle in theIndex._fileIndex - assert theIndex._fileIndex[pHandle]["T000000"]["level"] == "H0" - assert theIndex._fileIndex[pHandle]["T000000"]["title"] == "" - assert theIndex._fileIndex[pHandle]["T000000"]["layout"] == "NOTE" - assert theIndex._fileIndex[pHandle]["T000000"]["cCount"] == 36 - assert theIndex._fileIndex[pHandle]["T000000"]["wCount"] == 9 - assert theIndex._fileIndex[pHandle]["T000000"]["pCount"] == 1 - assert theIndex._fileIndex[pHandle]["T000000"]["synopsis"] == "" + assert theIndex._items[pHandle]["T000000"].references == {} + assert theIndex._items[pHandle]["T000000"].level == "H0" + assert theIndex._items[pHandle]["T000000"].title == "" + assert theIndex._items[pHandle]["T000000"].charCount == 36 + assert theIndex._items[pHandle]["T000000"].wordCount == 9 + assert theIndex._items[pHandle]["T000000"].paraCount == 1 + assert theIndex._items[pHandle]["T000000"].synopsis == "" assert theProject.closeProject() is True @@ -700,538 +682,6 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # END Test testCoreIndex_ExtractData -# @pytest.mark.core -# def testCoreIndex_CheckTagIndex(mockGUI): -# """Test the tag index checker. -# """ -# theProject = NWProject(mockGUI) -# theIndex = NWIndex(theProject) - -# # Valid Index -# theIndex._tagIndex = { -# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], -# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], -# } -# assert theIndex._checkTagIndex() is None - -# # Wrong Key Type -# theIndex._tagIndex = { -# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], -# 123456: [3, "bb2c23b3c42cc", "CHARACTER", "T000001"], -# } -# with pytest.raises(KeyError): -# theIndex._checkTagIndex() - -# # Wrong Length -# theIndex._tagIndex = { -# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], -# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "T000001", "Stuff"], -# } -# with pytest.raises(IndexError): -# theIndex._checkTagIndex() - -# # Wrong Type of Entry 0 -# theIndex._tagIndex = { -# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], -# "Jane": ["3", "bb2c23b3c42cc", "CHARACTER", "T000001"], -# } -# with pytest.raises(ValueError): -# theIndex._checkTagIndex() - -# # Wrong Type of Entry 1 -# theIndex._tagIndex = { -# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], -# "Jane": [3, 0xbb2c23b3c42cc, "CHARACTER", "T000001"], -# } -# with pytest.raises(ValueError): -# theIndex._checkTagIndex() - -# # Wrong Type of Entry 2 -# theIndex._tagIndex = { -# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], -# "Jane": [3, "bb2c23b3c42cc", "INVALID_CLASS", "T000001"], -# } -# with pytest.raises(ValueError): -# theIndex._checkTagIndex() - -# # Wrong Type of Entry 3 -# theIndex._tagIndex = { -# "John": [3, "14298de4d9524", "CHARACTER", "T000001"], -# "Jane": [3, "bb2c23b3c42cc", "CHARACTER", "INVALID"], -# } -# with pytest.raises(ValueError): -# theIndex._checkTagIndex() - -# # END Test testCoreIndex_CheckTagIndex - - -@pytest.mark.core -def testCoreIndex_CheckRefIndex(mockGUI): - """Test the reference index checker. - """ - theProject = NWProject(mockGUI) - theIndex = NWIndex(theProject) - - # Valid Index - theIndex._refIndex = { - "6a2d6d5f4f401": { - "T000000": [], - "T000001": [[3, "@pov", "Jane"], [4, "@location", "Earth"]], - } - } - assert theIndex._checkRefIndex() is None - - # Invalid Handle - theIndex._refIndex = { - "Ha2d6d5f4f401": { - "T000000": [], - "T000001": [[3, "@pov", "Jane"], [4, "@location", "Earth"]], - } - } - with pytest.raises(KeyError): - theIndex._checkRefIndex() - - # Invalid Title - theIndex._refIndex = { - "6a2d6d5f4f401": { - "T000000": [], - "INVALID": [[3, "@pov", "Jane"], [4, "@location", "Earth"]], - } - } - with pytest.raises(KeyError): - theIndex._checkRefIndex() - - # Wrong Length - theIndex._refIndex = { - "6a2d6d5f4f401": { - "T000000": [], - "T000001": [[3, "@pov", "Jane"], [4, "@location", "Earth", "Stuff"]], - } - } - with pytest.raises(IndexError): - theIndex._checkRefIndex() - - # Wrong Type of Entry 0 - theIndex._refIndex = { - "6a2d6d5f4f401": { - "T000000": [], - "T000001": [[3, "@pov", "Jane"], ["4", "@location", "Earth"]], - } - } - with pytest.raises(ValueError): - theIndex._checkRefIndex() - - # Wrong Type of Entry 1 - theIndex._refIndex = { - "6a2d6d5f4f401": { - "T000000": [], - "T000001": [[3, "@pov", "Jane"], [4, "@stuff", "Earth"]], - } - } - with pytest.raises(ValueError): - theIndex._checkRefIndex() - - # Wrong Type of Entry 2 - theIndex._refIndex = { - "6a2d6d5f4f401": { - "T000000": [], - "T000001": [[3, "@pov", "Jane"], [4, "@location", 123456]], - } - } - with pytest.raises(ValueError): - theIndex._checkRefIndex() - -# END Test testCoreIndex_CheckRefIndex - - -@pytest.mark.core -def testCoreIndex_CheckFileIndex(mockGUI): - """Test the file index checker. - """ - theProject = NWProject(mockGUI) - theIndex = NWIndex(theProject) - - # Valid Index - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - theIndex._fileIndex = theIndex._fileIndex.copy() - assert theIndex._checkFileIndex() is None - - # Invalid Handle - theIndex._fileIndex = { - "H3b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Invalid Title - theIndex._fileIndex = { - "53b69b83cdafc": { - "INVALID": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Wrong Length - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - "stuff": None - } - } - } - with pytest.raises(IndexError): - theIndex._checkFileIndex() - - # Missing Keys - # ============ - - # Missing 'level' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "stuff": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Missing 'title' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "stuff": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Missing 'layout' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "stuff": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Missing 'cCount' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "stuff": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Missing 'wCount' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "stuff": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Missing 'pCount' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "stuff": 2, - "synopsis": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Missing 'synopsis' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "stuff": "text", - } - } - } - with pytest.raises(KeyError): - theIndex._checkFileIndex() - - # Wrong Types - # =========== - - # Wrong Type for 'level' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "XX", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(ValueError): - theIndex._checkFileIndex() - - # Wrong Type for 'title' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": 12345678, - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(ValueError): - theIndex._checkFileIndex() - - # Wrong Type for 'layout' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "INVALID", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(ValueError): - theIndex._checkFileIndex() - - # Wrong Type for 'cCount' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": "72", - "wCount": 15, - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(ValueError): - theIndex._checkFileIndex() - - # Wrong Type for 'wCount' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": "15", - "pCount": 2, - "synopsis": "text", - } - } - } - with pytest.raises(ValueError): - theIndex._checkFileIndex() - - # Wrong Type for 'pCount' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": "2", - "synopsis": "text", - } - } - } - with pytest.raises(ValueError): - theIndex._checkFileIndex() - - # Wrong Type for 'synopsis' - theIndex._fileIndex = { - "53b69b83cdafc": { - "T000001": { - "level": "H1", - "title": "My Novel", - "layout": "DOCUMENT", - "cCount": 72, - "wCount": 15, - "pCount": 2, - "synopsis": 123456, - } - } - } - with pytest.raises(ValueError): - theIndex._checkFileIndex() - -# END Test testCoreIndex_CheckFileIndex - - -@pytest.mark.core -def testCoreIndex_CheckFileMeta(mockGUI): - """Test the file meta checker. - """ - theProject = NWProject(mockGUI) - theIndex = NWIndex(theProject) - - # Valid Index - theIndex._fileMeta = { - "53b69b83cdafc": ["H0", 72, 15, 2], - "974e400180a99": ["H0", 210, 40, 2], - } - assert theIndex._checkFileMeta() is None - - # Invalid Handle - theIndex._fileMeta = { - "53b69b83cdafc": ["H0", 72, 15, 2], - "h74e400180a99": ["H0", 210, 40, 2], - } - with pytest.raises(KeyError): - theIndex._checkFileMeta() - - # Wrong Length - theIndex._fileMeta = { - "53b69b83cdafc": ["H0", 72, 15, 2], - "974e400180a99": ["H0", 210, 40, 2, 8], - } - with pytest.raises(IndexError): - theIndex._checkFileMeta() - - # Content of Entry 0 - theIndex._fileMeta = { - "53b69b83cdafc": ["H0", 72, 15, 2], - "974e400180a99": ["XXX", 210, 40, 2], - } - with pytest.raises(ValueError): - theIndex._checkFileMeta() - - # Type of Entry 1 - theIndex._fileMeta = { - "53b69b83cdafc": ["H0", 72, 15, 2], - "974e400180a99": ["H0", "210", 40, 2], - } - with pytest.raises(ValueError): - theIndex._checkFileMeta() - - # Type of Entry 2 - theIndex._fileMeta = { - "53b69b83cdafc": ["H0", 72, 15, 2], - "974e400180a99": ["H0", 210, "40", 2], - } - with pytest.raises(ValueError): - theIndex._checkFileMeta() - - # Type of Entry 3 - theIndex._fileMeta = { - "53b69b83cdafc": ["H0", 72, 15, 2], - "974e400180a99": ["H0", 210, 40, "2"], - } - with pytest.raises(ValueError): - theIndex._checkFileMeta() - -# END Test testCoreIndex_CheckFileMeta - - @pytest.mark.core def testCoreIndex_CountWords(): """Test the word counter and the exclusion filers. diff --git a/tests/test_gui/test_gui_docviewer.py b/tests/test_gui/test_gui_docviewer.py index d3f245d1..0ed15742 100644 --- a/tests/test_gui/test_gui_docviewer.py +++ b/tests/test_gui/test_gui_docviewer.py @@ -47,8 +47,8 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum): # Rebuild the index nwGUI.mainMenu.aRebuildIndex.activate(QAction.Trigger) - assert nwGUI.theProject.index._tagIndex != {} - assert nwGUI.theProject.index._refIndex != {} + assert nwGUI.theProject.index._tags != {} + assert nwGUI.theProject.index._items != {} # Select a document in the project tree nwGUI.treeView.setSelectedHandle("88243afbe5ed8") From 347d34bf8342a2ffb57f2cb063cfab68bd4acdb3 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 29 May 2022 00:18:48 +0200 Subject: [PATCH 07/13] Clean up the index code a bit and add index validation --- novelwriter/core/index.py | 154 +++++++++++------- novelwriter/core/project.py | 6 +- .../coreIndex_LoadSave_tagsIndex.json | 4 +- 3 files changed, 97 insertions(+), 67 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index 95485bf9..d350f9fd 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -34,7 +34,9 @@ from novelwriter.enum import nwItemType, nwItemLayout from novelwriter.error import logException from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode from novelwriter.core.document import NWDoc -from novelwriter.common import checkInt, jsonEncode +from novelwriter.common import ( + checkInt, isHandle, isItemClass, isTitleTag, jsonEncode +) logger = logging.getLogger(__name__) @@ -74,13 +76,11 @@ class NWIndex(): def clearIndex(self): """Clear the index dictionaries and time stamps. """ + self._tags = {} + self._items = {} self._timeNovel = 0 self._timeNotes = 0 self._timeIndex = 0 - - self._tags = {} - self._items = {} - return def deleteHandle(self, tHandle): @@ -90,10 +90,8 @@ class NWIndex(): return logger.debug("Removing item '%s' from the index", tHandle) - for tTag in self._items[tHandle].allTags(): self._tags.pop(tTag, None) - self._items.pop(tHandle, None) return @@ -144,30 +142,36 @@ class NWIndex(): try: with open(indexFile, mode="r", encoding="utf-8") as inFile: theData = json.load(inFile) - except Exception: logger.error("Failed to load index file") logException() self._indexBroken = True return False - self._tags = theData.get("tagsIndex", {}) - for tHandle, tData in theData.get("itemIndex", {}).items(): - nwItem = self.theProject.tree[tHandle] - if nwItem is not None: - tItem = IndexItem(tHandle, nwItem) - tItem.unpackData(tData) - self._items[tHandle] = tItem + try: + self._validateTagsIndex(theData["tagsIndex"]) + self._validateItemIndex(theData["itemIndex"]) + except Exception: + logger.error("The index content is invalid") + logException() + self._indexBroken = True + return False - nowTime = round(time()) - self._timeNovel = nowTime - self._timeNotes = nowTime - self._timeIndex = nowTime + logger.debug("Checking index") + + # Check that all files are indexed + for fHandle in self.theProject.projFiles: + if fHandle not in self._items: + logger.warning("Item '%s' is not in the index", fHandle) + self.reIndexHandle(fHandle) + + nowTime = round(time()) + self._timeNovel = nowTime + self._timeNotes = nowTime + self._timeIndex = nowTime logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000) - self._checkIndex() - return True def saveIndex(self): @@ -214,11 +218,11 @@ class NWIndex(): logger.info("Not indexing non-file item '%s'", tHandle) return False + # Delete the old entry and create a new self.deleteHandle(tHandle) - - # Run word counter for the whole text self._items[tHandle] = IndexItem(tHandle, theItem) + # Run word counter for the whole text cC, wC, pC = countWords(theText) theItem.setCharCount(cC) theItem.setWordCount(wC) @@ -249,7 +253,7 @@ class NWIndex(): continue if aLine.startswith("#"): - isTitle = self._indexTitle(tHandle, aLine, nLine, itemLayout) + isTitle = self._indexTitle(tHandle, aLine, nLine) if isTitle and nLine > 0: if nTitle > 0: lastText = "\n".join(theLines[nTitle-1:nLine-1]) @@ -257,7 +261,7 @@ class NWIndex(): nTitle = nLine elif aLine.startswith("@"): - self._indexKeyword(tHandle, aLine, nLine, nTitle, itemClass) + self._indexKeyword(tHandle, aLine, nTitle, itemClass) elif aLine.startswith("%"): if nTitle > 0: @@ -274,7 +278,7 @@ class NWIndex(): lastText = "\n".join(theLines[nTitle-1:]) self._indexWordCounts(tHandle, lastText, nTitle) - # Index page with no titles and references + # Also count words on a page with no titles if nTitle == 0: self._indexWordCounts(tHandle, theText, nTitle) @@ -292,7 +296,7 @@ class NWIndex(): # Internal Indexers ## - def _indexTitle(self, tHandle, aLine, nLine, itemLayout): + def _indexTitle(self, tHandle, aLine, nLine): """Save information about the title and its location in the file to the index. """ @@ -341,7 +345,7 @@ class NWIndex(): self._items[tHandle].setHeadingSynopsis(sTitle, theText) return - def _indexKeyword(self, tHandle, aLine, nLine, nTitle, itemClass): + def _indexKeyword(self, tHandle, aLine, nTitle, itemClass): """Validate and save the information about a reference to a tag in another file. """ @@ -361,11 +365,9 @@ class NWIndex(): "heading": sTitle, "class": itemClass.name, } - if tHandle in self._items: - self._items[tHandle].setHeadingTag(sTitle, theBits[1]) + self._items[tHandle].setHeadingTag(sTitle, theBits[1]) else: - if tHandle in self._items: - self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0]) + self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0]) return @@ -627,26 +629,51 @@ class NWIndex(): return theHandles - ## - # Index Checkers - ## - - def _checkIndex(self): - """Check that the entries in the index are valid and contain the - elements it should. Also check that each file present in the - contents folder when the project was loaded are also present in - the fileMeta index. + def _validateTagsIndex(self, tagsIndex): + """Iterate through the tagsIndex loaded from cache and check + that it's valid. """ - logger.debug("Checking index") - tStart = time() + self._tags = {} + if not isinstance(tagsIndex, dict): + raise ValueError("tagsIndex is not a dict") - # If the index was ok, we check that project files are indexed - for fHandle in self.theProject.projFiles: - if fHandle not in self._items: - logger.warning("Item '%s' is not in the index", fHandle) - self.reIndexHandle(fHandle) + for tagKey, tagData in tagsIndex.items(): + if not isinstance(tagKey, str): + raise ValueError("tagsIndex keys must be a strings") + if "handle" not in tagData: + raise KeyError("A tagIndex item is missing a handle entry") + if "heading" not in tagData: + raise KeyError("A tagIndex item is missing a heading entry") + if "class" not in tagData: + raise KeyError("A tagIndex item is missing a class entry") + if not isHandle(tagData["handle"]): + raise ValueError("tagsIndex handle must be a handle") + if not isTitleTag(tagData["heading"]): + raise ValueError("tagsIndex heading must be a title tag") + if not isItemClass(tagData["class"]): + raise ValueError("tagsIndex handle must be an nwItemClass") - logger.verbose("Index check completed in %.3f ms", (time() - tStart)*1000) + self._tags = tagsIndex + + return + + def _validateItemIndex(self, itemIndex): + """Iterate through the itemIndex loaded from cache and check + that it's valid. + """ + self._items = {} + if not isinstance(itemIndex, dict): + raise ValueError("itemIndex is not a dict") + + for tHandle, tData in itemIndex.items(): + if not isHandle(tHandle): + raise ValueError("itemIndex keys must be handles") + + nwItem = self.theProject.tree[tHandle] + if nwItem is not None: + tItem = IndexItem(tHandle, nwItem) + tItem.unpackData(tData) + self._items[tHandle] = tItem return @@ -734,6 +761,10 @@ def countWords(theText): return charCount, wordCount, paraCount +# =============================================================================================== # +# Indexer Objects +# =============================================================================================== # + class IndexItem: def __init__(self, tHandle, tItem): @@ -773,20 +804,11 @@ class IndexItem: # Setters ## - def setLevel(self, level): - if level in H_VALID: - self._level = level - else: - self._level = "H0" - return - def updateLevel(self, level): """Set the level only if it is H0. """ - if level in H_VALID and self._level == "H0": + if self._level == "H0": self._level = level - else: - self._level = "H0" return def addHeading(self, tHeading): @@ -867,6 +889,8 @@ class IndexItem: self._level = data.get("level", "H0") references = data.get("references", {}) for sTitle, hData in data.get("headings", {}).items(): + if not isTitleTag(sTitle): + raise ValueError("The itemIndex contains an invalid title key") tHeading = IndexHeading(sTitle) tHeading.unpackData(hData) tHeading.unpackReferences(references.get(sTitle, {})) @@ -940,8 +964,6 @@ class IndexHeading: def setLevel(self, level): if level in H_VALID: self._level = level - else: - self._level = "H0" return def setCounts(self, charCount, wordCount, paraCount): @@ -1007,7 +1029,15 @@ class IndexHeading: """Unpack a set of references from a dictionary. """ for tagKey, refTypes in data.items(): - self._refs[tagKey] = set(refTypes) + if not isinstance(tagKey, str): + raise ValueError("itemIndex reference key must be a string") + if not isinstance(refTypes, list): + raise ValueError("itemIndex reference types must be a list") + for refType in refTypes: + if refType in nwKeyWords.VALID_KEYS: + self.addReference(tagKey, refType) + else: + raise ValueError("The itemIndex contains an invalid reference type") return # END Class IndexHeading diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py index 0a13cb16..460fb91f 100644 --- a/novelwriter/core/project.py +++ b/novelwriter/core/project.py @@ -123,15 +123,15 @@ class NWProject(): ## @property - def index(self) -> NWIndex: + def index(self): return self._projIndex @property - def tree(self) -> NWTree: + def tree(self): return self._projTree @property - def options(self) -> OptionState: + def options(self): return self._optState ## diff --git a/tests/reference/coreIndex_LoadSave_tagsIndex.json b/tests/reference/coreIndex_LoadSave_tagsIndex.json index 44adc7ad..fafdeb68 100644 --- a/tests/reference/coreIndex_LoadSave_tagsIndex.json +++ b/tests/reference/coreIndex_LoadSave_tagsIndex.json @@ -39,7 +39,7 @@ } }, "88243afbe5ed8": { - "level": "H0", + "level": "H3", "headings": { "T000001": {"level": "H3", "title": "Scene One", "tag": "", "cCount": 1197, "wCount": 174, "pCount": 2, "synopsis": "Aenean ut placerat velit. Etiam laoreet ullamcorper risus, eget lobortis enim scelerisque non. Suspendisse id maximus nunc, et mollis sapien. Curabitur vel semper sapien, non pulvinar dolor. Etiam finibus nisi vel mi molestie consectetur."}, "T000013": {"level": "H4", "title": "Scene One, Section Two", "tag": "", "cCount": 1561, "wCount": 230, "pCount": 2, "synopsis": ""} @@ -49,7 +49,7 @@ } }, "f96ec11c6a3da": { - "level": "H0", + "level": "H3", "headings": { "T000001": {"level": "H3", "title": "Scene Two", "tag": "", "cCount": 2034, "wCount": 299, "pCount": 3, "synopsis": "Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Integer sapien nulla, dictum at lacus a, dignissim consectetur dolor. Nunc vel eleifend lacus, eu dapibus orci."}, "T000015": {"level": "H4", "title": "Scene Two, Section Two", "tag": "", "cCount": 2009, "wCount": 301, "pCount": 3, "synopsis": ""} From 77e50a6cee6216de4d528db5e777dc0cf5b74f4a Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 29 May 2022 18:58:32 +0200 Subject: [PATCH 08/13] Move the item index into a wrapper class and combine the access functions --- novelwriter/core/index.py | 562 +++++++++++++++++---------- novelwriter/gui/noveltree.py | 4 +- novelwriter/gui/outline.py | 2 +- tests/test_core/test_core_index.py | 167 ++++---- tests/test_gui/test_gui_docviewer.py | 2 +- 5 files changed, 440 insertions(+), 297 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index d350f9fd..dcd38a1d 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -4,8 +4,9 @@ novelWriter – Project Index Data class for the project index of tags, headers and references File History: -Created: 2019-04-22 [0.0.1] countWords -Created: 2019-05-27 [0.1.4] NWIndex +Created: 2019-04-22 [0.0.1] countWords +Created: 2019-05-27 [0.1.4] NWIndex +Created: 2022-05-28 [1.7rc1] IndexItem, IndexHeading This file is a part of novelWriter Copyright 2018–2022, Veronica Berglyd Olsen @@ -56,7 +57,7 @@ class NWIndex(): # Indices self._tags = {} - self._items = {} + self._itemIndex = ItemIndex(theProject) # TimeStamps self._timeNovel = 0 @@ -65,6 +66,10 @@ class NWIndex(): return + ## + # Properties + ## + @property def indexBroken(self): return self._indexBroken @@ -77,7 +82,7 @@ class NWIndex(): """Clear the index dictionaries and time stamps. """ self._tags = {} - self._items = {} + self._itemIndex.clear() self._timeNovel = 0 self._timeNotes = 0 self._timeIndex = 0 @@ -86,13 +91,11 @@ class NWIndex(): def deleteHandle(self, tHandle): """Delete all entries of a given document handle. """ - if tHandle not in self._items: - return - logger.debug("Removing item '%s' from the index", tHandle) - for tTag in self._items[tHandle].allTags(): + for tTag in self._itemIndex.allItemTags(tHandle): self._tags.pop(tTag, None) - self._items.pop(tHandle, None) + + del self._itemIndex[tHandle] return @@ -150,7 +153,7 @@ class NWIndex(): try: self._validateTagsIndex(theData["tagsIndex"]) - self._validateItemIndex(theData["itemIndex"]) + self._itemIndex.unpackData(theData["itemIndex"]) except Exception: logger.error("The index content is invalid") logException() @@ -161,7 +164,7 @@ class NWIndex(): # Check that all files are indexed for fHandle in self.theProject.projFiles: - if fHandle not in self._items: + if fHandle not in self._itemIndex: logger.warning("Item '%s' is not in the index", fHandle) self.reIndexHandle(fHandle) @@ -183,11 +186,11 @@ class NWIndex(): tStart = time() try: - itemsIndex = {handle: item.packData() for handle, item in self._items.items()} + itemIndex = self._itemIndex.packData() with open(indexFile, mode="w+", encoding="utf-8") as outFile: outFile.write("{\n") outFile.write(f' "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n') - outFile.write(f' "itemIndex": {jsonEncode(itemsIndex, n=1, nmax=4)}\n') + outFile.write(f' "itemIndex": {jsonEncode(itemIndex, n=1, nmax=4)}\n') outFile.write("}\n") except Exception: @@ -220,7 +223,7 @@ class NWIndex(): # Delete the old entry and create a new self.deleteHandle(tHandle) - self._items[tHandle] = IndexItem(tHandle, theItem) + self._itemIndex.add(tHandle, theItem) # Run word counter for the whole text cC, wC, pC = countWords(theText) @@ -240,9 +243,6 @@ class NWIndex(): logger.debug("Not indexing inactive item '%s'", tHandle) return False - itemClass = theItem.itemClass - itemLayout = theItem.itemLayout - logger.debug("Indexing item with handle '%s'", tHandle) # Scan the text content @@ -261,7 +261,7 @@ class NWIndex(): nTitle = nLine elif aLine.startswith("@"): - self._indexKeyword(tHandle, aLine, nTitle, itemClass) + self._indexKeyword(tHandle, aLine, nTitle, theItem.itemClass) elif aLine.startswith("%"): if nTitle > 0: @@ -285,7 +285,7 @@ class NWIndex(): # Update timestamps for index changes nowTime = round(time()) self._timeIndex = nowTime - if itemLayout == nwItemLayout.NOTE: + if theItem.itemLayout == nwItemLayout.NOTE: self._timeNotes = nowTime else: self._timeNovel = nowTime @@ -296,7 +296,7 @@ class NWIndex(): # Internal Indexers ## - def _indexTitle(self, tHandle, aLine, nLine): + def _indexTitle(self, tHandle, aLine, nTitle): """Save information about the title and its location in the file to the index. """ @@ -321,28 +321,24 @@ class NWIndex(): else: return False - sTitle = f"T{nLine:06d}" - tItem = self._items[tHandle] - tItem.updateLevel(hDepth) - tItem.addHeading(IndexHeading(sTitle, hDepth, hText)) + sTitle = f"T{nTitle:06d}" + self._itemIndex.addItemHeading(tHandle, sTitle, hDepth, hText) return True def _indexWordCounts(self, tHandle, theText, nTitle): """Count text stats and save the counts to the index. """ - cC, wC, pC = countWords(theText) sTitle = f"T{nTitle:06d}" - if tHandle in self._items: - self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC) + cC, wC, pC = countWords(theText) + self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC) return def _indexSynopsis(self, tHandle, theText, nTitle): """Save the synopsis to the index. """ sTitle = f"T{nTitle:06d}" - if tHandle in self._items: - self._items[tHandle].setHeadingSynopsis(sTitle, theText) + self._itemIndex.setHeadingSynopsis(tHandle, sTitle, theText) return def _indexKeyword(self, tHandle, aLine, nTitle, itemClass): @@ -365,9 +361,9 @@ class NWIndex(): "heading": sTitle, "class": itemClass.name, } - self._items[tHandle].setHeadingTag(sTitle, theBits[1]) + self._itemIndex.setHeadingTag(tHandle, sTitle, theBits[1]) else: - self._items[tHandle].addHeadingReferences(sTitle, theBits[1:], theBits[0]) + self._itemIndex.addHeadingReferences(tHandle, sTitle, theBits[1:], theBits[0]) return @@ -447,35 +443,31 @@ class NWIndex(): # Extract Data ## - def novelStructure(self, skipExcluded=True): + def novelStructure(self, skipExcl=True): """Iterate over all titles in the novel, in the correct order as they appear in the tree view and in the respective document files, but skipping all note files. """ - for tHandle in self._listNovelHandles(skipExcluded): - for sTitle in self._items[tHandle].headings: - tKey = f"{tHandle}:{sTitle}" - yield tKey, tHandle, sTitle, self._items[tHandle][sTitle] + for tHandle, sTitle, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl): + tKey = f"{tHandle}:{sTitle}" + yield tKey, tHandle, sTitle, hItem + return - def getNovelWordCount(self, skipExcluded=True): + def getNovelWordCount(self, skipExcl=True): """Count the number of words in the novel project. """ wCount = 0 - for tHandle in self._listNovelHandles(skipExcluded): - for hItem in self._items[tHandle].entries: - wCount += hItem.wordCount - + for _, _, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl): + wCount += hItem.wordCount return wCount - def getNovelTitleCounts(self, skipExcluded=True): + def getNovelTitleCounts(self, skipExcl=True): """Count the number of titles in the novel project. """ hCount = [0, 0, 0, 0, 0] - for tHandle in self._listNovelHandles(skipExcluded): - for hItem in self._items[tHandle].entries: - iLevel = H_LEVEL.get(hItem.level, 0) - hCount[iLevel] += 1 - + for _, _, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl): + iLevel = H_LEVEL.get(hItem.level, 0) + hCount[iLevel] += 1 return hCount def getHandleWordCounts(self, tHandle): @@ -483,7 +475,7 @@ class NWIndex(): """ return [ (f"{tHandle}:{sTitle}", hItem.wordCount) - for sTitle, hItem in self._items.get(tHandle, {}).items() + for sTitle, hItem in self._itemIndex.iterItemHeaders(tHandle) ] def getHandleHeaders(self, tHandle): @@ -491,39 +483,34 @@ class NWIndex(): """ return [ (sTitle, hItem.level, hItem.title) - for sTitle, hItem in self._items.get(tHandle, {}).items() + for sTitle, hItem in self._itemIndex.iterItemHeaders(tHandle) ] def getHandleHeaderLevel(self, tHandle): """Get the header level of the first header of a handle. """ - if tHandle in self._items: - return self._items[tHandle].level - else: - return "H0" + return self._itemIndex.mainItemHeader(tHandle) - def getTableOfContents(self, maxDepth, skipExcluded=True): + def getTableOfContents(self, maxDepth, skipExcl=True): """Generate a table of contents up to a maximum depth. """ tOrder = [] tData = {} pKey = None - for tHandle in self._listNovelHandles(skipExcluded): - for sTitle in self._items[tHandle].headings: - tKey = f"{tHandle}:{sTitle}" - hItem = self._items[tHandle][sTitle] - iLevel = H_LEVEL.get(hItem.level, 0) - if iLevel > maxDepth: - if pKey in tData: - tData[pKey]["words"] += hItem.wordCount - else: - pKey = tKey - tOrder.append(tKey) - tData[tKey] = { - "level": iLevel, - "title": hItem.title, - "words": hItem.wordCount, - } + for tHandle, sTitle, hItem in self._itemIndex.iterNovelStructure(skipExcl=skipExcl): + tKey = f"{tHandle}:{sTitle}" + iLevel = H_LEVEL.get(hItem.level, 0) + if iLevel > maxDepth: + if pKey in tData: + tData[pKey]["words"] += hItem.wordCount + else: + pKey = tKey + tOrder.append(tKey) + tData[tKey] = { + "level": iLevel, + "title": hItem.title, + "words": hItem.wordCount, + } theToC = [( tKey, @@ -538,35 +525,26 @@ class NWIndex(): """Return the counts for a file, or a section of a file, starting at title sTitle if it is provided. """ - cC = 0 - wC = 0 - pC = 0 + tItem = self._itemIndex[tHandle] + if tItem is None: + return 0, 0, 0 if sTitle is None: - if tHandle in self._items: - tItem = self._items[tHandle].item - cC = tItem.charCount - wC = tItem.wordCount - pC = tItem.paraCount + cItem = tItem.item else: - if tHandle in self._items: - if sTitle in self._items[tHandle]: - hItem = self._items[tHandle][sTitle] - cC = hItem.charCount - wC = hItem.wordCount - pC = hItem.paraCount + cItem = tItem[sTitle] - return cC, wC, pC + if cItem is not None: + return cItem.charCount, cItem.wordCount, cItem.paraCount + + return 0, 0, 0 def getReferences(self, tHandle, sTitle=None): """Extract all references made in a file, and optionally title section. """ theRefs = {x: [] for x in nwKeyWords.KEY_CLASS} - if tHandle not in self._items: - return theRefs - - for rTitle, hItem in self._items[tHandle].items(): + for rTitle, hItem in self._itemIndex.iterItemHeaders(tHandle): if sTitle is None or sTitle == rTitle: for aTag, refTypes in hItem.references.items(): for refType in refTypes: @@ -578,28 +556,26 @@ class NWIndex(): def getNovelData(self, tHandle, sTitle): """Return the novel data of a given handle and title. """ - if tHandle in self._items: - if sTitle in self._items[tHandle]: - return self._items[tHandle][sTitle] + if tHandle in self._itemIndex: + return self._itemIndex[tHandle][sTitle] return None def getBackReferenceList(self, tHandle): """Build a list of files referring back to our file, specified by tHandle. """ - if tHandle is None or tHandle not in self._items: + if tHandle is None or tHandle not in self._itemIndex: return {} theRefs = {} - theTags = self._items[tHandle].allTags() + theTags = self._itemIndex.allItemTags(tHandle) if not theTags: return theRefs - for aHandle, tItem in self._items.items(): - for sTitle, hItem in tItem.items(): - for aTag in hItem.references: - if aTag in theTags and aHandle not in theRefs: - theRefs[aHandle] = sTitle + for aHandle, sTitle, hItem in self._itemIndex.iterAllHeaders(): + for aTag in hItem.references: + if aTag in theTags and aHandle not in theRefs: + theRefs[aHandle] = sTitle return theRefs @@ -613,22 +589,6 @@ class NWIndex(): # Internal Functions ## - def _listNovelHandles(self, skipExcluded): - """Return a list of all handles that exist in the novel index. - """ - theHandles = [] - for tItem in self.theProject.tree: - if tItem is None: - continue - if not tItem.isExported and skipExcluded: - continue - if tItem.itemLayout == nwItemLayout.NOTE: - continue - if tItem.itemHandle in self._items: - theHandles.append(tItem.itemHandle) - - return theHandles - def _validateTagsIndex(self, tagsIndex): """Iterate through the tagsIndex loaded from cache and check that it's valid. @@ -657,15 +617,172 @@ class NWIndex(): return - def _validateItemIndex(self, itemIndex): - """Iterate through the itemIndex loaded from cache and check - that it's valid. +# END Class NWIndex + + +# =============================================================================================== # +# Indexer Objects +# =============================================================================================== # + +class ItemIndex: + """A wrapper object holding the indexed items. + """ + + def __init__(self, theProject): + self.theProject = theProject + self._items = {} + return + + ## + # Methods + ## + + def clear(self): + """Clear the index. """ self._items = {} - if not isinstance(itemIndex, dict): + return + + def __contains__(self, tHandle): + """Check if an item exists in the index, + """ + return tHandle in self._items + + def __delitem__(self, tHandle): + """Delete an entry in the index. + """ + self._items.pop(tHandle, None) + return + + def __getitem__(self, tHandle): + """Return an item, or return None if it isn't found. + """ + return self._items.get(tHandle, None) + + def add(self, tHandle, tItem): + """Add a new item to the index. This will overwrite the item if + it already exists. + """ + self._items[tHandle] = IndexItem(tHandle, tItem) + return + + def mainItemHeader(self, tHandle): + """Return the primary item header for an item. + """ + if tHandle in self._items: + return self._items[tHandle].level + return "H0" + + def allItemTags(self, tHandle): + """Get all tags set for headings of an item. + """ + if tHandle in self._items: + return self._items[tHandle].allTags() + return [] + + def iterItemHeaders(self, tHandle): + """Iterate over all item headers of an item. + """ + if tHandle in self._items: + for sTitle, hItem in self._items[tHandle].items(): + yield sTitle, hItem + return + + def iterAllHeaders(self): + """Iterate through all items and headings in the index. + """ + for tHandle, tItem in self._items.items(): + for sTitle, hItem in tItem.items(): + yield tHandle, sTitle, hItem + return + + def iterNovelStructure(self, rootHandle=None, skipExcl=False): + """Iterate over all items and headers in the novel structure for + a given root handle, or for all if root handle is None. + """ + for tItem in self.theProject.tree: + if tItem is None: + continue + if tItem.itemLayout == nwItemLayout.NOTE: + continue + if skipExcl and not tItem.isExported: + continue + + tHandle = tItem.itemHandle + if tHandle not in self._items: + continue + + if rootHandle is None: + for sTitle, hItem in self._items[tHandle].items(): + yield tHandle, sTitle, hItem + elif tItem.rootHandle == rootHandle: + for sTitle, hItem in self._items[tHandle].items(): + yield tHandle, sTitle, hItem + else: + continue + + return + + ## + # Setters + ## + + def addItemHeading(self, tHandle, sTitle, hDepth, hText): + """Set the main heading level of an item. + """ + if tHandle in self._items: + tItem = self._items[tHandle] + tItem.updateLevel(hDepth) + tItem.addHeading(IndexHeading(sTitle, hDepth, hText)) + return + + def setHeadingCounts(self, tHandle, sTitle, cC, wC, pC): + """Set the character, word and paragraph counts of a heading + on a given item. + """ + if tHandle in self._items: + self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC) + return + + def setHeadingSynopsis(self, tHandle, sTitle, sText): + """Set the synopsis text for a heading on a given item. + """ + if tHandle in self._items: + self._items[tHandle].setHeadingSynopsis(sTitle, sText) + return + + def setHeadingTag(self, tHandle, sTitle, tagKey): + """Set the main tag for a heading on a given item. + """ + if tHandle in self._items: + self._items[tHandle].setHeadingTag(sTitle, tagKey) + return + + def addHeadingReferences(self, tHandle, sTitle, tagKeys, refType): + """Set the reference tags for a heading on a given item. + """ + if tHandle in self._items: + self._items[tHandle].addHeadingReferences(sTitle, tagKeys, refType) + return + + ## + # Pack/Unpack + ## + + def packData(self): + """Pack all the data of the index into a single dictionary. + """ + return {handle: item.packData() for handle, item in self._items.items()} + + def unpackData(self, data): + """Iterate through the itemIndex loaded from cache and check + that it's valid. This will raise errors if there is a problem. + """ + self._items = {} + if not isinstance(data, dict): raise ValueError("itemIndex is not a dict") - for tHandle, tData in itemIndex.items(): + for tHandle, tData in data.items(): if not isHandle(tHandle): raise ValueError("itemIndex keys must be handles") @@ -677,100 +794,14 @@ class NWIndex(): return -# END Class NWIndex +# END Class ItemIndex -# =============================================================================================== # -# Simple Word Counter -# =============================================================================================== # - -def countWords(theText): - """Count words in a piece of text, skipping special syntax and - comments. - """ - charCount = 0 - wordCount = 0 - paraCount = 0 - prevEmpty = True - - if not isinstance(theText, str): - return charCount, wordCount, paraCount - - # We need to treat dashes as word separators for counting words. - # The check+replace approach is much faster than direct replace for - # large texts, and a bit slower for small texts, but in the latter - # case it doesn't really matter. - if nwUnicode.U_ENDASH in theText: - theText = theText.replace(nwUnicode.U_ENDASH, " ") - if nwUnicode.U_EMDASH in theText: - theText = theText.replace(nwUnicode.U_EMDASH, " ") - - for aLine in theText.splitlines(): - - countPara = True - - if not aLine: - prevEmpty = True - continue - if aLine[0] == "@" or aLine[0] == "%": - continue - - if aLine[0] == "[": - if aLine.startswith(("[NEWPAGE]", "[NEW PAGE]", "[VSPACE]")): - continue - elif aLine.startswith("[VSPACE:") and aLine.endswith("]"): - continue - - elif aLine[0] == "#": - if aLine[:5] == "#### ": - aLine = aLine[5:] - countPara = False - elif aLine[:4] == "### ": - aLine = aLine[4:] - countPara = False - elif aLine[:3] == "## ": - aLine = aLine[3:] - countPara = False - elif aLine[:2] == "# ": - aLine = aLine[2:] - countPara = False - elif aLine[:3] == "#! ": - aLine = aLine[3:] - countPara = False - elif aLine[:4] == "##! ": - aLine = aLine[4:] - countPara = False - - elif aLine[0] == ">" or aLine[-1] == "<": - if aLine[:2] == ">>": - aLine = aLine[2:].lstrip(" ") - elif aLine[:1] == ">": - aLine = aLine[1:].lstrip(" ") - if aLine[-2:] == "<<": - aLine = aLine[:-2].rstrip(" ") - elif aLine[-1:] == "<": - aLine = aLine[:-1].rstrip(" ") - - wordCount += len(aLine.split()) - charCount += len(aLine) - if countPara and prevEmpty: - paraCount += 1 - - prevEmpty = not countPara - - return charCount, wordCount, paraCount - - -# =============================================================================================== # -# Indexer Objects -# =============================================================================================== # - class IndexItem: def __init__(self, tHandle, tItem): self._handle = tHandle self._item = tItem - self._level = "H0" self._headings = {} self._index = 0 @@ -780,6 +811,9 @@ class IndexItem: return + def __repr__(self): + return f"" + ## # Properties ## @@ -792,47 +826,50 @@ class IndexItem: def level(self): return self._level - @property - def headings(self): - return sorted(self._headings.keys()) - - @property - def entries(self): - return self._headings.values() - ## # Setters ## def updateLevel(self, level): - """Set the level only if it is H0. + """Set the level only if it has not already been set. """ if self._level == "H0": self._level = level return def addHeading(self, tHeading): + """Add a heading to the item. Also remove the placeholder entry + if it exists. + """ if H_NONE in self._headings: self._headings.pop(H_NONE) self._headings[tHeading.key] = tHeading return def setHeadingCounts(self, sTitle, charCount, wordCount, paraCount): + """Set the character, word and paragraph count of a heading. + """ if sTitle in self._headings: self._headings[sTitle].setCounts(charCount, wordCount, paraCount) return def setHeadingSynopsis(self, sTitle, synopText): + """Set the synopsis text of a heading. + """ if sTitle in self._headings: self._headings[sTitle].setSynopsis(synopText) return def setHeadingTag(self, sTitle, tagKey): + """Set the tag of a heading. + """ if sTitle in self._headings: self._headings[sTitle].setTag(tagKey) return def addHeadingReferences(self, sTitle, tagKeys, refType): + """Add a reference key and all its types to a heading. + """ if sTitle in self._headings: for tagKey in tagKeys: self._headings[sTitle].addReference(tagKey, refType) @@ -917,6 +954,9 @@ class IndexHeading: return + def __repr__(self): + return f"" + ## # Properties ## @@ -962,21 +1002,30 @@ class IndexHeading: ## def setLevel(self, level): + """Set the level of the header if it's a valid value. + """ if level in H_VALID: self._level = level return def setCounts(self, charCount, wordCount, paraCount): + """Set the character, word and paragraph count. Make sure the + value is an integer and is not smaller than 0. + """ self._charCount = max(0, checkInt(charCount, 0)) self._wordCount = max(0, checkInt(wordCount, 0)) self._paraCount = max(0, checkInt(paraCount, 0)) return def setSynopsis(self, synopText): + """Set the synopsis text and make sure it is a string. + """ self._synopsis = str(synopText) return def setTag(self, tagKey): + """Set the tag for references, and make sure it is a string. + """ self._tag = str(tagKey) return @@ -1041,3 +1090,84 @@ class IndexHeading: return # END Class IndexHeading + + +# =============================================================================================== # +# Simple Word Counter +# =============================================================================================== # + +def countWords(theText): + """Count words in a piece of text, skipping special syntax and + comments. + """ + charCount = 0 + wordCount = 0 + paraCount = 0 + prevEmpty = True + + if not isinstance(theText, str): + return charCount, wordCount, paraCount + + # We need to treat dashes as word separators for counting words. + # The check+replace approach is much faster than direct replace for + # large texts, and a bit slower for small texts, but in the latter + # case it doesn't really matter. + if nwUnicode.U_ENDASH in theText: + theText = theText.replace(nwUnicode.U_ENDASH, " ") + if nwUnicode.U_EMDASH in theText: + theText = theText.replace(nwUnicode.U_EMDASH, " ") + + for aLine in theText.splitlines(): + + countPara = True + + if not aLine: + prevEmpty = True + continue + if aLine[0] == "@" or aLine[0] == "%": + continue + + if aLine[0] == "[": + if aLine.startswith(("[NEWPAGE]", "[NEW PAGE]", "[VSPACE]")): + continue + elif aLine.startswith("[VSPACE:") and aLine.endswith("]"): + continue + + elif aLine[0] == "#": + if aLine[:5] == "#### ": + aLine = aLine[5:] + countPara = False + elif aLine[:4] == "### ": + aLine = aLine[4:] + countPara = False + elif aLine[:3] == "## ": + aLine = aLine[3:] + countPara = False + elif aLine[:2] == "# ": + aLine = aLine[2:] + countPara = False + elif aLine[:3] == "#! ": + aLine = aLine[3:] + countPara = False + elif aLine[:4] == "##! ": + aLine = aLine[4:] + countPara = False + + elif aLine[0] == ">" or aLine[-1] == "<": + if aLine[:2] == ">>": + aLine = aLine[2:].lstrip(" ") + elif aLine[:1] == ">": + aLine = aLine[1:].lstrip(" ") + if aLine[-2:] == "<<": + aLine = aLine[:-2].rstrip(" ") + elif aLine[-1:] == "<": + aLine = aLine[:-1].rstrip(" ") + + wordCount += len(aLine.split()) + charCount += len(aLine) + if countPara and prevEmpty: + paraCount += 1 + + prevEmpty = not countPara + + return charCount, wordCount, paraCount diff --git a/novelwriter/gui/noveltree.py b/novelwriter/gui/noveltree.py index 28edb2d9..a1cd11d3 100644 --- a/novelwriter/gui/noveltree.py +++ b/novelwriter/gui/noveltree.py @@ -251,9 +251,7 @@ class GuiNovelTree(QTreeWidget): currChapter = None currScene = None - for tKey, tHandle, sTitle, novIdx in self.theProject.index.novelStructure( - skipExcluded=True - ): + for tKey, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(skipExcl=True): tItem = self._createTreeItem(tHandle, sTitle, tKey, novIdx) self._treeMap[tKey] = tItem diff --git a/novelwriter/gui/outline.py b/novelwriter/gui/outline.py index e3886b87..83b04f47 100644 --- a/novelwriter/gui/outline.py +++ b/novelwriter/gui/outline.py @@ -389,7 +389,7 @@ class GuiOutline(QTreeWidget): currChapter = None currScene = None - for _, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(skipExcluded=True): + for _, tHandle, sTitle, novIdx in self.theProject.index.novelStructure(skipExcl=True): tItem = self._createTreeItem(tHandle, sTitle, novIdx) diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index eff85fec..e39981f6 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -69,19 +69,19 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): # Take a copy of the index tagIndex = str(theIndex._tags) - itemsIndex = str({handle: item.packData() for handle, item in theIndex._items.items()}) + itemsIndex = str(theIndex._itemIndex.packData()) # Delete a handle assert theIndex._tags.get("Bod", None) is not None - assert theIndex._items.get("4c4f28287af27", None) is not None + assert theIndex._itemIndex["4c4f28287af27"] is not None theIndex.deleteHandle("4c4f28287af27") assert theIndex._tags.get("Bod", None) is None - assert theIndex._items.get("4c4f28287af27", None) is None + assert theIndex._itemIndex["4c4f28287af27"] is None # Clear the index theIndex.clearIndex() assert theIndex._tags == {} - assert theIndex._items == {} + assert theIndex._itemIndex._items == {} # Make the load fail with monkeypatch.context() as mp: @@ -92,9 +92,7 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): assert theIndex.loadIndex() is True assert str(theIndex._tags) == tagIndex - assert str( - {handle: item.packData() for handle, item in theIndex._items.items()} - ) == itemsIndex + assert str(theIndex._itemIndex.packData()) == itemsIndex # Break the index and check that we notice # assert theIndex.indexBroken is False @@ -328,40 +326,40 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "##### Title Five\n\n" # Not interpreted as a title, the hashes are counted as a word "Paragraph Five.\n\n" )) - assert theIndex._items[nHandle]["T000001"].references == {} - assert theIndex._items[nHandle]["T000007"].references == {} - assert theIndex._items[nHandle]["T000013"].references == {} - assert theIndex._items[nHandle]["T000019"].references == {} + assert theIndex._itemIndex[nHandle]["T000001"].references == {} + assert theIndex._itemIndex[nHandle]["T000007"].references == {} + assert theIndex._itemIndex[nHandle]["T000013"].references == {} + assert theIndex._itemIndex[nHandle]["T000019"].references == {} - assert theIndex._items[nHandle]["T000001"].level == "H1" - assert theIndex._items[nHandle]["T000007"].level == "H2" - assert theIndex._items[nHandle]["T000013"].level == "H3" - assert theIndex._items[nHandle]["T000019"].level == "H4" + assert theIndex._itemIndex[nHandle]["T000001"].level == "H1" + assert theIndex._itemIndex[nHandle]["T000007"].level == "H2" + assert theIndex._itemIndex[nHandle]["T000013"].level == "H3" + assert theIndex._itemIndex[nHandle]["T000019"].level == "H4" - assert theIndex._items[nHandle]["T000001"].title == "Title One" - assert theIndex._items[nHandle]["T000007"].title == "Title Two" - assert theIndex._items[nHandle]["T000013"].title == "Title Three" - assert theIndex._items[nHandle]["T000019"].title == "Title Four" + assert theIndex._itemIndex[nHandle]["T000001"].title == "Title One" + assert theIndex._itemIndex[nHandle]["T000007"].title == "Title Two" + assert theIndex._itemIndex[nHandle]["T000013"].title == "Title Three" + assert theIndex._itemIndex[nHandle]["T000019"].title == "Title Four" - assert theIndex._items[nHandle]["T000001"].charCount == 23 - assert theIndex._items[nHandle]["T000007"].charCount == 23 - assert theIndex._items[nHandle]["T000013"].charCount == 27 - assert theIndex._items[nHandle]["T000019"].charCount == 56 + assert theIndex._itemIndex[nHandle]["T000001"].charCount == 23 + assert theIndex._itemIndex[nHandle]["T000007"].charCount == 23 + assert theIndex._itemIndex[nHandle]["T000013"].charCount == 27 + assert theIndex._itemIndex[nHandle]["T000019"].charCount == 56 - assert theIndex._items[nHandle]["T000001"].wordCount == 4 - assert theIndex._items[nHandle]["T000007"].wordCount == 4 - assert theIndex._items[nHandle]["T000013"].wordCount == 4 - assert theIndex._items[nHandle]["T000019"].wordCount == 9 + assert theIndex._itemIndex[nHandle]["T000001"].wordCount == 4 + assert theIndex._itemIndex[nHandle]["T000007"].wordCount == 4 + assert theIndex._itemIndex[nHandle]["T000013"].wordCount == 4 + assert theIndex._itemIndex[nHandle]["T000019"].wordCount == 9 - assert theIndex._items[nHandle]["T000001"].paraCount == 1 - assert theIndex._items[nHandle]["T000007"].paraCount == 1 - assert theIndex._items[nHandle]["T000013"].paraCount == 1 - assert theIndex._items[nHandle]["T000019"].paraCount == 3 + assert theIndex._itemIndex[nHandle]["T000001"].paraCount == 1 + assert theIndex._itemIndex[nHandle]["T000007"].paraCount == 1 + assert theIndex._itemIndex[nHandle]["T000013"].paraCount == 1 + assert theIndex._itemIndex[nHandle]["T000019"].paraCount == 3 - assert theIndex._items[nHandle]["T000001"].synopsis == "Synopsis One." - assert theIndex._items[nHandle]["T000007"].synopsis == "Synopsis Two." - assert theIndex._items[nHandle]["T000013"].synopsis == "Synopsis Three." - assert theIndex._items[nHandle]["T000019"].synopsis == "Synopsis Four." + assert theIndex._itemIndex[nHandle]["T000001"].synopsis == "Synopsis One." + assert theIndex._itemIndex[nHandle]["T000007"].synopsis == "Synopsis Two." + assert theIndex._itemIndex[nHandle]["T000013"].synopsis == "Synopsis Three." + assert theIndex._itemIndex[nHandle]["T000019"].synopsis == "Synopsis Four." # Note File assert theIndex.scanText(cHandle, ( @@ -370,13 +368,13 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "% synopsis: Synopsis One.\n\n" "Paragraph One.\n\n" )) - assert theIndex._items[cHandle]["T000001"].references == {} - assert theIndex._items[cHandle]["T000001"].level == "H1" - assert theIndex._items[cHandle]["T000001"].title == "Title One" - assert theIndex._items[cHandle]["T000001"].charCount == 23 - assert theIndex._items[cHandle]["T000001"].wordCount == 4 - assert theIndex._items[cHandle]["T000001"].paraCount == 1 - assert theIndex._items[cHandle]["T000001"].synopsis == "Synopsis One." + assert theIndex._itemIndex[cHandle]["T000001"].references == {} + assert theIndex._itemIndex[cHandle]["T000001"].level == "H1" + assert theIndex._itemIndex[cHandle]["T000001"].title == "Title One" + assert theIndex._itemIndex[cHandle]["T000001"].charCount == 23 + assert theIndex._itemIndex[cHandle]["T000001"].wordCount == 4 + assert theIndex._itemIndex[cHandle]["T000001"].paraCount == 1 + assert theIndex._itemIndex[cHandle]["T000001"].synopsis == "Synopsis One." # Valid and Invalid References assert theIndex.scanText(sHandle, ( @@ -387,7 +385,7 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "% synopsis: Synopsis One.\n\n" "Paragraph One.\n\n" )) - assert theIndex._items[sHandle]["T000001"].references == { + assert theIndex._itemIndex[sHandle]["T000001"].references == { "One": {"@pov"}, "Two": {"@char"} } @@ -398,25 +396,25 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "#! My Project\n\n" ">> By Jane Doe <<\n\n" )) - assert theIndex._items[cHandle]["T000001"].references == {} - assert theIndex._items[tHandle]["T000001"].level == "H1" - assert theIndex._items[tHandle]["T000001"].title == "My Project" - assert theIndex._items[tHandle]["T000001"].charCount == 21 - assert theIndex._items[tHandle]["T000001"].wordCount == 5 - assert theIndex._items[tHandle]["T000001"].paraCount == 1 - assert theIndex._items[tHandle]["T000001"].synopsis == "" + assert theIndex._itemIndex[cHandle]["T000001"].references == {} + assert theIndex._itemIndex[tHandle]["T000001"].level == "H1" + assert theIndex._itemIndex[tHandle]["T000001"].title == "My Project" + assert theIndex._itemIndex[tHandle]["T000001"].charCount == 21 + assert theIndex._itemIndex[tHandle]["T000001"].wordCount == 5 + assert theIndex._itemIndex[tHandle]["T000001"].paraCount == 1 + assert theIndex._itemIndex[tHandle]["T000001"].synopsis == "" assert theIndex.scanText(tHandle, ( "##! Prologue\n\n" "In the beginning there was time ...\n\n" )) - assert theIndex._items[cHandle]["T000001"].references == {} - assert theIndex._items[tHandle]["T000001"].level == "H2" - assert theIndex._items[tHandle]["T000001"].title == "Prologue" - assert theIndex._items[tHandle]["T000001"].charCount == 43 - assert theIndex._items[tHandle]["T000001"].wordCount == 8 - assert theIndex._items[tHandle]["T000001"].paraCount == 1 - assert theIndex._items[tHandle]["T000001"].synopsis == "" + assert theIndex._itemIndex[cHandle]["T000001"].references == {} + assert theIndex._itemIndex[tHandle]["T000001"].level == "H2" + assert theIndex._itemIndex[tHandle]["T000001"].title == "Prologue" + assert theIndex._itemIndex[tHandle]["T000001"].charCount == 43 + assert theIndex._itemIndex[tHandle]["T000001"].wordCount == 8 + assert theIndex._itemIndex[tHandle]["T000001"].paraCount == 1 + assert theIndex._itemIndex[tHandle]["T000001"].synopsis == "" # Page wo/Title # ============= @@ -425,25 +423,25 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): assert theIndex.scanText(pHandle, ( "This is a page with some text on it.\n\n" )) - assert theIndex._items[pHandle]["T000000"].references == {} - assert theIndex._items[pHandle]["T000000"].level == "H0" - assert theIndex._items[pHandle]["T000000"].title == "" - assert theIndex._items[pHandle]["T000000"].charCount == 36 - assert theIndex._items[pHandle]["T000000"].wordCount == 9 - assert theIndex._items[pHandle]["T000000"].paraCount == 1 - assert theIndex._items[pHandle]["T000000"].synopsis == "" + assert theIndex._itemIndex[pHandle]["T000000"].references == {} + assert theIndex._itemIndex[pHandle]["T000000"].level == "H0" + assert theIndex._itemIndex[pHandle]["T000000"].title == "" + assert theIndex._itemIndex[pHandle]["T000000"].charCount == 36 + assert theIndex._itemIndex[pHandle]["T000000"].wordCount == 9 + assert theIndex._itemIndex[pHandle]["T000000"].paraCount == 1 + assert theIndex._itemIndex[pHandle]["T000000"].synopsis == "" theProject.tree[pHandle]._layout = nwItemLayout.NOTE assert theIndex.scanText(pHandle, ( "This is a page with some text on it.\n\n" )) - assert theIndex._items[pHandle]["T000000"].references == {} - assert theIndex._items[pHandle]["T000000"].level == "H0" - assert theIndex._items[pHandle]["T000000"].title == "" - assert theIndex._items[pHandle]["T000000"].charCount == 36 - assert theIndex._items[pHandle]["T000000"].wordCount == 9 - assert theIndex._items[pHandle]["T000000"].paraCount == 1 - assert theIndex._items[pHandle]["T000000"].synopsis == "" + assert theIndex._itemIndex[pHandle]["T000000"].references == {} + assert theIndex._itemIndex[pHandle]["T000000"].level == "H0" + assert theIndex._itemIndex[pHandle]["T000000"].title == "" + assert theIndex._itemIndex[pHandle]["T000000"].charCount == 36 + assert theIndex._itemIndex[pHandle]["T000000"].wordCount == 9 + assert theIndex._itemIndex[pHandle]["T000000"].paraCount == 1 + assert theIndex._itemIndex[pHandle]["T000000"].synopsis == "" assert theProject.closeProject() is True @@ -488,13 +486,13 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): theProject.tree[nHandle].setExported(False) theKeys = [] - for aKey, _, _, _ in theIndex.novelStructure(skipExcluded=False): + for aKey, _, _, _ in theIndex.novelStructure(skipExcl=False): theKeys.append(aKey) assert theKeys == ["%s:T000001" % nHandle] theKeys = [] - for aKey, _, _, _ in theIndex.novelStructure(skipExcluded=True): + for aKey, _, _, _ in theIndex.novelStructure(skipExcl=True): theKeys.append(aKey) assert theKeys == [] @@ -625,12 +623,29 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): assert theIndex.scanText(sHandle, "### Scene One\n\n") assert theIndex.scanText(tHandle, "### Scene Two\n\n") - assert theIndex._listNovelHandles(False) == [nHandle, hHandle, sHandle, tHandle] - assert theIndex._listNovelHandles(True) == [hHandle, sHandle, tHandle] + assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=False)] == [ + (nHandle, "T000001"), + (nHandle, "T000011"), + (hHandle, "T000001"), + (sHandle, "T000001"), + (tHandle, "T000001"), + ] + + assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=True)] == [ + (hHandle, "T000001"), + (sHandle, "T000001"), + (tHandle, "T000001"), + ] # Add a fake handle to the tree and check that it's ignored theProject.tree._treeOrder.append("0000000000000") - assert theIndex._listNovelHandles(False) == [nHandle, hHandle, sHandle, tHandle] + assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=False)] == [ + (nHandle, "T000001"), + (nHandle, "T000011"), + (hHandle, "T000001"), + (sHandle, "T000001"), + (tHandle, "T000001"), + ] theProject.tree._treeOrder.remove("0000000000000") # Extract stats diff --git a/tests/test_gui/test_gui_docviewer.py b/tests/test_gui/test_gui_docviewer.py index 0ed15742..999504bf 100644 --- a/tests/test_gui/test_gui_docviewer.py +++ b/tests/test_gui/test_gui_docviewer.py @@ -48,7 +48,7 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum): # Rebuild the index nwGUI.mainMenu.aRebuildIndex.activate(QAction.Trigger) assert nwGUI.theProject.index._tags != {} - assert nwGUI.theProject.index._items != {} + assert nwGUI.theProject.index._itemIndex._items != {} # Select a document in the project tree nwGUI.treeView.setSelectedHandle("88243afbe5ed8") From d1b32f1ba178f17390eb62ea7d1d2a3df6a71b58 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 29 May 2022 19:24:41 +0200 Subject: [PATCH 09/13] Move the tags index into a wrapper class --- novelwriter/core/index.py | 121 +++++++++++++++++++++------ tests/test_core/test_core_index.py | 22 ++--- tests/test_gui/test_gui_docviewer.py | 2 +- 3 files changed, 107 insertions(+), 38 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index dcd38a1d..ad4b863c 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -56,7 +56,7 @@ class NWIndex(): self._indexBroken = False # Indices - self._tags = {} + self._tagsIndex = TagsIndex() self._itemIndex = ItemIndex(theProject) # TimeStamps @@ -81,7 +81,7 @@ class NWIndex(): def clearIndex(self): """Clear the index dictionaries and time stamps. """ - self._tags = {} + self._tagsIndex.clear() self._itemIndex.clear() self._timeNovel = 0 self._timeNotes = 0 @@ -93,7 +93,7 @@ class NWIndex(): """ logger.debug("Removing item '%s' from the index", tHandle) for tTag in self._itemIndex.allItemTags(tHandle): - self._tags.pop(tTag, None) + del self._tagsIndex[tTag] del self._itemIndex[tHandle] @@ -152,7 +152,7 @@ class NWIndex(): return False try: - self._validateTagsIndex(theData["tagsIndex"]) + self._tagsIndex.unpackData(theData["tagsIndex"]) self._itemIndex.unpackData(theData["itemIndex"]) except Exception: logger.error("The index content is invalid") @@ -186,10 +186,11 @@ class NWIndex(): tStart = time() try: + tagsIndex = self._tagsIndex.packData() itemIndex = self._itemIndex.packData() with open(indexFile, mode="w+", encoding="utf-8") as outFile: outFile.write("{\n") - outFile.write(f' "tagsIndex": {jsonEncode(self._tags, n=1, nmax=2)},\n') + outFile.write(f' "tagsIndex": {jsonEncode(tagsIndex, n=1, nmax=2)},\n') outFile.write(f' "itemIndex": {jsonEncode(itemIndex, n=1, nmax=4)}\n') outFile.write("}\n") @@ -356,11 +357,7 @@ class NWIndex(): sTitle = f"T{nTitle:06d}" if theBits[0] == nwKeyWords.TAG_KEY: - self._tags[theBits[1]] = { - "handle": tHandle, - "heading": sTitle, - "class": itemClass.name, - } + self._tagsIndex.add(theBits[1], tHandle, sTitle, itemClass) self._itemIndex.setHeadingTag(tHandle, sTitle, theBits[1]) else: self._itemIndex.addHeadingReferences(tHandle, sTitle, theBits[1:], theBits[0]) @@ -425,8 +422,8 @@ class NWIndex(): # For a tag, only the first value is accepted, the rest are ignored if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1: - if theBits[1] in self._tags: - isGood[1] = self._tags[theBits[1]].get("handle") == tItem.itemHandle + if theBits[1] in self._tagsIndex: + isGood[1] = self._tagsIndex.tagHandle(theBits[1]) == tItem.itemHandle else: isGood[1] = True return isGood @@ -434,8 +431,8 @@ class NWIndex(): # If we're still here, we check that the references exist theKey = nwKeyWords.KEY_CLASS[theBits[0]].name for n in range(1, nBits): - if theBits[n] in self._tags: - isGood[n] = theKey == self._tags[theBits[n]].get("class") + if theBits[n] in self._tagsIndex: + isGood[n] = self._tagsIndex.tagClass(theBits[n]) == theKey return isGood @@ -582,22 +579,98 @@ class NWIndex(): def getTagSource(self, theTag): """Return the source location of a given tag. """ - ref = self._tags.get(theTag, {}) - return ref.get("handle"), ref.get("heading", H_NONE) + tHandle = self._tagsIndex.tagHandle(theTag) + sTitle = self._tagsIndex.tagHeading(theTag) + return tHandle, sTitle + +# END Class NWIndex + + +# =============================================================================================== # +# Indexer Objects +# =============================================================================================== # + +class TagsIndex: + """A wrapper class that holds the reverse lookup tags index. + """ + + def __init__(self): + self._tags = {} + return ## - # Internal Functions + # Methods ## - def _validateTagsIndex(self, tagsIndex): + def clear(self): + """Clear the index. + """ + self._tags = {} + return + + def __contains__(self, tagKey): + """Check if a tag exists in the index, + """ + return tagKey in self._tags + + def __delitem__(self, tagKey): + """Delete an entry in the index. + """ + self._tags.pop(tagKey, None) + return + + def __getitem__(self, tagKey): + """Return a tag, or return None if it isn't found. + """ + return self._tags.get(tagKey, None) + + def add(self, tagKey, tHandle, sTitle, itemClass): + """Add a key to the index and set all values. + """ + self._tags[tagKey] = { + "handle": tHandle, "heading": sTitle, "class": itemClass.name + } + return + + def tagHandle(self, tagKey): + """Get the handle of a given tag. + """ + if tagKey in self._tags: + return self._tags.get(tagKey).get("handle") + return None + + def tagHeading(self, tagKey): + """Get the heading of a given tag. + """ + if tagKey in self._tags: + return self._tags.get(tagKey).get("heading") + return H_NONE + + def tagClass(self, tagKey): + """Get the class of a given tag. + """ + if tagKey in self._tags: + return self._tags.get(tagKey).get("class") + return None + + ## + # Pack/Unpack + ## + + def packData(self): + """Pack all the data of the tags into a single dictionary. + """ + return self._tags + + def unpackData(self, data): """Iterate through the tagsIndex loaded from cache and check that it's valid. """ self._tags = {} - if not isinstance(tagsIndex, dict): + if not isinstance(data, dict): raise ValueError("tagsIndex is not a dict") - for tagKey, tagData in tagsIndex.items(): + for tagKey, tagData in data.items(): if not isinstance(tagKey, str): raise ValueError("tagsIndex keys must be a strings") if "handle" not in tagData: @@ -613,17 +686,13 @@ class NWIndex(): if not isItemClass(tagData["class"]): raise ValueError("tagsIndex handle must be an nwItemClass") - self._tags = tagsIndex + self._tags = data return -# END Class NWIndex +# END Class TagsIndex -# =============================================================================================== # -# Indexer Objects -# =============================================================================================== # - class ItemIndex: """A wrapper object holding the indexed items. """ diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index e39981f6..76ffc70a 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -68,19 +68,19 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): assert theIndex.saveIndex() is True # Take a copy of the index - tagIndex = str(theIndex._tags) + tagIndex = str(theIndex._tagsIndex.packData()) itemsIndex = str(theIndex._itemIndex.packData()) # Delete a handle - assert theIndex._tags.get("Bod", None) is not None + assert theIndex._tagsIndex["Bod"] is not None assert theIndex._itemIndex["4c4f28287af27"] is not None theIndex.deleteHandle("4c4f28287af27") - assert theIndex._tags.get("Bod", None) is None + assert theIndex._tagsIndex["Bod"] is None assert theIndex._itemIndex["4c4f28287af27"] is None # Clear the index theIndex.clearIndex() - assert theIndex._tags == {} + assert theIndex._tagsIndex._tags == {} assert theIndex._itemIndex._items == {} # Make the load fail @@ -91,7 +91,7 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): # Make the load pass assert theIndex.loadIndex() is True - assert str(theIndex._tags) == tagIndex + assert str(theIndex._tagsIndex.packData()) == tagIndex assert str(theIndex._itemIndex.packData()) == itemsIndex # Break the index and check that we notice @@ -188,9 +188,9 @@ def testCoreIndex_CheckThese(nwMinimal, mockGUI): "@pov: Jane\n" "@invalid: John\n" # Checks for issue #688 )) - assert theIndex._tags == { - "Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"} - } + assert theIndex._tagsIndex.tagHandle("Jane") == cHandle + assert theIndex._tagsIndex.tagHeading("Jane") == "T000001" + assert theIndex._tagsIndex.tagClass("Jane") == "CHARACTER" assert theIndex.getNovelData(nHandle, "T000001").title == "Hello World!" assert theIndex.getReferences(nHandle, "T000001") == { "@char": [], @@ -301,9 +301,9 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): "This is a story about Jane Smith.\n\n" "Well, not really.\n" )) - assert theIndex._tags == { - "Jane": {"handle": cHandle, "heading": "T000001", "class": "CHARACTER"} - } + assert theIndex._tagsIndex.tagHandle("Jane") == cHandle + assert theIndex._tagsIndex.tagHeading("Jane") == "T000001" + assert theIndex._tagsIndex.tagClass("Jane") == "CHARACTER" assert theIndex.getNovelData(nHandle, "T000001").title == "Hello World!" # Title Indexing diff --git a/tests/test_gui/test_gui_docviewer.py b/tests/test_gui/test_gui_docviewer.py index 999504bf..1fa6b2c7 100644 --- a/tests/test_gui/test_gui_docviewer.py +++ b/tests/test_gui/test_gui_docviewer.py @@ -47,7 +47,7 @@ def testGuiViewer_Main(qtbot, monkeypatch, nwGUI, nwLipsum): # Rebuild the index nwGUI.mainMenu.aRebuildIndex.activate(QAction.Trigger) - assert nwGUI.theProject.index._tags != {} + assert nwGUI.theProject.index._tagsIndex._tags != {} assert nwGUI.theProject.index._itemIndex._items != {} # Select a document in the project tree From 865acfd7e92e362da479f2ff18f17462746e1989 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 29 May 2022 19:40:14 +0200 Subject: [PATCH 10/13] Clean up imports --- novelwriter/core/__init__.py | 3 +-- novelwriter/core/index.py | 2 +- tests/test_core/test_core_tohtml.py | 3 ++- tests/test_core/test_core_tomd.py | 3 ++- tests/test_core/test_core_toodt.py | 3 ++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/novelwriter/core/__init__.py b/novelwriter/core/__init__.py index 6e69917f..c91ca941 100644 --- a/novelwriter/core/__init__.py +++ b/novelwriter/core/__init__.py @@ -20,7 +20,7 @@ along with this program. If not, see . """ from novelwriter.core.document import NWDoc -from novelwriter.core.index import NWIndex, countWords +from novelwriter.core.index import countWords from novelwriter.core.project import NWProject from novelwriter.core.spellcheck import NWSpellEnchant from novelwriter.core.tohtml import ToHtml @@ -30,7 +30,6 @@ from novelwriter.core.tomd import ToMarkdown __all__ = [ "countWords", "NWDoc", - "NWIndex", "NWProject", "NWSpellEnchant", "ToHtml", diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index ad4b863c..c1cb1e01 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -46,7 +46,7 @@ H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4} H_NONE = "T000000" -class NWIndex(): +class NWIndex: def __init__(self, theProject): diff --git a/tests/test_core/test_core_tohtml.py b/tests/test_core/test_core_tohtml.py index 11d89572..12072e09 100644 --- a/tests/test_core/test_core_tohtml.py +++ b/tests/test_core/test_core_tohtml.py @@ -24,7 +24,8 @@ import pytest from tools import readFile -from novelwriter.core import NWProject, NWIndex, ToHtml +from novelwriter.core import NWProject, ToHtml +from novelwriter.core.index import NWIndex @pytest.mark.core diff --git a/tests/test_core/test_core_tomd.py b/tests/test_core/test_core_tomd.py index 51eea72b..c2235ff8 100644 --- a/tests/test_core/test_core_tomd.py +++ b/tests/test_core/test_core_tomd.py @@ -24,7 +24,8 @@ import pytest from tools import readFile -from novelwriter.core import NWProject, NWIndex, ToMarkdown +from novelwriter.core import NWProject, ToMarkdown +from novelwriter.core.index import NWIndex @pytest.mark.core diff --git a/tests/test_core/test_core_toodt.py b/tests/test_core/test_core_toodt.py index e2ccb4a5..febbc94f 100644 --- a/tests/test_core/test_core_toodt.py +++ b/tests/test_core/test_core_toodt.py @@ -28,7 +28,8 @@ from shutil import copyfile from tools import cmpFiles -from novelwriter.core import NWProject, NWIndex, ToOdt +from novelwriter.core import NWProject, ToOdt +from novelwriter.core.index import NWIndex from novelwriter.core.toodt import ODTParagraphStyle, ODTTextStyle, XMLParagraph, _mkTag XML_NS = [ From eed59a96be863ea14bd933c5562d4afa40eebb6f Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Mon, 30 May 2022 00:28:43 +0200 Subject: [PATCH 11/13] Make some fixes to the index class, and improve test coverage --- novelwriter/core/index.py | 53 +++-- novelwriter/guimain.py | 21 +- sample/content/5eaea4e8cdee8.nwd | 1 + sample/content/88706ddc78b1b.nwd | 2 +- sample/content/b3e74dbc1f584.nwd | 1 + sample/content/b8136a5a774a0.nwd | 2 +- tests/test_core/test_core_index.py | 332 ++++++++++++++++++++++++----- 7 files changed, 320 insertions(+), 92 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index c1cb1e01..cdd78032 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -7,6 +7,7 @@ File History: Created: 2019-04-22 [0.0.1] countWords Created: 2019-05-27 [0.1.4] NWIndex Created: 2022-05-28 [1.7rc1] IndexItem, IndexHeading +Created: 2022-05-29 [1.7rc1] TagsIndex, ItemIndex This file is a part of novelWriter Copyright 2018–2022, Veronica Berglyd Olsen @@ -43,10 +44,24 @@ logger = logging.getLogger(__name__) H_VALID = ("H0", "H1", "H2", "H3", "H4") H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4} -H_NONE = "T000000" +TT_NONE = "T000000" class NWIndex: + """This class holds the entire index for a given project. The index + contains the data that isn't stored in the project items themselves. + The content of the index is updated every time a file item is saved. + + The primary index data is contained in the ItemIndex class, which + contains an IndexItem representing each NWItem. Each IndexItem holds + an IndexHeading object for each heading of the item's text. + + A reverse index of all tags is contained in the TagsIndex class. + This is duplicate information used for quicker lookups from the tags + and back to items where they are defined. + + The index data is cached in a JSON file between writing sessions. + """ def __init__(self, theProject): @@ -104,10 +119,10 @@ class NWIndex: moved from the archive or trash folders back into the active project. """ - logger.debug("Re-indexing item '%s'", tHandle) if not self.theProject.tree.checkType(tHandle, nwItemType.FILE): return False + logger.debug("Re-indexing item '%s'", tHandle) theDoc = NWDoc(self.theProject, tHandle) theText = theDoc.readDocument() self.scanText(tHandle, theText if theText is not None else "") @@ -140,6 +155,7 @@ class NWIndex: indexFile = os.path.join(self.theProject.projMeta, nwFiles.INDEX_FILE) tStart = time() + self._indexBroken = False if os.path.isfile(indexFile): logger.debug("Loading index file") try: @@ -222,8 +238,9 @@ class NWIndex: logger.info("Not indexing non-file item '%s'", tHandle) return False - # Delete the old entry and create a new - self.deleteHandle(tHandle) + # Delete tags and create new item entry + for tTag in self._itemIndex.allItemTags(tHandle): + del self._tagsIndex[tTag] self._itemIndex.add(tHandle, theItem) # Run word counter for the whole text @@ -644,7 +661,7 @@ class TagsIndex: """ if tagKey in self._tags: return self._tags.get(tagKey).get("heading") - return H_NONE + return TT_NONE def tagClass(self, tagKey): """Get the class of a given tag. @@ -782,11 +799,11 @@ class ItemIndex: continue if rootHandle is None: - for sTitle, hItem in self._items[tHandle].items(): - yield tHandle, sTitle, hItem - elif tItem.rootHandle == rootHandle: - for sTitle, hItem in self._items[tHandle].items(): - yield tHandle, sTitle, hItem + for sTitle in self._items[tHandle].headings(): + yield tHandle, sTitle, self._items[tHandle][sTitle] + elif tItem.itemRoot == rootHandle: + for sTitle in self._items[tHandle].headings(): + yield tHandle, sTitle, self._items[tHandle][sTitle] else: continue @@ -876,7 +893,7 @@ class IndexItem: self._index = 0 # Add a placeholder heading - self._headings[H_NONE] = IndexHeading(H_NONE) + self._headings[TT_NONE] = IndexHeading(TT_NONE) return @@ -910,8 +927,8 @@ class IndexItem: """Add a heading to the item. Also remove the placeholder entry if it exists. """ - if H_NONE in self._headings: - self._headings.pop(H_NONE) + if TT_NONE in self._headings: + self._headings.pop(TT_NONE) self._headings[tHeading.key] = tHeading return @@ -957,6 +974,9 @@ class IndexItem: def items(self): return self._headings.items() + def headings(self): + return sorted(self._headings.keys()) + def allTags(self): """Return a list of all tags in the current item. """ @@ -1102,9 +1122,10 @@ class IndexHeading: """Add a record of a reference tag, and what keyword types it is associated with. """ - if tagKey not in self._refs: - self._refs[tagKey] = set() - self._refs[tagKey].add(refType) + if refType in nwKeyWords.VALID_KEYS: + if tagKey not in self._refs: + self._refs[tagKey] = set() + self._refs[tagKey].add(refType) return ## diff --git a/novelwriter/guimain.py b/novelwriter/guimain.py index 3a957577..3d61a65e 100644 --- a/novelwriter/guimain.py +++ b/novelwriter/guimain.py @@ -865,22 +865,13 @@ class GuiMain(QMainWindow): self.theProject.index.clearIndex() for tItem in self.theProject.tree: + if tItem is None: # pragma: no cover + continue # This is a bug trap - if tItem is not None: - self.setStatus(self.tr("Indexing: '{0}'").format(tItem.itemName)) - else: - self.setStatus(self.tr("Indexing: '{0}'").format(self.tr("Unknown item"))) - - if tItem is not None and tItem.itemType == nwItemType.FILE: - logger.verbose("Scanning '%s'", tItem.itemName) - self.theProject.index.reIndexHandle(tItem.itemHandle) - - # Get Word Counts - cC, wC, pC = self.theProject.index.getCounts(tItem.itemHandle) - tItem.setCharCount(cC) - tItem.setWordCount(wC) - tItem.setParaCount(pC) - self.treeView.propagateCount(tItem.itemHandle, wC, countChildren=True) + logger.verbose("Indexing '%s'", tItem.itemName) + if self.theProject.index.reIndexHandle(tItem.itemHandle): + # Update Word Counts + self.treeView.propagateCount(tItem.itemHandle, tItem.wordCount, countChildren=True) self.treeView.setTreeItemValues(tItem.itemHandle) tEnd = time() diff --git a/sample/content/5eaea4e8cdee8.nwd b/sample/content/5eaea4e8cdee8.nwd index 1a7f3c79..0f8ecc26 100644 --- a/sample/content/5eaea4e8cdee8.nwd +++ b/sample/content/5eaea4e8cdee8.nwd @@ -4,5 +4,6 @@ # Mars @tag: Mars +@location: Space It’s red. Dusty and red. diff --git a/sample/content/88706ddc78b1b.nwd b/sample/content/88706ddc78b1b.nwd index ce4d2123..0f140538 100644 --- a/sample/content/88706ddc78b1b.nwd +++ b/sample/content/88706ddc78b1b.nwd @@ -1,5 +1,5 @@ %%~name: Chapter Two -%%~path: e7ded148d6e4a/88706ddc78b1b +%%~path: 7031beac91f75/88706ddc78b1b %%~kind: NOVEL/DOCUMENT ## Where has John Gone? diff --git a/sample/content/b3e74dbc1f584.nwd b/sample/content/b3e74dbc1f584.nwd index 6931a299..c980865e 100644 --- a/sample/content/b3e74dbc1f584.nwd +++ b/sample/content/b3e74dbc1f584.nwd @@ -4,5 +4,6 @@ # Earth @tag: Earth +@location: Space Third planet from the sun, fairly dense, and with lots of people on it. \ No newline at end of file diff --git a/sample/content/b8136a5a774a0.nwd b/sample/content/b8136a5a774a0.nwd index 3c2c1854..636c7227 100644 --- a/sample/content/b8136a5a774a0.nwd +++ b/sample/content/b8136a5a774a0.nwd @@ -1,6 +1,6 @@ %%~name: Delete Me! %%~path: 98acd8c76c93a/b8136a5a774a0 -%%~kind: NOVEL/DOCUMENT +%%~kind: TRASH/DOCUMENT ### Delete Me! This scene is trash. \ No newline at end of file diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index 76ffc70a..b1233275 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -19,14 +19,14 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ -import pytest import os import json +import pytest from shutil import copyfile from mock import causeException -from tools import cmpFiles +from tools import buildTestProject, cmpFiles, writeFile from novelwriter.core.project import NWProject from novelwriter.core.index import NWIndex, countWords @@ -87,36 +87,58 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): with monkeypatch.context() as mp: mp.setattr(json, "load", causeException) assert theIndex.loadIndex() is False + assert theIndex.indexBroken is True # Make the load pass assert theIndex.loadIndex() is True + assert theIndex.indexBroken is False assert str(theIndex._tagsIndex.packData()) == tagIndex assert str(theIndex._itemIndex.packData()) == itemsIndex - # Break the index and check that we notice - # assert theIndex.indexBroken is False - # theIndex._tagIndex["Bod"].append("Stuff") - # theIndex._checkIndex() - # assert theIndex.indexBroken is True + # Check File + copyfile(projFile, testFile) + assert cmpFiles(testFile, compFile) + + # Write an emtpy index file and load it + writeFile(projFile, "{}") + assert theIndex.loadIndex() is False + assert theIndex.indexBroken is True + + # Write an index file that passes loading, but is still empty + writeFile(projFile, '{"tagsIndex": {}, "itemIndex": {}}') + assert theIndex.loadIndex() is True + assert theIndex.indexBroken is False + + # Check that the index is re-populated + assert "04468803b92e1" in theIndex._itemIndex + assert "2426c6f0ca922" in theIndex._itemIndex + assert "441420a886d82" in theIndex._itemIndex + assert "47666c91c7ccf" in theIndex._itemIndex + assert "4c4f28287af27" in theIndex._itemIndex + assert "846352075de7d" in theIndex._itemIndex + assert "88243afbe5ed8" in theIndex._itemIndex + assert "88d59a277361b" in theIndex._itemIndex + assert "8c58a65414c23" in theIndex._itemIndex + assert "db7e733775d4d" in theIndex._itemIndex + assert "eb103bc70c90c" in theIndex._itemIndex + assert "f8c0562e50f1b" in theIndex._itemIndex + assert "f96ec11c6a3da" in theIndex._itemIndex + assert "fb609cd8319dc" in theIndex._itemIndex + assert "7a992350f3eb6" in theIndex._itemIndex # Finalise assert theProject.closeProject() is True - copyfile(projFile, testFile) - assert cmpFiles(testFile, compFile) - # END Test testCoreIndex_LoadSave @pytest.mark.core -def testCoreIndex_ScanThis(nwMinimal, mockGUI): +def testCoreIndex_ScanThis(mockGUI): """Test the tag scanner function scanThis. """ theProject = NWProject(mockGUI) - assert theProject.openProject(nwMinimal) is True - - theIndex = NWIndex(theProject) + theIndex = theProject.index isValid, theBits, thePos = theIndex.scanThis("tag: this, and this") assert isValid is False @@ -161,15 +183,15 @@ def testCoreIndex_ScanThis(nwMinimal, mockGUI): @pytest.mark.core -def testCoreIndex_CheckThese(nwMinimal, mockGUI): +def testCoreIndex_CheckThese(mockGUI, fncDir, mockRnd): """Test the tag checker function checkThese. """ theProject = NWProject(mockGUI) - assert theProject.openProject(nwMinimal) is True + buildTestProject(theProject, fncDir) + theIndex = theProject.index - theIndex = NWIndex(theProject) - nHandle = theProject.newFile("Hello", "a508bb932959c") - cHandle = theProject.newFile("Jane", "afb3043c7b2b3") + nHandle = theProject.newFile("Hello", "0000000000010") + cHandle = theProject.newFile("Jane", "0000000000012") nItem = theProject.tree[nHandle] cItem = theProject.tree[cHandle] @@ -239,17 +261,16 @@ def testCoreIndex_CheckThese(nwMinimal, mockGUI): @pytest.mark.core -def testCoreIndex_ScanText(nwMinimal, mockGUI): +def testCoreIndex_ScanText(mockGUI, fncDir, mockRnd): """Check the index text scanner. """ theProject = NWProject(mockGUI) - assert theProject.openProject(nwMinimal) is True - - theIndex = NWIndex(theProject) + buildTestProject(theProject, fncDir) + theIndex = theProject.index # Some items for fail to scan tests - dHandle = theProject.newFolder("Folder", "a508bb932959c") - xHandle = theProject.newFile("No Layout", "a508bb932959c") + dHandle = theProject.newFolder("Folder", "0000000000010") + xHandle = theProject.newFile("No Layout", "0000000000010") xItem = theProject.tree[xHandle] xItem.setLayout(nwItemLayout.NO_LAYOUT) @@ -279,11 +300,11 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): assert theIndex.scanText(xHandle, "Hello World!") is False # Make some usable items - tHandle = theProject.newFile("Title", "a508bb932959c") - pHandle = theProject.newFile("Page", "a508bb932959c") - nHandle = theProject.newFile("Hello", "a508bb932959c") - cHandle = theProject.newFile("Jane", "afb3043c7b2b3") - sHandle = theProject.newFile("Scene", "a508bb932959c") + tHandle = theProject.newFile("Title", "0000000000010") + pHandle = theProject.newFile("Page", "0000000000010") + nHandle = theProject.newFile("Hello", "0000000000010") + cHandle = theProject.newFile("Jane", "0000000000012") + sHandle = theProject.newFile("Scene", "0000000000010") # Text Indexing # ============= @@ -449,18 +470,27 @@ def testCoreIndex_ScanText(nwMinimal, mockGUI): @pytest.mark.core -def testCoreIndex_ExtractData(nwMinimal, mockGUI): +def testCoreIndex_ExtractData(mockGUI, fncDir, mockRnd): """Check the index data extraction functions. """ theProject = NWProject(mockGUI) - assert theProject.openProject(nwMinimal) is True + buildTestProject(theProject, fncDir) - theIndex = NWIndex(theProject) - nHandle = theProject.newFile("Hello", "a508bb932959c") - cHandle = theProject.newFile("Jane", "afb3043c7b2b3") + theIndex = theProject.index + theIndex.reIndexHandle("0000000000010") + theIndex.reIndexHandle("0000000000011") + theIndex.reIndexHandle("0000000000012") + theIndex.reIndexHandle("0000000000013") + theIndex.reIndexHandle("0000000000014") + theIndex.reIndexHandle("0000000000015") + theIndex.reIndexHandle("0000000000016") + theIndex.reIndexHandle("0000000000017") + + nHandle = theProject.newFile("Hello", "0000000000010") + cHandle = theProject.newFile("Jane", "0000000000012") assert theIndex.getNovelData("", "") is None - assert theIndex.getNovelData("a508bb932959c", "") is None + assert theIndex.getNovelData("0000000000010", "") is None assert theIndex.scanText(cHandle, ( "# Jane Smith\n" @@ -480,7 +510,12 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): for aKey, _, _, _ in theIndex.novelStructure(): theKeys.append(aKey) - assert theKeys == ["%s:T000001" % nHandle] + assert theKeys == [ + "0000000000014:T000001", + "0000000000016:T000001", + "0000000000017:T000001", + "%s:T000001" % nHandle, + ] # Check that excluded files can be skipped theProject.tree[nHandle].setExported(False) @@ -489,19 +524,22 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): for aKey, _, _, _ in theIndex.novelStructure(skipExcl=False): theKeys.append(aKey) - assert theKeys == ["%s:T000001" % nHandle] + assert theKeys == [ + "0000000000014:T000001", + "0000000000016:T000001", + "0000000000017:T000001", + "%s:T000001" % nHandle, + ] theKeys = [] for aKey, _, _, _ in theIndex.novelStructure(skipExcl=True): theKeys.append(aKey) - assert theKeys == [] - - theKeys = [] - for aKey, _, _, _ in theIndex.novelStructure(): - theKeys.append(aKey) - - assert theKeys == [] + assert theKeys == [ + "0000000000014:T000001", + "0000000000016:T000001", + "0000000000017:T000001", + ] # The novel file should have the correct counts cC, wC, pC = theIndex.getCounts(nHandle) @@ -528,6 +566,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # None handle should return an empty dict assert theIndex.getBackReferenceList(None) == {} + # The Title Page file should have no references as it has no tag + assert theIndex.getBackReferenceList("0000000000014") == {} + # The character file should have a record of the reference from the novel file theRefs = theIndex.getBackReferenceList(cHandle) assert theRefs == {nHandle: "T000001"} @@ -542,6 +583,10 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # ========= # For whole text and sections + # Invalid handle or title should return 0s + assert theIndex.getCounts("stuff") == (0, 0, 0) + assert theIndex.getCounts(nHandle, "stuff") == (0, 0, 0) + # Get section counts for a novel file assert theIndex.scanText(nHandle, ( "# Hello World!\n" @@ -611,9 +656,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # Novel Stats # =========== - hHandle = theProject.newFile("Chapter", "a508bb932959c") - sHandle = theProject.newFile("Scene One", "a508bb932959c") - tHandle = theProject.newFile("Scene Two", "a508bb932959c") + hHandle = theProject.newFile("Chapter", "0000000000010") + sHandle = theProject.newFile("Scene One", "0000000000010") + tHandle = theProject.newFile("Scene Two", "0000000000010") theProject.tree[hHandle].itemLayout == nwItemLayout.DOCUMENT theProject.tree[sHandle].itemLayout == nwItemLayout.DOCUMENT @@ -624,6 +669,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): assert theIndex.scanText(tHandle, "### Scene Two\n\n") assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=False)] == [ + ("0000000000014", "T000001"), + ("0000000000016", "T000001"), + ("0000000000017", "T000001"), (nHandle, "T000001"), (nHandle, "T000011"), (hHandle, "T000001"), @@ -632,6 +680,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): ] assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=True)] == [ + ("0000000000014", "T000001"), + ("0000000000016", "T000001"), + ("0000000000017", "T000001"), (hHandle, "T000001"), (sHandle, "T000001"), (tHandle, "T000001"), @@ -640,6 +691,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # Add a fake handle to the tree and check that it's ignored theProject.tree._treeOrder.append("0000000000000") assert [(h, t) for h, t, _ in theIndex._itemIndex.iterNovelStructure(skipExcl=False)] == [ + ("0000000000014", "T000001"), + ("0000000000016", "T000001"), + ("0000000000017", "T000001"), (nHandle, "T000001"), (nHandle, "T000011"), (hHandle, "T000001"), @@ -649,25 +703,33 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): theProject.tree._treeOrder.remove("0000000000000") # Extract stats - assert theIndex.getNovelWordCount(False) == 34 - assert theIndex.getNovelWordCount(True) == 6 - assert theIndex.getNovelTitleCounts(False) == [0, 2, 1, 2, 0] - assert theIndex.getNovelTitleCounts(True) == [0, 0, 1, 2, 0] + assert theIndex.getNovelWordCount(skipExcl=False) == 43 + assert theIndex.getNovelWordCount(skipExcl=True) == 15 + assert theIndex.getNovelTitleCounts(skipExcl=False) == [0, 3, 2, 3, 0] + assert theIndex.getNovelTitleCounts(skipExcl=True) == [0, 1, 2, 3, 0] # Table of Contents - assert theIndex.getTableOfContents(0, True) == [] - assert theIndex.getTableOfContents(1, True) == [] - assert theIndex.getTableOfContents(2, True) == [ + assert theIndex.getTableOfContents(0, skipExcl=True) == [] + assert theIndex.getTableOfContents(1, skipExcl=True) == [ + ("0000000000014:T000001", 1, "New Novel", 15), + ] + assert theIndex.getTableOfContents(2, skipExcl=True) == [ + ("0000000000014:T000001", 1, "New Novel", 5), + ("0000000000016:T000001", 2, "New Chapter", 4), ("%s:T000001" % hHandle, 2, "Chapter One", 6), ] - assert theIndex.getTableOfContents(3, True) == [ + assert theIndex.getTableOfContents(3, skipExcl=True) == [ + ("0000000000014:T000001", 1, "New Novel", 5), + ("0000000000016:T000001", 2, "New Chapter", 2), + ("0000000000017:T000001", 3, "New Scene", 2), ("%s:T000001" % hHandle, 2, "Chapter One", 2), ("%s:T000001" % sHandle, 3, "Scene One", 2), ("%s:T000001" % tHandle, 3, "Scene Two", 2), ] - assert theIndex.getTableOfContents(0, False) == [] - assert theIndex.getTableOfContents(1, False) == [ + assert theIndex.getTableOfContents(0, skipExcl=False) == [] + assert theIndex.getTableOfContents(1, skipExcl=False) == [ + ("0000000000014:T000001", 1, "New Novel", 9), ("%s:T000001" % nHandle, 1, "Hello World!", 12), ("%s:T000011" % nHandle, 1, "Hello World!", 22), ] @@ -682,7 +744,9 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): ("%s:T000001" % nHandle, 12), ("%s:T000011" % nHandle, 16) ] - assert theProject.closeProject() + assert theIndex.saveIndex() is True + assert theProject.saveProject() is True + assert theProject.closeProject() is True # Header Record bHandle = "0000000000000" @@ -697,6 +761,156 @@ def testCoreIndex_ExtractData(nwMinimal, mockGUI): # END Test testCoreIndex_ExtractData +@pytest.mark.core +def testCoreIndex_ItemIndex(mockGUI, fncDir, mockRnd): + """Check the ItemIndex class. + """ + theProject = NWProject(mockGUI) + buildTestProject(theProject, fncDir) + + nHandle = "0000000000014" + cHandle = "0000000000016" + sHandle = "0000000000017" + + assert theProject.index.saveIndex() is True + itemIndex = theProject.index._itemIndex + + # The index should be empty + assert nHandle not in itemIndex + assert cHandle not in itemIndex + assert sHandle not in itemIndex + + # Unpack Data + # =========== + + # Data must be dictionary + with pytest.raises(ValueError): + itemIndex.unpackData("stuff") + + # Keys must be valid handles + with pytest.raises(ValueError): + itemIndex.unpackData({"stuff": "more stuff"}) + + # Unknown keys should be skipped + itemIndex.unpackData({"0000000000000": {}}) + assert itemIndex._items == {} + + # Known keys can be added, even witout data + itemIndex.unpackData({nHandle: {}}) + assert nHandle in itemIndex + itemIndex.clear() + + # Add Items + # ========= + assert cHandle not in itemIndex + + # Add the novel chapter file + itemIndex.add(cHandle, theProject.tree[cHandle]) + assert cHandle in itemIndex + assert itemIndex[cHandle].item == theProject.tree[cHandle] + assert itemIndex.mainItemHeader(cHandle) == "H0" + assert itemIndex.allItemTags(cHandle) == [] + assert list(itemIndex.iterItemHeaders(cHandle))[0][0] == "T000000" + + # Add a heading to the item, which should replace the T000000 heading + itemIndex.addItemHeading(cHandle, "T000001", "H2", "Chapter One") + assert itemIndex.mainItemHeader(cHandle) == "H2" + assert list(itemIndex.iterItemHeaders(cHandle))[0][0] == "T000001" + + # Set the remainig data values + itemIndex.setHeadingCounts(cHandle, "T000001", 60, 10, 2) + itemIndex.setHeadingSynopsis(cHandle, "T000001", "In the beginning ...") + itemIndex.setHeadingTag(cHandle, "T000001", "One") # Although it isn't allowed to have a tag + itemIndex.addHeadingReferences(cHandle, "T000001", ["Jane"], "@pov") + itemIndex.addHeadingReferences(cHandle, "T000001", ["Jane"], "@focus") + itemIndex.addHeadingReferences(cHandle, "T000001", ["Jane", "John"], "@char") + idxData = itemIndex.packData() + + assert idxData[cHandle]["level"] == "H2" + assert idxData[cHandle]["headings"]["T000001"] == { + "level": "H2", "title": "Chapter One", "tag": "One", + "cCount": 60, "wCount": 10, "pCount": 2, "synopsis": "In the beginning ...", + } + assert "@pov" in idxData[cHandle]["references"]["T000001"]["Jane"] + assert "@focus" in idxData[cHandle]["references"]["T000001"]["Jane"] + assert "@char" in idxData[cHandle]["references"]["T000001"]["Jane"] + assert "@char" in idxData[cHandle]["references"]["T000001"]["John"] + + # Add the other two files + itemIndex.add(nHandle, theProject.tree[nHandle]) + itemIndex.add(sHandle, theProject.tree[sHandle]) + itemIndex.addItemHeading(nHandle, "T000001", "H1", "Novel") + itemIndex.addItemHeading(sHandle, "T000001", "H3", "Scene One") + + # Data Extraction + # =============== + + # Get headers + allHeads = list(itemIndex.iterAllHeaders()) + assert allHeads[0][0] == cHandle + assert allHeads[1][0] == nHandle + assert allHeads[2][0] == sHandle + assert allHeads[0][1] == "T000001" + assert allHeads[1][1] == "T000001" + assert allHeads[2][1] == "T000001" + + # Ask for stuff that doesn't exist + assert itemIndex.mainItemHeader("blablabla") == "H0" + assert itemIndex.allItemTags("blablabla") == [] + + # Novel Structure + # =============== + + # Add a second novel + mHandle = theProject.newRoot(nwItemClass.NOVEL) + uHandle = theProject.newFile("Title Page", mHandle) + itemIndex.add(uHandle, theProject.tree[uHandle]) + itemIndex.addItemHeading(uHandle, "T000001", "H1", "Novel 2") + assert uHandle in itemIndex + + # Structure of all novels + nStruct = list(itemIndex.iterNovelStructure()) + assert len(nStruct) == 4 + assert nStruct[0][0] == nHandle + assert nStruct[1][0] == cHandle + assert nStruct[2][0] == sHandle + assert nStruct[3][0] == uHandle + + # Novel structure with root handle set + nStruct = list(itemIndex.iterNovelStructure(rootHandle="0000000000010")) + assert len(nStruct) == 3 + assert nStruct[0][0] == nHandle + assert nStruct[1][0] == cHandle + assert nStruct[2][0] == sHandle + + nStruct = list(itemIndex.iterNovelStructure(rootHandle=mHandle)) + assert len(nStruct) == 1 + assert nStruct[0][0] == uHandle + + # Inject garbage into tree + theProject.tree._treeOrder.append("stuff") + nStruct = list(itemIndex.iterNovelStructure()) + assert len(nStruct) == 4 + assert nStruct[0][0] == nHandle + assert nStruct[1][0] == cHandle + assert nStruct[2][0] == sHandle + assert nStruct[3][0] == uHandle + + # Skip excluded + theProject.tree[sHandle].setExported(False) + nStruct = list(itemIndex.iterNovelStructure(skipExcl=True)) + assert len(nStruct) == 3 + assert nStruct[0][0] == nHandle + assert nStruct[1][0] == cHandle + assert nStruct[2][0] == uHandle + + # Delete new item + del itemIndex[uHandle] + assert uHandle not in itemIndex + +# END Test testCoreIndex_ItemIndex + + @pytest.mark.core def testCoreIndex_CountWords(): """Test the word counter and the exclusion filers. From 22099c1b56d7a7316e5d14c851cab094b866fd54 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Fri, 3 Jun 2022 11:07:40 +0200 Subject: [PATCH 12/13] Make some minor improvements to handling of tags and references --- novelwriter/core/index.py | 44 +++++++++++++------ sample/content/636b6aa9b697b.nwd | 2 +- sample/content/88706ddc78b1b.nwd | 1 + sample/content/ae7339df26ded.nwd | 1 + sample/content/b3e74dbc1f584.nwd | 2 +- .../coreIndex_LoadSave_tagsIndex.json | 16 +++---- 6 files changed, 43 insertions(+), 23 deletions(-) diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index cdd78032..aba00ba5 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -227,7 +227,7 @@ class NWIndex: """Scan a piece of text associated with a handle. This will update the indices accordingly. This function takes the handle and text as separate inputs as we want to primarily scan the - files before we save them in which case we already have the + files before we save them, in which case we already have the text. """ theItem = self.theProject.tree[tHandle] @@ -238,9 +238,8 @@ class NWIndex: logger.info("Not indexing non-file item '%s'", tHandle) return False - # Delete tags and create new item entry - for tTag in self._itemIndex.allItemTags(tHandle): - del self._tagsIndex[tTag] + # Keep a record of existing tags, and create a new item entry + itemTags = dict.fromkeys(self._itemIndex.allItemTags(tHandle), False) self._itemIndex.add(tHandle, theItem) # Run word counter for the whole text @@ -279,7 +278,7 @@ class NWIndex: nTitle = nLine elif aLine.startswith("@"): - self._indexKeyword(tHandle, aLine, nTitle, theItem.itemClass) + self._indexKeyword(tHandle, aLine, nTitle, theItem.itemClass, itemTags) elif aLine.startswith("%"): if nTitle > 0: @@ -300,6 +299,12 @@ class NWIndex: if nTitle == 0: self._indexWordCounts(tHandle, theText, nTitle) + # Prune no longer used tags + for tTag, isActive in itemTags.items(): + if not isActive: + logger.verbose("Deleting removed tag '%s'", tTag) + del self._tagsIndex[tTag] + # Update timestamps for index changes nowTime = round(time()) self._timeIndex = nowTime @@ -359,9 +364,11 @@ class NWIndex: self._itemIndex.setHeadingSynopsis(tHandle, sTitle, theText) return - def _indexKeyword(self, tHandle, aLine, nTitle, itemClass): + def _indexKeyword(self, tHandle, aLine, nTitle, itemClass, itemTags): """Validate and save the information about a reference to a tag - in another file. + in another file, or the setting of a tag in the file. A record + of active tags is updated so that no longer used tags can be + pruned later. """ isValid, theBits, _ = self.scanThis(aLine) if not isValid or len(theBits) < 2: @@ -374,8 +381,10 @@ class NWIndex: sTitle = f"T{nTitle:06d}" if theBits[0] == nwKeyWords.TAG_KEY: - self._tagsIndex.add(theBits[1], tHandle, sTitle, itemClass) - self._itemIndex.setHeadingTag(tHandle, sTitle, theBits[1]) + tagName = theBits[1] + self._tagsIndex.add(tagName, tHandle, sTitle, itemClass) + self._itemIndex.setHeadingTag(tHandle, sTitle, tagName) + itemTags[tagName] = True else: self._itemIndex.addHeadingReferences(tHandle, sTitle, theBits[1:], theBits[0]) @@ -884,6 +893,11 @@ class ItemIndex: class IndexItem: + """This object represents the index data of a project item (NWItem). + It holds a record of all the headings in the text, and the meta data + associated with each heading. It also holds a pointer to the project + item. + """ def __init__(self, tHandle, tItem): self._handle = tHandle @@ -1027,6 +1041,10 @@ class IndexItem: class IndexHeading: + """This object represents a section of text in a project item + associated with a single (valid) heading. It holds a separate record + of all references made under each heading. + """ def __init__(self, key, level="H0", title=""): self._key = key @@ -1148,7 +1166,7 @@ class IndexHeading: def packReferences(self): """Pack references into a dictionary for saving to cache. """ - return {key: list(value) for key, value in self._refs.items()} + return {key: ",".join(value) for key, value in self._refs.items()} def unpackData(self, data): """Unpack a heading entry from a dictionary. @@ -1170,9 +1188,9 @@ class IndexHeading: for tagKey, refTypes in data.items(): if not isinstance(tagKey, str): raise ValueError("itemIndex reference key must be a string") - if not isinstance(refTypes, list): - raise ValueError("itemIndex reference types must be a list") - for refType in refTypes: + if not isinstance(refTypes, str): + raise ValueError("itemIndex reference types must be a string") + for refType in refTypes.split(","): if refType in nwKeyWords.VALID_KEYS: self.addReference(tagKey, refType) else: diff --git a/sample/content/636b6aa9b697b.nwd b/sample/content/636b6aa9b697b.nwd index 20a66690..a334e8ce 100644 --- a/sample/content/636b6aa9b697b.nwd +++ b/sample/content/636b6aa9b697b.nwd @@ -4,7 +4,7 @@ ### Making a Scene @pov: Jane -@char: John +@char: John, Jane @location: Earth A scene is defined by a level three heading, like the one at the top of this page. The scene will be assigned to the chapter preceding it in the project tree. The scene document can be sorted after the chapter document, or as a child of the chapter. Both result in the same output in the end, so it is a matter of preference. diff --git a/sample/content/88706ddc78b1b.nwd b/sample/content/88706ddc78b1b.nwd index 0f140538..ceaddd29 100644 --- a/sample/content/88706ddc78b1b.nwd +++ b/sample/content/88706ddc78b1b.nwd @@ -11,6 +11,7 @@ ### Jane Cannot Find John @pov: Jane +@focus: John @location: Space Jane has been looking all over for John. He’s nowhere to be found on Earth, so Jane goes to space. diff --git a/sample/content/ae7339df26ded.nwd b/sample/content/ae7339df26ded.nwd index 1eb7a65d..8b53f816 100644 --- a/sample/content/ae7339df26ded.nwd +++ b/sample/content/ae7339df26ded.nwd @@ -4,6 +4,7 @@ ### We Found John! @pov: John +@focus: John @location: Mars Jane has been searching for a while, and she finally found John on Mars. He was indeed in space! What was he doing on Mars anyway? Well, it turns out, he was farming potatoes. diff --git a/sample/content/b3e74dbc1f584.nwd b/sample/content/b3e74dbc1f584.nwd index c980865e..bb88600b 100644 --- a/sample/content/b3e74dbc1f584.nwd +++ b/sample/content/b3e74dbc1f584.nwd @@ -6,4 +6,4 @@ @tag: Earth @location: Space -Third planet from the sun, fairly dense, and with lots of people on it. \ No newline at end of file +Third planet from the sun, fairly dense, and with lots of people on it. diff --git a/tests/reference/coreIndex_LoadSave_tagsIndex.json b/tests/reference/coreIndex_LoadSave_tagsIndex.json index fafdeb68..60c59d86 100644 --- a/tests/reference/coreIndex_LoadSave_tagsIndex.json +++ b/tests/reference/coreIndex_LoadSave_tagsIndex.json @@ -35,7 +35,7 @@ "T000001": {"level": "H2", "title": "Chapter One", "tag": "", "cCount": 419, "wCount": 67, "pCount": 1, "synopsis": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque at aliquam quam."} }, "references": { - "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + "T000001": {"Bod": "@pov", "Main": "@plot", "Europe": "@location"} } }, "88243afbe5ed8": { @@ -45,7 +45,7 @@ "T000013": {"level": "H4", "title": "Scene One, Section Two", "tag": "", "cCount": 1561, "wCount": 230, "pCount": 2, "synopsis": ""} }, "references": { - "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + "T000001": {"Bod": "@pov", "Main": "@plot", "Europe": "@location"} } }, "f96ec11c6a3da": { @@ -55,7 +55,7 @@ "T000015": {"level": "H4", "title": "Scene Two, Section Two", "tag": "", "cCount": 2009, "wCount": 301, "pCount": 3, "synopsis": ""} }, "references": { - "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + "T000001": {"Bod": "@pov", "Main": "@plot", "Europe": "@location"} } }, "846352075de7d": { @@ -70,7 +70,7 @@ "T000001": {"level": "H2", "title": "Chapter Two", "tag": "", "cCount": 477, "wCount": 70, "pCount": 1, "synopsis": "Curabitur a elit posuere, varius ex et, convallis neque. Phasellus sagittis pharetra sem vitae dapibus. Curabitur varius lorem non pulvinar congue."} }, "references": { - "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + "T000001": {"Bod": "@pov", "Main": "@plot", "Europe": "@location"} } }, "eb103bc70c90c": { @@ -79,7 +79,7 @@ "T000001": {"level": "H3", "title": "Scene Three", "tag": "", "cCount": 3006, "wCount": 439, "pCount": 4, "synopsis": "Aenean ut libero ut lectus porttitor rhoncus vel et massa. Nam pretium, nibh et varius vehicula, urna metus blandit eros, euismod pharetra diam diam et libero. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos."} }, "references": { - "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + "T000001": {"Bod": "@pov", "Main": "@plot", "Europe": "@location"} } }, "f8c0562e50f1b": { @@ -88,7 +88,7 @@ "T000001": {"level": "H3", "title": "Scene Four", "tag": "", "cCount": 3839, "wCount": 563, "pCount": 6, "synopsis": "Nam tempor blandit magna laoreet aliquet. Vestibulum auctor posuere leo, ac gravida nisi rhoncus varius. Aenean posuere dolor vitae condimentum volutpat. Donec egestas volutpat risus, quis luctus justo."} }, "references": { - "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + "T000001": {"Bod": "@pov", "Main": "@plot", "Europe": "@location"} } }, "47666c91c7ccf": { @@ -97,7 +97,7 @@ "T000001": {"level": "H3", "title": "Scene Five", "tag": "", "cCount": 3644, "wCount": 543, "pCount": 5, "synopsis": "Praesent eget est porta, dictum ante in, egestas risus. Mauris risus mauris, consequat aliquam mauris et, feugiat iaculis ipsum. Aliquam arcu ipsum, fermentum ut arcu sed, lobortis euismod sem. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus."} }, "references": { - "T000001": {"Bod": ["@pov"], "Main": ["@plot"], "Europe": ["@location"]} + "T000001": {"Bod": "@pov", "Main": "@plot", "Europe": "@location"} } }, "4c4f28287af27": { @@ -106,7 +106,7 @@ "T000001": {"level": "H1", "title": "Nobody Owens", "tag": "Bod", "cCount": 1864, "wCount": 284, "pCount": 3, "synopsis": ""} }, "references": { - "T000001": {"Main": ["@plot"]} + "T000001": {"Main": "@plot"} } }, "2426c6f0ca922": { From 63068f1018248b605d19c511b2fe1274c91186f5 Mon Sep 17 00:00:00 2001 From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com> Date: Sun, 5 Jun 2022 13:17:24 +0200 Subject: [PATCH 13/13] Add full test coverage of index class --- novelwriter/constants.py | 10 +- novelwriter/core/index.py | 44 +++-- tests/test_core/test_core_index.py | 293 ++++++++++++++++++++++++++--- 3 files changed, 305 insertions(+), 42 deletions(-) diff --git a/novelwriter/constants.py b/novelwriter/constants.py index 30071606..dc62dbd3 100644 --- a/novelwriter/constants.py +++ b/novelwriter/constants.py @@ -34,7 +34,7 @@ def trConst(tString): return QCoreApplication.translate("Constant", tString) -class nwConst(): +class nwConst: # Date and Time Formats FMT_TSTAMP = "%Y-%m-%d %H:%M:%S" # Default format @@ -48,7 +48,7 @@ class nwConst(): # END Class nwConst -class nwRegEx(): +class nwRegEx: FMT_EI = r"(?" + ## # Properties ## @@ -124,8 +125,7 @@ class NWIndex: logger.debug("Re-indexing item '%s'", tHandle) theDoc = NWDoc(self.theProject, tHandle) - theText = theDoc.readDocument() - self.scanText(tHandle, theText if theText is not None else "") + self.scanText(tHandle, theDoc.readDocument() or "") return True @@ -316,7 +316,7 @@ class NWIndex: return True ## - # Internal Indexers + # Internal Indexer Helpers ## def _indexTitle(self, tHandle, aLine, nTitle): @@ -613,11 +613,13 @@ class NWIndex: # =============================================================================================== # -# Indexer Objects +# The Tags Index Object # =============================================================================================== # class TagsIndex: - """A wrapper class that holds the reverse lookup tags index. + """A wrapper class that holds the reverse lookup tags index. This is + just a simple wrapper around a single dictionary to keep tighter + control of the keys. """ def __init__(self): @@ -719,8 +721,16 @@ class TagsIndex: # END Class TagsIndex +# =============================================================================================== # +# The Item Index Objects +# =============================================================================================== # + class ItemIndex: - """A wrapper object holding the indexed items. + """A wrapper object holding the indexed items. This is a warapper + class around a single storage dictionary with a set of utility + functions for setting and accessing the index data. Each indexed + item is stored in an IndexItem object, which again holds an + IndexHeading object for each header of the text. """ def __init__(self, theProject): @@ -896,7 +906,8 @@ class IndexItem: """This object represents the index data of a project item (NWItem). It holds a record of all the headings in the text, and the meta data associated with each heading. It also holds a pointer to the project - item. + item. The main heading level of the item is also held here since it + must be reset each time the item is re-indexed. """ def __init__(self, tHandle, tItem): @@ -912,7 +923,7 @@ class IndexItem: return def __repr__(self): - return f"" + return f"" ## # Properties @@ -1062,7 +1073,7 @@ class IndexHeading: return def __repr__(self): - return f"" + return f"" ## # Properties @@ -1165,8 +1176,11 @@ class IndexHeading: def packReferences(self): """Pack references into a dictionary for saving to cache. + Multiple types are packed into a sorted, comma separated string. + It is sorted to prevent creating unnecessary diffs as the order + of a set is not guaranteed. """ - return {key: ",".join(value) for key, value in self._refs.items()} + return {key: ",".join(sorted(list(value))) for key, value in self._refs.items()} def unpackData(self, data): """Unpack a heading entry from a dictionary. @@ -1189,7 +1203,7 @@ class IndexHeading: if not isinstance(tagKey, str): raise ValueError("itemIndex reference key must be a string") if not isinstance(refTypes, str): - raise ValueError("itemIndex reference types must be a string") + raise ValueError("itemIndex reference type must be a string") for refType in refTypes.split(","): if refType in nwKeyWords.VALID_KEYS: self.addReference(tagKey, refType) diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index b1233275..78361bb6 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -29,7 +29,7 @@ from mock import causeException from tools import buildTestProject, cmpFiles, writeFile from novelwriter.core.project import NWProject -from novelwriter.core.index import NWIndex, countWords +from novelwriter.core.index import NWIndex, countWords, TagsIndex from novelwriter.enum import nwItemClass, nwItemLayout @@ -46,6 +46,8 @@ def testCoreIndex_LoadSave(monkeypatch, nwLipsum, mockGUI, outDir, refDir): assert theProject.openProject(nwLipsum) theIndex = NWIndex(theProject) + assert repr(theIndex) == "" + notIndexable = { "b3643d0f92e32": False, # Novel ROOT "45e6b01ca35c1": False, # Chapter One FOLDER @@ -761,6 +763,166 @@ def testCoreIndex_ExtractData(mockGUI, fncDir, mockRnd): # END Test testCoreIndex_ExtractData +@pytest.mark.core +def testCoreIndex_TagsIndex(): + """Check the TagsIndex class. + """ + tagsIndex = TagsIndex() + assert tagsIndex._tags == {} + + # Expected data + content = { + "Tag1": { + "handle": "0000000000001", + "heading": "T000001", + "class": nwItemClass.NOVEL.name, + }, + "Tag2": { + "handle": "0000000000002", + "heading": "T000002", + "class": nwItemClass.CHARACTER.name, + }, + "Tag3": { + "handle": "0000000000003", + "heading": "T000003", + "class": nwItemClass.PLOT.name, + }, + } + + # Add data + tagsIndex.add("Tag1", "0000000000001", "T000001", nwItemClass.NOVEL) + tagsIndex.add("Tag2", "0000000000002", "T000002", nwItemClass.CHARACTER) + tagsIndex.add("Tag3", "0000000000003", "T000003", nwItemClass.PLOT) + assert tagsIndex._tags == content + + # Get items + assert tagsIndex["Tag1"] == content["Tag1"] + assert tagsIndex["Tag2"] == content["Tag2"] + assert tagsIndex["Tag3"] == content["Tag3"] + assert tagsIndex["Tag4"] is None + + # Contains + assert "Tag1" in tagsIndex + assert "Tag2" in tagsIndex + assert "Tag3" in tagsIndex + assert "Tag4" not in tagsIndex + + # Read back handles + assert tagsIndex.tagHandle("Tag1") == "0000000000001" + assert tagsIndex.tagHandle("Tag2") == "0000000000002" + assert tagsIndex.tagHandle("Tag3") == "0000000000003" + assert tagsIndex.tagHandle("Tag4") is None + + # Read back headings + assert tagsIndex.tagHeading("Tag1") == "T000001" + assert tagsIndex.tagHeading("Tag2") == "T000002" + assert tagsIndex.tagHeading("Tag3") == "T000003" + assert tagsIndex.tagHeading("Tag4") == "T000000" + + # Read back classes + assert tagsIndex.tagClass("Tag1") == nwItemClass.NOVEL.name + assert tagsIndex.tagClass("Tag2") == nwItemClass.CHARACTER.name + assert tagsIndex.tagClass("Tag3") == nwItemClass.PLOT.name + assert tagsIndex.tagClass("Tag4") is None + + # Pack Data + assert tagsIndex.packData() == content + + # Delete the second key and a nomn-existant key + del tagsIndex["Tag2"] + del tagsIndex["Tag4"] + assert "Tag1" in tagsIndex + assert "Tag2" not in tagsIndex + assert "Tag3" in tagsIndex + assert "Tag4" not in tagsIndex + + # Clear and reload + tagsIndex.clear() + assert tagsIndex._tags == {} + assert tagsIndex.packData() == {} + + tagsIndex.unpackData(content) + assert tagsIndex._tags == content + assert tagsIndex.packData() == content + + # Unpack Errors + # ============= + tagsIndex.clear() + + # Invalid data type + with pytest.raises(ValueError): + tagsIndex.unpackData([]) + + # Invalid key + with pytest.raises(ValueError): + tagsIndex.unpackData({ + 1234: { + "handle": "0000000000001", + "heading": "T000001", + "class": "NOVEL", + } + }) + + # Missing handle + with pytest.raises(KeyError): + tagsIndex.unpackData({ + "Tag1": { + "heading": "T000001", + "class": "NOVEL", + } + }) + + # Missing heading + with pytest.raises(KeyError): + tagsIndex.unpackData({ + "Tag1": { + "handle": "0000000000001", + "class": "NOVEL", + } + }) + + # Missing class + with pytest.raises(KeyError): + tagsIndex.unpackData({ + "Tag1": { + "handle": "0000000000001", + "heading": "T000001", + } + }) + + # Invalid handle + with pytest.raises(ValueError): + tagsIndex.unpackData({ + "Tag1": { + "handle": "blablabla", + "heading": "T000001", + "class": "NOVEL", + } + }) + + # Invalid heading + with pytest.raises(ValueError): + tagsIndex.unpackData({ + "Tag1": { + "handle": "0000000000001", + "heading": "blabla", + "class": "NOVEL", + } + }) + + # Invalid class + with pytest.raises(ValueError): + tagsIndex.unpackData({ + "Tag1": { + "handle": "0000000000001", + "heading": "T000001", + "class": "blabla", + } + }) + +# END Test testCoreIndex_TagsIndex + + @pytest.mark.core def testCoreIndex_ItemIndex(mockGUI, fncDir, mockRnd): """Check the ItemIndex class. @@ -780,26 +942,6 @@ def testCoreIndex_ItemIndex(mockGUI, fncDir, mockRnd): assert cHandle not in itemIndex assert sHandle not in itemIndex - # Unpack Data - # =========== - - # Data must be dictionary - with pytest.raises(ValueError): - itemIndex.unpackData("stuff") - - # Keys must be valid handles - with pytest.raises(ValueError): - itemIndex.unpackData({"stuff": "more stuff"}) - - # Unknown keys should be skipped - itemIndex.unpackData({"0000000000000": {}}) - assert itemIndex._items == {} - - # Known keys can be added, even witout data - itemIndex.unpackData({nHandle: {}}) - assert nHandle in itemIndex - itemIndex.clear() - # Add Items # ========= assert cHandle not in itemIndex @@ -820,7 +962,7 @@ def testCoreIndex_ItemIndex(mockGUI, fncDir, mockRnd): # Set the remainig data values itemIndex.setHeadingCounts(cHandle, "T000001", 60, 10, 2) itemIndex.setHeadingSynopsis(cHandle, "T000001", "In the beginning ...") - itemIndex.setHeadingTag(cHandle, "T000001", "One") # Although it isn't allowed to have a tag + itemIndex.setHeadingTag(cHandle, "T000001", "One") itemIndex.addHeadingReferences(cHandle, "T000001", ["Jane"], "@pov") itemIndex.addHeadingReferences(cHandle, "T000001", ["Jane"], "@focus") itemIndex.addHeadingReferences(cHandle, "T000001", ["Jane", "John"], "@char") @@ -842,6 +984,37 @@ def testCoreIndex_ItemIndex(mockGUI, fncDir, mockRnd): itemIndex.addItemHeading(nHandle, "T000001", "H1", "Novel") itemIndex.addItemHeading(sHandle, "T000001", "H3", "Scene One") + # Check Item and Heading Direct Access + # ==================================== + + # Check repr strings + assert repr(itemIndex[nHandle]) == f"" + assert repr(itemIndex[nHandle]["T000001"]) == "" + + # Check content of a single item + assert "T000001" in itemIndex[nHandle] + assert itemIndex[cHandle].allTags() == ["One"] + + # Check the content of a single heading + assert itemIndex[cHandle]["T000001"].key == "T000001" + assert itemIndex[cHandle]["T000001"].level == "H2" + assert itemIndex[cHandle]["T000001"].title == "Chapter One" + assert itemIndex[cHandle]["T000001"].tag == "One" + assert itemIndex[cHandle]["T000001"].charCount == 60 + assert itemIndex[cHandle]["T000001"].wordCount == 10 + assert itemIndex[cHandle]["T000001"].paraCount == 2 + assert itemIndex[cHandle]["T000001"].synopsis == "In the beginning ..." + assert "Jane" in itemIndex[cHandle]["T000001"].references + assert "John" in itemIndex[cHandle]["T000001"].references + + # Check heading level setter + itemIndex[cHandle]["T000001"].setLevel("H3") # Change it + assert itemIndex[cHandle]["T000001"].level == "H3" + itemIndex[cHandle]["T000001"].setLevel("H2") # Set it back + assert itemIndex[cHandle]["T000001"].level == "H2" + itemIndex[cHandle]["T000001"].setLevel("H5") # Invalid level + assert itemIndex[cHandle]["T000001"].level == "H2" + # Data Extraction # =============== @@ -908,6 +1081,82 @@ def testCoreIndex_ItemIndex(mockGUI, fncDir, mockRnd): del itemIndex[uHandle] assert uHandle not in itemIndex + # Unpack Error Handling + # ===================== + + # Pack/unpack should restore state + content = itemIndex.packData() + itemIndex.clear() + itemIndex.unpackData(content) + assert itemIndex.packData() == content + itemIndex.clear() + + # Data must be dictionary + with pytest.raises(ValueError): + itemIndex.unpackData("stuff") + + # Keys must be valid handles + with pytest.raises(ValueError): + itemIndex.unpackData({"stuff": "more stuff"}) + + # Unknown keys should be skipped + itemIndex.unpackData({"0000000000000": {}}) + assert itemIndex._items == {} + + # Known keys can be added, even witout data + itemIndex.unpackData({nHandle: {}}) + assert nHandle in itemIndex + + # Title tags must be valid + with pytest.raises(ValueError): + itemIndex.unpackData({cHandle: {"headings": {"TTTTTTT": {}}}}) + + # Reference without a heading should be rejected + itemIndex.unpackData({ + cHandle: { + "headings": {"T000001": {}}, + "references": {"T000001": {}, "T000002": {}}, + } + }) + assert "T000001" in itemIndex[cHandle] + assert "T000002" not in itemIndex[cHandle] + itemIndex.clear() + + # Tag keys must be strings + with pytest.raises(ValueError): + itemIndex.unpackData({ + cHandle: { + "headings": {"T000001": {}}, + "references": {"T000001": {1234: "@pov"}}, + } + }) + + # Type must be strings + with pytest.raises(ValueError): + itemIndex.unpackData({ + cHandle: { + "headings": {"T000001": {}}, + "references": {"T000001": {"John": []}}, + } + }) + + # Types must be valid + with pytest.raises(ValueError): + itemIndex.unpackData({ + cHandle: { + "headings": {"T000001": {}}, + "references": {"T000001": {"John": "@pov,@char,@stuff"}}, + } + }) + + # This should pass + itemIndex.unpackData({ + cHandle: { + "headings": {"T000001": {}}, + "references": {"T000001": {"John": "@pov,@char"}}, + } + }) + # END Test testCoreIndex_ItemIndex