From efdf88be66bcc56bc8e5bc24e089c9485b18de69 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Mon, 5 Oct 2020 20:54:19 +0200
Subject: [PATCH 1/9] Drop the doceditor reloadText function as it resets undo
stack
---
nw/gui/doceditor.py | 26 ++++++++++++++++----------
nw/guimain.py | 4 ++--
2 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/nw/gui/doceditor.py b/nw/gui/doceditor.py
index 1410c7b4..14f1db91 100644
--- a/nw/gui/doceditor.py
+++ b/nw/gui/doceditor.py
@@ -223,21 +223,12 @@ class GuiDocEditor(QTextEdit):
# font changed, otherwise we just clear the editor entirely,
# which makes it read only.
if self.theHandle is not None:
- self.reloadText()
+ self.redrawText()
else:
self.clearEditor()
return True
- def reloadText(self):
- """Reloads the document currently being edited.
- """
- if self.theHandle is not None:
- tHandle = self.theHandle
- self.clearEditor()
- self.loadText(tHandle, showStatus=False)
- return
-
def loadText(self, tHandle, tLine=None, showStatus=True):
"""Load text from a document into the editor. If we have an io
error, we must handle this and clear the editor so that we don't
@@ -297,6 +288,21 @@ class GuiDocEditor(QTextEdit):
return True
+ def reHighLightText(self, forceBigDoc=False):
+ """Run the syntax highlighter again.
+ """
+ if not self.bigDoc or forceBigDoc:
+ qApp.setOverrideCursor(QCursor(Qt.WaitCursor))
+ self.hLight.rehighlight()
+ qApp.restoreOverrideCursor()
+ return
+
+ def redrawText(self):
+ """Redraw the text by marking the document content as "dirty".
+ """
+ self.qDocument.markContentsDirty(0, self.qDocument.characterCount())
+ return
+
def replaceText(self, theText):
"""Replaces the text of the current document with the provided
text. This also clears undo history.
diff --git a/nw/guimain.py b/nw/guimain.py
index 5f237bb7..62b2db74 100644
--- a/nw/guimain.py
+++ b/nw/guimain.py
@@ -716,13 +716,13 @@ class GuiMain(QMainWindow):
tEnd = time()
self.statusBar.setStatus("Indexing completed in %.1f ms" % ((tEnd - tStart)*1000.0))
- self.docEditor.reloadText()
-
qApp.restoreOverrideCursor()
if not beQuiet:
self.makeAlert("The project index has been successfully rebuilt.", nwAlert.INFO)
+ self.docEditor.reHighLightText()
+
return True
def rebuildOutline(self):
From 003d28f9dbac8c6001455fd62c5c69d60ad63cfa Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Mon, 5 Oct 2020 22:55:21 +0200
Subject: [PATCH 2/9] Updating the index now only rehighlights lines with meta
keywords
---
nw/gui/doceditor.py | 9 +++------
nw/gui/dochighlight.py | 29 +++++++++++++++++++++++++++++
nw/guimain.py | 3 +--
3 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/nw/gui/doceditor.py b/nw/gui/doceditor.py
index 14f1db91..88b6b4bf 100644
--- a/nw/gui/doceditor.py
+++ b/nw/gui/doceditor.py
@@ -288,13 +288,10 @@ class GuiDocEditor(QTextEdit):
return True
- def reHighLightText(self, forceBigDoc=False):
- """Run the syntax highlighter again.
+ def updateTagHighLighting(self, forceBigDoc=False):
+ """Rerun the syntax highlighter on all meta data lines.
"""
- if not self.bigDoc or forceBigDoc:
- qApp.setOverrideCursor(QCursor(Qt.WaitCursor))
- self.hLight.rehighlight()
- qApp.restoreOverrideCursor()
+ self.hLight.rehighlightByType(GuiDocHighlighter.BLOCK_META)
return
def redrawText(self):
diff --git a/nw/gui/dochighlight.py b/nw/gui/dochighlight.py
index 7b95ffeb..663732b4 100644
--- a/nw/gui/dochighlight.py
+++ b/nw/gui/dochighlight.py
@@ -39,6 +39,11 @@ logger = logging.getLogger(__name__)
class GuiDocHighlighter(QSyntaxHighlighter):
+ BLOCK_NONE = 0
+ BLOCK_TEXT = 1
+ BLOCK_META = 2
+ BLOCK_TITLE = 4
+
def __init__(self, theDoc, theParent):
QSyntaxHighlighter.__init__(self, theDoc)
@@ -229,6 +234,22 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self.theHandle = theHandle
return True
+ ##
+ # Methods
+ ##
+
+ def rehighlightByType(self, theType):
+ """Loop through all blocks and rehighlight those of a given
+ content type.
+ """
+ qDocument = self.document()
+ nBlocks = qDocument.blockCount()
+ for i in range(nBlocks):
+ theBlock = qDocument.findBlockByNumber(i)
+ if theBlock.userState() & theType == theType:
+ self.rehighlightBlock(theBlock)
+ return
+
##
# Highlight Block
##
@@ -239,10 +260,12 @@ class GuiDocHighlighter(QSyntaxHighlighter):
is significantly faster than running the regex checks used for
text paragraphs.
"""
+ self.setCurrentBlockState(self.BLOCK_NONE)
if self.theHandle is None or not theText:
return
if theText.startswith("@"): # Keywords and commands
+ self.setCurrentBlockState(self.BLOCK_META)
tItem = self.theParent.theProject.projTree[self.theHandle]
isValid, theBits, thePos = self.theIndex.scanThis(theText)
isGood = self.theIndex.checkThese(theBits, tItem)
@@ -266,22 +289,27 @@ class GuiDocHighlighter(QSyntaxHighlighter):
return
elif theText.startswith("# "): # Header 1
+ self.setCurrentBlockState(self.BLOCK_TITLE)
self.setFormat(0, 1, self.hStyles["header1h"])
self.setFormat(1, len(theText), self.hStyles["header1"])
elif theText.startswith("## "): # Header 2
+ self.setCurrentBlockState(self.BLOCK_TITLE)
self.setFormat(0, 2, self.hStyles["header2h"])
self.setFormat(2, len(theText), self.hStyles["header2"])
elif theText.startswith("### "): # Header 3
+ self.setCurrentBlockState(self.BLOCK_TITLE)
self.setFormat(0, 3, self.hStyles["header3h"])
self.setFormat(3, len(theText), self.hStyles["header3"])
elif theText.startswith("#### "): # Header 4
+ self.setCurrentBlockState(self.BLOCK_TITLE)
self.setFormat(0, 4, self.hStyles["header4h"])
self.setFormat(4, len(theText), self.hStyles["header4"])
elif theText.startswith("%"): # Comments
+ self.setCurrentBlockState(self.BLOCK_TEXT)
toCheck = theText[1:].lstrip()
synTag = toCheck[:9].lower()
tLen = len(theText)
@@ -294,6 +322,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
self.setFormat(0, tLen, self.hStyles["hidden"])
else: # Text Paragraph
+ self.setCurrentBlockState(self.BLOCK_TEXT)
for rX, xFmt in self.rxRules:
rxItt = rX.globalMatch(theText, 0)
while rxItt.hasNext():
diff --git a/nw/guimain.py b/nw/guimain.py
index 62b2db74..3d60f349 100644
--- a/nw/guimain.py
+++ b/nw/guimain.py
@@ -716,13 +716,12 @@ class GuiMain(QMainWindow):
tEnd = time()
self.statusBar.setStatus("Indexing completed in %.1f ms" % ((tEnd - tStart)*1000.0))
+ self.docEditor.updateTagHighLighting()
qApp.restoreOverrideCursor()
if not beQuiet:
self.makeAlert("The project index has been successfully rebuilt.", nwAlert.INFO)
- self.docEditor.reHighLightText()
-
return True
def rebuildOutline(self):
From 93851d84bf35542147ea0928956e02ffbad38bb4 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 01:14:14 +0200
Subject: [PATCH 3/9] Delayed cursor move for large documents speeds up loading
significantly
---
nw/gui/doceditor.py | 98 +++++++++++++++++++++++++++++++++------------
nw/guimain.py | 1 +
2 files changed, 73 insertions(+), 26 deletions(-)
diff --git a/nw/gui/doceditor.py b/nw/gui/doceditor.py
index 88b6b4bf..ab317efb 100644
--- a/nw/gui/doceditor.py
+++ b/nw/gui/doceditor.py
@@ -36,7 +36,7 @@ import logging
from time import time
from PyQt5.QtCore import (
- Qt, QSize, QThread, QTimer, pyqtSlot, QRegExp, QRegularExpression
+ Qt, QSize, QThread, QTimer, pyqtSlot, QRegExp, QRegularExpression, QPointF
)
from PyQt5.QtGui import (
QTextCursor, QTextOption, QKeySequence, QFont, QColor, QPalette,
@@ -69,21 +69,23 @@ class GuiDocEditor(QTextEdit):
self.theParent = theParent
self.theTheme = theParent.theTheme
self.theProject = theParent.theProject
- self.docChanged = False
- self.spellCheck = False
self.nwDocument = NWDoc(self.theProject, self.theParent)
- self.theHandle = None
- self.theDict = None
+
+ self.docChanged = False # Flag for changed status of document
+ self.spellCheck = False # Flag for spell checking enabled
+ self.theHandle = None # The handle of the open file
+ self.theDict = None # The current spell check dictionary
+ self.nonWord = "\"'" # Characters to not include in spell checking
# Document Variables
- self.charCount = 0
- self.wordCount = 0
- self.paraCount = 0
- self.lastEdit = 0
- self.lastFind = None
- self.bigDoc = False
- self.doReplace = False
- self.nonWord = "\"'"
+ self.charCount = 0 # Character count
+ self.wordCount = 0 # Word count
+ self.paraCount = 0 # Paragraph count
+ self.lastEdit = 0 # Time stamp of last edit
+ self.lastFind = None # Position of the last found search word
+ self.bigDoc = False # Flag for very large document size
+ self.doReplace = False # Switch to temporarily disable auto-replace
+ self.queuePos = None # Used for delayed change of cursor position
# Typography
self.typDQOpen = self.mainConf.fmtDoubleQuotes[0]
@@ -94,6 +96,7 @@ class GuiDocEditor(QTextEdit):
# Core Elements and Signals
self.qDocument = self.document()
self.qDocument.contentsChange.connect(self._docChange)
+ self.qDocument.documentLayout().documentSizeChanged.connect(self._docSizeChanged)
# Document Title
self.docHeader = GuiDocEditHeader(self)
@@ -161,8 +164,10 @@ class GuiDocEditor(QTextEdit):
self.wordCount = 0
self.paraCount = 0
self.lastEdit = 0
+ self.lastFind = None
self.bigDoc = False
self.doReplace = False
+ self.queuePos = None
self.setDocumentChanged(False)
self.docHeader.setTitleFromHandle(self.theHandle)
@@ -216,6 +221,12 @@ class GuiDocEditor(QTextEdit):
self.qDocument.setDefaultTextOption(theOpt)
+ # Refresh the tab stops
+ if self.mainConf.verQtValue >= 51000:
+ self.setTabStopDistance(self.mainConf.getTabWidth())
+ else:
+ self.setTabStopWidth(self.mainConf.getTabWidth())
+
# Initialise the syntax highlighter
self.hLight.initHighlighter()
@@ -249,7 +260,8 @@ class GuiDocEditor(QTextEdit):
# Check that the document is not too big for full, initial spell
# checking. If it is too big, we switch to only check as we type
- self._checkDocSize(len(theDoc))
+ docSize = len(theDoc)
+ self._checkDocSize(docSize)
spTemp = self.hLight.spellCheck
if self.bigDoc:
self.hLight.spellCheck = False
@@ -257,16 +269,12 @@ class GuiDocEditor(QTextEdit):
bfTime = time()
self._allowAutoReplace(False)
self.setPlainText(theDoc)
+ qApp.processEvents()
+
self._allowAutoReplace(True)
afTime = time()
logger.debug("Document highlighted in %.3f milliseconds" % (1000*(afTime-bfTime)))
- theItem = self.nwDocument.getCurrentItem()
- if tLine is None and theItem is not None:
- self.setCursorPosition(theItem.cursorPos)
- else:
- self.setCursorLine(tLine)
-
self.lastEdit = time()
self._runCounter()
self.wcTimer.start()
@@ -280,11 +288,18 @@ class GuiDocEditor(QTextEdit):
self.hLight.spellCheck = spTemp
qApp.restoreOverrideCursor()
- # Refresh the tab stops
- if self.mainConf.verQtValue >= 51000:
- self.setTabStopDistance(self.mainConf.getTabWidth())
+ theItem = self.nwDocument.getCurrentItem()
+ if tLine is None and theItem is not None:
+ # For large documents we queue the repositioning until the
+ # document layout has grown past the point we want to move
+ # the cursor to. This makes the loading significantly
+ # faster.
+ if docSize > 50000:
+ self.queuePos = theItem.cursorPos
+ else:
+ self.setCursorPosition(theItem.cursorPos)
else:
- self.setTabStopWidth(self.mainConf.getTabWidth())
+ self.setCursorLine(tLine)
return True
@@ -318,11 +333,10 @@ class GuiDocEditor(QTextEdit):
return False
docText = self.getText()
- cursPos = self.getCursorPosition()
theItem.setCharCount(self.charCount)
theItem.setWordCount(self.wordCount)
theItem.setParaCount(self.paraCount)
- theItem.setCursorPos(cursPos)
+ self.saveCursorPosition()
self.nwDocument.saveDocument(docText)
self.setDocumentChanged(False)
@@ -431,6 +445,15 @@ class GuiDocEditor(QTextEdit):
"""
return self.textCursor().selectionEnd()
+ def saveCursorPosition(self):
+ """Save the cursor position to the current project otem.
+ """
+ theItem = self.nwDocument.getCurrentItem()
+ if theItem is not None:
+ cursPos = self.getCursorPosition()
+ theItem.setCursorPos(cursPos)
+ return
+
def setCursorLine(self, theLine):
"""Move the cursor to a given line in the document.
"""
@@ -898,6 +921,29 @@ class GuiDocEditor(QTextEdit):
return
+ @pyqtSlot("QSizeF")
+ def _docSizeChanged(self, theSize):
+ """Called whenever the underlying document layout size changes.
+ This is used to queue the repositioning of the cursor for very
+ large documents to ensure the region where the cursor is being
+ moved to has been drawn before the move is made.
+ """
+ if self.queuePos is not None:
+ thePos = self.qDocument.documentLayout().hitTest(
+ QPointF(theSize.width(), theSize.height()), Qt.FuzzyHit
+ )
+ if self.queuePos <= thePos:
+ logger.verbose(
+ "Allowed cursor move to %d <= %d" % (self.queuePos, thePos)
+ )
+ self.setCursorPosition(self.queuePos)
+ self.queuePos = None
+ else:
+ logger.verbose(
+ "Denied cursor move to %d > %d" % (self.queuePos, thePos)
+ )
+ return
+
##
# Internal Functions
##
diff --git a/nw/guimain.py b/nw/guimain.py
index 3d60f349..5ee6e9b7 100644
--- a/nw/guimain.py
+++ b/nw/guimain.py
@@ -448,6 +448,7 @@ class GuiMain(QMainWindow):
"""Close the document and clear the editor and title field.
"""
if self.hasProject:
+ self.docEditor.saveCursorPosition()
if self.docEditor.docChanged:
self.saveDocument()
self.docEditor.clearEditor()
From dceb5e62c63ab304dbd9705fb9f8d716ed9cff40 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 01:37:30 +0200
Subject: [PATCH 4/9] Fixed a typo and downgraded some error messages in the
index class to info
---
nw/core/index.py | 12 ++++++------
nw/gui/doceditor.py | 2 +-
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/nw/core/index.py b/nw/core/index.py
index 8925b07e..f715d835 100644
--- a/nw/core/index.py
+++ b/nw/core/index.py
@@ -271,16 +271,16 @@ class NWIndex():
theRoot = self.theProject.projTree.getRootItem(tHandle)
if theItem is None:
- logger.error("Not indexing unknown item %s" % tHandle)
+ logger.info("Not indexing unknown item %s" % tHandle)
return False
if theItem.itemType != nwItemType.FILE:
- logger.error("Not indexing non-file item %s" % tHandle)
+ logger.info("Not indexing non-file item %s" % tHandle)
return False
if theItem.itemLayout == nwItemLayout.NO_LAYOUT:
- logger.error("Not indexing no-layout item %s" % tHandle)
+ logger.info("Not indexing no-layout item %s" % tHandle)
return False
if theItem.parHandle is None:
- logger.error("Not indexing orphaned item %s" % tHandle)
+ logger.info("Not indexing orphaned item %s" % tHandle)
return False
# Run word counter for the whole text
@@ -289,10 +289,10 @@ class NWIndex():
# If the file is archived or trashed, we don't index the file itself
if self.theProject.projTree.isTrashRoot(theItem.parHandle):
- logger.error("Not indexing trash item %s" % tHandle)
+ logger.info("Not indexing trash item %s" % tHandle)
return False
if theRoot.itemClass == nwItemClass.ARCHIVE:
- logger.error("Not indexing archived item %s" % tHandle)
+ logger.info("Not indexing archived item %s" % tHandle)
return False
itemClass = theItem.itemClass
diff --git a/nw/gui/doceditor.py b/nw/gui/doceditor.py
index ab317efb..319abf1e 100644
--- a/nw/gui/doceditor.py
+++ b/nw/gui/doceditor.py
@@ -446,7 +446,7 @@ class GuiDocEditor(QTextEdit):
return self.textCursor().selectionEnd()
def saveCursorPosition(self):
- """Save the cursor position to the current project otem.
+ """Save the cursor position to the current project item object.
"""
theItem = self.nwDocument.getCurrentItem()
if theItem is not None:
From b3b3c581f0984de89888615e18188e0ce49108c7 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 13:18:08 +0200
Subject: [PATCH 5/9] Speedup of word counter for very large documents
---
nw/core/tools.py | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/nw/core/tools.py b/nw/core/tools.py
index ce30de96..59c141d9 100644
--- a/nw/core/tools.py
+++ b/nw/core/tools.py
@@ -29,6 +29,8 @@
import logging
+from nw.constants import nwUnicode
+
logger = logging.getLogger(__name__)
# =============================================================================================== #
@@ -44,6 +46,15 @@ def countWords(theText):
paraCount = 0
prevEmpty = True
+ # We need to treat dashes as word separators for counting words.
+ # The check+replace apprach is much faster that direct replace for
+ # large texts, and a bit slower for small texts, but in the latter
+ # case it doesn't matter.
+ if nwUnicode.U_ENDASH in theText:
+ theText = theText.replace(nwUnicode.U_ENDASH, " ")
+ if nwUnicode.U_EMDASH in theText:
+ theText = theText.replace(nwUnicode.U_EMDASH, " ")
+
for aLine in theText.splitlines():
countPara = True
@@ -72,8 +83,7 @@ def countWords(theText):
charCount -= 2
countPara = False
- theBuff = aLine.replace("–", " ").replace("—", " ")
- wordCount += len(theBuff.split())
+ wordCount += len(aLine.split())
charCount += theLen
if countPara and prevEmpty:
paraCount += 1
From 3893892927456477b13e639484a833e232ce17a8 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 13:18:42 +0200
Subject: [PATCH 6/9] Enforce a maximum document size of the document editor of
5 MB
---
nw/constants/constants.py | 4 ++-
nw/gui/doceditor.py | 63 +++++++++++++++++++++++++++++++--------
2 files changed, 54 insertions(+), 13 deletions(-)
diff --git a/nw/constants/constants.py b/nw/constants/constants.py
index 358b860e..9a539b73 100644
--- a/nw/constants/constants.py
+++ b/nw/constants/constants.py
@@ -33,7 +33,9 @@ class nwConst():
fStampFmt = "%Y-%m-%d %H.%M.%S" # FileName safe format
dStampFmt = "%Y-%m-%d" # Date only format
- maxDepth = 30 # Maximum folder depth of a project
+ maxDepth = 30 # Maximum folder depth of a project
+ maxDocSize = 5000000 # Maxium size of a single document
+ maxBuildSize = 10000000 # Maxium size of a project build
# END Class nwConst
diff --git a/nw/gui/doceditor.py b/nw/gui/doceditor.py
index 319abf1e..7e83461a 100644
--- a/nw/gui/doceditor.py
+++ b/nw/gui/doceditor.py
@@ -52,7 +52,7 @@ from nw.core import NWDoc, NWSpellCheck, NWSpellSimple, countWords
from nw.gui.dochighlight import GuiDocHighlighter
from nw.common import transferCase
from nw.constants import (
- nwAlert, nwUnicode, nwDocAction, nwDocInsert, nwItemClass
+ nwConst, nwAlert, nwUnicode, nwDocAction, nwDocInsert, nwItemClass
)
logger = logging.getLogger(__name__)
@@ -255,12 +255,21 @@ class GuiDocEditor(QTextEdit):
self.clearEditor()
return False
+ docSize = len(theDoc)
+ if docSize > nwConst.maxDocSize:
+ self.theParent.makeAlert((
+ "The document you are trying to open is too big. "
+ "The document size is %.2f\u202fMB. "
+ "The maximum size allowed is %.2f\u202fMB."
+ ) % (docSize/1.0e6, nwConst.maxDocSize/1.0e6), nwAlert.ERROR)
+ self.clearEditor()
+ return False
+
qApp.setOverrideCursor(QCursor(Qt.WaitCursor))
self.hLight.setHandle(tHandle)
# Check that the document is not too big for full, initial spell
# checking. If it is too big, we switch to only check as we type
- docSize = len(theDoc)
self._checkDocSize(docSize)
spTemp = self.hLight.spellCheck
if self.bigDoc:
@@ -286,7 +295,6 @@ class GuiDocEditor(QTextEdit):
self.docFooter.setHandle(self.theHandle)
self.updateDocMargins()
self.hLight.spellCheck = spTemp
- qApp.restoreOverrideCursor()
theItem = self.nwDocument.getCurrentItem()
if tLine is None and theItem is not None:
@@ -301,6 +309,8 @@ class GuiDocEditor(QTextEdit):
else:
self.setCursorLine(tLine)
+ qApp.restoreOverrideCursor()
+
return True
def updateTagHighLighting(self, forceBigDoc=False):
@@ -319,10 +329,20 @@ class GuiDocEditor(QTextEdit):
"""Replaces the text of the current document with the provided
text. This also clears undo history.
"""
+ docSize = len(theText)
+ if docSize > nwConst.maxDocSize:
+ self.theParent.makeAlert((
+ "The text you are trying to add is too big. "
+ "The text size is %.2f\u202fMB. "
+ "The maximum size allowed is %.2f\u202fMB."
+ ) % (docSize/1.0e6, nwConst.maxDocSize/1.0e6), nwAlert.ERROR)
+ return False
+
self.setPlainText(theText)
self.setDocumentChanged(True)
self.updateDocMargins()
- return
+
+ return True
def saveText(self):
"""Save the text currently in the editor to the NWDoc object,
@@ -753,6 +773,13 @@ class GuiDocEditor(QTextEdit):
"""
self.lastEdit = time()
self.lastFind = None
+ if self.qDocument.characterCount() > nwConst.maxDocSize:
+ self.theParent.makeAlert((
+ "The document has grown too big and you cannot add more text to it. "
+ "The maximum size of a single novelWriter document is %.2f\u202fMB."
+ ) % (nwConst.maxDocSize/1.0e6), nwAlert.ERROR)
+ self.undo()
+ return
if not self.docChanged:
self.setDocumentChanged(True)
if not self.wcTimer.isActive():
@@ -1099,15 +1126,24 @@ class GuiDocEditor(QTextEdit):
"""Check if document size crosses the big document limit set in
config. If so, we will set the big document flag to True.
"""
- if theSize > self.mainConf.bigDocLimit*1000:
- logger.info(
- "The document size is %d > %d, big doc mode is enabled" % (
- theSize, self.mainConf.bigDocLimit*1000
+ newState = theSize > self.mainConf.bigDocLimit*1000
+
+ if newState != self.bigDoc:
+ if newState:
+ logger.info(
+ "The document size is {:n} > {:n}, big doc mode has been enabled".format(
+ theSize, self.mainConf.bigDocLimit*1000
+ )
)
- )
- self.bigDoc = True
- else:
- self.bigDoc = False
+ else:
+ logger.info(
+ "The document size is {:n} <= {:n}, big doc mode has been disabled".format(
+ theSize, self.mainConf.bigDocLimit*1000
+ )
+ )
+
+ self.bigDoc = newState
+
return
def _wrapSelection(self, tBefore, tAfter=None):
@@ -2227,6 +2263,9 @@ class GuiDocEditFooter(QWidget):
self.wordsText.setText("Words: {:n} ({:+n})".format(wCount, wDiff))
+ byteSize = self.docEditor.qDocument.characterCount()
+ self.wordsText.setToolTip("Document size is {:n} bytes".format(byteSize))
+
return
# END Class GuiDocEditFooter
From 23d5e33864a844436a19cb78d211816f49dfbc3e Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 14:29:33 +0200
Subject: [PATCH 7/9] Added large size handling to build tool
---
nw/core/tokenizer.py | 18 +++++++++++++++++-
nw/gui/build.py | 44 ++++++++++++++++++++++++++++++++++++++++----
2 files changed, 57 insertions(+), 5 deletions(-)
diff --git a/nw/core/tokenizer.py b/nw/core/tokenizer.py
index eb6b98d8..2f172ca7 100644
--- a/nw/core/tokenizer.py
+++ b/nw/core/tokenizer.py
@@ -33,7 +33,7 @@ from PyQt5.QtCore import QRegularExpression
from nw.core.document import NWDoc
from nw.core.tools import numberToWord, numberToRoman
-from nw.constants import nwItemLayout, nwItemType, nwRegEx
+from nw.constants import nwConst, nwItemLayout, nwItemType, nwRegEx
logger = logging.getLogger(__name__)
@@ -120,6 +120,9 @@ class Tokenizer():
self.isNote = False
self.isNovel = False
+ # Error Handling
+ self.errData = []
+
return
##
@@ -212,6 +215,14 @@ class Tokenizer():
theDocument = NWDoc(self.theProject, self.theParent)
self.theText = theDocument.openDocument(theHandle)
+ docSize = len(self.theText)
+ if docSize > nwConst.maxDocSize:
+ errVal = "Document '%s' is too big (%.2f MB). Skipping." % (
+ self.theItem.itemName, docSize/1.0e6
+ )
+ self.theText = "# ERROR\n\n%s\n\n" % errVal
+ self.errData.append(errVal)
+
self.isNone = self.theItem.itemLayout == nwItemLayout.NO_LAYOUT
self.isTitle = self.theItem.itemLayout == nwItemLayout.TITLE
self.isBook = self.theItem.itemLayout == nwItemLayout.BOOK
@@ -230,6 +241,11 @@ class Tokenizer():
"""
return self.theResult
+ def getResultSize(self):
+ """Return the size of the result from the conversion.
+ """
+ return len(self.theResult)
+
def getFilteredMarkdown(self):
"""Return the novelWriter markdown after the filters have been applied.
"""
diff --git a/nw/gui/build.py b/nw/gui/build.py
index 5d36e026..52ceb549 100644
--- a/nw/gui/build.py
+++ b/nw/gui/build.py
@@ -49,7 +49,7 @@ from nw.common import fuzzyTime, makeFileNameSafe
from nw.gui.custom import QSwitch
from nw.core import ToHtml
from nw.constants import (
- nwAlert, nwFiles, nwItemType, nwItemLayout, nwItemClass
+ nwConst, nwAlert, nwFiles, nwItemType, nwItemLayout, nwItemClass
)
logger = logging.getLogger(__name__)
@@ -77,7 +77,7 @@ class GuiBuildNovel(QDialog):
self.theTheme = theParent.theTheme
self.optState = self.theProject.optState
- self.htmlText = [] # List of html document
+ self.htmlText = [] # List of html documents
self.htmlStyle = [] # List of html styles
self.nwdText = [] # List of markdown documents
self.buildTime = 0 # The timestamp of the last build
@@ -494,7 +494,15 @@ class GuiBuildNovel(QDialog):
self.docView.clearStyleSheet()
else:
self.docView.setStyleSheet(self.htmlStyle)
- self.docView.setContent(self.htmlText, self.buildTime)
+
+ htmlSize = sum([len(x) for x in self.htmlText])
+ if htmlSize < nwConst.maxBuildSize:
+ self.docView.setContent(self.htmlText, self.buildTime)
+ else:
+ self.docView.setText(
+ "Failed to generate preview. The result is too big."
+ )
+ self._enableQtSave(False)
else:
self.htmlText = []
self.htmlStyle = []
@@ -554,6 +562,8 @@ class GuiBuildNovel(QDialog):
self.htmlStyle = []
self.nwdText = []
+ htmlSize = 0
+
for nItt, tItem in enumerate(self.theProject.projTree):
noteRoot = noteFiles
@@ -578,6 +588,7 @@ class GuiBuildNovel(QDialog):
makeHtml.doPostProcessing()
self.htmlText.append(makeHtml.getResult())
self.nwdText.append(makeHtml.getFilteredMarkdown())
+ htmlSize += makeHtml.getResultSize()
except Exception as e:
logger.error("Failed to generate html of document '%s'" % tItem.itemHandle)
@@ -591,6 +602,12 @@ class GuiBuildNovel(QDialog):
# Update progress bar, also for skipped items
self.buildProgress.setValue(nItt+1)
+ if makeHtml.errData:
+ self.theParent.makeAlert((
+ "There were problems when building the project:"
+ "
- %s"
+ ) % "
- ".join(makeHtml.errData), nwAlert.ERROR)
+
if replaceTabs:
htmlText = []
eightSpace = " "*8
@@ -615,7 +632,16 @@ class GuiBuildNovel(QDialog):
self.docView.clearStyleSheet()
else:
self.docView.setStyleSheet(self.htmlStyle)
- self.docView.setContent(self.htmlText, self.buildTime)
+
+ if htmlSize < nwConst.maxBuildSize:
+ self.docView.setContent(self.htmlText, self.buildTime)
+ self._enableQtSave(True)
+ else:
+ self.docView.setText(
+ "Failed to generate preview. The result is too big."
+ )
+ allowQtSave = False
+ self._enableQtSave(False)
self._saveCache()
@@ -962,6 +988,16 @@ class GuiBuildNovel(QDialog):
# Internal Functions
##
+ def _enableQtSave(self, theState):
+ """Set the enabled status of Save menu entries that depend on
+ the QTextDocument.
+ """
+ self.saveODT.setEnabled(theState)
+ self.savePDF.setEnabled(theState)
+ self.saveMD.setEnabled(theState)
+ self.saveTXT.setEnabled(theState)
+ return
+
def _saveSettings(self):
"""Save the various user settings.
"""
From 8d66875ebc6ad0c6fa2fcd9bd0f798a28b58cfa8 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 20:28:00 +0200
Subject: [PATCH 8/9] Performance improvements to Build and Writing Stats, and
making sure only one of each can be opened
---
nw/common.py | 10 ++++++++++
nw/gui/build.py | 35 +++++++++++++++++++++++------------
nw/gui/writingstats.py | 11 ++++++++---
nw/guimain.py | 17 +++++++++++++++--
4 files changed, 56 insertions(+), 17 deletions(-)
diff --git a/nw/common.py b/nw/common.py
index 12f140ef..2fef7874 100644
--- a/nw/common.py
+++ b/nw/common.py
@@ -29,6 +29,8 @@ import logging
from datetime import datetime
+from PyQt5.QtWidgets import qApp
+
from nw.constants import nwConst, nwUnicode
logger = logging.getLogger(__name__)
@@ -251,3 +253,11 @@ def makeFileNameSafe(theText):
if c.isalpha() or c.isdigit() or c == " ":
cleanName += c
return cleanName
+
+def getGuiItem(theName):
+ """Returns a QtWidget based on its objectName.
+ """
+ for qWidget in qApp.topLevelWidgets():
+ if qWidget.objectName() == theName:
+ return qWidget
+ return None
diff --git a/nw/gui/build.py b/nw/gui/build.py
index 52ceb549..6e571959 100644
--- a/nw/gui/build.py
+++ b/nw/gui/build.py
@@ -36,10 +36,10 @@ from datetime import datetime
from PyQt5.QtCore import Qt, QByteArray, QTimer
from PyQt5.QtPrintSupport import QPrinter, QPrintPreviewDialog
from PyQt5.QtGui import (
- QPalette, QColor, QTextDocumentWriter, QFont
+ QPalette, QColor, QTextDocumentWriter, QFont, QCursor
)
from PyQt5.QtWidgets import (
- QDialog, QVBoxLayout, QHBoxLayout, QTextBrowser, QPushButton, QLabel,
+ qApp, QDialog, QVBoxLayout, QHBoxLayout, QTextBrowser, QPushButton, QLabel,
QLineEdit, QGroupBox, QGridLayout, QProgressBar, QMenu, QAction,
QFileDialog, QFontDialog, QSpinBox, QScrollArea, QSplitter, QWidget,
QSizePolicy
@@ -483,7 +483,11 @@ class GuiBuildNovel(QDialog):
logger.debug("GuiBuildNovel initialisation complete")
- # Load from Cache
+ return
+
+ def viewCachedDoc(self):
+ """Load the previously generated document from cache.
+ """
if self._loadCache():
textFont = self.textFont.text()
textSize = self.textSize.value()
@@ -497,19 +501,22 @@ class GuiBuildNovel(QDialog):
htmlSize = sum([len(x) for x in self.htmlText])
if htmlSize < nwConst.maxBuildSize:
+ qApp.processEvents()
self.docView.setContent(self.htmlText, self.buildTime)
else:
self.docView.setText(
"Failed to generate preview. The result is too big."
)
self._enableQtSave(False)
+
else:
self.htmlText = []
self.htmlStyle = []
self.nwdText = []
self.buildTime = 0
+ return False
- return
+ return True
##
# Slots
@@ -578,6 +585,7 @@ class GuiBuildNovel(QDialog):
makeHtml.doConvert()
self.htmlText.append(makeHtml.getResult())
self.nwdText.append(makeHtml.getFilteredMarkdown())
+ htmlSize += makeHtml.getResultSize()
elif self._checkInclude(tItem, noteFiles, novelFiles, ignoreFlag):
makeHtml.setText(tItem.itemHandle)
@@ -640,7 +648,6 @@ class GuiBuildNovel(QDialog):
self.docView.setText(
"Failed to generate preview. The result is too big."
)
- allowQtSave = False
self._enableQtSave(False)
self._saveCache()
@@ -981,7 +988,8 @@ class GuiBuildNovel(QDialog):
"""Capture the user closing the window so we can save settings.
"""
self._saveSettings()
- QDialog.closeEvent(self, theEvent)
+ self.docView.clear()
+ theEvent.accept()
return
##
@@ -1099,6 +1107,12 @@ class GuiBuildNovelDocView(QTextBrowser):
theFont.setPointSize(self.mainConf.textSize)
self.setFont(theFont)
+ # Set the tab stops
+ if self.mainConf.verQtValue >= 51000:
+ self.setTabStopDistance(self.mainConf.getTabWidth())
+ else:
+ self.setTabStopWidth(self.mainConf.getTabWidth())
+
docPalette = self.palette()
docPalette.setColor(QPalette.Base, QColor(255, 255, 255))
docPalette.setColor(QPalette.Text, QColor(0, 0, 0))
@@ -1162,17 +1176,13 @@ class GuiBuildNovelDocView(QTextBrowser):
self.buildTime = timeStamp
sPos = self.verticalScrollBar().value()
-
- # Refresh the tab stops
- if self.mainConf.verQtValue >= 51000:
- self.setTabStopDistance(self.mainConf.getTabWidth())
- else:
- self.setTabStopWidth(self.mainConf.getTabWidth())
+ qApp.setOverrideCursor(QCursor(Qt.WaitCursor))
theText = theText.replace("\t", "!!tab!!")
theText = theText.replace("", "")
theText = theText.replace("", "")
self.setHtml(theText)
+ qApp.processEvents()
while self.find("!!tab!!"):
theCursor = self.textCursor()
@@ -1184,6 +1194,7 @@ class GuiBuildNovelDocView(QTextBrowser):
# Since we change the content while it may still be rendering, we mark
# the document dirty again to make sure it's re-rendered properly.
self.qDocument.markContentsDirty(0, self.qDocument.characterCount())
+ qApp.restoreOverrideCursor()
return
diff --git a/nw/gui/writingstats.py b/nw/gui/writingstats.py
index 165e7fd9..c467615c 100644
--- a/nw/gui/writingstats.py
+++ b/nw/gui/writingstats.py
@@ -33,7 +33,7 @@ import os
from datetime import datetime
from PyQt5.QtCore import Qt
-from PyQt5.QtGui import QPixmap
+from PyQt5.QtGui import QPixmap, QCursor
from PyQt5.QtWidgets import (
qApp, QDialog, QTreeWidget, QTreeWidgetItem, QDialogButtonBox, QGridLayout,
QLabel, QGroupBox, QMenu, QAction, QFileDialog, QSpinBox, QHBoxLayout
@@ -253,10 +253,15 @@ class GuiWritingStats(QDialog):
logger.debug("GuiWritingStats initialisation complete")
- qApp.processEvents()
+ return
+
+ def populateGUI(self):
+ """Populate list box with data from the log file.
+ """
+ qApp.setOverrideCursor(QCursor(Qt.WaitCursor))
self._loadLogFile()
self._updateListBox()
-
+ qApp.restoreOverrideCursor()
return
##
diff --git a/nw/guimain.py b/nw/guimain.py
index 5ee6e9b7..0cdef1ef 100644
--- a/nw/guimain.py
+++ b/nw/guimain.py
@@ -48,6 +48,7 @@ from nw.gui import (
)
from nw.core import NWProject, NWDoc, NWIndex
from nw.constants import nwItemType, nwItemClass, nwAlert
+from nw.common import getGuiItem
logger = logging.getLogger(__name__)
@@ -816,9 +817,15 @@ class GuiMain(QMainWindow):
logger.error("No project open")
return
- dlgBuild = GuiBuildNovel(self, self.theProject)
+ dlgBuild = getGuiItem("GuiBuildNovel")
+ if dlgBuild is None:
+ dlgBuild = GuiBuildNovel(self, self.theProject)
+
dlgBuild.setModal(False)
dlgBuild.show()
+ qApp.processEvents()
+ dlgBuild.viewCachedDoc()
+
return
def showWritingStatsDialog(self):
@@ -828,9 +835,15 @@ class GuiMain(QMainWindow):
logger.error("No project open")
return
- dlgStats = GuiWritingStats(self, self.theProject)
+ dlgStats = getGuiItem("GuiWritingStats")
+ if dlgStats is None:
+ dlgStats = GuiWritingStats(self, self.theProject)
+
dlgStats.setModal(False)
dlgStats.show()
+ qApp.processEvents()
+ dlgStats.populateGUI()
+
return
def showAboutNWDialog(self):
From bd33b28d2df757bf02b0101c718a07ee6d8e5e06 Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com>
Date: Tue, 6 Oct 2020 20:34:12 +0200
Subject: [PATCH 9/9] Fix dialog test method and make appropriate changes to
the project load tool
---
nw/gui/projload.py | 24 ++++++++++++++++++------
tests/test_dialogs.py | 43 ++++++++++++++++++++++++++++++++++++++-----
2 files changed, 56 insertions(+), 11 deletions(-)
diff --git a/nw/gui/projload.py b/nw/gui/projload.py
index 38356093..8f18f7bb 100644
--- a/nw/gui/projload.py
+++ b/nw/gui/projload.py
@@ -125,7 +125,7 @@ class GuiProjectLoad(QDialog):
self.buttonBox = QDialogButtonBox(QDialogButtonBox.Open | QDialogButtonBox.Cancel)
self.buttonBox.accepted.connect(self._doOpenRecent)
- self.buttonBox.rejected.connect(self._doClose)
+ self.buttonBox.rejected.connect(self._doCancel)
self.newButton = self.buttonBox.addButton("New", QDialogButtonBox.ActionRole)
self.newButton.clicked.connect(self._doNewProject)
@@ -153,7 +153,7 @@ class GuiProjectLoad(QDialog):
"""Close the dialog window with a recent project selected.
"""
logger.verbose("GuiProjectLoad open button clicked")
- self._saveDialogState()
+ self._saveSettings()
selItems = self.listBox.selectedItems()
if selItems:
@@ -194,11 +194,12 @@ class GuiProjectLoad(QDialog):
return
- def _doClose(self):
+ def _doCancel(self):
"""Close the dialog window without doing anything.
"""
logger.verbose("GuiProjectLoad close button clicked")
- self._saveDialogState()
+ self.openPath = None
+ self.openState = self.NONE_STATE
self.close()
return
@@ -206,7 +207,7 @@ class GuiProjectLoad(QDialog):
"""Create a new project.
"""
logger.verbose("GuiProjectLoad new project button clicked")
- self._saveDialogState()
+ self._saveSettings()
self.openPath = None
self.openState = self.NEW_STATE
self.accept()
@@ -230,11 +231,22 @@ class GuiProjectLoad(QDialog):
return
+ ##
+ # Events
+ ##
+
+ def closeEvent(self, theEvent):
+ """Capture the user closing the dialog so we can save settings.
+ """
+ self._saveSettings()
+ theEvent.accept()
+ return
+
##
# Internal Functions
##
- def _saveDialogState(self):
+ def _saveSettings(self):
"""Save the changes made to the dialog.
"""
colWidths = [0, 0, 0]
diff --git a/tests/test_dialogs.py b/tests/test_dialogs.py
index 93a5f43f..6fe0f2e4 100644
--- a/tests/test_dialogs.py
+++ b/tests/test_dialogs.py
@@ -114,7 +114,12 @@ def testProjectSettings(qtbot, monkeypatch, yesToAll, nwFuncTemp, nwTempGUI, nwR
projEdit._doSave()
# Open again, and check project settings
- projEdit = GuiProjectSettings(nwGUI, nwGUI.theProject)
+ nwGUI.mainMenu.aProjectSettings.activate(QAction.Trigger)
+ qtbot.waitUntil(lambda: getGuiItem("GuiProjectSettings") is not None, timeout=1000)
+
+ projEdit = getGuiItem("GuiProjectSettings")
+ assert isinstance(projEdit, GuiProjectSettings)
+
qtbot.addWidget(projEdit)
assert projEdit.tabMain.editName.text() == "Project Name"
assert projEdit.tabMain.editTitle.text() == "Project Title"
@@ -582,8 +587,13 @@ def testBuildTool(qtbot, yesToAll, nwTempBuild, nwLipsum, nwRef, nwTemp):
nwBuild._doClose()
# Re-open build dialog from cahce
- nwBuild = GuiBuildNovel(nwGUI, nwGUI.theProject)
+ nwGUI.mainMenu.aBuildProject.activate(QAction.Trigger)
+ qtbot.waitUntil(lambda: getGuiItem("GuiBuildNovel") is not None, timeout=1000)
+ nwBuild = getGuiItem("GuiBuildNovel")
+ assert isinstance(nwBuild, GuiBuildNovel)
+
+ assert nwBuild.viewCachedDoc()
assert nwBuild.htmlText == htmlText
assert nwBuild.htmlStyle == htmlStyle
assert nwBuild.nwdText == nwdText
@@ -659,7 +669,11 @@ def testMergeSplitTools(qtbot, monkeypatch, yesToAll, nwTempGUI, nwLipsum, nwRef
# Split By Scene
assert nwGUI.treeView.setSelectedHandle("73475cb40a568")
qtbot.wait(stepDelay)
- nwSplit = GuiDocSplit(nwGUI, nwGUI.theProject)
+ nwGUI.mainMenu.aSplitDoc.activate(QAction.Trigger)
+ qtbot.waitUntil(lambda: getGuiItem("GuiDocSplit") is not None, timeout=1000)
+
+ nwSplit = getGuiItem("GuiDocSplit")
+ assert isinstance(nwSplit, GuiDocSplit)
qtbot.wait(stepDelay)
nwSplit.splitLevel.setCurrentIndex(2)
qtbot.wait(stepDelay)
@@ -691,7 +705,11 @@ def testMergeSplitTools(qtbot, monkeypatch, yesToAll, nwTempGUI, nwLipsum, nwRef
# Split By Section
assert nwGUI.treeView.setSelectedHandle("73475cb40a568")
qtbot.wait(stepDelay)
- nwSplit = GuiDocSplit(nwGUI, nwGUI.theProject)
+ nwGUI.mainMenu.aSplitDoc.activate(QAction.Trigger)
+ qtbot.waitUntil(lambda: getGuiItem("GuiDocSplit") is not None, timeout=1000)
+
+ nwSplit = getGuiItem("GuiDocSplit")
+ assert isinstance(nwSplit, GuiDocSplit)
qtbot.wait(stepDelay)
nwSplit.splitLevel.setCurrentIndex(3)
qtbot.wait(stepDelay)
@@ -945,6 +963,7 @@ def testLoadProject(qtbot, monkeypatch, yesToAll, nwMinimal, nwTemp):
assert nwGUI.openProject(nwMinimal)
assert nwGUI.closeProject()
+ qtbot.wait(stepDelay)
monkeypatch.setattr(GuiProjectLoad, "exec_", lambda *args: None)
monkeypatch.setattr(GuiProjectLoad, "result", lambda *args: QDialog.Accepted)
nwGUI.mainMenu.aOpenProject.activate(QAction.Trigger)
@@ -954,36 +973,50 @@ def testLoadProject(qtbot, monkeypatch, yesToAll, nwMinimal, nwTemp):
assert isinstance(nwLoad, GuiProjectLoad)
nwLoad.show()
+ qtbot.wait(stepDelay)
recentCount = nwLoad.listBox.topLevelItemCount()
assert recentCount > 0
+ qtbot.wait(stepDelay)
selItem = nwLoad.listBox.topLevelItem(0)
selPath = selItem.data(nwLoad.C_NAME, Qt.UserRole)
assert isinstance(selItem, QTreeWidgetItem)
+ qtbot.wait(stepDelay)
nwLoad.selPath.setText("")
nwLoad.listBox.setCurrentItem(selItem)
nwLoad._doSelectRecent()
assert nwLoad.selPath.text() == selPath
+ qtbot.wait(stepDelay)
qtbot.mouseClick(nwLoad.buttonBox.button(QDialogButtonBox.Open), Qt.LeftButton)
assert nwLoad.openPath == selPath
assert nwLoad.openState == nwLoad.OPEN_STATE
# Just create a new project load from scratch for the rest of the test
del nwLoad
- nwLoad = GuiProjectLoad(nwGUI)
+
+ qtbot.wait(stepDelay)
+ nwGUI.mainMenu.aOpenProject.activate(QAction.Trigger)
+ qtbot.waitUntil(lambda: getGuiItem("GuiProjectLoad") is not None, timeout=1000)
+
+ qtbot.wait(stepDelay)
+ nwLoad = getGuiItem("GuiProjectLoad")
+ assert isinstance(nwLoad, GuiProjectLoad)
nwLoad.show()
+ qtbot.wait(stepDelay)
qtbot.mouseClick(nwLoad.buttonBox.button(QDialogButtonBox.Cancel), Qt.LeftButton)
assert nwLoad.openPath is None
assert nwLoad.openState == nwLoad.NONE_STATE
+ qtbot.wait(stepDelay)
nwLoad.show()
qtbot.mouseClick(nwLoad.newButton, Qt.LeftButton)
assert nwLoad.openPath is None
assert nwLoad.openState == nwLoad.NEW_STATE
+ qtbot.wait(stepDelay)
nwLoad.show()
nwLoad._keyPressDelete()
assert nwLoad.listBox.topLevelItemCount() == recentCount - 1