Merge pull request #154 from vkbo/hlight_improvements

Highlighter Improvements

nw/config.py

@@ -93,6 +93,7 @@ class Config:
         self.wordCountTimer  = 5.0
         self.showTabsNSpaces = False
         self.showLineEndings = False
+        self.bigDocLimit     = 800
 
         self.fmtApostrophe   = nwUnicode.U_RSQUO
         self.fmtSingleQuotes = [nwUnicode.U_LSQUO,nwUnicode.U_RSQUO]
@@ -322,6 +323,9 @@ class Config:
         self.showLineEndings = self._parseLine(
             cnfParse, cnfSec, "showlineendings", self.CNF_BOOL, self.showLineEndings
         )
+        self.bigDocLimit = self._parseLine(
+            cnfParse, cnfSec, "bigdoclimit", self.CNF_INT, self.bigDocLimit
+        )
 
         ## Backup
         cnfSec = "Backup"
@@ -412,6 +416,7 @@ class Config:
         cnfParse.set(cnfSec,"spellcheck",      str(self.spellLanguage))
         cnfParse.set(cnfSec,"showtabsnspaces", str(self.showTabsNSpaces))
         cnfParse.set(cnfSec,"showlineendings", str(self.showLineEndings))
+        cnfParse.set(cnfSec,"bigdoclimit",     str(self.bigDocLimit))
 
         ## Backup
         cnfSec = "Backup"

nw/gui/dialogs/configeditor.py

@@ -173,11 +173,24 @@ class GuiConfigEditGeneral(QWidget):
             self.spellToolList.setCurrentIndex(toolIdx)
         self._doUpdateSpellTool(0)
 
-        self.spellLangForm.addWidget(QLabel("Provider"), 0, 0)
+        self.spellBigDoc = QSpinBox(self)
-        self.spellLangForm.addWidget(self.spellToolList, 0, 1)
+        self.spellBigDoc.setMinimum(10)
-        self.spellLangForm.addWidget(QLabel("Language"), 1, 0)
+        self.spellBigDoc.setMaximum(10000)
-        self.spellLangForm.addWidget(self.spellLangList, 1, 1)
+        self.spellBigDoc.setSingleStep(10)
-        self.spellLangForm.setColumnStretch(2, 1)
+        self.spellBigDoc.setToolTip((
+            "Disable spell checking when loading large documents. "
+            "Spell checking will only run on paragraphs you edit."
+        ))
+        self.spellBigDoc.setValue(self.mainConf.bigDocLimit)
+
+        self.spellLangForm.addWidget(QLabel("Provider"),   0, 0)
+        self.spellLangForm.addWidget(self.spellToolList,   0, 1, 1, 3)
+        self.spellLangForm.addWidget(QLabel("Language"),   1, 0)
+        self.spellLangForm.addWidget(self.spellLangList,   1, 1, 1, 3)
+        self.spellLangForm.addWidget(QLabel("Size limit"), 2, 0)
+        self.spellLangForm.addWidget(self.spellBigDoc,     2, 1)
+        self.spellLangForm.addWidget(QLabel("kb"),         2, 2)
+        self.spellLangForm.setColumnStretch(4, 1)
 
         # AutoSave
         self.autoSave     = QGroupBox("Automatic Save", self)
@@ -251,6 +264,7 @@ class GuiConfigEditGeneral(QWidget):
         guiDark         = self.guiDarkIcons.isChecked()
         spellTool       = self.spellToolList.currentData()
         spellLanguage   = self.spellLangList.currentData()
+        bigDocLimit     = self.spellBigDoc.value()
         autoSaveDoc     = self.autoSaveDoc.value()
         autoSaveProj    = self.autoSaveProj.value()
         backupPath      = self.projBackupPath.text()
@@ -265,6 +279,7 @@ class GuiConfigEditGeneral(QWidget):
         self.mainConf.guiDark         = guiDark
         self.mainConf.spellTool       = spellTool
         self.mainConf.spellLanguage   = spellLanguage
+        self.mainConf.bigDocLimit     = bigDocLimit
         self.mainConf.autoSaveDoc     = autoSaveDoc
         self.mainConf.autoSaveProj    = autoSaveProj
         self.mainConf.backupPath      = backupPath
@@ -540,6 +555,7 @@ class GuiConfigEditEditor(QWidget):
         self.outerBox.addWidget(self.quoteStyle,  2, 1, 2, 1)
         self.outerBox.addWidget(self.showGuides,  4, 1)
         self.outerBox.setColumnStretch(2, 1)
+        self.outerBox.setRowStretch(5, 1)
         self.setLayout(self.outerBox)
 
         return

nw/gui/elements/doceditor.py

@@ -53,6 +53,7 @@ class GuiDocEditor(QTextEdit):
         self.wordCount = 0
         self.paraCount = 0
         self.lastEdit  = 0
+        self.bigDoc    = False
         self.nonWord   = "\"'"
 
         # Typography
@@ -114,17 +115,22 @@ class GuiDocEditor(QTextEdit):
         return
 
     def clearEditor(self):
+        """Clear the current document and reset all document related
+         flags and counters.
+        """
 
         self.nwDocument.clearDocument()
         self.setReadOnly(True)
         self.clear()
         self.wcTimer.stop()
 
-        self.theHandle    = None
+        self.theHandle = None
-        self.charCount    = 0
+        self.charCount = 0
-        self.wordCount    = 0
+        self.wordCount = 0
-        self.paraCount    = 0
+        self.paraCount = 0
-        self.lastEdit     = 0
+        self.lastEdit  = 0
+        self.bigDoc    = False
+
         self.hasSelection = False
 
         self.setDocumentChanged(False)
@@ -214,7 +220,19 @@ class GuiDocEditor(QTextEdit):
             return False
 
         self.hLight.setHandle(tHandle)
+
+        # Check that the document is not too big for full, initial spell
+        # checking. If it is too big, we switch to only check as we type
+        self._checkDocSize(len(theDoc))
+        spTemp = self.hLight.spellCheck
+        if self.bigDoc:
+            self.hLight.spellCheck = False
+
+        bfTime = time()
         self.setPlainText(theDoc)
+        afTime = time()
+        logger.debug("Document highlighted in %.3f milliseconds" % (1000*(afTime-bfTime)))
+
         self.setCursorPosition(self.nwDocument.theItem.cursorPos)
         self.lastEdit = time()
         self._runCounter()
@@ -227,6 +245,8 @@ class GuiDocEditor(QTextEdit):
         else:
             self.theParent.noticeBar.showNote("This document is read only.")
 
+        self.hLight.spellCheck = spTemp
+
         return True
 
     def replaceText(self, theText):
@@ -285,9 +305,11 @@ class GuiDocEditor(QTextEdit):
         # Updating root frame triggers a QTextDocument->contentsChange
         # signal, which we do not want as it re-runs the syntax
         # highlighter and spell checker, so we block it briefly.
+        # We then emit a signal that does not trigger re-highlighting.
         self.qDocument.blockSignals(True)
         self.qDocument.rootFrame().setFrameFormat(docFormat)
         self.qDocument.blockSignals(False)
+        self.qDocument.contentsChange.emit(0,0,0)
 
         return
 
@@ -353,18 +375,26 @@ class GuiDocEditor(QTextEdit):
         self.theParent.mainMenu.setSpellCheck(theMode)
         self.theProject.setSpellCheck(theMode)
         self.hLight.setSpellCheck(theMode)
-        self.hLight.rehighlight()
+        self.reHighlightDocument()
 
         logger.verbose("Spell check is set to %s" % str(theMode))
 
         return True
 
-    def updateSpellCheck(self):
+    def reHighlightDocument(self):
         """Rerun the highlighter to update spell checking status of the
-        currently loaded text.
+        currently loaded text. The fastest way to do this, at least as
+        of Qt 5.13, is to clear the text and put it back.
         """
+
         if self.spellCheck:
-            self.hLight.rehighlight()
+            theText = self.getText()
+            self.clear()
+            bfTime = time()
+            self.setPlainText(theText)
+            afTime = time()
+            logger.debug("Document re-highlighted in %.3f milliseconds" % (1000*(afTime-bfTime)))
+
         return True
 
     ##
@@ -596,6 +626,9 @@ class GuiDocEditor(QTextEdit):
         return
 
     def _docChange(self, thePos, charsRemoved, charsAdded):
+        """Triggered by QTextDocument->contentsChanged. This also
+        triggers the syntax highlighter.
+        """
         self.lastEdit = time()
         if not self.docChanged:
             self.setDocumentChanged(True)
@@ -687,9 +720,24 @@ class GuiDocEditor(QTextEdit):
         self.theParent.statusBar.setCounts(self.charCount,self.wordCount,self.paraCount)
         self.theParent.treeView.propagateCount(tHandle, self.wordCount)
         self.theParent.treeView.projectWordCount()
+        self._checkDocSize(self.charCount)
 
         return
 
+    def _checkDocSize(self, theSize):
+        """Check if document size crosses the big document limit set in
+        config. If so, we will set the big document flag to True.
+        """
+        if theSize > self.mainConf.bigDocLimit*1000:
+            logger.info(
+                "The document size is %d > %d, big doc mode is enabled" % (
+                theSize, self.mainConf.bigDocLimit*1000
+            ))
+            self.bigDoc = True
+        else:
+            self.bigDoc = False
+        return
+
     def _wrapSelection(self, tBefore, tAfter):
         """Wraps the selected text in whatever is in tBefore and tAfter.
         If there is no selection, the autoSelect setting decides the

nw/gui/mainmenu.py

@@ -618,7 +618,7 @@ class GuiMainMenu(QMenuBar):
         self.aReRunSpell = QAction("Re-Run Spell Check", self)
         self.aReRunSpell.setStatusTip("Run the spell checker on current document")
         self.aReRunSpell.setShortcut("F7")
-        self.aReRunSpell.triggered.connect(self.theParent.docEditor.updateSpellCheck)
+        self.aReRunSpell.triggered.connect(self.theParent.docEditor.reHighlightDocument)
         self.toolsMenu.addAction(self.aReRunSpell)
 
         # Tools > Separator

nw/gui/tools/dochighlight.py

@@ -101,47 +101,7 @@ class GuiDocHighlighter(QSyntaxHighlighter):
             "value"     : self._makeFormat(self.colVal),
         }
 
-        # Headers
         self.hRules = []
-        self.hRules.append((
-            r"^(#{1}) (.*)[^\n]", {
-                0 : self.hStyles["header1"],
-                1 : self.hStyles["header1h"],
-            }
-        ))
-        self.hRules.append((
-            r"^(#{2}) (.*)[^\n]", {
-                0 : self.hStyles["header2"],
-                1 : self.hStyles["header2h"],
-            }
-        ))
-        self.hRules.append((
-            r"^(#{3}) (.*)[^\n]", {
-                0 : self.hStyles["header3"],
-                1 : self.hStyles["header3h"],
-            }
-        ))
-        self.hRules.append((
-            r"^(#{4}) (.*)[^\n]", {
-                0 : self.hStyles["header4"],
-                1 : self.hStyles["header4h"],
-            }
-        ))
-
-        # Keyword/Value
-        # self.hRules.append((
-        #     r"^(@.+?)\s*:\s*(.+?)$", {
-        #         1 : self.hStyles["keyword"],
-        #         2 : self.hStyles["value"],
-        #     }
-        # ))
-
-        # Comments
-        self.hRules.append((
-            r"^%.*$", {
-                0 : self.hStyles["hidden"],
-            }
-        ))
 
         # Trailing Spaces, 2+
         self.hRules.append((
@@ -241,12 +201,16 @@ class GuiDocHighlighter(QSyntaxHighlighter):
     ##
 
     def highlightBlock(self, theText):
+        """Highlight a single block. Prefer to check first character for
+        all formats that are defined by their initial characters. This
+        is significantly faster than running the regex checks we use for
+        text paragraphs.
+        """
 
-        if self.theHandle is None:
+        if self.theHandle is None or not theText:
             return
 
-        if theText.startswith("@"):
+        if theText.startswith("@"): # Keywords and commands
-            # Highlighting of keywords and commands
             tItem = self.theParent.theProject.getItem(self.theHandle)
             isValid, theBits, thePos = self.theIndex.scanThis(theText)
             isGood = self.theIndex.checkThese(theBits, tItem)
@@ -265,11 +229,30 @@ class GuiDocHighlighter(QSyntaxHighlighter):
                         kwFmt.setUnderlineStyle(QTextCharFormat.SpellCheckUnderline)
                         self.setFormat(xPos, xLen, kwFmt)
 
-            # We're done, no need to continue
+            # We never want to run the spell checker on keyword/values,
+            # so we force a return here
             return
 
-        else:
+        elif theText.startswith("# "): # Header 1
-            # For other text, just use our regex rules
+            self.setFormat(0, 1, self.hStyles["header1h"])
+            self.setFormat(1, len(theText), self.hStyles["header1"])
+
+        elif theText.startswith("## "): # Header 2
+            self.setFormat(0, 2, self.hStyles["header2h"])
+            self.setFormat(2, len(theText), self.hStyles["header2"])
+
+        elif theText.startswith("### "): # Header 3
+            self.setFormat(0, 3, self.hStyles["header3h"])
+            self.setFormat(3, len(theText), self.hStyles["header3"])
+
+        elif theText.startswith("#### "): # Header 4
+            self.setFormat(0, 4, self.hStyles["header4h"])
+            self.setFormat(4, len(theText), self.hStyles["header4"])
+
+        elif theText.startswith("%"): # Comments
+            self.setFormat(0, len(theText), self.hStyles["hidden"])
+
+        else: # Text Paragraph
             for rX, xFmt in self.rxRules:
                 rxItt = rX.globalMatch(theText, 0)
                 while rxItt.hasNext():
@@ -303,6 +286,10 @@ class GuiDocHighlighter(QSyntaxHighlighter):
     ##
 
     def _makeFormat(self, fmtCol=None, fmtStyle=None, fmtSize=None):
+        """Generate a valid character format to be applied to the text
+        that is to be highlighted.
+        """
+
         theFormat = QTextCharFormat()
 
         if fmtCol is not None:

tests/reference/novelwriter.conf

@@ -1,5 +1,5 @@
 [Main]
-timestamp = 2019-11-09 14:38:32
+timestamp = 2019-11-19 21:49:29
 theme = default
 syntax = default_light
 guidark = False
@@ -36,6 +36,7 @@ spelltool = internal
 spellcheck = en
 showtabsnspaces = False
 showlineendings = False
+bigdoclimit = 800
 
 [Backup]
 backuppath =