Remove the difflib spellchecker (#875)
* Delete simple spell checker * Clean up the spell checker code * Update tests * Delete the bundled dictionaries * Update documentation * Use set() for user word list * Fix tests so they pass also without dictionaries installed
This commit is contained in:
Veronica Berglyd Olsen
committed by
GitHub
GitHub
parent
4b52f343c1
commit
f6845a9820
@@ -22,7 +22,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from novelwriter.core.document import NWDoc
|
||||
from novelwriter.core.index import NWIndex, countWords
|
||||
from novelwriter.core.project import NWProject
|
||||
from novelwriter.core.spellcheck import NWSpellCheck, NWSpellEnchant, NWSpellSimple
|
||||
from novelwriter.core.spellcheck import NWSpellEnchant
|
||||
from novelwriter.core.tohtml import ToHtml
|
||||
from novelwriter.core.toodt import ToOdt
|
||||
from novelwriter.core.tomd import ToMarkdown
|
||||
@@ -32,9 +32,7 @@ __all__ = [
|
||||
"NWDoc",
|
||||
"NWIndex",
|
||||
"NWProject",
|
||||
"NWSpellCheck",
|
||||
"NWSpellEnchant",
|
||||
"NWSpellSimple",
|
||||
"ToHtml",
|
||||
"ToOdt",
|
||||
"ToMarkdown",
|
||||
|
||||
+55
-208
@@ -24,115 +24,25 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
|
||||
import os
|
||||
import difflib
|
||||
import logging
|
||||
import novelwriter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =============================================================================================== #
|
||||
# SpellChecking SuperClass
|
||||
# =============================================================================================== #
|
||||
|
||||
class NWSpellCheck():
|
||||
|
||||
theDict = None
|
||||
projDict = []
|
||||
class NWSpellEnchant():
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.mainConf = novelwriter.CONFIG
|
||||
self.theDict = None
|
||||
self.projDict = set()
|
||||
self.projectDict = None
|
||||
self.spellLanguage = None
|
||||
return
|
||||
|
||||
def setLanguage(self, theLang, projectDict=None):
|
||||
"""Default function.
|
||||
"""
|
||||
return
|
||||
|
||||
def checkWord(self, theWord):
|
||||
"""Default function.
|
||||
"""
|
||||
return True
|
||||
|
||||
def suggestWords(self, theWord):
|
||||
"""Default function.
|
||||
"""
|
||||
return []
|
||||
|
||||
def addWord(self, newWord):
|
||||
"""Add a word to the project dictionary.
|
||||
"""
|
||||
if self.projectDict is not None and newWord not in self.projDict:
|
||||
newWord = newWord.strip()
|
||||
try:
|
||||
with open(self.projectDict, mode="a+", encoding="utf-8") as outFile:
|
||||
outFile.write("%s\n" % newWord)
|
||||
self.projDict.append(newWord)
|
||||
except Exception:
|
||||
logger.error("Failed to add word to project word list %s", str(self.projectDict))
|
||||
novelwriter.logException()
|
||||
return False
|
||||
return True
|
||||
return False
|
||||
|
||||
def listDictionaries(self):
|
||||
"""Default function.
|
||||
"""
|
||||
return []
|
||||
|
||||
def describeDict(self):
|
||||
"""Default function.
|
||||
"""
|
||||
return "", ""
|
||||
|
||||
##
|
||||
# Internal Functions
|
||||
##
|
||||
|
||||
def _readProjectDictionary(self, projectDict):
|
||||
"""Read the content of the project dictionary, and add it to the
|
||||
lookup lists.
|
||||
"""
|
||||
self.projDict = []
|
||||
self.projectDict = projectDict
|
||||
|
||||
if projectDict is None:
|
||||
return False
|
||||
|
||||
if not os.path.isfile(projectDict):
|
||||
return False
|
||||
|
||||
try:
|
||||
logger.debug("Loading project word list")
|
||||
with open(projectDict, mode="r", encoding="utf-8") as wordsFile:
|
||||
for theLine in wordsFile:
|
||||
theLine = theLine.strip()
|
||||
if len(theLine) > 0 and theLine not in self.projDict:
|
||||
self.projDict.append(theLine)
|
||||
logger.debug("Project word list contains %d words", len(self.projDict))
|
||||
|
||||
except Exception:
|
||||
logger.error("Failed to load project word list")
|
||||
novelwriter.logException()
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# END Class NWSpellCheck
|
||||
|
||||
|
||||
# =============================================================================================== #
|
||||
# Enchant Based SpellChecking
|
||||
# =============================================================================================== #
|
||||
|
||||
class NWSpellEnchant(NWSpellCheck):
|
||||
|
||||
def __init__(self):
|
||||
NWSpellCheck.__init__(self)
|
||||
logger.debug("Enchant spell checking activated")
|
||||
self.theBroker = None
|
||||
|
||||
logger.debug("Enchant spell checking activated")
|
||||
|
||||
return
|
||||
|
||||
def setLanguage(self, theLang, projectDict=None):
|
||||
@@ -173,11 +83,23 @@ class NWSpellEnchant(NWSpellCheck):
|
||||
return self.theDict.suggest(theWord)
|
||||
|
||||
def addWord(self, newWord):
|
||||
"""Wrapper function for pyenchant.
|
||||
"""Add a word to the project dictionary.
|
||||
"""
|
||||
self.theDict.add_to_session(newWord)
|
||||
NWSpellCheck.addWord(self, newWord)
|
||||
return
|
||||
|
||||
if self.projectDict is not None and newWord not in self.projDict:
|
||||
newWord = newWord.strip()
|
||||
try:
|
||||
with open(self.projectDict, mode="a+", encoding="utf-8") as outFile:
|
||||
outFile.write("%s\n" % newWord)
|
||||
self.projDict.add(newWord)
|
||||
except Exception:
|
||||
logger.error("Failed to add word to project word list %s", str(self.projectDict))
|
||||
novelwriter.logException()
|
||||
return False
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def listDictionaries(self):
|
||||
"""Wrapper function for pyenchant.
|
||||
@@ -207,6 +129,39 @@ class NWSpellEnchant(NWSpellCheck):
|
||||
|
||||
return spTag, spName
|
||||
|
||||
##
|
||||
# Internal Functions
|
||||
##
|
||||
|
||||
def _readProjectDictionary(self, projectDict):
|
||||
"""Read the content of the project dictionary, and add it to the
|
||||
lookup lists.
|
||||
"""
|
||||
self.projDict = set()
|
||||
self.projectDict = projectDict
|
||||
|
||||
if projectDict is None:
|
||||
return False
|
||||
|
||||
if not os.path.isfile(projectDict):
|
||||
return False
|
||||
|
||||
try:
|
||||
logger.debug("Loading project word list")
|
||||
with open(projectDict, mode="r", encoding="utf-8") as wordsFile:
|
||||
for theLine in wordsFile:
|
||||
theLine = theLine.strip()
|
||||
if len(theLine) > 0 and theLine not in self.projDict:
|
||||
self.projDict.add(theLine)
|
||||
logger.debug("Project word list contains %d words", len(self.projDict))
|
||||
|
||||
except Exception:
|
||||
logger.error("Failed to load project word list")
|
||||
novelwriter.logException()
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# END Class NWSpellEnchant
|
||||
|
||||
|
||||
@@ -226,111 +181,3 @@ class FakeEnchant:
|
||||
return
|
||||
|
||||
# END Class FakeEnchant
|
||||
|
||||
|
||||
# =============================================================================================== #
|
||||
# Fallback SpellChecking Using difflib
|
||||
# =============================================================================================== #
|
||||
|
||||
class NWSpellSimple(NWSpellCheck):
|
||||
"""Internal spell check tool that uses standard Python packages with
|
||||
no other external dependencies. This is the fallback spell checker
|
||||
when no other is available. This method is slower than enchant.
|
||||
"""
|
||||
theWords = set()
|
||||
|
||||
def __init__(self):
|
||||
NWSpellCheck.__init__(self)
|
||||
self.theLang = ""
|
||||
logger.debug("Simple spell checking activated")
|
||||
return
|
||||
|
||||
def setLanguage(self, theLang, projectDict=None):
|
||||
"""Load a dictionary as a list from the app assets folder.
|
||||
"""
|
||||
self.theLang = theLang
|
||||
self.theWords = set()
|
||||
dictFile = os.path.join(self.mainConf.dictPath, theLang+".dict")
|
||||
try:
|
||||
with open(dictFile, mode="r", encoding="utf-8") as wordsFile:
|
||||
for theLine in wordsFile:
|
||||
if len(theLine) == 0 or theLine.startswith("#"):
|
||||
continue
|
||||
self.theWords.add(theLine.strip().lower())
|
||||
|
||||
logger.debug("Spell check dictionary for language '%s' loaded", theLang)
|
||||
logger.debug("Dictionary contains %d words", len(self.theWords))
|
||||
self.spellLanguage = theLang
|
||||
|
||||
except Exception:
|
||||
logger.error("Failed to load spell check word list for language '%s'", theLang)
|
||||
novelwriter.logException()
|
||||
self.spellLanguage = None
|
||||
|
||||
self._readProjectDictionary(projectDict)
|
||||
for pWord in self.projDict:
|
||||
self.theWords.add(pWord)
|
||||
|
||||
return
|
||||
|
||||
def checkWord(self, theWord):
|
||||
"""Check if a word exists in the word list. Make sure to keep
|
||||
this function as fast as possible as it is called for every
|
||||
word by the syntax highlighter.
|
||||
"""
|
||||
theWord = theWord.replace(self.mainConf.fmtApostrophe, "'").lower()
|
||||
return theWord in self.theWords
|
||||
|
||||
def suggestWords(self, theWord):
|
||||
"""Get suggestions for correct word from difflib, and make sure
|
||||
the first character is upper case if that was also the case for
|
||||
the word be3ing checked. Also make sure the apostrophe is
|
||||
changed to the one in the dictionary, and then put back in the
|
||||
results.
|
||||
"""
|
||||
theWord = theWord.strip()
|
||||
if len(theWord) == 0:
|
||||
return []
|
||||
|
||||
theMatches = difflib.get_close_matches(theWord.lower(), self.theWords, n=10, cutoff=0.75)
|
||||
theOptions = []
|
||||
for aWord in theMatches:
|
||||
if len(aWord) == 0:
|
||||
continue
|
||||
if theWord[0].isupper():
|
||||
aWord = aWord[0].upper() + aWord[1:]
|
||||
aWord = aWord.replace("'", self.mainConf.fmtApostrophe)
|
||||
theOptions.append(aWord)
|
||||
|
||||
return theOptions
|
||||
|
||||
def addWord(self, newWord):
|
||||
"""Wrapper for the internal project dictionary feature.
|
||||
"""
|
||||
newWord = newWord.strip().lower()
|
||||
if newWord not in self.theWords:
|
||||
self.theWords.add(newWord)
|
||||
NWSpellCheck.addWord(self, newWord)
|
||||
return
|
||||
|
||||
def listDictionaries(self):
|
||||
"""Lists the dictionary files in the app assets folder.
|
||||
"""
|
||||
retList = []
|
||||
for dictFile in os.listdir(self.mainConf.dictPath):
|
||||
|
||||
fRoot, fExt = os.path.splitext(dictFile)
|
||||
if fExt != ".dict":
|
||||
continue
|
||||
|
||||
retList.append((fRoot, "difflib"))
|
||||
|
||||
return retList
|
||||
|
||||
def describeDict(self):
|
||||
"""Return the tag and provider of the currently loaded
|
||||
dictionary.
|
||||
"""
|
||||
return self.theLang, ""
|
||||
|
||||
# END Class NWSpellSimple
|
||||
|
||||
Reference in New Issue
Block a user