Change format of wordlist.txt file to json

This commit is contained in:
Veronica Berglyd Olsen
2023-06-13 17:31:16 +02:00
parent 9ac61105f3
commit 0003f8c24d
8 changed files with 200 additions and 178 deletions
+1 -1
View File
@@ -90,7 +90,7 @@ class nwFiles:
INDEX_FILE = "index.json"
OPTS_FILE = "options.json"
PROJ_DICT = "wordlist.txt"
SESS_STATS = "sessionStats.log"
DICT_FILE = "userdict.json"
SESS_FILE = "sessions.jsonl"
# END Class nwFiles
+124 -101
View File
@@ -1,7 +1,6 @@
"""
novelWriter Spell Check Classes
=================================
Wrapper classes for spell checking tools
File History:
Created: 2019-06-11 [0.1.5]
@@ -22,29 +21,37 @@ General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import json
import logging
from collections import namedtuple
from typing import TYPE_CHECKING, Iterator
from pathlib import Path
from novelwriter.error import logException
from novelwriter.constants import nwFiles
if TYPE_CHECKING: # pragma: no cover
from novelwriter.core.project import NWProject
logger = logging.getLogger(__name__)
class NWSpellEnchant:
"""Core: Enchant Spell Checking Wrapper
def __init__(self):
self._theDict = None
self._projDict = set()
self._projectDict = None
self._spellLanguage = None
self._theBroker = None
This is a rapper class for Enchant to keep the API consistent
between spell check tools.
"""
def __init__(self, project: NWProject):
self._project = project
self._dictObj = FakeEnchant()
self._userDict = UserDictionary(project)
self._language = None
self._broker = None
logger.debug("Enchant spell checking activated")
return
##
@@ -52,43 +59,43 @@ class NWSpellEnchant:
##
@property
def spellLanguage(self):
return self._spellLanguage
def spellLanguage(self) -> str | None:
return self._language
##
# Setters
##
def setLanguage(self, theLang, projectDict=None):
def setLanguage(self, language: str | None):
"""Load a dictionary for the language specified in the config.
If that fails, we load a mock dictionary so that lookups don't
crash. Note that enchant will allow loading an empty string as
a tag, but this will fail later on. See issue #1096.
"""
self._theBroker = None
self._theDict = None
self._spellLanguage = None
self._dictObj = FakeEnchant()
self._broker = None
self._language = None
try:
import enchant
if theLang and enchant.dict_exists(theLang):
self._theBroker = enchant.Broker()
self._theDict = self._theBroker.request_dict(theLang)
self._spellLanguage = theLang
logger.debug("Enchant spell checking for language '%s' loaded", theLang)
if language and enchant.dict_exists(language):
self._broker = enchant.Broker()
self._dictObj = self._broker.request_dict(language)
self._language = language
logger.debug("Enchant spell checking for language '%s' loaded", language)
else:
logger.warning("Enchant found no dictionary for language '%s'", theLang)
logger.warning("Enchant found no dictionary for language '%s'", language)
except Exception:
logger.error("Failed to load enchant spell checking for language '%s'", theLang)
logger.error("Failed to load enchant spell checking for language '%s'", language)
if self._theDict is None:
self._theDict = FakeEnchant()
if self._dictObj is None:
self._dictObj = FakeEnchant()
else:
self._readProjectDictionary(projectDict)
for pWord in self._projDict:
self._theDict.add_to_session(pWord)
self._userDict.load()
for pWord in self._userDict:
self._dictObj.add_to_session(pWord)
return
@@ -96,47 +103,38 @@ class NWSpellEnchant:
# Methods
##
def checkWord(self, theWord):
"""Wrapper function for pyenchant.
"""
def checkWord(self, word: str) -> bool:
"""Wrapper function for pyenchant."""
try:
return self._theDict.check(theWord)
return bool(self._dictObj.check(word))
except Exception:
return True
def suggestWords(self, theWord):
"""Wrapper function for pyenchant.
"""
def suggestWords(self, word: str) -> list[str]:
"""Wrapper function for pyenchant."""
try:
return self._theDict.suggest(theWord)
return self._dictObj.suggest(word)
except Exception:
return []
def addWord(self, newWord):
"""Add a word to the project dictionary.
"""
def addWord(self, word: str) -> bool:
"""Add a word to the project dictionary."""
word = word.strip()
if not word:
return False
try:
self._theDict.add_to_session(newWord)
self._dictObj.add_to_session(word)
except Exception:
return False
if self._projectDict is not None and newWord not in self._projDict:
newWord = newWord.strip()
try:
with open(self._projectDict, mode="a+", encoding="utf-8") as outFile:
outFile.write("%s\n" % newWord)
self._projDict.add(newWord)
except Exception:
logger.error("Failed to add word to project word list %s", str(self._projectDict))
logException()
return False
return True
added = self._userDict.add(word)
if added:
self._userDict.save()
return False
return added
def listDictionaries(self):
"""Wrapper function for pyenchant.
"""
def listDictionaries(self) -> list[tuple[str, str]]:
"""Wrapper function for pyenchant."""
retList = []
try:
import enchant
@@ -147,73 +145,98 @@ class NWSpellEnchant:
return retList
def describeDict(self):
def describeDict(self) -> tuple[str, str]:
"""Return the tag and provider of the currently loaded
dictionary.
"""
try:
spTag = self._theDict.tag
spName = self._theDict.provider.name
tag = self._dictObj.tag
name = self._dictObj.provider.name # type: ignore
except Exception:
logger.error("Failed to extract information about the dictionary")
logException()
spTag = ""
spName = ""
tag = ""
name = ""
return spTag, spName
##
# Internal Functions
##
def _readProjectDictionary(self, projectDict):
"""Read the content of the project dictionary, and add it to the
lookup lists.
"""
self._projDict = set()
self._projectDict = projectDict
if not isinstance(projectDict, Path):
return False
if not projectDict.exists():
return False
try:
logger.debug("Loading project word list")
with open(projectDict, mode="r", encoding="utf-8") as wordsFile:
for theLine in wordsFile:
theLine = theLine.strip()
if len(theLine) > 0 and theLine not in self._projDict:
self._projDict.add(theLine)
logger.debug("Project word list contains %d words", len(self._projDict))
except Exception:
logger.error("Failed to load project word list")
logException()
return False
return True
return tag, name
# END Class NWSpellEnchant
class FakeEnchant:
"""Fallback for when Enchant is selected, but not installed.
"""
"""Fallback for when Enchant is selected, but not installed."""
def __init__(self):
class FakeProvider:
name = ""
self.tag = ""
self.provider = namedtuple("provider", "name")
self.provider.name = ""
self.provider = FakeProvider()
return
def check(self, theWord):
def check(self, word: str) -> bool:
return True
def suggest(self, theWord):
def suggest(self, word) -> list[str]:
return []
def add_to_session(self, theWord):
def add_to_session(self, word: str):
return
# END Class FakeEnchant
class UserDictionary:
def __init__(self, project: NWProject):
self._project = project
self._words = set()
self._path = None
return
def __contains__(self, word: str) -> bool:
return word in self._words
def __iter__(self) -> Iterator[str]:
return iter(self._words)
def add(self, word: str) -> bool:
"""Add a word to the dictionary, and return True if it was
added, or False if it already existed.
"""
if word in self._words:
return False
self._words.add(word)
return True
def load(self):
"""Load the user's dictionary."""
self._path = self._project.storage.getMetaFile(nwFiles.DICT_FILE)
if not isinstance(self._path, Path):
return
try:
with open(self._path, mode="r", encoding="utf-8") as fObj:
data = json.load(fObj)
self._words = set(data.get("novelWriter.userDict", []))
except Exception:
logger.error("Failed to load user dictionary")
logException()
return
def save(self):
"""Save the user's dictionary."""
if self._path is None:
self._path = self._project.storage.getMetaFile(nwFiles.DICT_FILE)
if not isinstance(self._path, Path):
return
try:
with open(self._path, mode="w", encoding="utf-8") as fObj:
data = {"novelWriter.userDict": list(self._words)}
json.dump(data, fObj, indent=2)
except Exception:
logger.error("Failed to save user dictionary")
logException()
return
# END Class UserDictionary
+31 -1
View File
@@ -36,6 +36,7 @@ from novelwriter.common import minmax
from novelwriter.constants import nwFiles
from novelwriter.core.document import NWDocument
from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
from novelwriter.core.spellcheck import UserDictionary
if TYPE_CHECKING: # pragma: no cover
from novelwriter.core.project import NWProject
@@ -253,7 +254,7 @@ class NWStorage:
(baseMeta / nwFiles.BUILDS_FILE, f"meta/{nwFiles.BUILDS_FILE}"),
(baseMeta / nwFiles.INDEX_FILE, f"meta/{nwFiles.INDEX_FILE}"),
(baseMeta / nwFiles.OPTS_FILE, f"meta/{nwFiles.OPTS_FILE}"),
(baseMeta / nwFiles.PROJ_DICT, f"meta/{nwFiles.PROJ_DICT}"),
(baseMeta / nwFiles.DICT_FILE, f"meta/{nwFiles.DICT_FILE}"),
(baseMeta / nwFiles.SESS_FILE, f"meta/{nwFiles.SESS_FILE}"),
]
for contItem in baseCont.iterdir():
@@ -378,6 +379,10 @@ class NWStorage:
if sessLog.is_file():
self._convertOldLogFile(sessLog, path / "meta" / nwFiles.SESS_FILE)
wordList = path / "meta" / "wordlist.txt"
if wordList.is_file():
self._convertOldWordList(wordList)
remove = [
path / "meta" / "tagsIndex.json", # Renamed in 2.1 Beta 1
path / "meta" / "mainOptions.json", # Replaced in 0.5
@@ -413,6 +418,31 @@ class NWStorage:
return
def _convertOldWordList(self, wordList: Path) -> bool:
"""Convert the old word list plain text file to new format."""
if not wordList.exists():
# Nothing to convert
return True
userDict = UserDictionary(self._project)
try:
with open(wordList, mode="r", encoding="utf-8") as fObj:
for line in fObj:
word = line.strip()
if word:
userDict.add(word)
# Dave dictionary and clean up old file
userDict.save()
wordList.unlink()
except Exception:
logger.error("Failed to convert old word list file")
logException()
return False
return True
def _convertOldLogFile(self, sessLog: Path, sessJson: Path) -> bool:
"""Convert the old text log file format to the new JSON Lines
format.
+35 -63
View File
@@ -1,7 +1,6 @@
"""
novelWriter GUI User Wordlist
===============================
Class holding the user's wordlist dialog
File History:
Created: 2021-02-12 [1.2rc1]
@@ -22,28 +21,31 @@ General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import TYPE_CHECKING
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import (
QDialog, QDialogButtonBox, QVBoxLayout, QHBoxLayout, QListWidget,
QAbstractItemView, QPushButton, QLineEdit, QLabel
QAbstractItemView, QDialog, QDialogButtonBox, QHBoxLayout, QLabel,
QLineEdit, QListWidget, QListWidgetItem, QPushButton, QVBoxLayout
)
from novelwriter import CONFIG
from novelwriter.enum import nwAlert
from novelwriter.error import logException
from novelwriter.constants import nwFiles
from novelwriter.core.spellcheck import UserDictionary
if TYPE_CHECKING: # pragma: no cover
from novelwriter.guimain import GuiMain
logger = logging.getLogger(__name__)
class GuiWordList(QDialog):
def __init__(self, mainGui):
def __init__(self, mainGui: GuiMain):
super().__init__(parent=mainGui)
logger.debug("Create: GuiWordList")
@@ -120,67 +122,49 @@ class GuiWordList(QDialog):
# Slots
##
def _doAdd(self):
"""Add a new word to the word list.
"""
newWord = self.newEntry.text().strip()
if newWord == "":
def _doAdd(self) -> bool:
"""Add a new word to the word list."""
word = self.newEntry.text().strip()
if word == "":
self.mainGui.makeAlert(self.tr(
"Cannot add a blank word."
), nwAlert.ERROR)
return False
if self.listBox.findItems(newWord, Qt.MatchExactly):
if self.listBox.findItems(word, Qt.MatchExactly):
self.mainGui.makeAlert(self.tr(
"The word '{0}' is already in the word list."
).format(newWord), nwAlert.ERROR)
).format(word), nwAlert.ERROR)
return False
self.listBox.addItem(newWord)
self.listBox.addItem(word)
self.newEntry.setText("")
return True
def _doDelete(self):
"""Delete the selected item.
"""
"""Delete the selected item."""
selItem = self.listBox.selectedItems()
if selItem:
self.listBox.takeItem(self.listBox.row(selItem[0]))
return
def _doSave(self):
"""Save the new word list and close.
"""
"""Save the new word list and close."""
self._saveGuiSettings()
dctFile = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
if not isinstance(dctFile, Path):
return False
tmpFile = dctFile.with_suffix(".tmp")
try:
with open(tmpFile, mode="w", encoding="utf-8") as outFile:
for i in range(self.listBox.count()):
item = self.listBox.item(i)
if item is not None:
outFile.write(item.text() + "\n")
tmpFile.replace(dctFile)
except Exception:
logger.error("Could not save new word list")
logException()
self.reject()
return False
userDict = UserDictionary(self.theProject)
for i in range(self.listBox.count()):
item = self.listBox.item(i)
if isinstance(item, QListWidgetItem):
word = item.text().strip()
if word:
userDict.add(word)
userDict.save()
self.accept()
return True
def _doClose(self):
"""Close without saving the word list.
"""
"""Close without saving the word list."""
self._saveGuiSettings()
self.reject()
return
@@ -190,29 +174,17 @@ class GuiWordList(QDialog):
##
def _loadWordList(self):
"""Load the project's word list, if it exists.
"""
wordList = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
if not isinstance(wordList, Path):
return False
"""Load the project's word list, if it exists."""
userDict = UserDictionary(self.theProject)
userDict.load()
self.listBox.clear()
if not wordList.exists():
logger.debug("No project dictionary file found")
return False
with open(wordList, mode="r", encoding="utf-8") as inFile:
for inLine in inFile:
theWord = inLine.strip()
if len(theWord) == 0:
continue
self.listBox.addItem(theWord)
return True
for word in userDict:
if word:
self.listBox.addItem(word)
return
def _saveGuiSettings(self):
"""Save GUI settings.
"""
"""Save GUI settings."""
winWidth = CONFIG.rpxInt(self.width())
winHeight = CONFIG.rpxInt(self.height())
+3 -4
View File
@@ -52,7 +52,7 @@ from PyQt5.QtWidgets import (
from novelwriter import CONFIG
from novelwriter.enum import nwAlert, nwDocAction, nwDocInsert, nwDocMode, nwItemClass
from novelwriter.common import minmax, transferCase
from novelwriter.constants import nwConst, nwFiles, nwKeyWords, nwUnicode
from novelwriter.constants import nwConst, nwKeyWords, nwUnicode
from novelwriter.core.index import countWords
from novelwriter.core.spellcheck import NWSpellEnchant
from novelwriter.gui.dochighlight import GuiDocHighlighter
@@ -131,7 +131,7 @@ class GuiDocEditor(QTextEdit):
self.docSearch = GuiDocEditSearch(self)
# Syntax
self.spEnchant = NWSpellEnchant()
self.spEnchant = NWSpellEnchant(self.theProject)
self.highLight = GuiDocHighlighter(qDoc, self.mainGui, self.spEnchant)
# Context Menu
@@ -703,8 +703,7 @@ class GuiDocEditor(QTextEdit):
else:
theLang = self.theProject.data.spellLang
projDict = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
self.spEnchant.setLanguage(theLang, projDict)
self.spEnchant.setLanguage(theLang)
_, theProvider = self.spEnchant.describeDict()
self.spellDictionaryChanged.emit(str(theLang), str(theProvider))
+4 -4
View File
@@ -37,18 +37,18 @@ def testCoreSpell_FakeEnchant(monkeypatch):
mp.setitem(sys.modules, "enchant", None)
spChk = NWSpellEnchant()
spChk.setLanguage("en_US", "")
assert isinstance(spChk._theDict, FakeEnchant)
assert isinstance(spChk._dictObj, FakeEnchant)
# Request a non-existent dictionary
spChk = NWSpellEnchant()
spChk.setLanguage("whatchamajig", "")
assert isinstance(spChk._theDict, FakeEnchant)
assert isinstance(spChk._dictObj, FakeEnchant)
# Request an emety language string
# See issue https://github.com/vkbo/novelWriter/issues/1096
spChk = NWSpellEnchant()
spChk.setLanguage("", "")
assert isinstance(spChk._theDict, FakeEnchant)
assert isinstance(spChk._dictObj, FakeEnchant)
# FakeEnchant should handle requests
fkChk = FakeEnchant()
@@ -96,7 +96,7 @@ def testCoreSpell_Enchant(monkeypatch, fncPath):
assert spChk._readProjectDictionary(None) is False
assert spChk._readProjectDictionary(wList) is True
assert spChk._projectDict == wList
assert spChk._userDictPath == wList
# Cannot write to file
with monkeypatch.context() as mp:
+1 -2
View File
@@ -27,7 +27,6 @@ from PyQt5.QtWidgets import QDialog, QAction
from tools import buildTestProject, writeFile, readFile, getGuiItem
from mocked import causeOSError
from novelwriter.constants import nwFiles
from novelwriter.dialogs.wordlist import GuiWordList
@@ -43,7 +42,7 @@ def testDlgWordList_Dialog(qtbot, monkeypatch, nwGUI, projPath):
# Open project
nwGUI.openProject(projPath)
dictFile = projPath / "meta" / nwFiles.PROJ_DICT
dictFile = projPath / "meta" / "wordlist.txt"
# Load the dialog
nwGUI.mainMenu.aEditWordList.activate(QAction.Trigger)
+1 -2
View File
@@ -28,7 +28,6 @@ from tools import getGuiItem, writeFile, buildTestProject
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QAction, QFileDialog
from novelwriter.constants import nwFiles
from novelwriter.tools.writingstats import GuiWritingStats
@@ -40,7 +39,7 @@ def testToolWritingStats_Main(qtbot, monkeypatch, nwGUI, projPath, tstPaths):
buildTestProject(nwGUI, projPath)
qtbot.wait(100)
assert nwGUI.saveProject()
sessFile = projPath / "meta" / nwFiles.SESS_STATS
sessFile = projPath / "meta" / "sessionStats.log"
# Open the Writing Stats dialog
nwGUI.mainMenu.aWritingStats.activate(QAction.Trigger)