From 0003f8c24dd861ffc686c0477e665ef6a3c6f017 Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Tue, 13 Jun 2023 17:31:16 +0200
Subject: [PATCH] Change format of wordlist.txt file to json
---
novelwriter/constants.py | 2 +-
novelwriter/core/spellcheck.py | 225 +++++++++++---------
novelwriter/core/storage.py | 32 ++-
novelwriter/dialogs/wordlist.py | 98 +++------
novelwriter/gui/doceditor.py | 7 +-
tests/test_core/test_core_spellcheck.py | 8 +-
tests/test_dialogs/test_dlg_wordlist.py | 3 +-
tests/test_tools/test_tools_writingstats.py | 3 +-
8 files changed, 200 insertions(+), 178 deletions(-)
diff --git a/novelwriter/constants.py b/novelwriter/constants.py
index 4a45f051..00e99713 100644
--- a/novelwriter/constants.py
+++ b/novelwriter/constants.py
@@ -90,7 +90,7 @@ class nwFiles:
INDEX_FILE = "index.json"
OPTS_FILE = "options.json"
PROJ_DICT = "wordlist.txt"
- SESS_STATS = "sessionStats.log"
+ DICT_FILE = "userdict.json"
SESS_FILE = "sessions.jsonl"
# END Class nwFiles
diff --git a/novelwriter/core/spellcheck.py b/novelwriter/core/spellcheck.py
index 884de15f..b5cac3a2 100644
--- a/novelwriter/core/spellcheck.py
+++ b/novelwriter/core/spellcheck.py
@@ -1,7 +1,6 @@
"""
novelWriter – Spell Check Classes
=================================
-Wrapper classes for spell checking tools
File History:
Created: 2019-06-11 [0.1.5]
@@ -22,29 +21,37 @@ General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
"""
+from __future__ import annotations
+import json
import logging
-from collections import namedtuple
+from typing import TYPE_CHECKING, Iterator
from pathlib import Path
from novelwriter.error import logException
+from novelwriter.constants import nwFiles
+
+if TYPE_CHECKING: # pragma: no cover
+ from novelwriter.core.project import NWProject
logger = logging.getLogger(__name__)
class NWSpellEnchant:
+ """Core: Enchant Spell Checking Wrapper
- def __init__(self):
-
- self._theDict = None
- self._projDict = set()
- self._projectDict = None
- self._spellLanguage = None
- self._theBroker = None
+ This is a rapper class for Enchant to keep the API consistent
+ between spell check tools.
+ """
+ def __init__(self, project: NWProject):
+ self._project = project
+ self._dictObj = FakeEnchant()
+ self._userDict = UserDictionary(project)
+ self._language = None
+ self._broker = None
logger.debug("Enchant spell checking activated")
-
return
##
@@ -52,43 +59,43 @@ class NWSpellEnchant:
##
@property
- def spellLanguage(self):
- return self._spellLanguage
+ def spellLanguage(self) -> str | None:
+ return self._language
##
# Setters
##
- def setLanguage(self, theLang, projectDict=None):
+ def setLanguage(self, language: str | None):
"""Load a dictionary for the language specified in the config.
If that fails, we load a mock dictionary so that lookups don't
crash. Note that enchant will allow loading an empty string as
a tag, but this will fail later on. See issue #1096.
"""
- self._theBroker = None
- self._theDict = None
- self._spellLanguage = None
+ self._dictObj = FakeEnchant()
+ self._broker = None
+ self._language = None
try:
import enchant
- if theLang and enchant.dict_exists(theLang):
- self._theBroker = enchant.Broker()
- self._theDict = self._theBroker.request_dict(theLang)
- self._spellLanguage = theLang
- logger.debug("Enchant spell checking for language '%s' loaded", theLang)
+ if language and enchant.dict_exists(language):
+ self._broker = enchant.Broker()
+ self._dictObj = self._broker.request_dict(language)
+ self._language = language
+ logger.debug("Enchant spell checking for language '%s' loaded", language)
else:
- logger.warning("Enchant found no dictionary for language '%s'", theLang)
+ logger.warning("Enchant found no dictionary for language '%s'", language)
except Exception:
- logger.error("Failed to load enchant spell checking for language '%s'", theLang)
+ logger.error("Failed to load enchant spell checking for language '%s'", language)
- if self._theDict is None:
- self._theDict = FakeEnchant()
+ if self._dictObj is None:
+ self._dictObj = FakeEnchant()
else:
- self._readProjectDictionary(projectDict)
- for pWord in self._projDict:
- self._theDict.add_to_session(pWord)
+ self._userDict.load()
+ for pWord in self._userDict:
+ self._dictObj.add_to_session(pWord)
return
@@ -96,47 +103,38 @@ class NWSpellEnchant:
# Methods
##
- def checkWord(self, theWord):
- """Wrapper function for pyenchant.
- """
+ def checkWord(self, word: str) -> bool:
+ """Wrapper function for pyenchant."""
try:
- return self._theDict.check(theWord)
+ return bool(self._dictObj.check(word))
except Exception:
return True
- def suggestWords(self, theWord):
- """Wrapper function for pyenchant.
- """
+ def suggestWords(self, word: str) -> list[str]:
+ """Wrapper function for pyenchant."""
try:
- return self._theDict.suggest(theWord)
+ return self._dictObj.suggest(word)
except Exception:
return []
- def addWord(self, newWord):
- """Add a word to the project dictionary.
- """
+ def addWord(self, word: str) -> bool:
+ """Add a word to the project dictionary."""
+ word = word.strip()
+ if not word:
+ return False
try:
- self._theDict.add_to_session(newWord)
+ self._dictObj.add_to_session(word)
except Exception:
return False
- if self._projectDict is not None and newWord not in self._projDict:
- newWord = newWord.strip()
- try:
- with open(self._projectDict, mode="a+", encoding="utf-8") as outFile:
- outFile.write("%s\n" % newWord)
- self._projDict.add(newWord)
- except Exception:
- logger.error("Failed to add word to project word list %s", str(self._projectDict))
- logException()
- return False
- return True
+ added = self._userDict.add(word)
+ if added:
+ self._userDict.save()
- return False
+ return added
- def listDictionaries(self):
- """Wrapper function for pyenchant.
- """
+ def listDictionaries(self) -> list[tuple[str, str]]:
+ """Wrapper function for pyenchant."""
retList = []
try:
import enchant
@@ -147,73 +145,98 @@ class NWSpellEnchant:
return retList
- def describeDict(self):
+ def describeDict(self) -> tuple[str, str]:
"""Return the tag and provider of the currently loaded
dictionary.
"""
try:
- spTag = self._theDict.tag
- spName = self._theDict.provider.name
+ tag = self._dictObj.tag
+ name = self._dictObj.provider.name # type: ignore
except Exception:
logger.error("Failed to extract information about the dictionary")
logException()
- spTag = ""
- spName = ""
+ tag = ""
+ name = ""
- return spTag, spName
-
- ##
- # Internal Functions
- ##
-
- def _readProjectDictionary(self, projectDict):
- """Read the content of the project dictionary, and add it to the
- lookup lists.
- """
- self._projDict = set()
- self._projectDict = projectDict
-
- if not isinstance(projectDict, Path):
- return False
-
- if not projectDict.exists():
- return False
-
- try:
- logger.debug("Loading project word list")
- with open(projectDict, mode="r", encoding="utf-8") as wordsFile:
- for theLine in wordsFile:
- theLine = theLine.strip()
- if len(theLine) > 0 and theLine not in self._projDict:
- self._projDict.add(theLine)
- logger.debug("Project word list contains %d words", len(self._projDict))
-
- except Exception:
- logger.error("Failed to load project word list")
- logException()
- return False
-
- return True
+ return tag, name
# END Class NWSpellEnchant
class FakeEnchant:
- """Fallback for when Enchant is selected, but not installed.
- """
+ """Fallback for when Enchant is selected, but not installed."""
def __init__(self):
+
+ class FakeProvider:
+ name = ""
+
self.tag = ""
- self.provider = namedtuple("provider", "name")
- self.provider.name = ""
+ self.provider = FakeProvider()
+
return
- def check(self, theWord):
+ def check(self, word: str) -> bool:
return True
- def suggest(self, theWord):
+ def suggest(self, word) -> list[str]:
return []
- def add_to_session(self, theWord):
+ def add_to_session(self, word: str):
return
# END Class FakeEnchant
+
+
+class UserDictionary:
+
+ def __init__(self, project: NWProject):
+ self._project = project
+ self._words = set()
+ self._path = None
+ return
+
+ def __contains__(self, word: str) -> bool:
+ return word in self._words
+
+ def __iter__(self) -> Iterator[str]:
+ return iter(self._words)
+
+ def add(self, word: str) -> bool:
+ """Add a word to the dictionary, and return True if it was
+ added, or False if it already existed.
+ """
+ if word in self._words:
+ return False
+ self._words.add(word)
+ return True
+
+ def load(self):
+ """Load the user's dictionary."""
+ self._path = self._project.storage.getMetaFile(nwFiles.DICT_FILE)
+ if not isinstance(self._path, Path):
+ return
+ try:
+ with open(self._path, mode="r", encoding="utf-8") as fObj:
+ data = json.load(fObj)
+ self._words = set(data.get("novelWriter.userDict", []))
+ except Exception:
+ logger.error("Failed to load user dictionary")
+ logException()
+ return
+
+ def save(self):
+ """Save the user's dictionary."""
+ if self._path is None:
+ self._path = self._project.storage.getMetaFile(nwFiles.DICT_FILE)
+ if not isinstance(self._path, Path):
+ return
+ try:
+ with open(self._path, mode="w", encoding="utf-8") as fObj:
+ data = {"novelWriter.userDict": list(self._words)}
+ json.dump(data, fObj, indent=2)
+ except Exception:
+ logger.error("Failed to save user dictionary")
+ logException()
+ return
+
+# END Class UserDictionary
diff --git a/novelwriter/core/storage.py b/novelwriter/core/storage.py
index 78d8d666..c56195bc 100644
--- a/novelwriter/core/storage.py
+++ b/novelwriter/core/storage.py
@@ -36,6 +36,7 @@ from novelwriter.common import minmax
from novelwriter.constants import nwFiles
from novelwriter.core.document import NWDocument
from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
+from novelwriter.core.spellcheck import UserDictionary
if TYPE_CHECKING: # pragma: no cover
from novelwriter.core.project import NWProject
@@ -253,7 +254,7 @@ class NWStorage:
(baseMeta / nwFiles.BUILDS_FILE, f"meta/{nwFiles.BUILDS_FILE}"),
(baseMeta / nwFiles.INDEX_FILE, f"meta/{nwFiles.INDEX_FILE}"),
(baseMeta / nwFiles.OPTS_FILE, f"meta/{nwFiles.OPTS_FILE}"),
- (baseMeta / nwFiles.PROJ_DICT, f"meta/{nwFiles.PROJ_DICT}"),
+ (baseMeta / nwFiles.DICT_FILE, f"meta/{nwFiles.DICT_FILE}"),
(baseMeta / nwFiles.SESS_FILE, f"meta/{nwFiles.SESS_FILE}"),
]
for contItem in baseCont.iterdir():
@@ -378,6 +379,10 @@ class NWStorage:
if sessLog.is_file():
self._convertOldLogFile(sessLog, path / "meta" / nwFiles.SESS_FILE)
+ wordList = path / "meta" / "wordlist.txt"
+ if wordList.is_file():
+ self._convertOldWordList(wordList)
+
remove = [
path / "meta" / "tagsIndex.json", # Renamed in 2.1 Beta 1
path / "meta" / "mainOptions.json", # Replaced in 0.5
@@ -413,6 +418,31 @@ class NWStorage:
return
+ def _convertOldWordList(self, wordList: Path) -> bool:
+ """Convert the old word list plain text file to new format."""
+ if not wordList.exists():
+ # Nothing to convert
+ return True
+
+ userDict = UserDictionary(self._project)
+ try:
+ with open(wordList, mode="r", encoding="utf-8") as fObj:
+ for line in fObj:
+ word = line.strip()
+ if word:
+ userDict.add(word)
+
+ # Dave dictionary and clean up old file
+ userDict.save()
+ wordList.unlink()
+
+ except Exception:
+ logger.error("Failed to convert old word list file")
+ logException()
+ return False
+
+ return True
+
def _convertOldLogFile(self, sessLog: Path, sessJson: Path) -> bool:
"""Convert the old text log file format to the new JSON Lines
format.
diff --git a/novelwriter/dialogs/wordlist.py b/novelwriter/dialogs/wordlist.py
index f2e7ddfe..118b94a4 100644
--- a/novelwriter/dialogs/wordlist.py
+++ b/novelwriter/dialogs/wordlist.py
@@ -1,7 +1,6 @@
"""
novelWriter – GUI User Wordlist
===============================
-Class holding the user's wordlist dialog
File History:
Created: 2021-02-12 [1.2rc1]
@@ -22,28 +21,31 @@ General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
"""
+from __future__ import annotations
import logging
-from pathlib import Path
+from typing import TYPE_CHECKING
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import (
- QDialog, QDialogButtonBox, QVBoxLayout, QHBoxLayout, QListWidget,
- QAbstractItemView, QPushButton, QLineEdit, QLabel
+ QAbstractItemView, QDialog, QDialogButtonBox, QHBoxLayout, QLabel,
+ QLineEdit, QListWidget, QListWidgetItem, QPushButton, QVBoxLayout
)
from novelwriter import CONFIG
from novelwriter.enum import nwAlert
-from novelwriter.error import logException
-from novelwriter.constants import nwFiles
+from novelwriter.core.spellcheck import UserDictionary
+
+if TYPE_CHECKING: # pragma: no cover
+ from novelwriter.guimain import GuiMain
logger = logging.getLogger(__name__)
class GuiWordList(QDialog):
- def __init__(self, mainGui):
+ def __init__(self, mainGui: GuiMain):
super().__init__(parent=mainGui)
logger.debug("Create: GuiWordList")
@@ -120,67 +122,49 @@ class GuiWordList(QDialog):
# Slots
##
- def _doAdd(self):
- """Add a new word to the word list.
- """
- newWord = self.newEntry.text().strip()
- if newWord == "":
+ def _doAdd(self) -> bool:
+ """Add a new word to the word list."""
+ word = self.newEntry.text().strip()
+ if word == "":
self.mainGui.makeAlert(self.tr(
"Cannot add a blank word."
), nwAlert.ERROR)
return False
- if self.listBox.findItems(newWord, Qt.MatchExactly):
+ if self.listBox.findItems(word, Qt.MatchExactly):
self.mainGui.makeAlert(self.tr(
"The word '{0}' is already in the word list."
- ).format(newWord), nwAlert.ERROR)
+ ).format(word), nwAlert.ERROR)
return False
- self.listBox.addItem(newWord)
+ self.listBox.addItem(word)
self.newEntry.setText("")
return True
def _doDelete(self):
- """Delete the selected item.
- """
+ """Delete the selected item."""
selItem = self.listBox.selectedItems()
if selItem:
self.listBox.takeItem(self.listBox.row(selItem[0]))
return
def _doSave(self):
- """Save the new word list and close.
- """
+ """Save the new word list and close."""
self._saveGuiSettings()
-
- dctFile = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
- if not isinstance(dctFile, Path):
- return False
-
- tmpFile = dctFile.with_suffix(".tmp")
- try:
- with open(tmpFile, mode="w", encoding="utf-8") as outFile:
- for i in range(self.listBox.count()):
- item = self.listBox.item(i)
- if item is not None:
- outFile.write(item.text() + "\n")
-
- tmpFile.replace(dctFile)
-
- except Exception:
- logger.error("Could not save new word list")
- logException()
- self.reject()
- return False
-
+ userDict = UserDictionary(self.theProject)
+ for i in range(self.listBox.count()):
+ item = self.listBox.item(i)
+ if isinstance(item, QListWidgetItem):
+ word = item.text().strip()
+ if word:
+ userDict.add(word)
+ userDict.save()
self.accept()
-
return True
def _doClose(self):
- """Close without saving the word list.
- """
+ """Close without saving the word list."""
self._saveGuiSettings()
self.reject()
return
@@ -190,29 +174,17 @@ class GuiWordList(QDialog):
##
def _loadWordList(self):
- """Load the project's word list, if it exists.
- """
- wordList = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
- if not isinstance(wordList, Path):
- return False
-
+ """Load the project's word list, if it exists."""
+ userDict = UserDictionary(self.theProject)
+ userDict.load()
self.listBox.clear()
- if not wordList.exists():
- logger.debug("No project dictionary file found")
- return False
-
- with open(wordList, mode="r", encoding="utf-8") as inFile:
- for inLine in inFile:
- theWord = inLine.strip()
- if len(theWord) == 0:
- continue
- self.listBox.addItem(theWord)
-
- return True
+ for word in userDict:
+ if word:
+ self.listBox.addItem(word)
+ return
def _saveGuiSettings(self):
- """Save GUI settings.
- """
+ """Save GUI settings."""
winWidth = CONFIG.rpxInt(self.width())
winHeight = CONFIG.rpxInt(self.height())
diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py
index 499ddf2a..1074e549 100644
--- a/novelwriter/gui/doceditor.py
+++ b/novelwriter/gui/doceditor.py
@@ -52,7 +52,7 @@ from PyQt5.QtWidgets import (
from novelwriter import CONFIG
from novelwriter.enum import nwAlert, nwDocAction, nwDocInsert, nwDocMode, nwItemClass
from novelwriter.common import minmax, transferCase
-from novelwriter.constants import nwConst, nwFiles, nwKeyWords, nwUnicode
+from novelwriter.constants import nwConst, nwKeyWords, nwUnicode
from novelwriter.core.index import countWords
from novelwriter.core.spellcheck import NWSpellEnchant
from novelwriter.gui.dochighlight import GuiDocHighlighter
@@ -131,7 +131,7 @@ class GuiDocEditor(QTextEdit):
self.docSearch = GuiDocEditSearch(self)
# Syntax
- self.spEnchant = NWSpellEnchant()
+ self.spEnchant = NWSpellEnchant(self.theProject)
self.highLight = GuiDocHighlighter(qDoc, self.mainGui, self.spEnchant)
# Context Menu
@@ -703,8 +703,7 @@ class GuiDocEditor(QTextEdit):
else:
theLang = self.theProject.data.spellLang
- projDict = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
- self.spEnchant.setLanguage(theLang, projDict)
+ self.spEnchant.setLanguage(theLang)
_, theProvider = self.spEnchant.describeDict()
self.spellDictionaryChanged.emit(str(theLang), str(theProvider))
diff --git a/tests/test_core/test_core_spellcheck.py b/tests/test_core/test_core_spellcheck.py
index 1782adb4..9e06c803 100644
--- a/tests/test_core/test_core_spellcheck.py
+++ b/tests/test_core/test_core_spellcheck.py
@@ -37,18 +37,18 @@ def testCoreSpell_FakeEnchant(monkeypatch):
mp.setitem(sys.modules, "enchant", None)
spChk = NWSpellEnchant()
spChk.setLanguage("en_US", "")
- assert isinstance(spChk._theDict, FakeEnchant)
+ assert isinstance(spChk._dictObj, FakeEnchant)
# Request a non-existent dictionary
spChk = NWSpellEnchant()
spChk.setLanguage("whatchamajig", "")
- assert isinstance(spChk._theDict, FakeEnchant)
+ assert isinstance(spChk._dictObj, FakeEnchant)
# Request an emety language string
# See issue https://github.com/vkbo/novelWriter/issues/1096
spChk = NWSpellEnchant()
spChk.setLanguage("", "")
- assert isinstance(spChk._theDict, FakeEnchant)
+ assert isinstance(spChk._dictObj, FakeEnchant)
# FakeEnchant should handle requests
fkChk = FakeEnchant()
@@ -96,7 +96,7 @@ def testCoreSpell_Enchant(monkeypatch, fncPath):
assert spChk._readProjectDictionary(None) is False
assert spChk._readProjectDictionary(wList) is True
- assert spChk._projectDict == wList
+ assert spChk._userDictPath == wList
# Cannot write to file
with monkeypatch.context() as mp:
diff --git a/tests/test_dialogs/test_dlg_wordlist.py b/tests/test_dialogs/test_dlg_wordlist.py
index 8781a82d..dfc4280f 100644
--- a/tests/test_dialogs/test_dlg_wordlist.py
+++ b/tests/test_dialogs/test_dlg_wordlist.py
@@ -27,7 +27,6 @@ from PyQt5.QtWidgets import QDialog, QAction
from tools import buildTestProject, writeFile, readFile, getGuiItem
from mocked import causeOSError
-from novelwriter.constants import nwFiles
from novelwriter.dialogs.wordlist import GuiWordList
@@ -43,7 +42,7 @@ def testDlgWordList_Dialog(qtbot, monkeypatch, nwGUI, projPath):
# Open project
nwGUI.openProject(projPath)
- dictFile = projPath / "meta" / nwFiles.PROJ_DICT
+ dictFile = projPath / "meta" / "wordlist.txt"
# Load the dialog
nwGUI.mainMenu.aEditWordList.activate(QAction.Trigger)
diff --git a/tests/test_tools/test_tools_writingstats.py b/tests/test_tools/test_tools_writingstats.py
index af2e4f0c..8ef82fcb 100644
--- a/tests/test_tools/test_tools_writingstats.py
+++ b/tests/test_tools/test_tools_writingstats.py
@@ -28,7 +28,6 @@ from tools import getGuiItem, writeFile, buildTestProject
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QAction, QFileDialog
-from novelwriter.constants import nwFiles
from novelwriter.tools.writingstats import GuiWritingStats
@@ -40,7 +39,7 @@ def testToolWritingStats_Main(qtbot, monkeypatch, nwGUI, projPath, tstPaths):
buildTestProject(nwGUI, projPath)
qtbot.wait(100)
assert nwGUI.saveProject()
- sessFile = projPath / "meta" / nwFiles.SESS_STATS
+ sessFile = projPath / "meta" / "sessionStats.log"
# Open the Writing Stats dialog
nwGUI.mainMenu.aWritingStats.activate(QAction.Trigger)