Change format of wordlist.txt file to json

novelwriter/constants.py

@@ -90,7 +90,7 @@ class nwFiles:
     INDEX_FILE  = "index.json"
     OPTS_FILE   = "options.json"
     PROJ_DICT   = "wordlist.txt"
-    SESS_STATS  = "sessionStats.log"
+    DICT_FILE   = "userdict.json"
     SESS_FILE   = "sessions.jsonl"
 
 # END Class nwFiles

novelwriter/core/spellcheck.py

@@ -1,7 +1,6 @@
 """
 novelWriter – Spell Check Classes
 =================================
-Wrapper classes for spell checking tools
 
 File History:
 Created: 2019-06-11 [0.1.5]
@@ -22,29 +21,37 @@ General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
+from __future__ import annotations
 
+import json
 import logging
 
-from collections import namedtuple
+from typing import TYPE_CHECKING, Iterator
 from pathlib import Path
 
 from novelwriter.error import logException
+from novelwriter.constants import nwFiles
+
+if TYPE_CHECKING:  # pragma: no cover
+    from novelwriter.core.project import NWProject
 
 logger = logging.getLogger(__name__)
 
 
 class NWSpellEnchant:
+    """Core: Enchant Spell Checking Wrapper
 
-    def __init__(self):
+    This is a rapper class for Enchant to keep the API consistent
-
+    between spell check tools.
-        self._theDict = None
+    """
-        self._projDict = set()
-        self._projectDict = None
-        self._spellLanguage = None
-        self._theBroker = None
 
+    def __init__(self, project: NWProject):
+        self._project = project
+        self._dictObj = FakeEnchant()
+        self._userDict = UserDictionary(project)
+        self._language = None
+        self._broker = None
         logger.debug("Enchant spell checking activated")
-
         return
 
     ##
@@ -52,43 +59,43 @@ class NWSpellEnchant:
     ##
 
     @property
-    def spellLanguage(self):
+    def spellLanguage(self) -> str | None:
-        return self._spellLanguage
+        return self._language
 
     ##
     #  Setters
     ##
 
-    def setLanguage(self, theLang, projectDict=None):
+    def setLanguage(self, language: str | None):
         """Load a dictionary for the language specified in the config.
         If that fails, we load a mock dictionary so that lookups don't
         crash. Note that enchant will allow loading an empty string as
         a tag, but this will fail later on. See issue #1096.
         """
-        self._theBroker = None
+        self._dictObj = FakeEnchant()
-        self._theDict = None
+        self._broker = None
-        self._spellLanguage = None
+        self._language = None
 
         try:
             import enchant
 
-            if theLang and enchant.dict_exists(theLang):
+            if language and enchant.dict_exists(language):
-                self._theBroker = enchant.Broker()
+                self._broker = enchant.Broker()
-                self._theDict = self._theBroker.request_dict(theLang)
+                self._dictObj = self._broker.request_dict(language)
-                self._spellLanguage = theLang
+                self._language = language
-                logger.debug("Enchant spell checking for language '%s' loaded", theLang)
+                logger.debug("Enchant spell checking for language '%s' loaded", language)
             else:
-                logger.warning("Enchant found no dictionary for language '%s'", theLang)
+                logger.warning("Enchant found no dictionary for language '%s'", language)
 
         except Exception:
-            logger.error("Failed to load enchant spell checking for language '%s'", theLang)
+            logger.error("Failed to load enchant spell checking for language '%s'", language)
 
-        if self._theDict is None:
+        if self._dictObj is None:
-            self._theDict = FakeEnchant()
+            self._dictObj = FakeEnchant()
         else:
-            self._readProjectDictionary(projectDict)
+            self._userDict.load()
-            for pWord in self._projDict:
+            for pWord in self._userDict:
-                self._theDict.add_to_session(pWord)
+                self._dictObj.add_to_session(pWord)
 
         return
 
@@ -96,47 +103,38 @@ class NWSpellEnchant:
     #  Methods
     ##
 
-    def checkWord(self, theWord):
+    def checkWord(self, word: str) -> bool:
-        """Wrapper function for pyenchant.
+        """Wrapper function for pyenchant."""
-        """
         try:
-            return self._theDict.check(theWord)
+            return bool(self._dictObj.check(word))
         except Exception:
             return True
 
-    def suggestWords(self, theWord):
+    def suggestWords(self, word: str) -> list[str]:
-        """Wrapper function for pyenchant.
+        """Wrapper function for pyenchant."""
-        """
         try:
-            return self._theDict.suggest(theWord)
+            return self._dictObj.suggest(word)
         except Exception:
             return []
 
-    def addWord(self, newWord):
+    def addWord(self, word: str) -> bool:
-        """Add a word to the project dictionary.
+        """Add a word to the project dictionary."""
-        """
+        word = word.strip()
+        if not word:
+            return False
         try:
-            self._theDict.add_to_session(newWord)
+            self._dictObj.add_to_session(word)
         except Exception:
             return False
 
-        if self._projectDict is not None and newWord not in self._projDict:
+        added = self._userDict.add(word)
-            newWord = newWord.strip()
+        if added:
-            try:
+            self._userDict.save()
-                with open(self._projectDict, mode="a+", encoding="utf-8") as outFile:
-                    outFile.write("%s\n" % newWord)
-                self._projDict.add(newWord)
-            except Exception:
-                logger.error("Failed to add word to project word list %s", str(self._projectDict))
-                logException()
-                return False
-            return True
 
-        return False
+        return added
 
-    def listDictionaries(self):
+    def listDictionaries(self) -> list[tuple[str, str]]:
-        """Wrapper function for pyenchant.
+        """Wrapper function for pyenchant."""
-        """
         retList = []
         try:
             import enchant
@@ -147,73 +145,98 @@ class NWSpellEnchant:
 
         return retList
 
-    def describeDict(self):
+    def describeDict(self) -> tuple[str, str]:
         """Return the tag and provider of the currently loaded
         dictionary.
         """
         try:
-            spTag = self._theDict.tag
+            tag = self._dictObj.tag
-            spName = self._theDict.provider.name
+            name = self._dictObj.provider.name  # type: ignore
         except Exception:
             logger.error("Failed to extract information about the dictionary")
             logException()
-            spTag = ""
+            tag = ""
-            spName = ""
+            name = ""
 
-        return spTag, spName
+        return tag, name
-
-    ##
-    #  Internal Functions
-    ##
-
-    def _readProjectDictionary(self, projectDict):
-        """Read the content of the project dictionary, and add it to the
-        lookup lists.
-        """
-        self._projDict = set()
-        self._projectDict = projectDict
-
-        if not isinstance(projectDict, Path):
-            return False
-
-        if not projectDict.exists():
-            return False
-
-        try:
-            logger.debug("Loading project word list")
-            with open(projectDict, mode="r", encoding="utf-8") as wordsFile:
-                for theLine in wordsFile:
-                    theLine = theLine.strip()
-                    if len(theLine) > 0 and theLine not in self._projDict:
-                        self._projDict.add(theLine)
-            logger.debug("Project word list contains %d words", len(self._projDict))
-
-        except Exception:
-            logger.error("Failed to load project word list")
-            logException()
-            return False
-
-        return True
 
 # END Class NWSpellEnchant
 
 
 class FakeEnchant:
-    """Fallback for when Enchant is selected, but not installed.
+    """Fallback for when Enchant is selected, but not installed."""
-    """
     def __init__(self):
+
+        class FakeProvider:
+            name = ""
+
         self.tag = ""
-        self.provider = namedtuple("provider", "name")
+        self.provider = FakeProvider()
-        self.provider.name = ""
+
         return
 
-    def check(self, theWord):
+    def check(self, word: str) -> bool:
         return True
 
-    def suggest(self, theWord):
+    def suggest(self, word) -> list[str]:
         return []
 
-    def add_to_session(self, theWord):
+    def add_to_session(self, word: str):
         return
 
 # END Class FakeEnchant
+
+
+class UserDictionary:
+
+    def __init__(self, project: NWProject):
+        self._project = project
+        self._words = set()
+        self._path = None
+        return
+
+    def __contains__(self, word: str) -> bool:
+        return word in self._words
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._words)
+
+    def add(self, word: str) -> bool:
+        """Add a word to the dictionary, and return True if it was
+        added, or False if it already existed.
+        """
+        if word in self._words:
+            return False
+        self._words.add(word)
+        return True
+
+    def load(self):
+        """Load the user's dictionary."""
+        self._path = self._project.storage.getMetaFile(nwFiles.DICT_FILE)
+        if not isinstance(self._path, Path):
+            return
+        try:
+            with open(self._path, mode="r", encoding="utf-8") as fObj:
+                data = json.load(fObj)
+            self._words = set(data.get("novelWriter.userDict", []))
+        except Exception:
+            logger.error("Failed to load user dictionary")
+            logException()
+        return
+
+    def save(self):
+        """Save the user's dictionary."""
+        if self._path is None:
+            self._path = self._project.storage.getMetaFile(nwFiles.DICT_FILE)
+        if not isinstance(self._path, Path):
+            return
+        try:
+            with open(self._path, mode="w", encoding="utf-8") as fObj:
+                data = {"novelWriter.userDict": list(self._words)}
+                json.dump(data, fObj, indent=2)
+        except Exception:
+            logger.error("Failed to save user dictionary")
+            logException()
+        return
+
+# END Class UserDictionary

novelwriter/core/storage.py

@@ -36,6 +36,7 @@ from novelwriter.common import minmax
 from novelwriter.constants import nwFiles
 from novelwriter.core.document import NWDocument
 from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter
+from novelwriter.core.spellcheck import UserDictionary
 
 if TYPE_CHECKING:  # pragma: no cover
     from novelwriter.core.project import NWProject
@@ -253,7 +254,7 @@ class NWStorage:
             (baseMeta / nwFiles.BUILDS_FILE, f"meta/{nwFiles.BUILDS_FILE}"),
             (baseMeta / nwFiles.INDEX_FILE,  f"meta/{nwFiles.INDEX_FILE}"),
             (baseMeta / nwFiles.OPTS_FILE,   f"meta/{nwFiles.OPTS_FILE}"),
-            (baseMeta / nwFiles.PROJ_DICT,   f"meta/{nwFiles.PROJ_DICT}"),
+            (baseMeta / nwFiles.DICT_FILE,   f"meta/{nwFiles.DICT_FILE}"),
             (baseMeta / nwFiles.SESS_FILE,   f"meta/{nwFiles.SESS_FILE}"),
         ]
         for contItem in baseCont.iterdir():
@@ -378,6 +379,10 @@ class NWStorage:
         if sessLog.is_file():
             self._convertOldLogFile(sessLog, path / "meta" / nwFiles.SESS_FILE)
 
+        wordList = path / "meta" / "wordlist.txt"
+        if wordList.is_file():
+            self._convertOldWordList(wordList)
+
         remove = [
             path / "meta" / "tagsIndex.json",          # Renamed in 2.1 Beta 1
             path / "meta" / "mainOptions.json",        # Replaced in 0.5
@@ -413,6 +418,31 @@ class NWStorage:
 
         return
 
+    def _convertOldWordList(self, wordList: Path) -> bool:
+        """Convert the old word list plain text file to new format."""
+        if not wordList.exists():
+            # Nothing to convert
+            return True
+
+        userDict = UserDictionary(self._project)
+        try:
+            with open(wordList, mode="r", encoding="utf-8") as fObj:
+                for line in fObj:
+                    word = line.strip()
+                    if word:
+                        userDict.add(word)
+
+            # Dave dictionary and clean up old file
+            userDict.save()
+            wordList.unlink()
+
+        except Exception:
+            logger.error("Failed to convert old word list file")
+            logException()
+            return False
+
+        return True
+
     def _convertOldLogFile(self, sessLog: Path, sessJson: Path) -> bool:
         """Convert the old text log file format to the new JSON Lines
         format.

novelwriter/dialogs/wordlist.py

@@ -1,7 +1,6 @@
 """
 novelWriter – GUI User Wordlist
 ===============================
-Class holding the user's wordlist dialog
 
 File History:
 Created: 2021-02-12 [1.2rc1]
@@ -22,28 +21,31 @@ General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
+from __future__ import annotations
 
 import logging
 
-from pathlib import Path
+from typing import TYPE_CHECKING
 
 from PyQt5.QtCore import Qt
 from PyQt5.QtWidgets import (
-    QDialog, QDialogButtonBox, QVBoxLayout, QHBoxLayout, QListWidget,
+    QAbstractItemView, QDialog, QDialogButtonBox, QHBoxLayout, QLabel,
-    QAbstractItemView, QPushButton, QLineEdit, QLabel
+    QLineEdit, QListWidget, QListWidgetItem, QPushButton, QVBoxLayout
 )
 
 from novelwriter import CONFIG
 from novelwriter.enum import nwAlert
-from novelwriter.error import logException
+from novelwriter.core.spellcheck import UserDictionary
-from novelwriter.constants import nwFiles
+
+if TYPE_CHECKING:  # pragma: no cover
+    from novelwriter.guimain import GuiMain
 
 logger = logging.getLogger(__name__)
 
 
 class GuiWordList(QDialog):
 
-    def __init__(self, mainGui):
+    def __init__(self, mainGui: GuiMain):
         super().__init__(parent=mainGui)
 
         logger.debug("Create: GuiWordList")
@@ -120,67 +122,49 @@ class GuiWordList(QDialog):
     #  Slots
     ##
 
-    def _doAdd(self):
+    def _doAdd(self) -> bool:
-        """Add a new word to the word list.
+        """Add a new word to the word list."""
-        """
+        word = self.newEntry.text().strip()
-        newWord = self.newEntry.text().strip()
+        if word == "":
-        if newWord == "":
             self.mainGui.makeAlert(self.tr(
                 "Cannot add a blank word."
             ), nwAlert.ERROR)
             return False
 
-        if self.listBox.findItems(newWord, Qt.MatchExactly):
+        if self.listBox.findItems(word, Qt.MatchExactly):
             self.mainGui.makeAlert(self.tr(
                 "The word '{0}' is already in the word list."
-            ).format(newWord), nwAlert.ERROR)
+            ).format(word), nwAlert.ERROR)
             return False
 
-        self.listBox.addItem(newWord)
+        self.listBox.addItem(word)
         self.newEntry.setText("")
 
         return True
 
     def _doDelete(self):
-        """Delete the selected item.
+        """Delete the selected item."""
-        """
         selItem = self.listBox.selectedItems()
         if selItem:
             self.listBox.takeItem(self.listBox.row(selItem[0]))
         return
 
     def _doSave(self):
-        """Save the new word list and close.
+        """Save the new word list and close."""
-        """
         self._saveGuiSettings()
-
+        userDict = UserDictionary(self.theProject)
-        dctFile = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
+        for i in range(self.listBox.count()):
-        if not isinstance(dctFile, Path):
+            item = self.listBox.item(i)
-            return False
+            if isinstance(item, QListWidgetItem):
-
+                word = item.text().strip()
-        tmpFile = dctFile.with_suffix(".tmp")
+                if word:
-        try:
+                    userDict.add(word)
-            with open(tmpFile, mode="w", encoding="utf-8") as outFile:
+        userDict.save()
-                for i in range(self.listBox.count()):
-                    item = self.listBox.item(i)
-                    if item is not None:
-                        outFile.write(item.text() + "\n")
-
-            tmpFile.replace(dctFile)
-
-        except Exception:
-            logger.error("Could not save new word list")
-            logException()
-            self.reject()
-            return False
-
         self.accept()
-
         return True
 
     def _doClose(self):
-        """Close without saving the word list.
+        """Close without saving the word list."""
-        """
         self._saveGuiSettings()
         self.reject()
         return
@@ -190,29 +174,17 @@ class GuiWordList(QDialog):
     ##
 
     def _loadWordList(self):
-        """Load the project's word list, if it exists.
+        """Load the project's word list, if it exists."""
-        """
+        userDict = UserDictionary(self.theProject)
-        wordList = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
+        userDict.load()
-        if not isinstance(wordList, Path):
-            return False
-
         self.listBox.clear()
-        if not wordList.exists():
+        for word in userDict:
-            logger.debug("No project dictionary file found")
+            if word:
-            return False
+                self.listBox.addItem(word)
-
+        return
-        with open(wordList, mode="r", encoding="utf-8") as inFile:
-            for inLine in inFile:
-                theWord = inLine.strip()
-                if len(theWord) == 0:
-                    continue
-                self.listBox.addItem(theWord)
-
-        return True
 
     def _saveGuiSettings(self):
-        """Save GUI settings.
+        """Save GUI settings."""
-        """
         winWidth  = CONFIG.rpxInt(self.width())
         winHeight = CONFIG.rpxInt(self.height())
 

novelwriter/gui/doceditor.py

@@ -52,7 +52,7 @@ from PyQt5.QtWidgets import (
 from novelwriter import CONFIG
 from novelwriter.enum import nwAlert, nwDocAction, nwDocInsert, nwDocMode, nwItemClass
 from novelwriter.common import minmax, transferCase
-from novelwriter.constants import nwConst, nwFiles, nwKeyWords, nwUnicode
+from novelwriter.constants import nwConst, nwKeyWords, nwUnicode
 from novelwriter.core.index import countWords
 from novelwriter.core.spellcheck import NWSpellEnchant
 from novelwriter.gui.dochighlight import GuiDocHighlighter
@@ -131,7 +131,7 @@ class GuiDocEditor(QTextEdit):
         self.docSearch = GuiDocEditSearch(self)
 
         # Syntax
-        self.spEnchant = NWSpellEnchant()
+        self.spEnchant = NWSpellEnchant(self.theProject)
         self.highLight = GuiDocHighlighter(qDoc, self.mainGui, self.spEnchant)
 
         # Context Menu
@@ -703,8 +703,7 @@ class GuiDocEditor(QTextEdit):
         else:
             theLang = self.theProject.data.spellLang
 
-        projDict = self.theProject.storage.getMetaFile(nwFiles.PROJ_DICT)
+        self.spEnchant.setLanguage(theLang)
-        self.spEnchant.setLanguage(theLang, projDict)
         _, theProvider = self.spEnchant.describeDict()
 
         self.spellDictionaryChanged.emit(str(theLang), str(theProvider))

tests/test_core/test_core_spellcheck.py

@@ -37,18 +37,18 @@ def testCoreSpell_FakeEnchant(monkeypatch):
         mp.setitem(sys.modules, "enchant", None)
         spChk = NWSpellEnchant()
         spChk.setLanguage("en_US", "")
-        assert isinstance(spChk._theDict, FakeEnchant)
+        assert isinstance(spChk._dictObj, FakeEnchant)
 
     # Request a non-existent dictionary
     spChk = NWSpellEnchant()
     spChk.setLanguage("whatchamajig", "")
-    assert isinstance(spChk._theDict, FakeEnchant)
+    assert isinstance(spChk._dictObj, FakeEnchant)
 
     # Request an emety language string
     # See issue https://github.com/vkbo/novelWriter/issues/1096
     spChk = NWSpellEnchant()
     spChk.setLanguage("", "")
-    assert isinstance(spChk._theDict, FakeEnchant)
+    assert isinstance(spChk._dictObj, FakeEnchant)
 
     # FakeEnchant should handle requests
     fkChk = FakeEnchant()
@@ -96,7 +96,7 @@ def testCoreSpell_Enchant(monkeypatch, fncPath):
 
     assert spChk._readProjectDictionary(None) is False
     assert spChk._readProjectDictionary(wList) is True
-    assert spChk._projectDict == wList
+    assert spChk._userDictPath == wList
 
     # Cannot write to file
     with monkeypatch.context() as mp:

tests/test_dialogs/test_dlg_wordlist.py

@@ -27,7 +27,6 @@ from PyQt5.QtWidgets import QDialog, QAction
 from tools import buildTestProject, writeFile, readFile, getGuiItem
 from mocked import causeOSError
 
-from novelwriter.constants import nwFiles
 from novelwriter.dialogs.wordlist import GuiWordList
 
 
@@ -43,7 +42,7 @@ def testDlgWordList_Dialog(qtbot, monkeypatch, nwGUI, projPath):
 
     # Open project
     nwGUI.openProject(projPath)
-    dictFile = projPath / "meta" / nwFiles.PROJ_DICT
+    dictFile = projPath / "meta" / "wordlist.txt"
 
     # Load the dialog
     nwGUI.mainMenu.aEditWordList.activate(QAction.Trigger)

tests/test_tools/test_tools_writingstats.py

@@ -28,7 +28,6 @@ from tools import getGuiItem, writeFile, buildTestProject
 from PyQt5.QtCore import Qt
 from PyQt5.QtWidgets import QAction, QFileDialog
 
-from novelwriter.constants import nwFiles
 from novelwriter.tools.writingstats import GuiWritingStats
 
 
@@ -40,7 +39,7 @@ def testToolWritingStats_Main(qtbot, monkeypatch, nwGUI, projPath, tstPaths):
     buildTestProject(nwGUI, projPath)
     qtbot.wait(100)
     assert nwGUI.saveProject()
-    sessFile = projPath / "meta" / nwFiles.SESS_STATS
+    sessFile = projPath / "meta" / "sessionStats.log"
 
     # Open the Writing Stats dialog
     nwGUI.mainMenu.aWritingStats.activate(QAction.Trigger)