Rename package from 'nw' to 'novelwriter' (#868)

* Rename nw folder to novelwriter
* Rename nw to novelwriter in auxiliary files
* Rename nw to novelwriter in main app source
* Rename nw to novelwriter in tests
* Make setup script for pdf docs less spammy

novelwriter/core/__init__.py

@@ -0,0 +1,41 @@
+"""
+novelWriter – Core Init
+=======================
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+from novelwriter.core.document import NWDoc
+from novelwriter.core.index import NWIndex, countWords
+from novelwriter.core.project import NWProject
+from novelwriter.core.spellcheck import NWSpellCheck, NWSpellEnchant, NWSpellSimple
+from novelwriter.core.tohtml import ToHtml
+from novelwriter.core.toodt import ToOdt
+from novelwriter.core.tomd import ToMarkdown
+
+__all__ = [
+    "countWords",
+    "NWDoc",
+    "NWIndex",
+    "NWProject",
+    "NWSpellCheck",
+    "NWSpellEnchant",
+    "NWSpellSimple",
+    "ToHtml",
+    "ToOdt",
+    "ToMarkdown",
+]

novelwriter/core/document.py

@@ -0,0 +1,244 @@
+"""
+novelWriter – Project Document
+==============================
+Data class for a single novelWriter document
+
+File History:
+Created: 2018-09-29 [0.0.1]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import os
+import logging
+
+from novelwriter.enum import nwItemLayout, nwItemClass
+from novelwriter.common import isHandle
+
+logger = logging.getLogger(__name__)
+
+
+class NWDoc():
+
+    def __init__(self, theProject, theHandle):
+
+        self.theProject = theProject
+
+        # Internal Variables
+        self._theItem   = None  # The currently open item
+        self._docHandle = None  # The handle of the currently open item
+        self._fileLoc   = None  # The file location of the currently open item
+        self._docMeta   = {}    # The meta data of the currently open item
+        self._docError  = ""    # The latest encountered IO error
+
+        if isHandle(theHandle):
+            self._docHandle = theHandle
+
+        if self._docHandle is not None:
+            self._theItem = self.theProject.projTree[theHandle]
+
+        return
+
+    ##
+    #  Class Methods
+    ##
+
+    def readDocument(self, isOrphan=False):
+        """Read a document from set handle, capturing potential file
+        system errors and parse meta data. If the document doesn't exist
+        on disk, return an empty string. If something went wrong, return
+        None.
+        """
+        self._docError = ""
+        if self._docHandle is None:
+            logger.error("No document handle set")
+            return None
+
+        if self._theItem is None and not isOrphan:
+            logger.error("Unknown novelWriter document")
+            return None
+
+        docFile = self._docHandle+".nwd"
+        logger.debug("Opening document: %s", docFile)
+
+        docPath = os.path.join(self.theProject.projContent, docFile)
+        self._fileLoc = docPath
+
+        theText = ""
+        self._docMeta = {}
+        if os.path.isfile(docPath):
+            try:
+                with open(docPath, mode="r", encoding="utf-8") as inFile:
+
+                    # Check the first <= 10 lines for metadata
+                    for i in range(10):
+                        inLine = inFile.readline()
+                        if inLine.startswith(r"%%~"):
+                            self._parseMeta(inLine)
+                        else:
+                            theText = inLine
+                            break
+
+                    # Load the rest of the file
+                    theText += inFile.read()
+
+            except Exception as e:
+                self._docError = str(e)
+                return None
+
+        else:
+            # The document file does not exist, so we assume it's a new
+            # document and initialise an empty text string.
+            logger.debug("The requested document does not exist.")
+            return ""
+
+        return theText
+
+    def writeDocument(self, docText):
+        """Write the document. The file is saved via a temp file in case
+        of save failure. Returns True if successful, False if not.
+        """
+        self._docError = ""
+        if self._docHandle is None:
+            logger.error("No document handle set")
+            return False
+
+        self.theProject.ensureFolderStructure()
+
+        docFile = self._docHandle+".nwd"
+        logger.debug("Saving document: %s", docFile)
+
+        docPath = os.path.join(self.theProject.projContent, docFile)
+        docTemp = os.path.join(self.theProject.projContent, docFile+"~")
+
+        # DocMeta line
+        if self._theItem is None:
+            docMeta = ""
+        else:
+            docMeta = (
+                f"%%~name: {self._theItem.itemName}\n"
+                f"%%~path: {self._theItem.itemParent}/{self._theItem.itemHandle}\n"
+                f"%%~kind: {self._theItem.itemClass.name}/{self._theItem.itemLayout.name}\n"
+            )
+
+        try:
+            with open(docTemp, mode="w", encoding="utf-8") as outFile:
+                outFile.write(docMeta)
+                outFile.write(docText)
+        except Exception as e:
+            self._docError = str(e)
+            return False
+
+        # If we're here, the file was successfully saved, so we can
+        # replace the temp file with the actual file
+        if os.path.isfile(docPath):
+            os.unlink(docPath)
+        os.rename(docTemp, docPath)
+
+        return True
+
+    def deleteDocument(self):
+        """Permanently delete a document source file and related files
+        from the project data folder.
+        """
+        self._docError = ""
+        if self._docHandle is None:
+            logger.error("No document handle set")
+            return False
+
+        docFile = self._docHandle+".nwd"
+
+        chkList = []
+        chkList.append(os.path.join(self.theProject.projContent, docFile))
+        chkList.append(os.path.join(self.theProject.projContent, docFile+"~"))
+
+        for chkFile in chkList:
+            if os.path.isfile(chkFile):
+                try:
+                    os.unlink(chkFile)
+                    logger.debug("Deleted: %s", chkFile)
+                except Exception as e:
+                    self._docError = str(e)
+                    return False
+
+        return True
+
+    ##
+    #  Getters
+    ##
+
+    def getFileLocation(self):
+        """Return the file location of the current file.
+        """
+        return self._fileLoc
+
+    def getCurrentItem(self):
+        """Return a pointer to the currently open item.
+        """
+        return self._theItem
+
+    def getMeta(self):
+        """Parses the document meta tag and returns the path and name as
+        a list and a string.
+        """
+        theName = self._docMeta.get("name", "")
+        theParent = self._docMeta.get("parent", None)
+        theClass = self._docMeta.get("class", None)
+        theLayout = self._docMeta.get("layout", None)
+
+        return theName, theParent, theClass, theLayout
+
+    def getError(self):
+        """Return the last recorded exception.
+        """
+        return self._docError
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _parseMeta(self, metaLine):
+        """Parse a line from the document statting with the characters
+        %%~ that may contain meta data.
+        """
+        if metaLine.startswith("%%~name:"):
+            self._docMeta["name"] = metaLine[8:].strip()
+
+        elif metaLine.startswith("%%~path:"):
+            metaVal = metaLine[8:].strip()
+            metaBits = metaVal.split("/")
+            if len(metaBits) == 2:
+                if isHandle(metaBits[0]):
+                    self._docMeta["parent"] = metaBits[0]
+                if isHandle(metaBits[1]):
+                    self._docMeta["handle"] = metaBits[1]
+
+        elif metaLine.startswith("%%~kind:"):
+            metaVal = metaLine[8:].strip()
+            metaBits = metaVal.split("/")
+            if len(metaBits) == 2:
+                if metaBits[0] in nwItemClass.__members__:
+                    self._docMeta["class"] = nwItemClass[metaBits[0]]
+                if metaBits[1] in nwItemLayout.__members__:
+                    self._docMeta["layout"] = nwItemLayout[metaBits[1]]
+
+        else:
+            logger.debug("Ignoring meta data: '%s'", metaLine.strip())
+
+        return
+
+# END Class NWDoc

novelwriter/core/index.py

@@ -0,0 +1,925 @@
+"""
+novelWriter – Project Index
+===========================
+Data class for the project index of tags, headers and references
+
+File History:
+Created: 2019-04-22 [0.0.1] countWords
+Created: 2019-05-27 [0.1.4] NWIndex
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import os
+import json
+import logging
+import novelwriter
+
+from time import time
+
+from novelwriter.enum import nwItemType, nwItemClass, nwItemLayout
+from novelwriter.constants import nwFiles, nwKeyWords, nwUnicode
+from novelwriter.core.document import NWDoc
+from novelwriter.common import (
+    isHandle, isTitleTag, isItemClass, isItemLayout, jsonEncode
+)
+
+logger = logging.getLogger(__name__)
+
+H_VALID = ("H0", "H1", "H2", "H3", "H4")
+H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4}
+
+
+class NWIndex():
+
+    def __init__(self, theProject):
+
+        # Internal
+        self.mainConf    = novelwriter.CONFIG
+        self.theProject  = theProject
+        self.indexBroken = False
+
+        # Indices
+        self._tagIndex  = {}
+        self._refIndex  = {}
+        self._fileIndex = {}
+        self._fileMeta  = {}
+
+        # TimeStamps
+        self._timeNovel = 0
+        self._timeNotes = 0
+        self._timeIndex = 0
+
+        return
+
+    ##
+    #  Public Methods
+    ##
+
+    def clearIndex(self):
+        """Clear the index dictionaries and time stamps.
+        """
+        self._tagIndex  = {}
+        self._refIndex  = {}
+        self._fileIndex = {}
+        self._fileMeta  = {}
+        self._timeNovel = 0
+        self._timeNotes = 0
+        self._timeIndex = 0
+        return
+
+    def deleteHandle(self, tHandle):
+        """Delete all entries of a given document handle.
+        """
+        logger.debug("Removing item '%s' from the index", tHandle)
+
+        delTags = []
+        for tTag in self._tagIndex:
+            if self._tagIndex[tTag][1] == tHandle:
+                delTags.append(tTag)
+
+        for tTag in delTags:
+            self._tagIndex.pop(tTag, None)
+
+        self._refIndex.pop(tHandle, None)
+        self._fileIndex.pop(tHandle, None)
+        self._fileMeta.pop(tHandle, None)
+
+        return
+
+    def reIndexHandle(self, tHandle):
+        """Put a file back into the index. This is used when files are
+        moved from the archive or trash folders back into the active
+        project.
+        """
+        logger.debug("Re-indexing item '%s'", tHandle)
+
+        tItem = self.theProject.projTree[tHandle]
+        if tItem is None:
+            return False
+        if tItem.itemType != nwItemType.FILE:
+            return False
+
+        theDoc = NWDoc(self.theProject, tHandle)
+        theText = theDoc.readDocument()
+        if theText:
+            self.scanText(tHandle, theText)
+
+        return True
+
+    def novelChangedSince(self, checkTime):
+        """Check if the novel index has changed since a given time.
+        """
+        return self._timeNovel > checkTime
+
+    def notesChangedSince(self, checkTime):
+        """Check if the notes index has changed since a given time.
+        """
+        return self._timeNotes > checkTime
+
+    def indexChangedSince(self, checkTime):
+        """Check if the index has changed since a given time.
+        """
+        return self._timeIndex > checkTime
+
+    ##
+    #  Load and Save Index to/from File
+    ##
+
+    def loadIndex(self):
+        """Load index from last session from the project meta folder.
+        """
+        theData = {}
+        indexFile = os.path.join(self.theProject.projMeta, nwFiles.INDEX_FILE)
+        tStart = time()
+
+        if os.path.isfile(indexFile):
+            logger.debug("Loading index file")
+            try:
+                with open(indexFile, mode="r", encoding="utf-8") as inFile:
+                    theData = json.load(inFile)
+
+            except Exception:
+                logger.error("Failed to load index file")
+                novelwriter.logException()
+                self.indexBroken = True
+                return False
+
+            self._tagIndex = theData.get("tagIndex", {})
+            self._refIndex = theData.get("refIndex", {})
+            self._fileIndex = theData.get("fileIndex", {})
+            self._fileMeta = theData.get("fileMeta", {})
+
+            nowTime = round(time())
+            self._timeNovel = nowTime
+            self._timeNotes = nowTime
+            self._timeIndex = nowTime
+
+        logger.verbose("Index loaded in %.3f ms", (time() - tStart)*1000)
+
+        self.checkIndex()
+
+        return True
+
+    def saveIndex(self):
+        """Save the current index as a json file in the project meta
+        data folder.
+        """
+        logger.debug("Saving index file")
+        indexFile = os.path.join(self.theProject.projMeta, nwFiles.INDEX_FILE)
+        tStart = time()
+
+        try:
+            with open(indexFile, mode="w+", encoding="utf-8") as outFile:
+                outFile.write("{\n")
+                outFile.write(f'"tagIndex": {jsonEncode(self._tagIndex, nmax=1)},\n')
+                outFile.write(f'"refIndex": {jsonEncode(self._refIndex, nmax=2)},\n')
+                outFile.write(f'"fileIndex": {jsonEncode(self._fileIndex, nmax=2)},\n')
+                outFile.write(f'"fileMeta": {jsonEncode(self._fileMeta, nmax=1)}\n')
+                outFile.write("}\n")
+
+        except Exception:
+            logger.error("Failed to save index file")
+            novelwriter.logException()
+            return False
+
+        logger.verbose("Index saved in %.3f ms", (time() - tStart)*1000)
+
+        return True
+
+    def checkIndex(self):
+        """Check that the entries in the index are valid and contain the
+        elements it should.
+        """
+        logger.debug("Checking index")
+        tStart = time()
+
+        try:
+            self._checkTagIndex()
+            self._checkRefIndex()
+            self._checkFileIndex()
+            self._checkFileMeta()
+            self.indexBroken = False
+
+        except Exception:
+            logger.error("Error while checking index")
+            novelwriter.logException()
+            self.indexBroken = True
+
+        logger.verbose("Index check took %.3f ms", (time() - tStart)*1000)
+        logger.debug("Index check complete")
+
+        if self.indexBroken:
+            self.clearIndex()
+
+        return
+
+    ##
+    #  Index Building
+    ##
+
+    def scanText(self, tHandle, theText):
+        """Scan a piece of text associated with a handle. This will
+        update the indices accordingly. This function takes the handle
+        and text as separate inputs as we want to primarily scan the
+        files before we save them, unless we're rebuilding the index.
+        """
+        theItem = self.theProject.projTree[tHandle]
+        theRoot = self.theProject.projTree.getRootItem(tHandle)
+
+        if theItem is None:
+            logger.info("Not indexing unknown item '%s'", tHandle)
+            return False
+        if theItem.itemType != nwItemType.FILE:
+            logger.info("Not indexing non-file item '%s'", tHandle)
+            return False
+        if theItem.itemLayout == nwItemLayout.NO_LAYOUT:
+            logger.info("Not indexing no-layout item '%s'", tHandle)
+            return False
+        if theItem.itemParent is None:
+            logger.info("Not indexing orphaned item '%s'", tHandle)
+            return False
+
+        # Run word counter for the whole text
+        cC, wC, pC = countWords(theText)
+        self._fileMeta[tHandle] = ["H0", cC, wC, pC]
+
+        # If the file is archived or trashed, we don't index the file itself
+        if self.theProject.projTree.isTrashRoot(theItem.itemParent):
+            logger.debug("Not indexing trash item '%s'", tHandle)
+            return False
+        if theRoot.itemClass == nwItemClass.ARCHIVE:
+            logger.debug("Not indexing archived item '%s'", tHandle)
+            return False
+
+        itemClass  = theItem.itemClass
+        itemLayout = theItem.itemLayout
+
+        logger.debug("Indexing item with handle '%s'", tHandle)
+
+        # Delete or reset old entries for the file
+        self._refIndex.pop(tHandle, None)
+        self._fileIndex[tHandle] = {}
+
+        # Also clear references to file in tag index
+        clearTags = []
+        for aTag in self._tagIndex:
+            if self._tagIndex[aTag][1] == tHandle:
+                clearTags.append(aTag)
+        for aTag in clearTags:
+            self._tagIndex.pop(aTag)
+
+        # Scan the text content
+        nLine = 0
+        nTitle = 0
+        theLines = theText.splitlines()
+        for aLine in theLines:
+            nLine += 1
+            nChar = len(aLine.strip())
+            if nChar == 0:
+                continue
+
+            if aLine.startswith("#"):
+                isTitle = self._indexTitle(tHandle, aLine, nLine, itemLayout)
+                if isTitle and nLine > 0:
+                    if nTitle > 0:
+                        lastText = "\n".join(theLines[nTitle-1:nLine-1])
+                        self._indexWordCounts(tHandle, lastText, nTitle)
+                    nTitle = nLine
+
+            elif aLine.startswith("@"):
+                self._indexKeyword(tHandle, aLine, nLine, nTitle, itemClass)
+
+            elif aLine.startswith("%"):
+                if nTitle > 0:
+                    toCheck = aLine[1:].lstrip()
+                    synTag = toCheck[:9].lower()
+                    tLen = len(aLine)
+                    cLen = len(toCheck)
+                    cOff = tLen - cLen
+                    if synTag == "synopsis:":
+                        self._indexSynopsis(tHandle, aLine[cOff+9:].strip(), nTitle)
+
+        # Count words for remaining text after last heading
+        if nTitle > 0:
+            lastText = "\n".join(theLines[nTitle-1:])
+            self._indexWordCounts(tHandle, lastText, nTitle)
+
+        # Index page with no titles and references
+        if nTitle == 0:
+            self._indexPage(tHandle, itemLayout)
+            self._indexWordCounts(tHandle, theText, nTitle)
+
+        # Update timestamps for index changes
+        nowTime = round(time())
+        self._timeIndex = nowTime
+        if itemLayout == nwItemLayout.NOTE:
+            self._timeNotes = nowTime
+        else:
+            self._timeNovel = nowTime
+
+        return True
+
+    ##
+    #  Internal Indexers
+    ##
+
+    def _indexTitle(self, tHandle, aLine, nLine, itemLayout):
+        """Save information about the title and its location in the
+        file to the index.
+        """
+        if aLine.startswith("# "):
+            hDepth = "H1"
+            hText = aLine[2:].strip()
+        elif aLine.startswith("## "):
+            hDepth = "H2"
+            hText = aLine[3:].strip()
+        elif aLine.startswith("### "):
+            hDepth = "H3"
+            hText = aLine[4:].strip()
+        elif aLine.startswith("#### "):
+            hDepth = "H4"
+            hText = aLine[5:].strip()
+        elif aLine.startswith("#! "):
+            hDepth = "H1"
+            hText = aLine[2:].strip()
+        elif aLine.startswith("##! "):
+            hDepth = "H2"
+            hText = aLine[4:].strip()
+        else:
+            return False
+
+        sTitle = "T%06d" % nLine
+        self._fileIndex[tHandle][sTitle] = {
+            "level": hDepth,
+            "title": hText,
+            "layout": itemLayout.name,
+            "cCount": 0,
+            "wCount": 0,
+            "pCount": 0,
+            "synopsis": "",
+        }
+
+        if self._fileMeta[tHandle][0] == "H0":
+            self._fileMeta[tHandle][0] = hDepth
+
+        return True
+
+    def _indexPage(self, tHandle, itemLayout):
+        """Index a page with no title.
+        """
+        self._fileIndex[tHandle]["T000000"] = {
+            "level": "H0",
+            "title": "",
+            "layout": itemLayout.name,
+            "cCount": 0,
+            "wCount": 0,
+            "pCount": 0,
+            "synopsis": "",
+        }
+
+        return
+
+    def _indexWordCounts(self, tHandle, theText, nTitle):
+        """Count text stats and save the counts to the index.
+        """
+        cC, wC, pC = countWords(theText)
+        sTitle = "T%06d" % nTitle
+        if tHandle in self._fileIndex:
+            if sTitle in self._fileIndex[tHandle]:
+                self._fileIndex[tHandle][sTitle]["cCount"] = cC
+                self._fileIndex[tHandle][sTitle]["wCount"] = wC
+                self._fileIndex[tHandle][sTitle]["pCount"] = pC
+        return
+
+    def _indexSynopsis(self, tHandle, theText, nTitle):
+        """Save the synopsis to the index.
+        """
+        sTitle = "T%06d" % nTitle
+        if tHandle in self._fileIndex:
+            if sTitle in self._fileIndex[tHandle]:
+                self._fileIndex[tHandle][sTitle]["synopsis"] = theText
+        return
+
+    def _indexKeyword(self, tHandle, aLine, nLine, nTitle, itemClass):
+        """Validate and save the information about a reference to a tag
+        in another file.
+        """
+        isValid, theBits, _ = self.scanThis(aLine)
+        if not isValid or len(theBits) < 2:
+            logger.warning("Skipping keyword with %d value(s) in '%s'", len(theBits), tHandle)
+            return
+
+        if theBits[0] not in nwKeyWords.VALID_KEYS:
+            logger.warning("Skipping invalid keyword '%s' in '%s'", theBits[0], tHandle)
+            return
+
+        sTitle = "T%06d" % nTitle
+        if theBits[0] == nwKeyWords.TAG_KEY:
+            self._tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name, sTitle]
+
+        else:
+            if tHandle not in self._refIndex:
+                self._refIndex[tHandle] = {}
+            if sTitle not in self._refIndex[tHandle]:
+                self._refIndex[tHandle][sTitle] = []
+            for aVal in theBits[1:]:
+                self._refIndex[tHandle][sTitle].append([nLine, theBits[0], aVal])
+
+        return
+
+    ##
+    #  Check @ Lines
+    ##
+
+    def scanThis(self, aLine):
+        """Scan a line starting with @ to check that it's valid. Then
+        split it up into its elements and positions as two arrays.
+        """
+        theBits = []  # The elements of the string
+        thePos  = []  # The absolute position of each element
+
+        aLine = aLine.rstrip()  # Remove all trailing white spaces
+        nChar = len(aLine)
+        if nChar < 2:
+            return False, theBits, thePos
+        if aLine[0] != "@":
+            return False, theBits, thePos
+
+        cKey, _, cVals = aLine.partition(":")
+        sKey = cKey.strip()
+        if sKey == "@":
+            return False, theBits, thePos
+
+        cPos = 0
+        theBits.append(sKey)
+        thePos.append(cPos)
+        cPos += len(cKey) + 1
+
+        if not cVals:
+            # No values, so we're done
+            return True, theBits, thePos
+
+        for cVal in cVals.split(","):
+            sVal = cVal.strip()
+            rLen = len(cVal.lstrip())
+            tLen = len(cVal)
+            theBits.append(sVal)
+            thePos.append(cPos + tLen - rLen)
+            cPos += tLen + 1
+
+        return True, theBits, thePos
+
+    def checkThese(self, theBits, tItem):
+        """Check the tags against the index to see if they are valid
+        tags. This is needed for syntax highlighting.
+        """
+        nBits = len(theBits)
+        isGood = [False]*nBits
+        if nBits == 0:
+            return []
+
+        # Check that the key is valid
+        isGood[0] = theBits[0] in nwKeyWords.VALID_KEYS
+        if not isGood[0] or nBits == 1:
+            return isGood
+
+        # For a tag, only the first value is accepted, the rest are ignored
+        if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1:
+            if theBits[1] in self._tagIndex:
+                isGood[1] = self._tagIndex[theBits[1]][1] == tItem.itemHandle
+            else:
+                isGood[1] = True
+            return isGood
+
+        # If we're still here, we check that the references exist
+        theKey = nwKeyWords.KEY_CLASS[theBits[0]].name
+        for n in range(1, nBits):
+            if theBits[n] in self._tagIndex:
+                isGood[n] = theKey == self._tagIndex[theBits[n]][2]
+
+        return isGood
+
+    ##
+    #  Extract Data
+    ##
+
+    def novelStructure(self, skipExcluded=True):
+        """Iterate over all titles in the novel, in the correct order as
+        they appear in the tree view and in the respective document
+        files, but skipping all note files.
+        """
+        for tHandle in self._listNovelHandles(skipExcluded):
+            for sTitle in sorted(self._fileIndex[tHandle]):
+                tKey = "%s:%s" % (tHandle, sTitle)
+                yield tKey, tHandle, sTitle, self._fileIndex[tHandle][sTitle]
+
+    def getNovelWordCount(self, skipExcluded=True):
+        """Count the number of words in the novel project.
+        """
+        wCount = 0
+        for tHandle in self._listNovelHandles(skipExcluded):
+            for sTitle in self._fileIndex[tHandle]:
+                wCount += self._fileIndex[tHandle][sTitle]["wCount"]
+
+        return wCount
+
+    def getNovelTitleCounts(self, skipExcluded=True):
+        """Count the number of titles in the novel project.
+        """
+        hCount = [0, 0, 0, 0, 0]
+        for tHandle in self._listNovelHandles(skipExcluded):
+            for sTitle in self._fileIndex[tHandle]:
+                theData = self._fileIndex[tHandle][sTitle]
+                iLevel = H_LEVEL.get(theData["level"], 0)
+                hCount[iLevel] += 1
+
+        return hCount
+
+    def getHandleWordCounts(self, tHandle):
+        """Get all header word counts for a specific handle.
+        """
+        theCounts = []
+        hRecord = self._fileIndex.get(tHandle, None)
+        if hRecord is None:
+            return theCounts
+
+        for sTitle, sData in hRecord.items():
+            theCounts.append(("%s:%s" % (tHandle, sTitle), sData["wCount"]))
+
+        return theCounts
+
+    def getHandleHeaders(self, tHandle):
+        """Get all headers for a specific handle.
+        """
+        theHeaders = []
+        hRecord = self._fileIndex.get(tHandle, None)
+        if hRecord is None:
+            return theHeaders
+
+        for sTitle, sData in hRecord.items():
+            theHeaders.append((sTitle, sData["level"], sData["title"]))
+
+        return theHeaders
+
+    def getHandleHeaderLevel(self, tHandle):
+        """Get the header level of the first header of a handle.
+        """
+        if tHandle in self._fileMeta:
+            return self._fileMeta[tHandle][0]
+        return "H0"
+
+    def getTableOfContents(self, maxDepth, skipExcluded=True):
+        """Generate a table of contents up to a maxiumum depth.
+        """
+        tOrder = []
+        tData = {}
+        pKey = None
+        for tHandle in self._listNovelHandles(skipExcluded):
+            for sTitle in sorted(self._fileIndex[tHandle]):
+                tKey = "%s:%s" % (tHandle, sTitle)
+                theData = self._fileIndex[tHandle][sTitle]
+                iLevel = H_LEVEL.get(theData["level"], 0)
+                if iLevel > maxDepth:
+                    if pKey in tData:
+                        theData["wCount"]
+                        tData[pKey]["words"] += theData["wCount"]
+                else:
+                    pKey = tKey
+                    tOrder.append(tKey)
+                    tData[tKey] = {
+                        "level": iLevel,
+                        "title": theData["title"],
+                        "words": theData["wCount"],
+                    }
+
+        theToC = []
+        for tKey in tOrder:
+            theToC.append((
+                tKey,
+                tData[tKey]["level"],
+                tData[tKey]["title"],
+                tData[tKey]["words"],
+            ))
+
+        return theToC
+
+    def getCounts(self, tHandle, sTitle=None):
+        """Returns the counts for a file, or a section of a file
+        starting at title sTitle if it is provided.
+        """
+        cC = 0
+        wC = 0
+        pC = 0
+
+        if sTitle is None:
+            if tHandle in self._fileMeta:
+                cC = self._fileMeta[tHandle][1]
+                wC = self._fileMeta[tHandle][2]
+                pC = self._fileMeta[tHandle][3]
+        else:
+            if tHandle in self._fileIndex:
+                if sTitle in self._fileIndex[tHandle]:
+                    cC = self._fileIndex[tHandle][sTitle]["cCount"]
+                    wC = self._fileIndex[tHandle][sTitle]["wCount"]
+                    pC = self._fileIndex[tHandle][sTitle]["pCount"]
+
+        return cC, wC, pC
+
+    def getReferences(self, tHandle, sTitle=None):
+        """Extract all references made in a file, and optionally title
+        section. sTitle must be a string.
+        """
+        theRefs = {}
+        for tKey in nwKeyWords.KEY_CLASS:
+            theRefs[tKey] = []
+
+        if tHandle not in self._refIndex:
+            return theRefs
+
+        for refTitle in self._refIndex[tHandle]:
+            for aTag in self._refIndex[tHandle][refTitle]:
+                if len(aTag) == 3 and (sTitle is None or sTitle == refTitle):
+                    if aTag[1] in theRefs:
+                        theRefs[aTag[1]].append(aTag[2])
+
+        return theRefs
+
+    def getNovelData(self, tHandle, sTitle):
+        """Return the novel data of a given handle and title.
+        """
+        if tHandle in self._fileIndex:
+            if sTitle in self._fileIndex[tHandle]:
+                return self._fileIndex[tHandle][sTitle]
+        return None
+
+    def getBackReferenceList(self, tHandle):
+        """Build a list of files referring back to our file, specified
+        by tHandle.
+        """
+        theRefs = {}
+        if tHandle is None:
+            return theRefs
+
+        theTags = set()
+        for tTag in self._tagIndex:
+            if tHandle == self._tagIndex[tTag][1]:
+                theTags.add(tTag)
+
+        if theTags:
+            for tHandle in self._refIndex:
+                for sTitle in self._refIndex[tHandle]:
+                    for _, _, tTag in self._refIndex[tHandle][sTitle]:
+                        if tTag in theTags and tHandle not in theRefs:
+                            theRefs[tHandle] = sTitle
+
+        return theRefs
+
+    def getTagSource(self, theTag):
+        """Return the source location of a given tag.
+        """
+        if theTag in self._tagIndex:
+            theRef = self._tagIndex[theTag]
+            if len(theRef) == 4:
+                return theRef[1], theRef[0], theRef[3]
+        return None, 0, "T000000"
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _listNovelHandles(self, skipExcluded):
+        """Return a list of all handles that exist in the novel index.
+        """
+        theHandles = []
+        for tItem in self.theProject.projTree:
+            if tItem is None:
+                continue
+            if not tItem.isExported and skipExcluded:
+                continue
+            if tItem.itemLayout == nwItemLayout.NOTE:
+                continue
+            if tItem.itemHandle in self._fileIndex:
+                theHandles.append(tItem.itemHandle)
+
+        return theHandles
+
+    ##
+    #  Index Checkers
+    ##
+
+    def _checkTagIndex(self):
+        """Scan the tag index for errors.
+        Warning: This function raises exceptions.
+        """
+        for tTag in self._tagIndex:
+            if not isinstance(tTag, str):
+                raise KeyError("tagIndex key is not a string")
+
+            tEntry = self._tagIndex[tTag]
+            if len(tEntry) != 4:
+                raise IndexError("tagIndex[a] expected 4 values")
+            if not isinstance(tEntry[0], int):
+                raise ValueError("tagIndex[a][0] is not an integer")
+            if not isHandle(tEntry[1]):
+                raise ValueError("tagIndex[a][1] is not a handle")
+            if not isItemClass(tEntry[2]):
+                raise ValueError("tagIndex[a][2] is not an nwItemClass")
+            if not isTitleTag(tEntry[3]):
+                raise ValueError("tagIndex[a][3] is not a title tag")
+
+        return
+
+    def _checkRefIndex(self):
+        """Scan the reference index for errors.
+        Warning: This function raises exceptions.
+        """
+        for tHandle in self._refIndex:
+            if not isHandle(tHandle):
+                raise KeyError("refIndex key is not a handle")
+
+            hEntry = self._refIndex[tHandle]
+            for sTitle in hEntry:
+                if not isTitleTag(sTitle):
+                    raise KeyError("refIndex[a] key is not a title tag")
+
+                sEntry = hEntry[sTitle]
+                for tEntry in sEntry:
+                    if len(tEntry) != 3:
+                        raise IndexError("refIndex[a][b][i] expected 3 values")
+                    if not isinstance(tEntry[0], int):
+                        raise ValueError("refIndex[a][b][i][0] is not an integer")
+                    if not tEntry[1] in nwKeyWords.VALID_KEYS:
+                        raise ValueError("refIndex[a][b][i][1] is not a keyword")
+                    if not isinstance(tEntry[2], str):
+                        raise ValueError("refIndex[a][b][i][2] is not a string")
+
+        return
+
+    def _checkFileIndex(self):
+        """Scan the file index for errors.
+        Warning: This function raises exceptions.
+        """
+        for tHandle in self._fileIndex:
+            if not isHandle(tHandle):
+                raise KeyError("fileIndex key is not a handle")
+
+            hEntry = self._fileIndex[tHandle]
+            for sTitle in self._fileIndex[tHandle]:
+                if not isTitleTag(sTitle):
+                    raise KeyError("fileIndex[a] key is not a title tag")
+
+                sEntry = hEntry[sTitle]
+                if len(sEntry) != 7:
+                    raise IndexError("fileIndex[a][b] expected 7 values")
+
+                if "level" not in sEntry:
+                    raise KeyError("fileIndex[a][b] has no 'level' key")
+                if "title" not in sEntry:
+                    raise KeyError("fileIndex[a][b] has no 'title' key")
+                if "layout" not in sEntry:
+                    raise KeyError("fileIndex[a][b] has no 'layout' key")
+                if "cCount" not in sEntry:
+                    raise KeyError("fileIndex[a][b] has no 'cCount' key")
+                if "wCount" not in sEntry:
+                    raise KeyError("fileIndex[a][b] has no 'wCount' key")
+                if "pCount" not in sEntry:
+                    raise KeyError("fileIndex[a][b] has no 'pCount' key")
+                if "synopsis" not in sEntry:
+                    raise KeyError("fileIndex[a][b] has no 'synopsis' key")
+
+                if not sEntry["level"] in H_VALID:
+                    raise ValueError("fileIndex[a][b][level] is not a header level")
+                if not isinstance(sEntry["title"], str):
+                    raise ValueError("fileIndex[a][b][title] is not a string")
+                if not isItemLayout(sEntry["layout"]):
+                    raise ValueError("fileIndex[a][b][layout] is not an nwItemLayout")
+                if not isinstance(sEntry["cCount"], int):
+                    raise ValueError("fileIndex[a][b][cCount] is not an integer")
+                if not isinstance(sEntry["wCount"], int):
+                    raise ValueError("fileIndex[a][b][wCount] is not an integer")
+                if not isinstance(sEntry["pCount"], int):
+                    raise ValueError("fileIndex[a][b][pCount] is not an integer")
+                if not isinstance(sEntry["synopsis"], str):
+                    raise ValueError("fileIndex[a][b][synopsis] is not a string")
+
+        return
+
+    def _checkFileMeta(self):
+        """Scan the text counts index for errors.
+        Warning: This function raises exceptions.
+        """
+        for tHandle in self._fileMeta:
+            if not isHandle(tHandle):
+                raise KeyError("fileMeta key is not a handle")
+
+            tEntry = self._fileMeta[tHandle]
+            if len(tEntry) != 4:
+                raise IndexError("fileMeta[a] expected 4 values")
+            if not tEntry[0] in H_VALID:
+                raise ValueError("fileMeta[a][0] is not a header level")
+            if not isinstance(tEntry[1], int):
+                raise ValueError("fileMeta[a][1] is not an integer")
+            if not isinstance(tEntry[2], int):
+                raise ValueError("fileMeta[a][2] is not an integer")
+            if not isinstance(tEntry[3], int):
+                raise ValueError("fileMeta[a][3] is not an integer")
+
+        return
+
+# END Class NWIndex
+
+
+# =============================================================================================== #
+#  Simple Word Counter
+# =============================================================================================== #
+
+def countWords(theText):
+    """Count words in a piece of text, skipping special syntax and
+    comments.
+    """
+    charCount = 0
+    wordCount = 0
+    paraCount = 0
+    prevEmpty = True
+
+    if not isinstance(theText, str):
+        return charCount, wordCount, paraCount
+
+    # We need to treat dashes as word separators for counting words.
+    # The check+replace apprach is much faster that direct replace for
+    # large texts, and a bit slower for small texts, but in the latter
+    # case it doesn't matter.
+    if nwUnicode.U_ENDASH in theText:
+        theText = theText.replace(nwUnicode.U_ENDASH, " ")
+    if nwUnicode.U_EMDASH in theText:
+        theText = theText.replace(nwUnicode.U_EMDASH, " ")
+
+    for aLine in theText.splitlines():
+
+        countPara = True
+
+        if not aLine:
+            prevEmpty = True
+            continue
+        if aLine[0] == "@" or aLine[0] == "%":
+            continue
+
+        if aLine[0] == "[":
+            if aLine.startswith(("[NEWPAGE]", "[NEW PAGE]", "[VSPACE]")):
+                continue
+            elif aLine.startswith("[VSPACE:") and aLine.endswith("]"):
+                continue
+
+        elif aLine[0] == "#":
+            if aLine[:5] == "#### ":
+                aLine = aLine[5:]
+                countPara = False
+            elif aLine[:4] == "### ":
+                aLine = aLine[4:]
+                countPara = False
+            elif aLine[:3] == "## ":
+                aLine = aLine[3:]
+                countPara = False
+            elif aLine[:2] == "# ":
+                aLine = aLine[2:]
+                countPara = False
+            elif aLine[:3] == "#! ":
+                aLine = aLine[3:]
+                countPara = False
+            elif aLine[:4] == "##! ":
+                aLine = aLine[4:]
+                countPara = False
+
+        elif aLine[0] == ">" or aLine[-1] == "<":
+            if aLine[:2] == ">>":
+                aLine = aLine[2:].lstrip(" ")
+            elif aLine[:1] == ">":
+                aLine = aLine[1:].lstrip(" ")
+            if aLine[-2:] == "<<":
+                aLine = aLine[:-2].rstrip(" ")
+            elif aLine[-1:] == "<":
+                aLine = aLine[:-1].rstrip(" ")
+
+        wordCount += len(aLine.split())
+        charCount += len(aLine)
+        if countPara and prevEmpty:
+            paraCount += 1
+
+        prevEmpty = not countPara
+
+    return charCount, wordCount, paraCount

novelwriter/core/item.py

@@ -0,0 +1,336 @@
+"""
+novelWriter – Project Item Class
+================================
+Data class for a project tree item
+
+File History:
+Created: 2018-10-27 [0.0.1]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import logging
+
+from lxml import etree
+
+from novelwriter.enum import nwItemType, nwItemClass, nwItemLayout
+from novelwriter.common import (
+    checkInt, isHandle, isItemClass, isItemLayout, isItemType
+)
+from novelwriter.constants import nwLabels, nwLists, trConst
+
+logger = logging.getLogger(__name__)
+
+
+class NWItem():
+
+    def __init__(self, theProject):
+
+        self.theProject = theProject
+
+        self.itemName   = ""
+        self.itemHandle = None
+        self.itemParent = None
+        self.itemOrder  = 0
+        self.itemType   = nwItemType.NO_TYPE
+        self.itemClass  = nwItemClass.NO_CLASS
+        self.itemLayout = nwItemLayout.NO_LAYOUT
+        self.itemStatus = None
+        self.isExpanded = False
+        self.isExported = True
+
+        # Document Meta Data
+        self.charCount = 0  # Current character count
+        self.wordCount = 0  # Current word count
+        self.paraCount = 0  # Current paragraph count
+        self.initCount = 0  # Initial word count
+        self.cursorPos = 0  # Last cursor position
+
+        return
+
+    ##
+    #  XML Pack/Unpack
+    ##
+
+    def packXML(self, xParent):
+        """Packs all the data in the class instance into an XML object.
+        """
+        xPack = etree.SubElement(xParent, "item", attrib={
+            "handle": str(self.itemHandle),
+            "order":  str(self.itemOrder),
+            "parent": str(self.itemParent),
+        })
+        self._subPack(xPack, "name",   text=str(self.itemName))
+        self._subPack(xPack, "type",   text=str(self.itemType.name))
+        self._subPack(xPack, "class",  text=str(self.itemClass.name))
+        self._subPack(xPack, "status", text=str(self.itemStatus))
+        if self.itemType == nwItemType.FILE:
+            self._subPack(xPack, "exported",  text=str(self.isExported))
+            self._subPack(xPack, "layout",    text=str(self.itemLayout.name))
+            self._subPack(xPack, "charCount", text=str(self.charCount), none=False)
+            self._subPack(xPack, "wordCount", text=str(self.wordCount), none=False)
+            self._subPack(xPack, "paraCount", text=str(self.paraCount), none=False)
+            self._subPack(xPack, "cursorPos", text=str(self.cursorPos), none=False)
+        else:
+            self._subPack(xPack, "expanded", text=str(self.isExpanded))
+
+        return
+
+    def unpackXML(self, xItem):
+        """Sets the values from an XML entry of type 'item'.
+        """
+        if xItem.tag != "item":
+            logger.error("XML entry is not an NWItem")
+            return False
+
+        if "handle" in xItem.attrib:
+            self.setHandle(xItem.attrib["handle"])
+        else:
+            logger.error("XML item entry does not have a handle")
+            return False
+
+        if "parent" in xItem.attrib:
+            self.setParent(xItem.attrib["parent"])
+
+        if "order" in xItem.attrib:
+            self.setOrder(xItem.attrib["order"])
+
+        tmpStatus = ""
+        for xValue in xItem:
+            if xValue.tag == "name":
+                self.setName(xValue.text)
+            elif xValue.tag == "type":
+                self.setType(xValue.text)
+            elif xValue.tag == "class":
+                self.setClass(xValue.text)
+            elif xValue.tag == "layout":
+                self.setLayout(xValue.text)
+            elif xValue.tag == "status":
+                tmpStatus = xValue.text
+            elif xValue.tag == "expanded":
+                self.setExpanded(xValue.text)
+            elif xValue.tag == "exported":
+                self.setExported(xValue.text)
+            elif xValue.tag == "charCount":
+                self.setCharCount(xValue.text)
+            elif xValue.tag == "wordCount":
+                self.setWordCount(xValue.text)
+            elif xValue.tag == "paraCount":
+                self.setParaCount(xValue.text)
+            elif xValue.tag == "cursorPos":
+                self.setCursorPos(xValue.text)
+            else:
+                # Sliently skip as we may otherwise cause orphaned
+                # items if an otherwise valid file is opened by a
+                # version of novelWriter that doesn't know the tag.
+                logger.error("Unknown tag '%s'", xValue.tag)
+
+        # Guarantees that <status> is parsed after <class>
+        self.setStatus(tmpStatus)
+
+        return True
+
+    @staticmethod
+    def _subPack(xParent, name, attrib=None, text=None, none=True):
+        """Packs the values into an xml element.
+        """
+        if not none and (text is None or text == "None"):
+            return None
+        xAttr = {} if attrib is None else attrib
+        xSub = etree.SubElement(xParent, name, attrib=xAttr)
+        if text is not None:
+            xSub.text = text
+
+        return
+
+    ##
+    #  Methods
+    ##
+
+    def describeMe(self, hLevel=None):
+        """Return a string description of the item.
+        """
+        descKey = "none"
+        if self.itemType == nwItemType.ROOT:
+            descKey = "root"
+        elif self.itemType == nwItemType.FOLDER:
+            descKey = "folder"
+        elif self.itemType == nwItemType.FILE:
+            if self.itemLayout == nwItemLayout.DOCUMENT:
+                if hLevel == "H1":
+                    descKey = "doc_h1"
+                elif hLevel == "H2":
+                    descKey = "doc_h2"
+                elif hLevel == "H3":
+                    descKey = "doc_h3"
+                else:
+                    descKey = "document"
+            elif self.itemLayout == nwItemLayout.NOTE:
+                descKey = "note"
+
+        return trConst(nwLabels.ITEM_DESCRIPTION.get(descKey, ""))
+
+    ##
+    #  Set Item Values
+    ##
+
+    def setName(self, theName):
+        """Set the item name.
+        """
+        if isinstance(theName, str):
+            self.itemName = theName.strip()
+        else:
+            self.itemName = ""
+        return
+
+    def setHandle(self, theHandle):
+        """Set the item handle, and ensure it is valid.
+        """
+        if isinstance(theHandle, str):
+            if isHandle(theHandle):
+                self.itemHandle = theHandle
+            else:
+                self.itemHandle = None
+        else:
+            self.itemHandle = None
+        return
+
+    def setParent(self, theParent):
+        """Set the parent handle, and ensure that it is valid.
+        """
+        if theParent is None:
+            self.itemParent = None
+        elif isinstance(theParent, str):
+            if isHandle(theParent):
+                self.itemParent = theParent
+            else:
+                self.itemParent = None
+        else:
+            self.itemParent = None
+        return
+
+    def setOrder(self, theOrder):
+        """Set the item order, and ensure that it is valid. This value
+        is purely a meta value, not actually used by novelWriter.
+        """
+        self.itemOrder = checkInt(theOrder, 0)
+        return
+
+    def setType(self, theType):
+        """Set the item type from either a proper nwItemType, or set it
+        from a string representing a nwItemType.
+        """
+        if isinstance(theType, nwItemType):
+            self.itemType = theType
+        elif isItemType(theType):
+            self.itemType = nwItemType[theType]
+        else:
+            logger.error("Unrecognised item type '%s'", theType)
+            self.itemType = nwItemType.NO_TYPE
+        return
+
+    def setClass(self, theClass):
+        """Set the item class from either a proper nwItemClass, or set
+        it from a string representing a nwItemClass.
+        """
+        if isinstance(theClass, nwItemClass):
+            self.itemClass = theClass
+        elif isItemClass(theClass):
+            self.itemClass = nwItemClass[theClass]
+        else:
+            logger.error("Unrecognised item class '%s'", theClass)
+            self.itemClass = nwItemClass.NO_CLASS
+        return
+
+    def setLayout(self, theLayout):
+        """Set the item layout from either a proper nwItemLayout, or set
+        it from a string representing a nwItemLayout.
+        """
+        if isinstance(theLayout, nwItemLayout):
+            self.itemLayout = theLayout
+        elif isItemLayout(theLayout):
+            self.itemLayout = nwItemLayout[theLayout]
+        elif theLayout in nwLists.DEP_LAYOUT:
+            self.itemLayout = nwItemLayout.DOCUMENT
+        else:
+            logger.error("Unrecognised item layout '%s'", theLayout)
+            self.itemLayout = nwItemLayout.NO_LAYOUT
+        return
+
+    def setStatus(self, theStatus):
+        """Set the item status by looking it up in the valid status
+        items of the current project.
+        """
+        if self.itemClass in nwLists.CLS_NOVEL:
+            self.itemStatus = self.theProject.statusItems.checkEntry(theStatus)
+        else:
+            self.itemStatus = self.theProject.importItems.checkEntry(theStatus)
+        return
+
+    def setExpanded(self, expState):
+        """Save the expanded status of an item in the project tree.
+        """
+        if isinstance(expState, str):
+            self.isExpanded = (expState == str(True))
+        else:
+            self.isExpanded = (expState is True)
+        return
+
+    def setExported(self, expState):
+        """Save the export flag.
+        """
+        if isinstance(expState, str):
+            self.isExported = (expState == str(True))
+        else:
+            self.isExported = (expState is True)
+        return
+
+    ##
+    #  Set Document Meta Data
+    ##
+
+    def setCharCount(self, theCount):
+        """Set the character count, and ensure that it is an integer.
+        """
+        self.charCount = checkInt(theCount, 0)
+        return
+
+    def setWordCount(self, theCount):
+        """Set the word count, and ensure that it is an integer.
+        """
+        self.wordCount = checkInt(theCount, 0)
+        return
+
+    def setParaCount(self, theCount):
+        """Set the paragraph count, and ensure that it is an integer.
+        """
+        self.paraCount = checkInt(theCount, 0)
+        return
+
+    def setCursorPos(self, thePosition):
+        """Set the cursor position, and ensure that it is an integer.
+        """
+        self.cursorPos = checkInt(thePosition, 0)
+        return
+
+    def saveInitialCount(self):
+        """Set the initial word count.
+        """
+        self.initCount = self.wordCount
+        return
+
+# END Class NWItem

novelwriter/core/options.py

@@ -0,0 +1,270 @@
+"""
+novelWriter – Project Options Cache
+===================================
+Data class for user-defined GUI project options
+
+File History:
+Created:   2019-10-21 [0.3.1]
+Rewritten: 2020-02-19 [0.4.5]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import os
+import json
+import logging
+import novelwriter
+
+from novelwriter.constants import nwFiles
+
+logger = logging.getLogger(__name__)
+
+
+class OptionState():
+
+    def __init__(self, theProject):
+
+        self.theProject = theProject
+        self.theState = {}
+        self.validMap = {
+            "GuiWritingStats": {
+                "winWidth",
+                "winHeight",
+                "widthCol0",
+                "widthCol1",
+                "widthCol2",
+                "widthCol3",
+                "sortCol",
+                "sortOrder",
+                "incNovel",
+                "incNotes",
+                "hideZeros",
+                "hideNegative",
+                "groupByDay",
+                "showIdleTime",
+                "histMax",
+            },
+            "GuiDocSplit": {
+                "spLevel",
+            },
+            "GuiBuildNovel": {
+                "winWidth",
+                "winHeight",
+                "boxWidth",
+                "docWidth",
+                "addNovel",
+                "addNotes",
+                "ignoreFlag",
+                "justifyText",
+                "excludeBody",
+                "textFont",
+                "textSize",
+                "lineHeight",
+                "noStyling",
+                "incSynopsis",
+                "incComments",
+                "incKeywords",
+                "incBodyText",
+                "replaceTabs",
+                "replaceUCode",
+            },
+            "GuiOutline": {
+                "headerOrder",
+                "columnWidth",
+                "columnHidden",
+            },
+            "GuiProjectSettings": {
+                "winWidth",
+                "winHeight",
+                "replaceColW",
+                "statusColW",
+                "importColW",
+            },
+            "GuiProjectDetails": {
+                "winWidth",
+                "winHeight",
+                "widthCol0",
+                "widthCol1",
+                "widthCol2",
+                "widthCol3",
+                "widthCol4",
+                "wordsPerPage",
+                "countFrom",
+                "clearDouble",
+            },
+            "GuiWordList": {
+                "winWidth",
+                "winHeight",
+            }
+        }
+
+        return
+
+    ##
+    #  Load and Save Cache
+    ##
+
+    def loadSettings(self):
+        """Load the options dictionary from the project settings file.
+        """
+        if self.theProject.projMeta is None:
+            return False
+
+        stateFile = os.path.join(self.theProject.projMeta, nwFiles.OPTS_FILE)
+        theState = {}
+
+        if os.path.isfile(stateFile):
+            logger.debug("Loading GUI options file")
+            try:
+                with open(stateFile, mode="r", encoding="utf-8") as inFile:
+                    theState = json.load(inFile)
+            except Exception:
+                logger.error("Failed to load GUI options file")
+                novelwriter.logException()
+                return False
+
+        # Filter out unused variables
+        for aGroup in theState:
+            if aGroup in self.validMap:
+                self.theState[aGroup] = {}
+                for anOpt in theState[aGroup]:
+                    if anOpt in self.validMap[aGroup]:
+                        self.theState[aGroup][anOpt] = theState[aGroup][anOpt]
+
+        return True
+
+    def saveSettings(self):
+        """Save the options dictionary to the project settings file.
+        """
+        if self.theProject.projMeta is None:
+            return False
+
+        stateFile = os.path.join(self.theProject.projMeta, nwFiles.OPTS_FILE)
+        logger.debug("Saving GUI options file")
+
+        try:
+            with open(stateFile, mode="w+", encoding="utf-8") as outFile:
+                json.dump(self.theState, outFile, indent=2)
+        except Exception:
+            logger.error("Failed to save GUI options file")
+            novelwriter.logException()
+            return False
+
+        return True
+
+    ##
+    #  Setters
+    ##
+
+    def setValue(self, setGroup, setName, setValue):
+        """Saves a value, with a given group and name.
+        """
+        if setGroup not in self.validMap:
+            logger.error("Unknown option group '%s'", setGroup)
+            return False
+
+        if setName not in self.validMap[setGroup]:
+            logger.error("Unknown option name '%s'", setName)
+            return False
+
+        if setGroup not in self.theState:
+            self.theState[setGroup] = {}
+
+        self.theState[setGroup][setName] = setValue
+
+        return True
+
+    ##
+    #  Getters
+    ##
+
+    def getValue(self, getGroup, getName, defaultValue):
+        """Return an arbitrary type value, if it exists. Otherwise,
+        return the default value.
+        """
+        if getGroup in self.theState:
+            if getName in self.theState[getGroup]:
+                return self.theState[getGroup][getName]
+        return defaultValue
+
+    def getString(self, getGroup, getName, defaultValue):
+        """Return the value as a string, if it exists. Otherwise, return
+        the default value.
+        """
+        if getGroup in self.theState:
+            if getName in self.theState[getGroup]:
+                return str(self.theState[getGroup][getName])
+        return defaultValue
+
+    def getInt(self, getGroup, getName, defaultValue):
+        """Return the value as an int, if it exists. Otherwise, return
+        the default value.
+        """
+        if getGroup in self.theState:
+            if getName in self.theState[getGroup]:
+                try:
+                    return int(self.theState[getGroup][getName])
+                except Exception as e:
+                    logger.warning(str(e))
+                    return defaultValue
+        return defaultValue
+
+    def getFloat(self, getGroup, getName, defaultValue):
+        """Return the value as a float, if it exists. Otherwise, return
+        the default value.
+        """
+        if getGroup in self.theState:
+            if getName in self.theState[getGroup]:
+                try:
+                    return float(self.theState[getGroup][getName])
+                except Exception as e:
+                    logger.warning(str(e))
+                    return defaultValue
+        return defaultValue
+
+    def getBool(self, getGroup, getName, defaultValue):
+        """Return the value as a bool, if it exists. Otherwise, return
+        the default value.
+        """
+        if getGroup in self.theState:
+            if getName in self.theState[getGroup]:
+                return bool(self.theState[getGroup][getName])
+        return defaultValue
+
+    ##
+    #  Validators
+    ##
+
+    def validIntRange(self, theValue, intA, intB, intDefault):
+        """Check that an int is in a given range. If it isn't, return
+        the default value.
+        """
+        if isinstance(theValue, int):
+            if theValue >= intA and theValue <= intB:
+                return theValue
+        return intDefault
+
+    def validIntTuple(self, theValue, theTuple, intDefault):
+        """Check that an int is an element of a tuple. If it isn't,
+        return the default value.
+        """
+        if isinstance(theValue, int):
+            if theValue in theTuple:
+                return theValue
+        return intDefault
+
+# END Class OptionState

novelwriter/core/spellcheck.py

@@ -0,0 +1,336 @@
+"""
+novelWriter – Spell Check Classes
+=================================
+Wrapper classes for spell checking tools
+
+File History:
+Created: 2019-06-11 [0.1.5]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import os
+import difflib
+import logging
+import novelwriter
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================================== #
+#  SpellChecking SuperClass
+# =============================================================================================== #
+
+class NWSpellCheck():
+
+    theDict = None
+    projDict = []
+
+    def __init__(self):
+        self.mainConf = novelwriter.CONFIG
+        self.projectDict = None
+        self.spellLanguage = None
+        return
+
+    def setLanguage(self, theLang, projectDict=None):
+        """Default function.
+        """
+        return
+
+    def checkWord(self, theWord):
+        """Default function.
+        """
+        return True
+
+    def suggestWords(self, theWord):
+        """Default function.
+        """
+        return []
+
+    def addWord(self, newWord):
+        """Add a word to the project dictionary.
+        """
+        if self.projectDict is not None and newWord not in self.projDict:
+            newWord = newWord.strip()
+            try:
+                with open(self.projectDict, mode="a+", encoding="utf-8") as outFile:
+                    outFile.write("%s\n" % newWord)
+                self.projDict.append(newWord)
+            except Exception:
+                logger.error("Failed to add word to project word list %s", str(self.projectDict))
+                novelwriter.logException()
+                return False
+            return True
+        return False
+
+    def listDictionaries(self):
+        """Default function.
+        """
+        return []
+
+    def describeDict(self):
+        """Default function.
+        """
+        return "", ""
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _readProjectDictionary(self, projectDict):
+        """Read the content of the project dictionary, and add it to the
+        lookup lists.
+        """
+        self.projDict = []
+        self.projectDict = projectDict
+
+        if projectDict is None:
+            return False
+
+        if not os.path.isfile(projectDict):
+            return False
+
+        try:
+            logger.debug("Loading project word list")
+            with open(projectDict, mode="r", encoding="utf-8") as wordsFile:
+                for theLine in wordsFile:
+                    theLine = theLine.strip()
+                    if len(theLine) > 0 and theLine not in self.projDict:
+                        self.projDict.append(theLine)
+            logger.debug("Project word list contains %d words", len(self.projDict))
+
+        except Exception:
+            logger.error("Failed to load project word list")
+            novelwriter.logException()
+            return False
+
+        return True
+
+# END Class NWSpellCheck
+
+
+# =============================================================================================== #
+#  Enchant Based SpellChecking
+# =============================================================================================== #
+
+class NWSpellEnchant(NWSpellCheck):
+
+    def __init__(self):
+        NWSpellCheck.__init__(self)
+        logger.debug("Enchant spell checking activated")
+        self.theBroker = None
+        return
+
+    def setLanguage(self, theLang, projectDict=None):
+        """Load a dictionary for the language specified in the config.
+        If that fails, we load a mock dictionary so that lookups don't
+        crash.
+        """
+        try:
+            import enchant
+            if self.theBroker is not None:
+                logger.debug("Deleting old pyenchant broker")
+                del self.theBroker
+
+            self.theBroker = enchant.Broker()
+            self.theDict = self.theBroker.request_dict(theLang)
+            self.spellLanguage = theLang
+            logger.debug("Enchant spell checking for language '%s' loaded", theLang)
+
+        except Exception:
+            logger.error("Failed to load enchant spell checking for language '%s'", theLang)
+            self.theDict = FakeEnchant()
+            self.spellLanguage = None
+
+        self._readProjectDictionary(projectDict)
+        for pWord in self.projDict:
+            self.theDict.add_to_session(pWord)
+
+        return
+
+    def checkWord(self, theWord):
+        """Wrapper function for pyenchant.
+        """
+        return self.theDict.check(theWord)
+
+    def suggestWords(self, theWord):
+        """Wrapper function for pyenchant.
+        """
+        return self.theDict.suggest(theWord)
+
+    def addWord(self, newWord):
+        """Wrapper function for pyenchant.
+        """
+        self.theDict.add_to_session(newWord)
+        NWSpellCheck.addWord(self, newWord)
+        return
+
+    def listDictionaries(self):
+        """Wrapper function for pyenchant.
+        """
+        retList = []
+        try:
+            import enchant
+            for spTag, spProvider in enchant.list_dicts():
+                retList.append((spTag, spProvider.name))
+        except Exception:
+            logger.error("Failed to list languages for enchant spell checking")
+
+        return retList
+
+    def describeDict(self):
+        """Return the tag and provider of the currently loaded
+        dictionary.
+        """
+        try:
+            spTag = self.theDict.tag
+            spName = self.theDict.provider.name
+        except Exception:
+            logger.error("Failed to extract information about the dictionary")
+            novelwriter.logException()
+            spTag = ""
+            spName = ""
+
+        return spTag, spName
+
+# END Class NWSpellEnchant
+
+
+class FakeEnchant:
+    """Fallback for when Enchant is selected, but not installed.
+    """
+    def __init__(self):
+        return
+
+    def check(self, theWord):
+        return True
+
+    def suggest(self, theWord):
+        return []
+
+    def add_to_session(self, theWord):
+        return
+
+# END Class FakeEnchant
+
+
+# =============================================================================================== #
+#  Fallback SpellChecking Using difflib
+# =============================================================================================== #
+
+class NWSpellSimple(NWSpellCheck):
+    """Internal spell check tool that uses standard Python packages with
+    no other external dependencies. This is the fallback spell checker
+    when no other is available. This method is slower than enchant.
+    """
+    theWords = set()
+
+    def __init__(self):
+        NWSpellCheck.__init__(self)
+        self.theLang = ""
+        logger.debug("Simple spell checking activated")
+        return
+
+    def setLanguage(self, theLang, projectDict=None):
+        """Load a dictionary as a list from the app assets folder.
+        """
+        self.theLang = theLang
+        self.theWords = set()
+        dictFile = os.path.join(self.mainConf.dictPath, theLang+".dict")
+        try:
+            with open(dictFile, mode="r", encoding="utf-8") as wordsFile:
+                for theLine in wordsFile:
+                    if len(theLine) == 0 or theLine.startswith("#"):
+                        continue
+                    self.theWords.add(theLine.strip().lower())
+
+            logger.debug("Spell check dictionary for language '%s' loaded", theLang)
+            logger.debug("Dictionary contains %d words", len(self.theWords))
+            self.spellLanguage = theLang
+
+        except Exception:
+            logger.error("Failed to load spell check word list for language '%s'", theLang)
+            novelwriter.logException()
+            self.spellLanguage = None
+
+        self._readProjectDictionary(projectDict)
+        for pWord in self.projDict:
+            self.theWords.add(pWord)
+
+        return
+
+    def checkWord(self, theWord):
+        """Check if a word exists in the word list. Make sure to keep
+        this function as fast as possible as it is called for every
+        word by the syntax highlighter.
+        """
+        theWord = theWord.replace(self.mainConf.fmtApostrophe, "'").lower()
+        return theWord in self.theWords
+
+    def suggestWords(self, theWord):
+        """Get suggestions for correct word from difflib, and make sure
+        the first character is upper case if that was also the case for
+        the word be3ing checked. Also make sure the apostrophe is
+        changed to the one in the dictionary, and then put back in the
+        results.
+        """
+        theWord = theWord.strip()
+        if len(theWord) == 0:
+            return []
+
+        theMatches = difflib.get_close_matches(theWord.lower(), self.theWords, n=10, cutoff=0.75)
+        theOptions = []
+        for aWord in theMatches:
+            if len(aWord) == 0:
+                continue
+            if theWord[0].isupper():
+                aWord = aWord[0].upper() + aWord[1:]
+            aWord = aWord.replace("'", self.mainConf.fmtApostrophe)
+            theOptions.append(aWord)
+
+        return theOptions
+
+    def addWord(self, newWord):
+        """Wrapper for the internal project dictionary feature.
+        """
+        newWord = newWord.strip().lower()
+        if newWord not in self.theWords:
+            self.theWords.add(newWord)
+        NWSpellCheck.addWord(self, newWord)
+        return
+
+    def listDictionaries(self):
+        """Lists the dictionary files in the app assets folder.
+        """
+        retList = []
+        for dictFile in os.listdir(self.mainConf.dictPath):
+
+            fRoot, fExt = os.path.splitext(dictFile)
+            if fExt != ".dict":
+                continue
+
+            retList.append((fRoot, "difflib"))
+
+        return retList
+
+    def describeDict(self):
+        """Return the tag and provider of the currently loaded
+        dictionary.
+        """
+        return self.theLang, ""
+
+# END Class NWSpellSimple

novelwriter/core/status.py

@@ -0,0 +1,187 @@
+"""
+novelWriter – Project Item Status Class
+=======================================
+Data class for the status/importance settings of a project item
+
+File History:
+Created: 2019-05-19 [0.1.3]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import logging
+
+from lxml import etree
+
+from novelwriter.common import checkInt
+
+logger = logging.getLogger(__name__)
+
+
+class NWStatus():
+
+    def __init__(self):
+
+        self._theLabels = []
+        self._theColours = []
+        self._theCounts = []
+        self._theMap = {}
+        self._theLength = 0
+        self._theIndex = 0
+
+        return
+
+    def addEntry(self, theLabel, theColours):
+        """Add a status entry to the status object, but ensure it isn't
+        a duplicate.
+        """
+        theLabel = theLabel.strip()
+        if self.lookupEntry(theLabel) is None:
+            self._theLabels.append(theLabel)
+            self._theColours.append(theColours)
+            self._theCounts.append(0)
+            self._theMap[theLabel] = self._theLength
+            self._theLength += 1
+        return True
+
+    def lookupEntry(self, theLabel):
+        """Look up a status entry in the object lists, and return it if
+        it exists.
+        """
+        if theLabel is None:
+            return None
+        theLabel = theLabel.strip()
+        if theLabel in self._theMap.keys():
+            return self._theMap[theLabel]
+        return None
+
+    def checkEntry(self, theStatus):
+        """Check if a status value is valid, and returns the safe
+        reference to be used internally.
+        """
+        if isinstance(theStatus, str):
+            theStatus = theStatus.strip()
+            if self.lookupEntry(theStatus) is not None:
+                return theStatus
+        theStatus = checkInt(theStatus, 0, False)
+        if theStatus >= 0 and theStatus < self._theLength:
+            return self._theLabels[theStatus]
+        return self._theLabels[0]
+
+    def setNewEntries(self, newList):
+        """Update the list of entries after they have been modified by
+        the GUI tool.
+        """
+        replaceMap = {}
+
+        if newList is not None:
+            self._theLabels = []
+            self._theColours = []
+            self._theCounts = []
+            self._theMap = {}
+            self._theLength = 0
+            self._theIndex = 0
+
+            for nName, nR, nG, nB, oName in newList:
+                self.addEntry(nName, (nR, nG, nB))
+                if nName != oName and oName is not None:
+                    replaceMap[oName] = nName
+
+        return replaceMap
+
+    def resetCounts(self):
+        """Clear the counts of references to the status entries.
+        """
+        self._theCounts = [0]*self._theLength
+        return
+
+    def countEntry(self, theLabel):
+        """Increment the counter for a given label. This should be used
+        together with resetCounts in a loop over project items.
+        """
+        theIndex = self.lookupEntry(theLabel)
+        if theIndex is not None:
+            self._theCounts[theIndex] += 1
+        return
+
+    def packXML(self, xParent):
+        """Pack the status entries into an XML object for saving to the
+        main project file.
+        """
+        for n in range(self._theLength):
+            xSub = etree.SubElement(xParent, "entry", attrib={
+                "blue":  str(self._theColours[n][2]),
+                "green": str(self._theColours[n][1]),
+                "red":   str(self._theColours[n][0]),
+            })
+            xSub.text = self._theLabels[n]
+        return True
+
+    def unpackXML(self, xParent):
+        """Unpack an XML tree and set the class values.
+        """
+        theLabels  = []
+        theColours = []
+
+        for xChild in xParent:
+            theLabels.append(xChild.text)
+            cR = checkInt(xChild.attrib.get("red", 0), 0, False)
+            cG = checkInt(xChild.attrib.get("green", 0), 0, False)
+            cB = checkInt(xChild.attrib.get("blue", 0), 0, False)
+            theColours.append((cR, cG, cB))
+
+        if len(theLabels) > 0:
+            self._theLabels  = []
+            self._theColours = []
+            self._theCounts  = []
+            self._theMap     = {}
+            self._theLength  = 0
+            self._theIndex   = 0
+
+            for n in range(len(theLabels)):
+                self.addEntry(theLabels[n], theColours[n])
+
+        return True
+
+    ##
+    #  Iterator Bits
+    ##
+
+    def __getitem__(self, n):
+        """Return an entry by its index.
+        """
+        if n >= 0 and n < self._theLength:
+            return self._theLabels[n], self._theColours[n], self._theCounts[n]
+        return None, None, None
+
+    def __iter__(self):
+        """Initialise the iterator.
+        """
+        self._theIndex = 0
+        return self
+
+    def __next__(self):
+        """Return the next entry for the iterator.
+        """
+        if self._theIndex < self._theLength:
+            theLabel, theColour, theCount = self.__getitem__(self._theIndex)
+            self._theIndex += 1
+            return theLabel, theColour, theCount
+        else:
+            raise StopIteration
+
+# END Class NWStatus

novelwriter/core/tohtml.py

@@ -0,0 +1,453 @@
+"""
+novelWriter – HTML Text Converter
+=================================
+Extends the Tokenizer class to generate HTML output
+
+File History:
+Created: 2019-05-07 [0.0.1]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import logging
+
+from novelwriter.core.tokenizer import Tokenizer
+from novelwriter.constants import nwKeyWords, nwLabels, nwHtmlUnicode
+
+logger = logging.getLogger(__name__)
+
+
+class ToHtml(Tokenizer):
+
+    M_PREVIEW = 0  # Tweak output for the DocViewer
+    M_EXPORT  = 1  # Tweak output for saving to HTML or printing
+    M_EBOOK   = 2  # Tweak output for converting to epub
+
+    def __init__(self, theProject):
+        Tokenizer.__init__(self, theProject)
+
+        self.genMode = self.M_EXPORT
+        self.cssStyles = True
+        self.fullHTML = []
+
+        # Internals
+        self._trMap = {}
+        self.setReplaceUnicode(False)
+
+        return
+
+    ##
+    #  Setters
+    ##
+
+    def setPreview(self, doComments, doSynopsis):
+        """If we're using this class to generate markdown preview, we
+        need to make a few changes to formatting, which is managed by
+        these flags.
+        """
+        self.genMode = self.M_PREVIEW
+        self.doKeywords = True
+        self.doComments = doComments
+        self.doSynopsis = doSynopsis
+
+        return
+
+    def setStyles(self, cssStyles):
+        """Enable/disable CSS styling. Some elements may still have
+        class tags.
+        """
+        self.cssStyles = cssStyles
+        return
+
+    def setReplaceUnicode(self, doReplace):
+        """Set the translation map to either minimal or full unicode for
+        html entities replacement.
+        """
+        # Control characters must always be replaced
+        # Angle brackets are replaced later as they are also used in
+        # formatting codes
+        self._trMap = str.maketrans({"&": "&amp;"})
+        if doReplace:
+            # Extend to all relevant Unicode characters
+            self._trMap.update(str.maketrans(nwHtmlUnicode.U_TO_H))
+
+        return
+
+    ##
+    #  Class Methods
+    ##
+
+    def getFullResultSize(self):
+        """Return the size of the full HTML result.
+        """
+        return sum([len(x) for x in self.fullHTML])
+
+    def doPreProcessing(self):
+        """Extend the auto-replace to also properly encode some unicode
+        characters into their respective HTML entities.
+        """
+        Tokenizer.doPreProcessing(self)
+        self.theText = self.theText.translate(self._trMap)
+        return
+
+    def doConvert(self):
+        """Convert the list of text tokens into a HTML document saved
+        to theResult.
+        """
+        if self.genMode == self.M_PREVIEW:
+            htmlTags = {  # HTML4 + CSS2
+                self.FMT_B_B: "<b>",
+                self.FMT_B_E: "</b>",
+                self.FMT_I_B: "<i>",
+                self.FMT_I_E: "</i>",
+                self.FMT_D_B: "<span style='text-decoration: line-through;'>",
+                self.FMT_D_E: "</span>",
+            }
+        else:
+            htmlTags = {  # HTML5
+                self.FMT_B_B: "<strong>",
+                self.FMT_B_E: "</strong>",
+                self.FMT_I_B: "<em>",
+                self.FMT_I_E: "</em>",
+                self.FMT_D_B: "<del>",
+                self.FMT_D_E: "</del>",
+            }
+
+        if self.isNovel and self.genMode != self.M_PREVIEW:
+            # For story files, we bump the titles one level up
+            h1Cl = " class='title'"
+            h1 = "h1"
+            h2 = "h1"
+            h3 = "h2"
+            h4 = "h3"
+        else:
+            h1Cl = ""
+            h1 = "h1"
+            h2 = "h2"
+            h3 = "h3"
+            h4 = "h4"
+
+        self.theResult = ""
+
+        thisPar = []
+        parStyle = None
+        tmpResult = []
+
+        for tType, tLine, tText, tFormat, tStyle in self.theTokens:
+
+            # Replace < and > before adding html tags
+            tText = tText.replace("<", "&lt;").replace(">", "&gt;")
+
+            # Styles
+            aStyle = []
+            if tStyle is not None and self.cssStyles:
+                if tStyle & self.A_LEFT:
+                    aStyle.append("text-align: left;")
+                elif tStyle & self.A_RIGHT:
+                    aStyle.append("text-align: right;")
+                elif tStyle & self.A_CENTRE:
+                    aStyle.append("text-align: center;")
+                elif tStyle & self.A_JUSTIFY:
+                    aStyle.append("text-align: justify;")
+
+                if tStyle & self.A_PBB:
+                    aStyle.append("page-break-before: always;")
+
+                if tStyle & self.A_PBA:
+                    aStyle.append("page-break-after: always;")
+
+                if tStyle & self.A_Z_BTMMRG:
+                    aStyle.append("margin-bottom: 0;")
+                if tStyle & self.A_Z_TOPMRG:
+                    aStyle.append("margin-top: 0;")
+
+                if tStyle & self.A_IND_L:
+                    aStyle.append("margin-left: %dpx;" % self.mainConf.tabWidth)
+                if tStyle & self.A_IND_R:
+                    aStyle.append("margin-right: %dpx;" % self.mainConf.tabWidth)
+
+            if len(aStyle) > 0:
+                hStyle = " style='%s'" % (" ".join(aStyle))
+            else:
+                hStyle = ""
+
+            if self.linkHeaders:
+                aNm = "<a name='T%06d'></a>" % tLine
+            else:
+                aNm = ""
+
+            # Process Text Type
+            if tType == self.T_EMPTY:
+                if parStyle is None:
+                    parStyle = ""
+                if len(thisPar) > 1 and self.cssStyles:
+                    parClass = " class='break'"
+                else:
+                    parClass = ""
+                if len(thisPar) > 0:
+                    tTemp = "<br/>".join(thisPar)
+                    tmpResult.append("<p%s%s>%s</p>\n" % (parClass, parStyle, tTemp.rstrip()))
+                thisPar = []
+                parStyle = None
+
+            elif tType == self.T_TITLE:
+                tHead = tText.replace(r"\\", "<br/>")
+                tmpResult.append("<h1 class='title'%s>%s%s</h1>\n" % (hStyle, aNm, tHead))
+
+            elif tType == self.T_UNNUM:
+                tHead = tText.replace(r"\\", "<br/>")
+                tmpResult.append("<%s%s>%s%s</%s>\n" % (h2, hStyle, aNm, tHead, h2))
+
+            elif tType == self.T_HEAD1:
+                tHead = tText.replace(r"\\", "<br/>")
+                tmpResult.append("<%s%s%s>%s%s</%s>\n" % (h1, h1Cl, hStyle, aNm, tHead, h1))
+
+            elif tType == self.T_HEAD2:
+                tHead = tText.replace(r"\\", "<br/>")
+                tmpResult.append("<%s%s>%s%s</%s>\n" % (h2, hStyle, aNm, tHead, h2))
+
+            elif tType == self.T_HEAD3:
+                tHead = tText.replace(r"\\", "<br/>")
+                tmpResult.append("<%s%s>%s%s</%s>\n" % (h3, hStyle, aNm, tHead, h3))
+
+            elif tType == self.T_HEAD4:
+                tHead = tText.replace(r"\\", "<br/>")
+                tmpResult.append("<%s%s>%s%s</%s>\n" % (h4, hStyle, aNm, tHead, h4))
+
+            elif tType == self.T_SEP:
+                tmpResult.append("<p class='sep'>%s</p>\n" % tText)
+
+            elif tType == self.T_SKIP:
+                tmpResult.append("<p class='skip'>&nbsp;</p>\n")
+
+            elif tType == self.T_TEXT:
+                tTemp = tText
+                if parStyle is None:
+                    parStyle = hStyle
+                for xPos, xLen, xFmt in reversed(tFormat):
+                    tTemp = tTemp[:xPos] + htmlTags[xFmt] + tTemp[xPos+xLen:]
+                thisPar.append(tTemp.rstrip())
+
+            elif tType == self.T_SYNOPSIS and self.doSynopsis:
+                tmpResult.append(self._formatSynopsis(tText))
+
+            elif tType == self.T_COMMENT and self.doComments:
+                tmpResult.append(self._formatComments(tText))
+
+            elif tType == self.T_KEYWORD and self.doKeywords:
+                tTemp = "<p%s>%s</p>\n" % (hStyle, self._formatKeywords(tText))
+                tmpResult.append(tTemp)
+
+        self.theResult = "".join(tmpResult)
+        tmpResult = []
+
+        if self.genMode != self.M_PREVIEW:
+            self.fullHTML.append(self.theResult)
+
+        return
+
+    def saveHTML5(self, savePath):
+        """Save the data to an .html file.
+        """
+        with open(savePath, mode="w", encoding="utf-8") as outFile:
+            theStyle = self.getStyleSheet()
+            theStyle.append("article {width: 800px; margin: 40px auto;}")
+            bodyText = "".join(self.fullHTML)
+            bodyText = bodyText.replace("\t", "&#09;").rstrip()
+
+            theHtml = (
+                "<!DOCTYPE html>\n"
+                "<html>\n"
+                "<head>\n"
+                "<meta charset='utf-8'>\n"
+                "<title>{projTitle:s}</title>\n"
+                "</head>\n"
+                "<style>\n"
+                "{htmlStyle:s}\n"
+                "</style>\n"
+                "<body>\n"
+                "<article>\n"
+                "{bodyText:s}\n"
+                "</article>\n"
+                "</body>\n"
+                "</html>\n"
+            ).format(
+                projTitle=self.theProject.projName,
+                htmlStyle="\n".join(theStyle),
+                bodyText=bodyText,
+            )
+            outFile.write(theHtml)
+
+        return
+
+    def replaceTabs(self, nSpaces=8, spaceChar="&nbsp;"):
+        """Replace tabs with spaces in the html.
+        """
+        htmlText = []
+        eightSpace = spaceChar*nSpaces
+        for aLine in self.fullHTML:
+            htmlText.append(aLine.replace("\t", eightSpace))
+
+        self.fullHTML = htmlText
+        return
+
+    def getStyleSheet(self):
+        """Generate a stylesheet appropriate for the current settings.
+        """
+        theStyles = []
+        if not self.cssStyles:
+            return theStyles
+
+        mScale = self.lineHeight/1.15
+        textAlign = "justify" if self.doJustify else "left"
+
+        theStyles.append("body {font-family: '%s'; font-size: %dpt;}" % (
+            self.textFont, self.textSize
+        ))
+        theStyles.append((
+            "p {"
+            "text-align: %s; line-height: %d%%; "
+            "margin-top: %.2fem; margin-bottom: %.2fem;"
+            "}"
+        ) % (
+            textAlign,
+            round(100 * self.lineHeight),
+            mScale * self.marginText[0],
+            mScale * self.marginText[1],
+        ))
+        theStyles.append((
+            "h1 {"
+            "color: rgb(66, 113, 174); "
+            "page-break-after: avoid; "
+            "margin-top: %.2fem; "
+            "margin-bottom: %.2fem;"
+            "}"
+        ) % (
+            mScale * self.marginHead1[0], mScale * self.marginHead1[1]
+        ))
+        theStyles.append((
+            "h2 {"
+            "color: rgb(66, 113, 174); "
+            "page-break-after: avoid; "
+            "margin-top: %.2fem; "
+            "margin-bottom: %.2fem;"
+            "}"
+        ) % (
+            mScale * self.marginHead2[0], mScale * self.marginHead2[1]
+        ))
+        theStyles.append((
+            "h3 {"
+            "color: rgb(50, 50, 50); "
+            "page-break-after: avoid; "
+            "margin-top: %.2fem; "
+            "margin-bottom: %.2fem;"
+            "}"
+        ) % (
+            mScale * self.marginHead3[0], mScale * self.marginHead3[1]
+        ))
+        theStyles.append((
+            "h4 {"
+            "color: rgb(50, 50, 50); "
+            "page-break-after: avoid; "
+            "margin-top: %.2fem; "
+            "margin-bottom: %.2fem;"
+            "}"
+        ) % (
+            mScale * self.marginHead4[0], mScale * self.marginHead4[1]
+        ))
+        theStyles.append((
+            ".title {"
+            "font-size: 2.5em; "
+            "margin-top: %.2fem; "
+            "margin-bottom: %.2fem;"
+            "}"
+        ) % (
+            mScale * self.marginTitle[0], mScale * self.marginTitle[1]
+        ))
+        theStyles.append((
+            ".sep, .skip {"
+            "text-align: center; "
+            "margin-top: %.2fem; "
+            "margin-bottom: %.2fem;}"
+        ) % (
+            mScale, mScale
+        ))
+
+        theStyles.append("a {color: rgb(66, 113, 174);}")
+        theStyles.append(".tags {color: rgb(245, 135, 31); font-weight: bold;}")
+        theStyles.append(".break {text-align: left;}")
+        theStyles.append(".synopsis {font-style: italic;}")
+        theStyles.append(".comment {font-style: italic; color: rgb(100, 100, 100);}")
+
+        return theStyles
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _formatSynopsis(self, tText):
+        """Apply HTML formatting to synopsis.
+        """
+        if self.genMode == self.M_PREVIEW:
+            sSynop = self._trSynopsis
+            return f"<p class='comment'><span class='synopsis'>{sSynop}:</span> {tText}</p>\n"
+        else:
+            sSynop = self._localLookup("Synopsis")
+            return f"<p class='synopsis'><strong>{sSynop}:</strong> {tText}</p>\n"
+
+    def _formatComments(self, tText):
+        """Apply HTML formatting to comments.
+        """
+        if self.genMode == self.M_PREVIEW:
+            return f"<p class='comment'>{tText}</p>\n"
+        else:
+            sComm = self._localLookup("Comment")
+            return f"<p class='comment'><strong>{sComm}:</strong> {tText}</p>\n"
+
+    def _formatKeywords(self, tText):
+        """Apply HTML formatting to keywords.
+        """
+        isValid, theBits, thePos = self.theParent.theIndex.scanThis("@"+tText)
+        if not isValid or not theBits:
+            return ""
+
+        retText = ""
+        refTags = []
+        if theBits[0] in nwLabels.KEY_NAME:
+            retText += "<span class='tags'>%s:</span> " % nwLabels.KEY_NAME[theBits[0]]
+            if len(theBits) > 1:
+                if theBits[0] == nwKeyWords.TAG_KEY:
+                    retText += "<a name='tag_%s'>%s</a>" % (
+                        theBits[1], theBits[1]
+                    )
+                else:
+                    if self.genMode == self.M_PREVIEW:
+                        for tTag in theBits[1:]:
+                            refTags.append("<a href='#%s=%s'>%s</a>" % (
+                                theBits[0][1:], tTag, tTag
+                            ))
+                        retText += ", ".join(refTags)
+                    else:
+                        for tTag in theBits[1:]:
+                            refTags.append("<a href='#tag_%s'>%s</a>" % (
+                                tTag, tTag
+                            ))
+                        retText += ", ".join(refTags)
+
+        return retText
+
+# END Class ToHtml

novelwriter/core/tokenizer.py

@@ -0,0 +1,737 @@
+"""
+novelWriter – Text Tokenizer
+============================
+Splits a piece of novelWriter markdown text into its elements
+
+File History:
+Created: 2019-05-05 [0.0.1]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import re
+import logging
+import novelwriter
+
+from operator import itemgetter
+from functools import partial
+
+from PyQt5.QtCore import QCoreApplication, QRegularExpression
+
+from novelwriter.enum import nwItemLayout, nwItemType
+from novelwriter.common import numberToRoman, checkInt
+from novelwriter.constants import nwConst, nwRegEx, nwUnicode
+from novelwriter.core.document import NWDoc
+
+logger = logging.getLogger(__name__)
+
+
+class Tokenizer():
+
+    # In-Text Format
+    FMT_B_B = 1  # Begin bold
+    FMT_B_E = 2  # End bold
+    FMT_I_B = 3  # Begin italics
+    FMT_I_E = 4  # End italics
+    FMT_D_B = 5  # Begin strikeout
+    FMT_D_E = 6  # End strikeout
+
+    # Block Type
+    T_EMPTY    = 1   # Empty line (new paragraph)
+    T_SYNOPSIS = 2   # Synopsis comment
+    T_COMMENT  = 3   # Comment line
+    T_KEYWORD  = 4   # Command line
+    T_TITLE    = 5   # Title
+    T_UNNUM    = 6   # Unnumbered
+    T_HEAD1    = 7   # Header 1
+    T_HEAD2    = 8   # Header 2
+    T_HEAD3    = 9   # Header 3
+    T_HEAD4    = 10  # Header 4
+    T_TEXT     = 11  # Text line
+    T_SEP      = 12  # Scene separator
+    T_SKIP     = 13  # Paragraph break
+
+    # Block Style
+    A_NONE     = 0x0000  # No special style
+    A_LEFT     = 0x0001  # Left aligned
+    A_RIGHT    = 0x0002  # Right aligned
+    A_CENTRE   = 0x0004  # Centred
+    A_JUSTIFY  = 0x0008  # Justified
+    A_PBB      = 0x0010  # Page break before
+    A_PBA      = 0x0020  # Page break after
+    A_Z_TOPMRG = 0x0040  # Zero top margin
+    A_Z_BTMMRG = 0x0080  # Zero bottom margin
+    A_IND_L    = 0x0100  # Left indentation
+    A_IND_R    = 0x0200  # Right indentation
+
+    def __init__(self, theProject):
+
+        self.theProject = theProject
+        self.theParent  = theProject.theParent
+        self.mainConf   = novelwriter.CONFIG
+
+        # Data Variables
+        self.theText   = ""    # The raw text to be tokenized
+        self.theHandle = None  # The handle associated with the text
+        self.theItem   = None  # The NWItem associated with the handle
+        self.theTokens = []    # The list of the processed tokens
+        self.theResult = ""    # The result of the last document
+
+        self.keepMarkdown = False  # Whether to keep the markdown text
+        self.theMarkdown  = []     # The result novelWriter markdown of all documents
+
+        # User Settings
+        self.textFont    = "Serif"  # Output text font
+        self.textSize    = 11       # Output text size
+        self.textFixed   = False    # Fixed width text
+        self.lineHeight  = 1.15     # Line height in units of em
+        self.blockIndent = 4.00     # Block indent in units of em
+        self.doJustify   = False    # Justify text
+        self.doBodyText  = True     # Include body text
+        self.doSynopsis  = False    # Also process synopsis comments
+        self.doComments  = False    # Also process comments
+        self.doKeywords  = False    # Also process keywords like tags and references
+
+        # Margins
+        self.marginTitle = (1.000, 0.500)
+        self.marginHead1 = (1.000, 0.500)
+        self.marginHead2 = (0.834, 0.500)
+        self.marginHead3 = (0.584, 0.500)
+        self.marginHead4 = (0.584, 0.500)
+        self.marginText  = (0.000, 0.584)
+        self.marginMeta  = (0.000, 0.584)
+
+        # Title Formats
+        self.fmtTitle   = "%title%"  # Formatting for titles
+        self.fmtChapter = "%title%"  # Formatting for numbered chapters
+        self.fmtUnNum   = "%title%"  # Formatting for unnumbered chapters
+        self.fmtScene   = "%title%"  # Formatting for scenes
+        self.fmtSection = "%title%"  # Formatting for sections
+
+        self.hideScene   = False  # Do not include scene headers
+        self.hideSection = False  # Do not include section headers
+
+        self.linkHeaders = False  # Add an anchor before headers
+
+        # Instance Variables
+        self.numChapter  = 0      # Counter for chapter numbers
+        self.numChScene  = 0      # Counter for scene number within chapter
+        self.numAbsScene = 0      # Counter for scene number within novel
+        self.firstScene  = False  # Flag to indicate that the first scene of the chapter
+
+        # This File
+        self.isNone  = False
+        self.isNovel = False
+        self.isNote  = False
+        self.isFirst = True
+
+        # Error Handling
+        self.errData = []
+
+        # Function Mapping
+        self._localLookup = self.theProject.localLookup
+        self.tr = partial(QCoreApplication.translate, "Tokenizer")
+
+        # Cached Translations
+        self._trSynopsis = self.tr("Synopsis")
+
+        return
+
+    ##
+    #  Setters
+    ##
+
+    def setTitleFormat(self, fmtTitle):
+        self.fmtTitle = fmtTitle
+        return
+
+    def setChapterFormat(self, fmtChapter):
+        self.fmtChapter = fmtChapter
+        return
+
+    def setUnNumberedFormat(self, fmtUnNum):
+        self.fmtUnNum = fmtUnNum
+        return
+
+    def setSceneFormat(self, fmtScene, hideScene):
+        self.fmtScene  = fmtScene
+        self.hideScene = hideScene
+        return
+
+    def setSectionFormat(self, fmtSection, hideSection):
+        self.fmtSection  = fmtSection
+        self.hideSection = hideSection
+        return
+
+    def setFont(self, textFont, textSize, textFixed=False):
+        self.textFont  = textFont
+        self.textSize  = round(int(textSize))
+        self.textFixed = textFixed
+        return
+
+    def setLineHeight(self, lineHeight):
+        self.lineHeight = min(max(float(lineHeight), 0.5), 5.0)
+        return
+
+    def setBlockIndent(self, blockIndent):
+        self.blockIndent = min(max(float(blockIndent), 0.0), 10.0)
+        return
+
+    def setJustify(self, doJustify):
+        self.doJustify = doJustify
+        return
+
+    def setTitleMargins(self, mUpper, mLower):
+        self.marginTitle = (float(mUpper), float(mLower))
+        return
+
+    def setHead1Margins(self, mUpper, mLower):
+        self.marginHead1 = (float(mUpper), float(mLower))
+        return
+
+    def setHead2Margins(self, mUpper, mLower):
+        self.marginHead2 = (float(mUpper), float(mLower))
+        return
+
+    def setHead3Margins(self, mUpper, mLower):
+        self.marginHead3 = (float(mUpper), float(mLower))
+        return
+
+    def setHead4Margins(self, mUpper, mLower):
+        self.marginHead4 = (float(mUpper), float(mLower))
+        return
+
+    def setTextMargins(self, mUpper, mLower):
+        self.marginText = (float(mUpper), float(mLower))
+        return
+
+    def setMetaMargins(self, mUpper, mLower):
+        self.marginMeta = (float(mUpper), float(mLower))
+        return
+
+    def setLinkHeaders(self, linkHeaders):
+        self.linkHeaders = linkHeaders
+        return
+
+    def setBodyText(self, doBodyText):
+        self.doBodyText = doBodyText
+        return
+
+    def setSynopsis(self, doSynopsis):
+        self.doSynopsis = doSynopsis
+        return
+
+    def setComments(self, doComments):
+        self.doComments = doComments
+        return
+
+    def setKeywords(self, doKeywords):
+        self.doKeywords = doKeywords
+        return
+
+    def setKeepMarkdown(self, keepMarkdown):
+        self.keepMarkdown = keepMarkdown
+        return
+
+    ##
+    #  Class Methods
+    ##
+
+    def addRootHeading(self, theHandle):
+        """Add a heading at the start of a new root folder.
+        """
+        theItem = self.theProject.projTree[theHandle]
+        if theItem is None:
+            return False
+
+        if theItem.itemType != nwItemType.ROOT:
+            return False
+
+        if self.isFirst:
+            textAlign = self.A_CENTRE
+            self.isFirst = False
+        else:
+            textAlign = self.A_PBB | self.A_CENTRE
+
+        theTitle = "%s: %s" % (self._localLookup("Notes"), theItem.itemName)
+        self.theTokens = []
+        self.theTokens.append((
+            self.T_TITLE, 0, theTitle, None, textAlign
+        ))
+        if self.keepMarkdown:
+            self.theMarkdown.append("# %s\n\n" % theTitle)
+
+        return True
+
+    def setText(self, theHandle, theText=None):
+        """Set the text for the tokenizer from a handle. If theText is
+        not set, load it from the file.
+        """
+        self.theHandle = theHandle
+        self.theItem   = self.theProject.projTree[theHandle]
+        if self.theItem is None:
+            return False
+
+        self.theText = ""
+        if theText is not None:
+            # If the text is set, just use that
+            self.theText = theText
+        else:
+            # Otherwise, load it from file
+            theDoc  = NWDoc(self.theProject, theHandle)
+            theText = theDoc.readDocument()
+            if theText:
+                self.theText = theText
+
+        docSize = len(self.theText)
+        if docSize > nwConst.MAX_DOCSIZE:
+            errVal = self.tr("Document '{0}' is too big ({1} MB). Skipping.").format(
+                self.theItem.itemName, f"{docSize/1.0e6:.2f}"
+            )
+            self.theText = "# %s\n\n%s\n\n" % (self.tr("ERROR"), errVal)
+            self.errData.append(errVal)
+
+        self.isNone  = self.theItem.itemLayout == nwItemLayout.NO_LAYOUT
+        self.isNovel = self.theItem.itemLayout == nwItemLayout.DOCUMENT
+        self.isNote  = self.theItem.itemLayout == nwItemLayout.NOTE
+
+        return True
+
+    def doPreProcessing(self):
+        """Run trough the various replace doctionaries.
+        """
+        # Process the user's auto-replace dictionary
+        if len(self.theProject.autoReplace) > 0:
+            repDict = {}
+            for aKey, aVal in self.theProject.autoReplace.items():
+                repDict["<%s>" % aKey] = aVal
+            xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
+            self.theText = xRep.sub(lambda x: repDict[x.group(0)], self.theText)
+
+        # Process the character translation map
+        trDict = {nwUnicode.U_MAPOSS: nwUnicode.U_RSQUO}
+        self.theText = self.theText.translate(str.maketrans(trDict))
+
+        return
+
+    def doPostProcessing(self):
+        """Do some postprocessing. Overloaded by subclasses. This just
+        does the standard escaped characters.
+        """
+        escapeDict = {
+            r"\*": "*",
+            r"\~": "~",
+            r"\_": "_",
+        }
+        escReplace = re.compile(
+            "|".join([re.escape(k) for k in escapeDict.keys()]), flags=re.DOTALL
+        )
+        self.theResult = escReplace.sub(
+            lambda x: escapeDict[x.group(0)], self.theResult
+        )
+        return
+
+    def tokenizeText(self):
+        """Scan the text for either lines starting with specific
+        characters that indicate headers, comments, commands etc, or
+        just contain plain text. In the case of plain text, apply the
+        same RegExes that the syntax highlighter uses and save the
+        locations of these formatting tags into the token array.
+
+        The format of the token list is an entry with a five-tuple for
+        each line in the file. The tuple is as follows:
+          1: The type of the block, self.T_*
+          2: The line in file where this block occurred
+          3: The text content of the block, without leading tags
+          4: The internal formatting map of the text, self.FMT_*
+          5: The style of the block, self.A_*
+        """
+        # RegExes for adding formatting tags within text lines
+        rxFormats = [
+            (QRegularExpression(nwRegEx.FMT_EI), [None, self.FMT_I_B, None, self.FMT_I_E]),
+            (QRegularExpression(nwRegEx.FMT_EB), [None, self.FMT_B_B, None, self.FMT_B_E]),
+            (QRegularExpression(nwRegEx.FMT_ST), [None, self.FMT_D_B, None, self.FMT_D_E]),
+        ]
+
+        self.theTokens = []
+        tmpMarkdown = []
+        nLine = 0
+        breakNext = False
+        for aLine in self.theText.splitlines():
+            nLine += 1
+            sLine = aLine.strip()
+
+            # Check for blank lines
+            if len(sLine) == 0:
+                self.theTokens.append((
+                    self.T_EMPTY, nLine, "", None, self.A_NONE
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("\n")
+
+                continue
+
+            if breakNext:
+                sAlign = self.A_PBB
+                breakNext = False
+            else:
+                sAlign = self.A_NONE
+
+            # Check Line Format
+            # =================
+
+            if aLine[0] == "[":
+                # Parse special formatting line
+
+                if sLine in ("[NEWPAGE]", "[NEW PAGE]"):
+                    breakNext = True
+                    continue
+
+                elif sLine == "[VSPACE]":
+                    self.theTokens.append(
+                        (self.T_SKIP, nLine, "", None, sAlign)
+                    )
+                    continue
+
+                elif sLine.startswith("[VSPACE:") and sLine.endswith("]"):
+                    nSkip = checkInt(sLine[8:-1], 0)
+                    if nSkip >= 1:
+                        self.theTokens.append(
+                            (self.T_SKIP, nLine, "", None, sAlign)
+                        )
+                    if nSkip > 1:
+                        self.theTokens += (nSkip - 1) * [
+                            (self.T_SKIP, nLine, "", None, self.A_NONE)
+                        ]
+                    continue
+
+            elif aLine[0] == "%":
+                cLine = aLine[1:].lstrip()
+                synTag = cLine[:9].lower()
+                if synTag == "synopsis:":
+                    self.theTokens.append((
+                        self.T_SYNOPSIS, nLine, cLine[9:].strip(), None, sAlign
+                    ))
+                    if self.doSynopsis and self.keepMarkdown:
+                        tmpMarkdown.append("%s\n" % aLine)
+                else:
+                    self.theTokens.append((
+                        self.T_COMMENT, nLine, aLine[1:].strip(), None, sAlign
+                    ))
+                    if self.doComments and self.keepMarkdown:
+                        tmpMarkdown.append("%s\n" % aLine)
+
+            elif aLine[0] == "@":
+                self.theTokens.append((
+                    self.T_KEYWORD, nLine, aLine[1:].strip(), None, sAlign
+                ))
+                if self.doKeywords and self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+            elif aLine[:2] == "# ":
+                if self.isNovel:
+                    sAlign |= self.A_CENTRE
+
+                if self.isNovel and not self.isFirst:
+                    sAlign |= self.A_PBB
+
+                self.theTokens.append((
+                    self.T_HEAD1, nLine, aLine[2:].strip(), None, sAlign
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+            elif aLine[:3] == "## ":
+                if self.isNovel and not self.isFirst:
+                    sAlign |= self.A_PBB
+
+                self.theTokens.append((
+                    self.T_HEAD2, nLine, aLine[3:].strip(), None, sAlign
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+            elif aLine[:4] == "### ":
+                self.theTokens.append((
+                    self.T_HEAD3, nLine, aLine[4:].strip(), None, sAlign
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+            elif aLine[:5] == "#### ":
+                self.theTokens.append((
+                    self.T_HEAD4, nLine, aLine[5:].strip(), None, sAlign
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+            elif aLine[:3] == "#! ":
+                if self.isNovel:
+                    tStyle = self.T_TITLE
+                else:
+                    tStyle = self.T_HEAD1
+
+                if self.isNovel and not self.isFirst:
+                    sAlign |= self.A_PBB
+
+                self.theTokens.append((
+                    tStyle, nLine, aLine[3:].strip(), None, sAlign | self.A_CENTRE
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+            elif aLine[:4] == "##! ":
+                if self.isNovel:
+                    tStyle = self.T_UNNUM
+                else:
+                    tStyle = self.T_HEAD2
+
+                if self.isNovel and not self.isFirst:
+                    sAlign |= self.A_PBB
+
+                self.theTokens.append((
+                    tStyle, nLine, aLine[4:].strip(), None, sAlign
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+            else:
+                if not self.doBodyText:
+                    # Skip all body text
+                    continue
+
+                # Check Alignment and Indentation
+                alnLeft = False
+                alnRight = False
+                indLeft = False
+                indRight = False
+                if aLine.startswith(">>"):
+                    alnRight = True
+                    aLine = aLine[2:].lstrip(" ")
+                elif aLine.startswith(">"):
+                    indLeft = True
+                    aLine = aLine[1:].lstrip(" ")
+
+                if aLine.endswith("<<"):
+                    alnLeft = True
+                    aLine = aLine[:-2].rstrip(" ")
+                elif aLine.endswith("<"):
+                    indRight = True
+                    aLine = aLine[:-1].rstrip(" ")
+
+                if alnLeft and alnRight:
+                    sAlign |= self.A_CENTRE
+                elif alnLeft:
+                    sAlign |= self.A_LEFT
+                elif alnRight:
+                    sAlign |= self.A_RIGHT
+
+                if indLeft:
+                    sAlign |= self.A_IND_L
+                if indRight:
+                    sAlign |= self.A_IND_R
+
+                # Otherwise we use RegEx to find formatting tags within a line of text
+                fmtPos = []
+                for theRX, theKeys in rxFormats:
+                    rxThis = theRX.globalMatch(aLine, 0)
+                    while rxThis.hasNext():
+                        rxMatch = rxThis.next()
+                        for n in range(1, len(theKeys)):
+                            if theKeys[n] is not None:
+                                xPos = rxMatch.capturedStart(n)
+                                xLen = rxMatch.capturedLength(n)
+                                fmtPos.append([xPos, xLen, theKeys[n]])
+
+                # Save the line as is, but append the array of formatting locations
+                # sorted by position
+                fmtPos = sorted(fmtPos, key=itemgetter(0))
+                self.theTokens.append((
+                    self.T_TEXT, nLine, aLine, fmtPos, sAlign
+                ))
+                if self.keepMarkdown:
+                    tmpMarkdown.append("%s\n" % aLine)
+
+        # If we have content, turn off the first page flag
+        if self.isFirst and self.theTokens:
+            self.isFirst = False
+
+        # Always add an empty line at the end of the file
+        self.theTokens.append((
+            self.T_EMPTY, nLine, "", None, self.A_NONE
+        ))
+        if self.keepMarkdown:
+            tmpMarkdown.append("\n")
+
+        if self.keepMarkdown:
+            self.theMarkdown.append("".join(tmpMarkdown))
+
+        # Second Pass
+        # ===========
+        # Some items need a second pass
+
+        pToken = (self.T_EMPTY, 0, "", None, self.A_NONE)
+        nToken = (self.T_EMPTY, 0, "", None, self.A_NONE)
+        tCount = len(self.theTokens)
+        for n, tToken in enumerate(self.theTokens):
+
+            if n > 0:
+                pToken = self.theTokens[n-1]
+            if n < tCount - 1:
+                nToken = self.theTokens[n+1]
+
+            if tToken[0] == self.T_KEYWORD:
+                aStyle = tToken[4]
+                if pToken[0] == self.T_KEYWORD:
+                    aStyle |= self.A_Z_TOPMRG
+                if nToken[0] == self.T_KEYWORD:
+                    aStyle |= self.A_Z_BTMMRG
+                self.theTokens[n] = (
+                    tToken[0], tToken[1], tToken[2], tToken[3], aStyle
+                )
+
+        return
+
+    def doHeaders(self):
+        """Apply formatting to the text headers for novel files. This
+        also applies chapter and scene numbering.
+        """
+        if not self.isNovel:
+            return False
+
+        for n, tToken in enumerate(self.theTokens):
+
+            # In case we see text before a scene, we reset the flag
+            if tToken[0] == self.T_TEXT:
+                self.firstScene = False
+
+            elif tToken[0] == self.T_HEAD1:
+                # Partition
+
+                tTemp = self._formatHeading(self.fmtTitle, tToken[2])
+                self.theTokens[n] = (
+                    tToken[0], tToken[1], tTemp, None, tToken[4]
+                )
+
+            elif tToken[0] in (self.T_HEAD2, self.T_UNNUM):
+                # Chapter
+
+                # Numbered or Unnumbered
+                if tToken[0] == self.T_UNNUM:
+                    tTemp = self._formatHeading(self.fmtUnNum, tToken[2])
+                else:
+                    self.numChapter += 1
+                    tTemp = self._formatHeading(self.fmtChapter, tToken[2])
+
+                # Format the chapter header
+                self.theTokens[n] = (
+                    tToken[0], tToken[1], tTemp, None, tToken[4]
+                )
+
+                # Set scene variables
+                self.firstScene = True
+                self.numChScene = 0
+
+            elif tToken[0] == self.T_HEAD3:
+                # Scene
+
+                self.numChScene += 1
+                self.numAbsScene += 1
+
+                tTemp = self._formatHeading(self.fmtScene, tToken[2])
+                if tTemp == "" and self.hideScene:
+                    self.theTokens[n] = (
+                        self.T_EMPTY, tToken[1], "", None, self.A_NONE
+                    )
+                elif tTemp == "" and not self.hideScene:
+                    if self.firstScene:
+                        self.theTokens[n] = (
+                            self.T_EMPTY, tToken[1], "", None, self.A_NONE
+                        )
+                    else:
+                        self.theTokens[n] = (
+                            self.T_SKIP, tToken[1], "", None, self.A_NONE
+                        )
+                elif tTemp == self.fmtScene:
+                    if self.firstScene:
+                        self.theTokens[n] = (
+                            self.T_EMPTY, tToken[1], "", None, self.A_NONE
+                        )
+                    else:
+                        self.theTokens[n] = (
+                            self.T_SEP, tToken[1], tTemp, None, self.A_CENTRE
+                        )
+                else:
+                    self.theTokens[n] = (
+                        tToken[0], tToken[1], tTemp, None, self.A_NONE
+                    )
+
+                # Definitely no longer the first scene
+                self.firstScene = False
+
+            elif tToken[0] == self.T_HEAD4:
+                # Section
+
+                tTemp = self._formatHeading(self.fmtSection, tToken[2])
+                if tTemp == "" and self.hideSection:
+                    self.theTokens[n] = (
+                        self.T_EMPTY, tToken[1], "", None, self.A_NONE
+                    )
+                elif tTemp == "" and not self.hideSection:
+                    self.theTokens[n] = (
+                        self.T_SKIP, tToken[1], "", None, self.A_NONE
+                    )
+                elif tTemp == self.fmtSection:
+                    self.theTokens[n] = (
+                        self.T_SEP, tToken[1], tTemp, None, self.A_CENTRE
+                    )
+                else:
+                    self.theTokens[n] = (
+                        tToken[0], tToken[1], tTemp, None, self.A_NONE
+                    )
+
+        return True
+
+    def saveRawMarkdown(self, savePath):
+        """Save the data to a plain text file.
+        """
+        with open(savePath, mode="w", encoding="utf-8") as outFile:
+            for nwdPage in self.theMarkdown:
+                outFile.write(nwdPage)
+        return
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _formatHeading(self, theTitle, theText):
+        """Replaces the %keyword% strings.
+        """
+        theTitle = theTitle.replace(r"%title%", theText)
+        theTitle = theTitle.replace(r"%ch%", str(self.numChapter))
+        theTitle = theTitle.replace(r"%sc%", str(self.numChScene))
+        theTitle = theTitle.replace(r"%sca%", str(self.numAbsScene))
+        if r"%chw%" in theTitle:
+            theTitle = theTitle.replace(r"%chw%", self._localLookup(self.numChapter))
+        if r"%chi%" in theTitle:
+            theTitle = theTitle.replace(r"%chi%", numberToRoman(self.numChapter, True))
+        if r"%chI%" in theTitle:
+            theTitle = theTitle.replace(r"%chI%", numberToRoman(self.numChapter, False))
+
+        return theTitle[:1].upper() + theTitle[1:]
+
+# END Class Tokenizer

novelwriter/core/tomd.py

@@ -0,0 +1,204 @@
+"""
+novelWriter – Markdown Text Converter
+=====================================
+Extends the Tokenizer class to generate Makrdown output
+
+File History:
+Created: 2021-02-06 [1.2a0]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import logging
+
+from novelwriter.constants import nwLabels
+from novelwriter.core.tokenizer import Tokenizer
+
+logger = logging.getLogger(__name__)
+
+
+class ToMarkdown(Tokenizer):
+
+    M_STD = 0  # Standard Markdown
+    M_GH  = 1  # GitHub Markdown
+
+    def __init__(self, theProject):
+        Tokenizer.__init__(self, theProject)
+
+        self.genMode = self.M_STD
+        self.fullMD = []
+
+        return
+
+    ##
+    #  Setters
+    ##
+
+    def setStandardMarkdown(self):
+        self.genMode = self.M_STD
+        return
+
+    def setGitHubMarkdown(self):
+        self.genMode = self.M_GH
+        return
+
+    ##
+    #  Class Methods
+    ##
+
+    def getFullResultSize(self):
+        """Return the size of the full Markdown result.
+        """
+        return sum([len(x) for x in self.fullMD])
+
+    def doConvert(self):
+        """Convert the list of text tokens into a HTML document saved
+        to theResult.
+        """
+        if self.genMode == self.M_STD:
+            # Standard
+            mdTags = {
+                self.FMT_B_B: "**",
+                self.FMT_B_E: "**",
+                self.FMT_I_B: "_",
+                self.FMT_I_E: "_",
+                self.FMT_D_B: "",
+                self.FMT_D_E: "",
+            }
+        else:
+            # GitHub
+            mdTags = {
+                self.FMT_B_B: "**",
+                self.FMT_B_E: "**",
+                self.FMT_I_B: "_",
+                self.FMT_I_E: "_",
+                self.FMT_D_B: "~~",
+                self.FMT_D_E: "~~",
+            }
+
+        self.theResult = ""
+
+        thisPar = []
+        tmpResult = []
+
+        for tType, _, tText, tFormat, tStyle in self.theTokens:
+
+            # Process Text Type
+            if tType == self.T_EMPTY:
+                if len(thisPar) > 0:
+                    tTemp = "  \n".join(thisPar)
+                    tmpResult.append("%s\n\n" % tTemp.rstrip(" "))
+                thisPar = []
+
+            elif tType == self.T_TITLE:
+                tHead = tText.replace(r"\\", "\n")
+                tmpResult.append("# %s\n\n" % tHead)
+
+            elif tType == self.T_UNNUM:
+                tHead = tText.replace(r"\\", "\n")
+                tmpResult.append("## %s\n\n" % tHead)
+
+            elif tType == self.T_HEAD1:
+                tHead = tText.replace(r"\\", "\n")
+                tmpResult.append("# %s\n\n" % tHead)
+
+            elif tType == self.T_HEAD2:
+                tHead = tText.replace(r"\\", "\n")
+                tmpResult.append("## %s\n\n" % tHead)
+
+            elif tType == self.T_HEAD3:
+                tHead = tText.replace(r"\\", "\n")
+                tmpResult.append("### %s\n\n" % tHead)
+
+            elif tType == self.T_HEAD4:
+                tHead = tText.replace(r"\\", "\n")
+                tmpResult.append("#### %s\n\n" % tHead)
+
+            elif tType == self.T_SEP:
+                tmpResult.append("%s\n\n" % tText)
+
+            elif tType == self.T_SKIP:
+                tmpResult.append("\n\n\n")
+
+            elif tType == self.T_TEXT:
+                tTemp = tText
+                for xPos, xLen, xFmt in reversed(tFormat):
+                    tTemp = tTemp[:xPos] + mdTags[xFmt] + tTemp[xPos+xLen:]
+                thisPar.append(tTemp.rstrip())
+
+            elif tType == self.T_SYNOPSIS and self.doSynopsis:
+                tmpResult.append("**%s:** %s\n\n" % (self._localLookup("Synopsis"), tText))
+
+            elif tType == self.T_COMMENT and self.doComments:
+                tmpResult.append("**%s:** %s\n\n" % (self._localLookup("Comment"), tText))
+
+            elif tType == self.T_KEYWORD and self.doKeywords:
+                tmpResult.append(self._formatKeywords(tText, tStyle))
+
+        self.theResult = "".join(tmpResult)
+        tmpResult = []
+
+        self.fullMD.append(self.theResult)
+
+        return
+
+    def saveMarkdown(self, savePath):
+        """Save the data to a plain text file.
+        """
+        with open(savePath, mode="w", encoding="utf-8") as outFile:
+            theText = "".join(self.fullMD)
+            outFile.write(theText)
+
+        return
+
+    def replaceTabs(self, nSpaces=8, spaceChar=" "):
+        """Replace tabs with spaces.
+        """
+        fullMD = []
+        eightSpace = spaceChar*nSpaces
+        for aPage in self.fullMD:
+            fullMD.append(aPage.replace("\t", eightSpace))
+
+        self.fullMD = fullMD
+        return
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _formatKeywords(self, tText, tStyle):
+        """Apply Markdown formatting to keywords.
+        """
+        isValid, theBits, _ = self.theParent.theIndex.scanThis("@"+tText)
+        if not isValid or not theBits:
+            return ""
+
+        retText = ""
+        if theBits[0] in nwLabels.KEY_NAME:
+            retText += "**%s:** " % nwLabels.KEY_NAME[theBits[0]]
+
+            if len(theBits) > 1:
+                retText += ", ".join(theBits[1:])
+
+        if tStyle & self.A_Z_BTMMRG:
+            retText += "  \n"
+        else:
+            retText += "\n\n"
+
+        return retText
+
+# END Class ToMarkdown

novelwriter/core/tree.py

@@ -0,0 +1,488 @@
+"""
+novelWriter – Project Tree Class
+================================
+Data class for the project's tree of project items
+
+File History:
+Created: 2020-05-07 [0.4.5]
+
+This file is a part of novelWriter
+Copyright 2018–2021, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import os
+import logging
+import novelwriter
+
+from time import time
+from lxml import etree
+from hashlib import sha256
+
+from novelwriter.enum import nwItemType, nwItemClass, nwItemLayout
+from novelwriter.common import checkHandle
+from novelwriter.constants import nwConst, nwFiles
+from novelwriter.core.item import NWItem
+
+logger = logging.getLogger(__name__)
+
+
+class NWTree():
+
+    def __init__(self, theProject):
+
+        self.theProject = theProject
+
+        self._projTree    = {}     # Holds all the items of the project
+        self._treeOrder   = []     # The order of the tree items on the tree view
+        self._treeRoots   = []     # The root items of the tree
+        self._trashRoot   = None   # The handle of the trash root folder
+        self._archRoot    = None   # The handle of the archive root folder
+        self._theIndex    = 0      # The current iterator index
+        self._treeChanged = False  # True if tree structure has changed
+
+        self._handleSeed  = None   # Used for generating handles for testing
+        self._handleCount = 0      # A counter that is added to the handle generator
+
+        return
+
+    ##
+    #  Class Methods
+    ##
+
+    def clear(self):
+        """Clear the item tree entirely.
+        """
+        self._projTree  = {}
+        self._treeOrder = []
+        self._treeRoots = []
+        self._trashRoot = None
+        self._archRoot  = None
+        self._theIndex  = 0
+        self._treeChanged = False
+        return
+
+    def handles(self):
+        """Returns a copy of the list of all the active handles.
+        """
+        return self._treeOrder.copy()
+
+    def append(self, tHandle, pHandle, nwItem):
+        """Add a new item to the end of the tree.
+        """
+        tHandle = checkHandle(tHandle, None, True)
+        pHandle = checkHandle(pHandle, None, True)
+        if tHandle is None:
+            tHandle = self._makeHandle()
+
+        if tHandle in self._projTree:
+            logger.warning("Duplicate handle '%s' detected, skipping", tHandle)
+            return False
+
+        logger.verbose("Adding item '%s' with parent '%s'", str(tHandle), str(pHandle))
+
+        nwItem.setHandle(tHandle)
+        nwItem.setParent(pHandle)
+
+        if nwItem.itemType == nwItemType.ROOT:
+            logger.verbose("Item '%s' is a root item", str(tHandle))
+            self._treeRoots.append(tHandle)
+            if nwItem.itemClass == nwItemClass.ARCHIVE:
+                logger.verbose("Item '%s' is the archive folder", str(tHandle))
+                self._archRoot = tHandle
+
+        if nwItem.itemType == nwItemType.TRASH:
+            if self._trashRoot is None:
+                logger.verbose("Item '%s' is the trash folder", str(tHandle))
+                self._trashRoot = tHandle
+            else:
+                logger.error("Only one trash folder allowed")
+                return False
+
+        self._projTree[tHandle] = nwItem
+        self._treeOrder.append(tHandle)
+        self._setTreeChanged(True)
+
+        return True
+
+    def packXML(self, xParent):
+        """Pack the content of the tree into the provided XML object. In
+        the order defined by the _treeOrder list.
+        """
+        xContent = etree.SubElement(xParent, "content", attrib={
+            "count": str(len(self._treeOrder))}
+        )
+        for tHandle in self._treeOrder:
+            tItem = self.__getitem__(tHandle)
+            tItem.packXML(xContent)
+        return
+
+    def unpackXML(self, xContent):
+        """Iterate through all items of a content XML object and add
+        them to the project tree.
+        """
+        if xContent.tag != "content":
+            logger.error("XML entry is not a NWTree")
+            return False
+
+        self.clear()
+        for xItem in xContent:
+            nwItem = NWItem(self.theProject)
+            if nwItem.unpackXML(xItem):
+                self.append(nwItem.itemHandle, nwItem.itemParent, nwItem)
+                nwItem.saveInitialCount()
+
+        return True
+
+    def writeToCFile(self):
+        """Write the convenience table of contents file in the root of
+        the project directory.
+        """
+        tocList = []
+        tocLen = 0
+        for tHandle in self._treeOrder:
+            tItem = self.__getitem__(tHandle)
+            if tItem is None:
+                continue
+            tFile = tHandle+".nwd"
+            if os.path.isfile(os.path.join(self.theProject.projContent, tFile)):
+                tocLine = "%-25s  %-9s  %-8s  %s" % (
+                    os.path.join("content", tFile),
+                    tItem.itemClass.name,
+                    tItem.itemLayout.name,
+                    tItem.itemName,
+                )
+                tocList.append(tocLine)
+                tocLen = max(tocLen, len(tocLine))
+
+        try:
+            # Dump the text
+            tocText = os.path.join(self.theProject.projPath, nwFiles.TOC_TXT)
+            with open(tocText, mode="w", encoding="utf-8") as outFile:
+                outFile.write("\n")
+                outFile.write("Table of Contents\n")
+                outFile.write("=================\n")
+                outFile.write("\n")
+                outFile.write("%-25s  %-9s  %-8s  %s\n" % (
+                    "File Name", "Class", "Layout", "Document Label"
+                ))
+                outFile.write("-"*max(tocLen, 62) + "\n")
+                outFile.write("\n".join(tocList))
+                outFile.write("\n")
+
+        except Exception:
+            logger.error("Could not write ToC file")
+            novelwriter.logException()
+            return False
+
+        return True
+
+    def sumWords(self):
+        """Loops over all entries and adds up the word counts.
+        """
+        noteWords = 0
+        novelWords = 0
+        for tHandle in self._treeOrder:
+            tItem = self.__getitem__(tHandle)
+            if tItem is None:
+                continue
+            if tItem.itemLayout == nwItemLayout.NO_LAYOUT:
+                pass
+            elif tItem.itemLayout == nwItemLayout.NOTE:
+                noteWords += tItem.wordCount
+            else:
+                novelWords += tItem.wordCount
+        return novelWords, noteWords
+
+    ##
+    #  Tree Structure Methods
+    ##
+
+    def trashRoot(self):
+        """Returns the handle of the trash folder, or None if there
+        isn't one.
+        """
+        if self._trashRoot:
+            return self._trashRoot
+        return None
+
+    def isTrashRoot(self, tHandle):
+        """Check if a handle is the trash folder.
+        """
+        if self._trashRoot is None:
+            return False
+        return tHandle == self._trashRoot
+
+    def archiveRoot(self):
+        """Returns the handle of the archive folder, or None if there
+        isn't one.
+        """
+        if self._archRoot:
+            return self._archRoot
+        return None
+
+    def findRoot(self, theClass):
+        """Find the root item for a given class.
+        Note: This returns the first item for class CUSTOM.
+        """
+        for aRoot in self._treeRoots:
+            tItem = self.__getitem__(aRoot)
+            if tItem is None:
+                continue
+            if theClass == tItem.itemClass:
+                return tItem.itemHandle
+        return None
+
+    def checkRootUnique(self, theClass):
+        """Checks if there already is a root entry of class 'theClass'
+        in the root of the project tree. CUSTOM class is skipped as it
+        is not required to be unique.
+        """
+        if theClass == nwItemClass.CUSTOM:
+            return True
+        for aRoot in self._treeRoots:
+            tItem = self.__getitem__(aRoot)
+            if tItem is None:
+                continue
+            if theClass == tItem.itemClass:
+                return False
+        return True
+
+    def getRootItem(self, tHandle):
+        """Iterate upwards in the tree until we find the item with
+        parent None, the root item. We do this with a for loop with a
+        maximum depth to make infinite loops impossible.
+        """
+        tItem = self.__getitem__(tHandle)
+        if tItem is not None:
+            for i in range(nwConst.MAX_DEPTH + 1):
+                if tItem.itemParent is None:
+                    return tItem
+                else:
+                    tHandle = tItem.itemParent
+                    tItem = self.__getitem__(tHandle)
+        return None
+
+    def getItemPath(self, tHandle):
+        """Iterate upwards in the tree until we find the item with
+        parent None, the root item, and return the list of handles.
+        We do this with a for loop with a maximum depth to make
+        infinite loops impossible.
+        """
+        tTree = []
+        tItem = self.__getitem__(tHandle)
+        if tItem is not None:
+            tTree.append(tHandle)
+            for i in range(nwConst.MAX_DEPTH + 1):
+                if tItem.itemParent is None:
+                    return tTree
+                else:
+                    tHandle = tItem.itemParent
+                    tItem   = self.__getitem__(tHandle)
+                    if tItem is None:
+                        return tTree
+                    else:
+                        tTree.append(tHandle)
+        return tTree
+
+    ##
+    #  Setters
+    ##
+
+    def setOrder(self, newOrder):
+        """Reorders the tree based on a list of items.
+        """
+        tmpOrder = []
+
+        # Add all known elements to a new temp list
+        for tHandle in newOrder:
+            if tHandle in self._projTree:
+                tmpOrder.append(tHandle)
+            else:
+                logger.error("Handle '%s' in new tree order is not in project tree", tHandle)
+
+        # Do a reverse lookup to check for items that will be lost
+        # This is mainly for debugging purposes
+        for tHandle in self._treeOrder:
+            if tHandle not in tmpOrder:
+                logger.warning("Handle '%s' in old tree order is not in new tree order", tHandle)
+
+        # Save the temp list
+        self._treeOrder = tmpOrder
+        self._setTreeChanged(True)
+        logger.verbose("Project tree order updated")
+
+        return
+
+    def setSeed(self, theSeed):
+        """Used for debugging!
+        Sets a seed for generating handles so that they always come out
+        in a predictable order.
+        """
+        self._handleSeed = theSeed
+        return
+
+    def setFileItemLayout(self, tHandle, itemLayout):
+        """Set the nwItemLayout for a specific file.
+        """
+        tItem = self.__getitem__(tHandle)
+        if tItem is None:
+            return False
+        if tItem.itemType != nwItemType.FILE:
+            logger.error("Item %s is not a file", tHandle)
+            return False
+        if not isinstance(itemLayout, nwItemLayout):
+            return False
+
+        tItem.setLayout(itemLayout)
+
+        return True
+
+    ##
+    #  Getters
+    ##
+
+    def countTypes(self):
+        """Count the number of files, folders and roots in the project.
+        """
+        nRoot = 0
+        nFolder = 0
+        nFile = 0
+
+        for tHandle in self._treeOrder:
+            tItem = self.__getitem__(tHandle)
+            if tItem is None:
+                continue
+            elif tItem.itemType == nwItemType.ROOT:
+                nRoot += 1
+            elif tItem.itemType == nwItemType.FOLDER:
+                nFolder += 1
+            elif tItem.itemType == nwItemType.FILE:
+                nFile += 1
+
+        return nRoot, nFolder, nFile
+
+    ##
+    #  Meta Methods
+    ##
+
+    def __len__(self):
+        """Return the length counter. Does not check that it is correct!
+        """
+        return len(self._treeOrder)
+
+    def __bool__(self):
+        """Returns True if the tree has any entries.
+        """
+        return len(self._treeOrder) > 0
+
+    ##
+    #  Item Access Methods
+    ##
+
+    def __getitem__(self, tHandle):
+        """Return a project item based on its handle. Returns None if
+        the handle doesn't exist in the project.
+        """
+        if tHandle in self._projTree:
+            return self._projTree[tHandle]
+        logger.error("No tree item with handle '%s'", str(tHandle))
+        return None
+
+    def __delitem__(self, tHandle):
+        """Remove an item from the internal lists and dictionaries.
+        """
+        if tHandle in self._treeOrder and tHandle in self._projTree:
+            self._treeOrder.remove(tHandle)
+            del self._projTree[tHandle]
+        else:
+            logger.warning("Failed to delete item '%s': item not found", tHandle)
+            return
+
+        if tHandle in self._treeRoots:
+            self._treeRoots.remove(tHandle)
+        if tHandle == self._trashRoot:
+            self._trashRoot = None
+        if tHandle == self._archRoot:
+            self._archRoot = None
+
+        self._setTreeChanged(True)
+
+        return
+
+    def __contains__(self, tHandle):
+        """Checks if a handle exists in the tree.
+        """
+        return tHandle in self._treeOrder
+
+    ##
+    #  Iterator Methods
+    ##
+
+    def __iter__(self):
+        """Initiates the iterator.
+        """
+        self._theIndex = 0
+        return self
+
+    def __next__(self):
+        """Returns the item from the next entry in the _treeOrder list.
+        """
+        if self._theIndex < len(self._treeOrder):
+            theItem = self.__getitem__(self._treeOrder[self._theIndex])
+            self._theIndex += 1
+            return theItem
+        else:
+            raise StopIteration
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _setTreeChanged(self, theState):
+        """Set the changed flag to theState, and if being set to True,
+        propagate that state change to the parent NWProject class.
+        """
+        self._treeChanged = theState
+        if theState:
+            self.theProject.setProjectChanged(True)
+        return
+
+    def _makeHandle(self, addSeed=""):
+        """Generate a unique item handle. In the event that the key
+        already exists, salt the seed and generate a new handle.
+        A key collision is very unlikely to be caused by the truncation
+        of the sha256 hash to 13 characters. Assuming it is near-random,
+        it will on average happen every 4.5^15 times. However, the clock
+        seed is likely to occasionally generate a collision if the
+        handle requests come faster than the clock resolution.
+        """
+        if self._handleSeed is None:
+            newSeed = "%s_%d_%s" % (str(time()), self._handleCount, addSeed)
+            self._handleCount += 1
+        else:
+            # This is used for debugging
+            newSeed = str(self._handleSeed)
+            self._handleSeed += 1
+
+        logger.verbose("Generating handle with seed '%s'", newSeed)
+        itemHandle = sha256(newSeed.encode()).hexdigest()[0:13]
+        if itemHandle in self._projTree:
+            logger.warning("Duplicate handle encountered! Retrying ...")
+            itemHandle = self._makeHandle(addSeed+"!")
+
+        return itemHandle
+
+# END Class NWTree