From 5befd244e68c0a8e7cbc6fbb80c9ce54c88a93e3 Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Thu, 21 Mar 2024 16:42:58 +0100
Subject: [PATCH] Move outline generation to Tokenizer class instead
---
novelwriter/core/docbuild.py | 22 +++++++--
novelwriter/core/index.py | 8 ++--
novelwriter/core/project.py | 4 +-
novelwriter/core/sessions.py | 4 +-
novelwriter/core/status.py | 8 ++--
novelwriter/core/tohtml.py | 23 ++-------
novelwriter/core/tokenizer.py | 79 +++++++++++++++++++++----------
novelwriter/core/tree.py | 4 +-
novelwriter/gui/docviewer.py | 10 ++--
novelwriter/gui/editordocument.py | 4 +-
novelwriter/tools/manuscript.py | 30 +++++-------
11 files changed, 109 insertions(+), 87 deletions(-)
diff --git a/novelwriter/core/docbuild.py b/novelwriter/core/docbuild.py
index ce5afdd7..3c1d6951 100644
--- a/novelwriter/core/docbuild.py
+++ b/novelwriter/core/docbuild.py
@@ -52,7 +52,10 @@ class NWBuildDocument:
manuscript, based on a build definition object (BuildSettings).
"""
- __slots__ = ("_project", "_build", "_queue", "_error", "_cache", "_count", "_preview")
+ __slots__ = (
+ "_project", "_build", "_queue", "_error", "_cache", "_count",
+ "_outline", "_preview"
+ )
def __init__(self, project: NWProject, build: BuildSettings) -> None:
self._project = project
@@ -61,6 +64,7 @@ class NWBuildDocument:
self._error = None
self._cache = None
self._count = False
+ self._outline = False
self._preview = False
return
@@ -86,13 +90,21 @@ class NWBuildDocument:
##
def setCountEnabled(self, state: bool) -> None:
- """Turn on/off stats counting for builds."""
+ """Turn on/off stats for builds."""
self._count = state
return
+ def setBuildOutline(self, state: bool) -> None:
+ """Turn on/off outline for builds."""
+ self._outline = state
+ return
+
def setPreviewMode(self, state: bool) -> None:
- """Set the preview mode of the build. Implies count mode."""
+ """Set the preview mode of the build. This also enables stats
+ count and outline mode.
+ """
self._preview = state
+ self._outline = state
self._count = state
return
@@ -364,12 +376,16 @@ class NWBuildDocument:
bldObj.doConvert()
if self._count:
bldObj.countStats()
+ if self._outline:
+ bldObj.buildOutline()
elif tItem.isFileType():
bldObj.setText(tHandle)
bldObj.doPreProcessing()
bldObj.tokenizeText()
if self._count:
bldObj.countStats()
+ if self._outline:
+ bldObj.buildOutline()
if convert:
bldObj.doConvert()
else:
diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py
index 66e2130b..7160ea03 100644
--- a/novelwriter/core/index.py
+++ b/novelwriter/core/index.py
@@ -33,7 +33,7 @@ import logging
from time import time
from typing import TYPE_CHECKING
from pathlib import Path
-from collections.abc import Generator, ItemsView, Iterable
+from collections.abc import ItemsView, Iterable
from novelwriter import SHARED
from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
@@ -523,7 +523,7 @@ class NWIndex:
return tItem[sTitle]
return None
- def iterItemHeadings(self, tHandle: str) -> Generator[str, IndexHeading]:
+ def iterItemHeadings(self, tHandle: str) -> Iterable[tuple[str, IndexHeading]]:
"""Get all headings for a specific item."""
if tItem := self._itemIndex[tHandle]:
yield from tItem.items()
@@ -531,7 +531,7 @@ class NWIndex:
def novelStructure(
self, rootHandle: str | None = None, activeOnly: bool = True
- ) -> Generator[tuple[str, str, str, IndexHeading]]:
+ ) -> Iterable[tuple[str, str, str, IndexHeading]]:
"""Iterate over all titles in the novel, in the correct order as
they appear in the tree view and in the respective document
files, but skipping all note files.
@@ -673,7 +673,7 @@ class NWIndex:
def getTagsData(
self, activeOnly: bool = True
- ) -> Generator[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
+ ) -> Iterable[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
"""Return all known tags."""
for tag, data in self._tagsIndex.items():
iItem = self._itemIndex[data.get("handle")]
diff --git a/novelwriter/core/project.py b/novelwriter/core/project.py
index b69c29bc..80aee83f 100644
--- a/novelwriter/core/project.py
+++ b/novelwriter/core/project.py
@@ -31,7 +31,7 @@ from time import time
from typing import TYPE_CHECKING
from pathlib import Path
from functools import partial
-from collections.abc import Generator
+from collections.abc import Iterable
from PyQt5.QtCore import QCoreApplication
@@ -517,7 +517,7 @@ class NWProject:
# Class Methods
##
- def iterProjectItems(self) -> Generator[NWItem]:
+ def iterProjectItems(self) -> Iterable[NWItem]:
"""This function ensures that the item tree loaded is sent to
the GUI tree view in such a way that the tree can be built. That
is, the parent item must be sent before its child. In principle,
diff --git a/novelwriter/core/sessions.py b/novelwriter/core/sessions.py
index 5356c6d4..2f79ad20 100644
--- a/novelwriter/core/sessions.py
+++ b/novelwriter/core/sessions.py
@@ -29,7 +29,7 @@ import logging
from time import time
from typing import TYPE_CHECKING
from pathlib import Path
-from collections.abc import Generator
+from collections.abc import Iterable
from novelwriter.error import logException
from novelwriter.common import formatTimeStamp
@@ -110,7 +110,7 @@ class NWSessionLog:
return True
- def iterRecords(self) -> Generator[dict]:
+ def iterRecords(self) -> Iterable[dict]:
"""Iterate through all records in the log."""
sessFile = self._project.storage.getMetaFile(nwFiles.SESS_FILE)
if isinstance(sessFile, Path) and sessFile.is_file():
diff --git a/novelwriter/core/status.py b/novelwriter/core/status.py
index ad8db0df..4b6911f2 100644
--- a/novelwriter/core/status.py
+++ b/novelwriter/core/status.py
@@ -28,10 +28,10 @@ import random
import logging
from typing import TYPE_CHECKING, Literal
-from collections.abc import Generator, ItemsView, Iterator, KeysView, ValuesView
+from collections.abc import ItemsView, Iterable, Iterator, KeysView, ValuesView
from PyQt5.QtGui import QIcon, QPainter, QPainterPath, QPixmap, QColor
-from PyQt5.QtCore import QRectF, Qt
+from PyQt5.QtCore import QRectF
from novelwriter import CONFIG
from novelwriter.common import minmax, simplified
@@ -193,7 +193,7 @@ class NWStatus:
self._store[key]["count"] += 1
return
- def pack(self) -> Generator[tuple[str, dict]]:
+ def pack(self) -> Iterable[tuple[str, dict]]:
"""Pack the status entries into a dictionary."""
for key, data in self._store.items():
yield (data["name"], {
@@ -248,7 +248,7 @@ class NWStatus:
def _createIcon(self, red: int, green: int, blue: int) -> QIcon:
"""Generate an icon for a status label."""
pixmap = QPixmap(self._iPX, self._iPX)
- pixmap.fill(Qt.transparent)
+ pixmap.fill(QColor(0, 0, 0, 0))
painter = QPainter(pixmap)
painter.setRenderHint(QPainter.Antialiasing)
diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py
index ad548aa6..72539c70 100644
--- a/novelwriter/core/tohtml.py
+++ b/novelwriter/core/tohtml.py
@@ -55,7 +55,6 @@ class ToHtml(Tokenizer):
self._genMode = self.M_EXPORT
self._cssStyles = True
self._fullHTML: list[str] = []
- self._navMap: dict[str, str] = {}
# Internals
self._trMap = {}
@@ -71,10 +70,6 @@ class ToHtml(Tokenizer):
def fullHTML(self) -> list[str]:
return self._fullHTML
- @property
- def navigationMap(self) -> dict[str, str]:
- return self._navMap
-
##
# Setters
##
@@ -174,6 +169,8 @@ class ToHtml(Tokenizer):
pStyle = None
lines = []
+ tHandle = self._handle
+
for tType, nHead, tText, tFormat, tStyle in self._tokens:
# Replace < and > with HTML entities
@@ -232,11 +229,9 @@ class ToHtml(Tokenizer):
else:
hStyle = ""
- if self._linkHeadings and self._handle:
- tHH = f"{self._handle}:T{nHead:04d}"
- aNm = f""
+ if self._linkHeadings and tHandle:
+ aNm = f""
else:
- tHH = ""
aNm = ""
# Process Text Type
@@ -256,32 +251,22 @@ class ToHtml(Tokenizer):
elif tType == self.T_TITLE:
tHead = tText.replace(nwHeadFmt.BR, "
")
lines.append(f"{aNm}{tHead}
\n")
- if tHH:
- self._navMap[tHH] = f"TT:{tHead}"
elif tType == self.T_HEAD1:
tHead = tText.replace(nwHeadFmt.BR, "
")
lines.append(f"<{h1}{h1Cl}{hStyle}>{aNm}{tHead}\n")
- if tHH:
- self._navMap[tHH] = f"H1:{tHead}"
elif tType == self.T_HEAD2:
tHead = tText.replace(nwHeadFmt.BR, "
")
lines.append(f"<{h2}{hStyle}>{aNm}{tHead}\n")
- if tHH:
- self._navMap[tHH] = f"H2:{tHead}"
elif tType == self.T_HEAD3:
tHead = tText.replace(nwHeadFmt.BR, "
")
lines.append(f"<{h3}{hStyle}>{aNm}{tHead}\n")
- if tHH:
- self._navMap[tHH] = f"H3:{tHead}"
elif tType == self.T_HEAD4:
tHead = tText.replace(nwHeadFmt.BR, "
")
lines.append(f"<{h4}{hStyle}>{aNm}{tHead}\n")
- if tHH:
- self._navMap[tHH] = f"H4:{tHead}"
elif tType == self.T_SEP:
lines.append(f"{tText}
\n")
diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py
index dd30cfc6..9125ba01 100644
--- a/novelwriter/core/tokenizer.py
+++ b/novelwriter/core/tokenizer.py
@@ -35,13 +35,13 @@ from functools import partial
from PyQt5.QtCore import QCoreApplication, QRegularExpression
-from novelwriter.enum import nwComment, nwItemLayout
from novelwriter.common import formatTimeStamp, numberToRoman, checkInt
from novelwriter.constants import (
nwHeadFmt, nwKeyWords, nwLabels, nwRegEx, nwShortcode, nwUnicode, trConst
)
from novelwriter.core.index import processComment
from novelwriter.core.project import NWProject
+from novelwriter.enum import nwComment, nwItemLayout
logger = logging.getLogger(__name__)
@@ -120,13 +120,14 @@ class Tokenizer(ABC):
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
- self._counts = {} # Counter data
self._keepMarkdown = False # Whether to keep the markdown text
self._allMarkdown = [] # The result novelWriter markdown of all documents
- # Processed Tokens
+ # Processed Tokens and Meta Data
self._tokens: list[tuple[int, int, str, list[tuple[int, int]], int]] = []
+ self._counts: dict[str, int] = {}
+ self._outline: dict[str, str] = {}
# User Settings
self._textFont = "Serif" # Output text font
@@ -226,6 +227,11 @@ class Tokenizer(ABC):
"""The collected stats about the text."""
return self._counts
+ @property
+ def textOutline(self) -> dict[str, str]:
+ """The generated outline of the text."""
+ return self._outline
+
@property
def errData(self) -> list[str]:
"""The error data."""
@@ -392,43 +398,42 @@ class Tokenizer(ABC):
def doConvert(self) -> None:
raise NotImplementedError
- def addRootHeading(self, tHandle: str) -> bool:
+ def addRootHeading(self, tHandle: str) -> None:
"""Add a heading at the start of a new root folder."""
- tItem = self._project.tree[tHandle]
- if not tItem or not tItem.isRootType():
- return False
+ self._text = ""
+ self._handle = None
- if self._isFirst:
- textAlign = self.A_CENTRE
- self._isFirst = False
- else:
- textAlign = self.A_PBB | self.A_CENTRE
+ if (tItem := self._project.tree[tHandle]) and tItem.isRootType():
+ self._handle = tHandle
+ if self._isFirst:
+ textAlign = self.A_CENTRE
+ self._isFirst = False
+ else:
+ textAlign = self.A_PBB | self.A_CENTRE
- trNotes = self._localLookup("Notes")
- title = f"{trNotes}: {tItem.itemName}"
- self._tokens = []
- self._tokens.append((
- self.T_TITLE, 0, title, [], textAlign
- ))
- if self._keepMarkdown:
- self._allMarkdown.append(f"# {title}\n\n")
+ trNotes = self._localLookup("Notes")
+ title = f"{trNotes}: {tItem.itemName}"
+ self._tokens = []
+ self._tokens.append((
+ self.T_TITLE, 1, title, [], textAlign
+ ))
+ if self._keepMarkdown:
+ self._allMarkdown.append(f"#! {title}\n\n")
- return True
+ return
def setText(self, tHandle: str, text: str | None = None) -> None:
"""Set the text for the tokenizer from a handle. If text is not
- set, its is loaded from the file.
+ set, it's is loaded from the file.
"""
self._text = ""
self._handle = None
if nwItem := self._project.tree[tHandle]:
if text is None:
text = self._project.storage.getDocument(tHandle).readDocument() or ""
-
self._text = text
self._handle = tHandle
self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT
-
return
def doPreProcessing(self) -> None:
@@ -798,7 +803,29 @@ class Tokenizer(ABC):
return
- def countStats(self) -> dict[str, int]:
+ def buildOutline(self) -> None:
+ """Build an outline of the text up to level 3 headings."""
+ tHandle = self._handle or ""
+ isNovel = self._isNovel
+ for tType, nHead, tText, _, _ in self._tokens:
+ if tType == self.T_TITLE:
+ prefix = "TT"
+ elif tType == self.T_HEAD1:
+ prefix = "PT" if isNovel else "H1"
+ elif tType == self.T_HEAD2:
+ prefix = "CH" if isNovel else "H2"
+ elif tType == self.T_HEAD3:
+ prefix = "SC" if isNovel else "H3"
+ else:
+ continue
+
+ key = f"{tHandle}:T{nHead:04d}"
+ text = tText.replace(nwHeadFmt.BR, " ").replace("&", "&")
+ self._outline[key] = f"{prefix}|{text}"
+
+ return
+
+ def countStats(self) -> None:
"""Count stats on the tokenized text."""
titleCount = self._counts.get("titleCount", 0)
paragraphCount = self._counts.get("paragraphCount", 0)
@@ -905,7 +932,7 @@ class Tokenizer(ABC):
self._counts["textWordChars"] = textWordChars
self._counts["titleWordChars"] = titleWordChars
- return {}
+ return
def saveRawMarkdown(self, path: str | Path) -> None:
"""Save the raw text to a plain text file."""
diff --git a/novelwriter/core/tree.py b/novelwriter/core/tree.py
index 4dea1a95..f206fdb2 100644
--- a/novelwriter/core/tree.py
+++ b/novelwriter/core/tree.py
@@ -28,7 +28,7 @@ import logging
from typing import TYPE_CHECKING, Literal, overload
from pathlib import Path
-from collections.abc import Generator, Iterator
+from collections.abc import Iterable, Iterator
from novelwriter.enum import nwItemClass, nwItemLayout, nwItemType
from novelwriter.error import logException
@@ -387,7 +387,7 @@ class NWTree:
rootClasses.add(nwItem.itemClass)
return rootClasses
- def iterRoots(self, itemClass: nwItemClass | None) -> Generator[tuple[str, NWItem]]:
+ def iterRoots(self, itemClass: nwItemClass | None) -> Iterable[tuple[str, NWItem]]:
"""Iterate over all root items of a given class in order."""
for tHandle in self._order:
nwItem = self.__getitem__(tHandle)
diff --git a/novelwriter/gui/docviewer.py b/novelwriter/gui/docviewer.py
index ac123ab9..36b9d80d 100644
--- a/novelwriter/gui/docviewer.py
+++ b/novelwriter/gui/docviewer.py
@@ -41,13 +41,13 @@ from PyQt5.QtWidgets import (
)
from novelwriter import CONFIG, SHARED
+from novelwriter.constants import nwHeaders, nwUnicode
+from novelwriter.core.tohtml import ToHtml
from novelwriter.enum import nwItemType, nwDocAction, nwDocMode
from novelwriter.error import logException
-from novelwriter.constants import nwHeaders, nwUnicode
+from novelwriter.extensions.eventfilters import WheelEventFilter
from novelwriter.extensions.modified import NIconToolButton
from novelwriter.gui.theme import STYLES_MIN_TOOLBUTTON
-from novelwriter.core.tohtml import ToHtml
-from novelwriter.extensions.eventfilters import WheelEventFilter
logger = logging.getLogger(__name__)
@@ -633,7 +633,7 @@ class GuiDocViewHeader(QWidget):
# Internal Variables
self._docHandle = None
- self._docOutline: dict[int, tuple[str, int]] = {}
+ self._docOutline: dict[str, tuple[str, int]] = {}
iPx = SHARED.theme.baseIconSize
mPx = CONFIG.pxInt(4)
@@ -730,7 +730,7 @@ class GuiDocViewHeader(QWidget):
self.refreshButton.setVisible(False)
return
- def setOutline(self, data: dict[int, tuple[str, int]]) -> None:
+ def setOutline(self, data: dict[str, tuple[str, int]]) -> None:
"""Set the document outline dataset."""
tHandle = self._docHandle
if data != self._docOutline and tHandle:
diff --git a/novelwriter/gui/editordocument.py b/novelwriter/gui/editordocument.py
index 421e89a0..04babe3f 100644
--- a/novelwriter/gui/editordocument.py
+++ b/novelwriter/gui/editordocument.py
@@ -23,10 +23,10 @@ along with this program. If not, see .
"""
from __future__ import annotations
-from collections.abc import Generator
import logging
from time import time
+from collections.abc import Iterable
from PyQt5.QtGui import QTextBlock, QTextCursor, QTextDocument
from PyQt5.QtCore import QObject, pyqtSlot
@@ -114,7 +114,7 @@ class GuiTextDocument(QTextDocument):
return word, cPos, cLen, SHARED.spelling.suggestWords(word)
return "", -1, -1, []
- def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Generator[QTextBlock]:
+ def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Iterable[QTextBlock]:
"""Iterate over all text blocks of a given type."""
count = 0
for i in range(self.blockCount()):
diff --git a/novelwriter/tools/manuscript.py b/novelwriter/tools/manuscript.py
index 906b37d5..d9d7342e 100644
--- a/novelwriter/tools/manuscript.py
+++ b/novelwriter/tools/manuscript.py
@@ -58,14 +58,6 @@ if TYPE_CHECKING: # pragma: no cover
logger = logging.getLogger(__name__)
-OUTLINE_MAP = {
- "TT": 0,
- "H1": 1,
- "H2": 2,
- "H3": 3,
- "H4": 4,
-}
-
class GuiManuscript(QDialog):
"""GUI Tools: Manuscript Tool
@@ -368,7 +360,7 @@ class GuiManuscript(QDialog):
"uuid": build.buildID,
"time": int(time()),
"stats": buildObj.textStats,
- "outline": buildObj.navigationMap,
+ "outline": buildObj.textOutline,
"styles": buildObj.getStyleSheet(),
"html": buildObj.fullHTML,
}
@@ -708,22 +700,24 @@ class _OutlineWidget(QWidget):
root = self.listView.invisibleRootItem()
parent = root
indent = False
- for anchor, text in data.items():
- level = OUTLINE_MAP.get(text[:2], -1)
- text = text[3:]
- if 0 <= level < 4:
+ for anchor, entry in data.items():
+ prefix, _, text = entry.partition("|")
+ if prefix in ("TT", "PT", "CH", "SC", "H1", "H2"):
item = QTreeWidgetItem([text])
item.setData(0, self.D_LINE, anchor)
- if level == 0:
+ if prefix == "TT":
item.setFont(0, tFont)
item.setForeground(0, tBrush)
- elif level == 1:
+ root.addChild(item)
+ parent = root
+ elif prefix == "PT":
item.setFont(0, hFont)
-
- if level < 3:
+ root.addChild(item)
+ parent = root
+ elif prefix in ("CH", "H1"):
root.addChild(item)
parent = item
- elif parent:
+ elif prefix in ("SC", "H2"):
parent.addChild(item)
indent = True