Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move outline generation to Tokenizer class instead

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-03-21 16:42:58 +01:00
parent 144ba66d9d
commit 5befd244e6
11 changed files with 109 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/docbuild.py
+19 -3
View File
@@ -52,7 +52,10 @@ class NWBuildDocument:
manuscript, based on a build definition object (BuildSettings).
"""
__slots__ = ("_project", "_build", "_queue", "_error", "_cache", "_count", "_preview")
__slots__ = (
"_project", "_build", "_queue", "_error", "_cache", "_count",
"_outline", "_preview"
)
def __init__(self, project: NWProject, build: BuildSettings) -> None:
self._project = project
@@ -61,6 +64,7 @@ class NWBuildDocument:
self._error = None
self._cache = None
self._count = False
self._outline = False
self._preview = False
return
@@ -86,13 +90,21 @@ class NWBuildDocument:
##
def setCountEnabled(self, state: bool) -> None:
"""Turn on/off stats counting for builds."""
"""Turn on/off stats for builds."""
self._count = state
return
def setBuildOutline(self, state: bool) -> None:
"""Turn on/off outline for builds."""
self._outline = state
return
def setPreviewMode(self, state: bool) -> None:
"""Set the preview mode of the build. Implies count mode."""
"""Set the preview mode of the build. This also enables stats
count and outline mode.
"""
self._preview = state
self._outline = state
self._count = state
return
@@ -364,12 +376,16 @@ class NWBuildDocument:
bldObj.doConvert()
if self._count:
bldObj.countStats()
if self._outline:
bldObj.buildOutline()
elif tItem.isFileType():
bldObj.setText(tHandle)
bldObj.doPreProcessing()
bldObj.tokenizeText()
if self._count:
bldObj.countStats()
if self._outline:
bldObj.buildOutline()
if convert:
bldObj.doConvert()
else:
novelwriter/core/index.py
+4 -4
View File
@@ -33,7 +33,7 @@ import logging
from time import time
from typing import TYPE_CHECKING
from pathlib import Path
from collections.abc import Generator, ItemsView, Iterable
from collections.abc import ItemsView, Iterable
from novelwriter import SHARED
from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
@@ -523,7 +523,7 @@ class NWIndex:
return tItem[sTitle]
return None
def iterItemHeadings(self, tHandle: str) -> Generator[str, IndexHeading]:
def iterItemHeadings(self, tHandle: str) -> Iterable[tuple[str, IndexHeading]]:
"""Get all headings for a specific item."""
if tItem := self._itemIndex[tHandle]:
yield from tItem.items()
@@ -531,7 +531,7 @@ class NWIndex:
def novelStructure(
self, rootHandle: str | None = None, activeOnly: bool = True
) -> Generator[tuple[str, str, str, IndexHeading]]:
) -> Iterable[tuple[str, str, str, IndexHeading]]:
"""Iterate over all titles in the novel, in the correct order as
they appear in the tree view and in the respective document
files, but skipping all note files.
@@ -673,7 +673,7 @@ class NWIndex:
def getTagsData(
self, activeOnly: bool = True
) -> Generator[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
) -> Iterable[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
"""Return all known tags."""
for tag, data in self._tagsIndex.items():
iItem = self._itemIndex[data.get("handle")]
novelwriter/core/project.py
+2 -2
View File
@@ -31,7 +31,7 @@ from time import time
from typing import TYPE_CHECKING
from pathlib import Path
from functools import partial
from collections.abc import Generator
from collections.abc import Iterable
from PyQt5.QtCore import QCoreApplication
@@ -517,7 +517,7 @@ class NWProject:
# Class Methods
##
def iterProjectItems(self) -> Generator[NWItem]:
def iterProjectItems(self) -> Iterable[NWItem]:
"""This function ensures that the item tree loaded is sent to
the GUI tree view in such a way that the tree can be built. That
is, the parent item must be sent before its child. In principle,
novelwriter/core/sessions.py
+2 -2
View File
@@ -29,7 +29,7 @@ import logging
from time import time
from typing import TYPE_CHECKING
from pathlib import Path
from collections.abc import Generator
from collections.abc import Iterable
from novelwriter.error import logException
from novelwriter.common import formatTimeStamp
@@ -110,7 +110,7 @@ class NWSessionLog:
return True
def iterRecords(self) -> Generator[dict]:
def iterRecords(self) -> Iterable[dict]:
"""Iterate through all records in the log."""
sessFile = self._project.storage.getMetaFile(nwFiles.SESS_FILE)
if isinstance(sessFile, Path) and sessFile.is_file():
novelwriter/core/status.py
+4 -4
View File
@@ -28,10 +28,10 @@ import random
import logging
from typing import TYPE_CHECKING, Literal
from collections.abc import Generator, ItemsView, Iterator, KeysView, ValuesView
from collections.abc import ItemsView, Iterable, Iterator, KeysView, ValuesView
from PyQt5.QtGui import QIcon, QPainter, QPainterPath, QPixmap, QColor
from PyQt5.QtCore import QRectF, Qt
from PyQt5.QtCore import QRectF
from novelwriter import CONFIG
from novelwriter.common import minmax, simplified
@@ -193,7 +193,7 @@ class NWStatus:
self._store[key]["count"] += 1
return
def pack(self) -> Generator[tuple[str, dict]]:
def pack(self) -> Iterable[tuple[str, dict]]:
"""Pack the status entries into a dictionary."""
for key, data in self._store.items():
yield (data["name"], {
@@ -248,7 +248,7 @@ class NWStatus:
def _createIcon(self, red: int, green: int, blue: int) -> QIcon:
"""Generate an icon for a status label."""
pixmap = QPixmap(self._iPX, self._iPX)
pixmap.fill(Qt.transparent)
pixmap.fill(QColor(0, 0, 0, 0))
painter = QPainter(pixmap)
painter.setRenderHint(QPainter.Antialiasing)
novelwriter/core/tohtml.py
+4 -19
View File
@@ -55,7 +55,6 @@ class ToHtml(Tokenizer):
self._genMode = self.M_EXPORT
self._cssStyles = True
self._fullHTML: list[str] = []
self._navMap: dict[str, str] = {}
# Internals
self._trMap = {}
@@ -71,10 +70,6 @@ class ToHtml(Tokenizer):
def fullHTML(self) -> list[str]:
return self._fullHTML
@property
def navigationMap(self) -> dict[str, str]:
return self._navMap
##
# Setters
##
@@ -174,6 +169,8 @@ class ToHtml(Tokenizer):
pStyle = None
lines = []
tHandle = self._handle
for tType, nHead, tText, tFormat, tStyle in self._tokens:
# Replace < and > with HTML entities
@@ -232,11 +229,9 @@ class ToHtml(Tokenizer):
else:
hStyle = ""
if self._linkHeadings and self._handle:
tHH = f"{self._handle}:T{nHead:04d}"
aNm = f"<a name='{tHH}'></a>"
if self._linkHeadings and tHandle:
aNm = f"<a name='{tHandle}:T{nHead:04d}'></a>"
else:
tHH = ""
aNm = ""
# Process Text Type
@@ -256,32 +251,22 @@ class ToHtml(Tokenizer):
elif tType == self.T_TITLE:
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
lines.append(f"<h1 class='title'{hStyle}>{aNm}{tHead}</h1>\n")
if tHH:
self._navMap[tHH] = f"TT:{tHead}"
elif tType == self.T_HEAD1:
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
lines.append(f"<{h1}{h1Cl}{hStyle}>{aNm}{tHead}</{h1}>\n")
if tHH:
self._navMap[tHH] = f"H1:{tHead}"
elif tType == self.T_HEAD2:
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
lines.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
if tHH:
self._navMap[tHH] = f"H2:{tHead}"
elif tType == self.T_HEAD3:
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
lines.append(f"<{h3}{hStyle}>{aNm}{tHead}</{h3}>\n")
if tHH:
self._navMap[tHH] = f"H3:{tHead}"
elif tType == self.T_HEAD4:
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
lines.append(f"<{h4}{hStyle}>{aNm}{tHead}</{h4}>\n")
if tHH:
self._navMap[tHH] = f"H4:{tHead}"
elif tType == self.T_SEP:
lines.append(f"<p class='sep'{hStyle}>{tText}</p>\n")
novelwriter/core/tokenizer.py
+53 -26
View File
@@ -35,13 +35,13 @@ from functools import partial
from PyQt5.QtCore import QCoreApplication, QRegularExpression
from novelwriter.enum import nwComment, nwItemLayout
from novelwriter.common import formatTimeStamp, numberToRoman, checkInt
from novelwriter.constants import (
nwHeadFmt, nwKeyWords, nwLabels, nwRegEx, nwShortcode, nwUnicode, trConst
)
from novelwriter.core.index import processComment
from novelwriter.core.project import NWProject
from novelwriter.enum import nwComment, nwItemLayout
logger = logging.getLogger(__name__)
@@ -120,13 +120,14 @@ class Tokenizer(ABC):
self._text = "" # The raw text to be tokenized
self._handle = None # The item handle currently being processed
self._result = "" # The result of the last document
self._counts = {} # Counter data
self._keepMarkdown = False # Whether to keep the markdown text
self._allMarkdown = [] # The result novelWriter markdown of all documents
# Processed Tokens
# Processed Tokens and Meta Data
self._tokens: list[tuple[int, int, str, list[tuple[int, int]], int]] = []
self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {}
# User Settings
self._textFont = "Serif" # Output text font
@@ -226,6 +227,11 @@ class Tokenizer(ABC):
"""The collected stats about the text."""
return self._counts
@property
def textOutline(self) -> dict[str, str]:
"""The generated outline of the text."""
return self._outline
@property
def errData(self) -> list[str]:
"""The error data."""
@@ -392,43 +398,42 @@ class Tokenizer(ABC):
def doConvert(self) -> None:
raise NotImplementedError
def addRootHeading(self, tHandle: str) -> bool:
def addRootHeading(self, tHandle: str) -> None:
"""Add a heading at the start of a new root folder."""
tItem = self._project.tree[tHandle]
if not tItem or not tItem.isRootType():
return False
self._text = ""
self._handle = None
if self._isFirst:
textAlign = self.A_CENTRE
self._isFirst = False
else:
textAlign = self.A_PBB | self.A_CENTRE
if (tItem := self._project.tree[tHandle]) and tItem.isRootType():
self._handle = tHandle
if self._isFirst:
textAlign = self.A_CENTRE
self._isFirst = False
else:
textAlign = self.A_PBB | self.A_CENTRE
trNotes = self._localLookup("Notes")
title = f"{trNotes}: {tItem.itemName}"
self._tokens = []
self._tokens.append((
self.T_TITLE, 0, title, [], textAlign
))
if self._keepMarkdown:
self._allMarkdown.append(f"# {title}\n\n")
trNotes = self._localLookup("Notes")
title = f"{trNotes}: {tItem.itemName}"
self._tokens = []
self._tokens.append((
self.T_TITLE, 1, title, [], textAlign
))
if self._keepMarkdown:
self._allMarkdown.append(f"#! {title}\n\n")
return True
return
def setText(self, tHandle: str, text: str | None = None) -> None:
"""Set the text for the tokenizer from a handle. If text is not
set, its is loaded from the file.
set, it's is loaded from the file.
"""
self._text = ""
self._handle = None
if nwItem := self._project.tree[tHandle]:
if text is None:
text = self._project.storage.getDocument(tHandle).readDocument() or ""
self._text = text
self._handle = tHandle
self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT
return
def doPreProcessing(self) -> None:
@@ -798,7 +803,29 @@ class Tokenizer(ABC):
return
def countStats(self) -> dict[str, int]:
def buildOutline(self) -> None:
"""Build an outline of the text up to level 3 headings."""
tHandle = self._handle or ""
isNovel = self._isNovel
for tType, nHead, tText, _, _ in self._tokens:
if tType == self.T_TITLE:
prefix = "TT"
elif tType == self.T_HEAD1:
prefix = "PT" if isNovel else "H1"
elif tType == self.T_HEAD2:
prefix = "CH" if isNovel else "H2"
elif tType == self.T_HEAD3:
prefix = "SC" if isNovel else "H3"
else:
continue
key = f"{tHandle}:T{nHead:04d}"
text = tText.replace(nwHeadFmt.BR, " ").replace("&amp;", "&")
self._outline[key] = f"{prefix}|{text}"
return
def countStats(self) -> None:
"""Count stats on the tokenized text."""
titleCount = self._counts.get("titleCount", 0)
paragraphCount = self._counts.get("paragraphCount", 0)
@@ -905,7 +932,7 @@ class Tokenizer(ABC):
self._counts["textWordChars"] = textWordChars
self._counts["titleWordChars"] = titleWordChars
return {}
return
def saveRawMarkdown(self, path: str | Path) -> None:
"""Save the raw text to a plain text file."""
novelwriter/core/tree.py
+2 -2
View File
@@ -28,7 +28,7 @@ import logging
from typing import TYPE_CHECKING, Literal, overload
from pathlib import Path
from collections.abc import Generator, Iterator
from collections.abc import Iterable, Iterator
from novelwriter.enum import nwItemClass, nwItemLayout, nwItemType
from novelwriter.error import logException
@@ -387,7 +387,7 @@ class NWTree:
rootClasses.add(nwItem.itemClass)
return rootClasses
def iterRoots(self, itemClass: nwItemClass | None) -> Generator[tuple[str, NWItem]]:
def iterRoots(self, itemClass: nwItemClass | None) -> Iterable[tuple[str, NWItem]]:
"""Iterate over all root items of a given class in order."""
for tHandle in self._order:
nwItem = self.__getitem__(tHandle)
novelwriter/gui/docviewer.py
+5 -5
View File
@@ -41,13 +41,13 @@ from PyQt5.QtWidgets import (
)
from novelwriter import CONFIG, SHARED
from novelwriter.constants import nwHeaders, nwUnicode
from novelwriter.core.tohtml import ToHtml
from novelwriter.enum import nwItemType, nwDocAction, nwDocMode
from novelwriter.error import logException
from novelwriter.constants import nwHeaders, nwUnicode
from novelwriter.extensions.eventfilters import WheelEventFilter
from novelwriter.extensions.modified import NIconToolButton
from novelwriter.gui.theme import STYLES_MIN_TOOLBUTTON
from novelwriter.core.tohtml import ToHtml
from novelwriter.extensions.eventfilters import WheelEventFilter
logger = logging.getLogger(__name__)
@@ -633,7 +633,7 @@ class GuiDocViewHeader(QWidget):
# Internal Variables
self._docHandle = None
self._docOutline: dict[int, tuple[str, int]] = {}
self._docOutline: dict[str, tuple[str, int]] = {}
iPx = SHARED.theme.baseIconSize
mPx = CONFIG.pxInt(4)
@@ -730,7 +730,7 @@ class GuiDocViewHeader(QWidget):
self.refreshButton.setVisible(False)
return
def setOutline(self, data: dict[int, tuple[str, int]]) -> None:
def setOutline(self, data: dict[str, tuple[str, int]]) -> None:
"""Set the document outline dataset."""
tHandle = self._docHandle
if data != self._docOutline and tHandle:
novelwriter/gui/editordocument.py
+2 -2
View File
@@ -23,10 +23,10 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
from collections.abc import Generator
import logging
from time import time
from collections.abc import Iterable
from PyQt5.QtGui import QTextBlock, QTextCursor, QTextDocument
from PyQt5.QtCore import QObject, pyqtSlot
@@ -114,7 +114,7 @@ class GuiTextDocument(QTextDocument):
return word, cPos, cLen, SHARED.spelling.suggestWords(word)
return "", -1, -1, []
def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Generator[QTextBlock]:
def iterBlockByType(self, cType: int, maxCount: int = 1000) -> Iterable[QTextBlock]:
"""Iterate over all text blocks of a given type."""
count = 0
for i in range(self.blockCount()):
novelwriter/tools/manuscript.py
+12 -18
View File
@@ -58,14 +58,6 @@ if TYPE_CHECKING: # pragma: no cover
logger = logging.getLogger(__name__)
OUTLINE_MAP = {
"TT": 0,
"H1": 1,
"H2": 2,
"H3": 3,
"H4": 4,
}
class GuiManuscript(QDialog):
"""GUI Tools: Manuscript Tool
@@ -368,7 +360,7 @@ class GuiManuscript(QDialog):
"uuid": build.buildID,
"time": int(time()),
"stats": buildObj.textStats,
"outline": buildObj.navigationMap,
"outline": buildObj.textOutline,
"styles": buildObj.getStyleSheet(),
"html": buildObj.fullHTML,
}
@@ -708,22 +700,24 @@ class _OutlineWidget(QWidget):
root = self.listView.invisibleRootItem()
parent = root
indent = False
for anchor, text in data.items():
level = OUTLINE_MAP.get(text[:2], -1)
text = text[3:]
if 0 <= level < 4:
for anchor, entry in data.items():
prefix, _, text = entry.partition("|")
if prefix in ("TT", "PT", "CH", "SC", "H1", "H2"):
item = QTreeWidgetItem([text])
item.setData(0, self.D_LINE, anchor)
if level == 0:
if prefix == "TT":
item.setFont(0, tFont)
item.setForeground(0, tBrush)
elif level == 1:
root.addChild(item)
parent = root
elif prefix == "PT":
item.setFont(0, hFont)
if level < 3:
root.addChild(item)
parent = root
elif prefix in ("CH", "H1"):
root.addChild(item)
parent = item
elif parent:
elif prefix in ("SC", "H2"):
parent.addChild(item)
indent = True