""" novelWriter – Text Tokenizer ============================ File History: Created: 2019-05-05 [0.0.1] Tokenizer Created: 2023-05-23 [2.1b1] HeadingFormatter This file is a part of novelWriter Copyright (C) 2019 Veronica Berglyd Olsen and novelWriter contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . """ from __future__ import annotations import logging import re from abc import ABC, abstractmethod from typing import TYPE_CHECKING, NamedTuple from PyQt6.QtCore import QLocale from PyQt6.QtGui import QColor, QFont from novelwriter import CONFIG from novelwriter.common import checkInt, fontMatcher, numberToRoman from novelwriter.constants import ( nwHeadFmt, nwKeyWords, nwLabels, nwShortcode, nwStats, nwStyles, nwUnicode, trConst ) from novelwriter.enum import nwComment, nwItemLayout from novelwriter.formats.shared import ( BlockFmt, BlockTyp, T_Block, T_Formats, T_Note, TextDocumentTheme, TextFmt ) from novelwriter.text.comments import processComment from novelwriter.text.patterns import REGEX_PATTERNS, DialogParser if TYPE_CHECKING: from pathlib import Path from novelwriter.core.project import NWProject logger = logging.getLogger(__name__) class ComStyle(NamedTuple): label: str = "" labelClass: str = "" textClass: str = "" COMMENT_STYLE = { nwComment.PLAIN: ComStyle("Comment", "comment", "comment"), nwComment.IGNORE: ComStyle(), nwComment.SYNOPSIS: ComStyle("Synopsis", "modifier", "note"), nwComment.SHORT: ComStyle("Short Description", "modifier", "note"), nwComment.NOTE: ComStyle("Note", "modifier", "note"), nwComment.FOOTNOTE: ComStyle("", "modifier", "note"), nwComment.COMMENT: ComStyle(), nwComment.STORY: ComStyle("Story Structure", "modifier", "note"), } HEADINGS = [ BlockTyp.TITLE, BlockTyp.PART, BlockTyp.HEAD1, BlockTyp.HEAD2, BlockTyp.HEAD3, BlockTyp.HEAD4, ] SKIP_INDENT = [*HEADINGS, BlockTyp.SEP, BlockTyp.SKIP] B_EMPTY: T_Block = (BlockTyp.EMPTY, "", "", [], BlockFmt.NONE) class Tokenizer(ABC): """Core: Text Tokenizer Abstract Base Class This is the base class for all document build classes. It parses the novelWriter markup format and generates a registry of tokens and text that can be further processed into other output formats by subclasses. """ def __init__(self, project: NWProject) -> None: self._project = project # Data Variables self._text = "" # The raw text to be tokenized self._handle = None # The item handle currently being processed self._keepRaw = False # Whether to keep the raw text, used by ToRaw self._noTokens = False # Disable tokenization if they're not needed # Blocks and Meta Data (Per Document) self._blocks: list[T_Block] = [] self._footnotes: dict[str, T_Note] = {} # Blocks and Meta Data (Per Instance) self._raw: list[str] = [] self._pages: list[str] = [] self._counts: dict[str, int] = {} self._outline: dict[str, str] = {} # User Settings self._dLocale = CONFIG.locale # The document locale self._textFont = QFont("Serif", 11) # Output text font self._lineHeight = 1.15 # Line height in units of em self._colorHeads = True # Colourise headings self._scaleHeads = True # Scale headings to larger font size self._boldHeads = True # Bold headings self._blockIndent = 4.00 # Block indent in units of em self._firstIndent = False # Enable first line indent self._firstWidth = 1.40 # First line indent in units of em self._indentFirst = False # Indent first paragraph self._doJustify = False # Justify text self._doBodyText = True # Include body text self._doComments = set() # Comment styles to allow self._doKeywords = False # Also process keywords like tags and references self._keepBreaks = True # Keep line breaks in paragraphs self._defaultAlign = "left" # The default text alignment self._skipKeywords: set[str] = set() # Keywords to ignore # Defaults self._doComments.add(nwComment.FOOTNOTE) # Other Setting self._theme = TextDocumentTheme() self._classes: dict[str, QColor] = {} # Margins self._marginTitle = nwStyles.T_MARGIN["H0"] self._marginHead1 = nwStyles.T_MARGIN["H1"] self._marginHead2 = nwStyles.T_MARGIN["H2"] self._marginHead3 = nwStyles.T_MARGIN["H3"] self._marginHead4 = nwStyles.T_MARGIN["H4"] self._marginText = nwStyles.T_MARGIN["TT"] self._marginMeta = nwStyles.T_MARGIN["MT"] self._marginFoot = nwStyles.T_MARGIN["FT"] self._marginSep = nwStyles.T_MARGIN["SP"] # Title Formats self._fmtPart = nwHeadFmt.TITLE # Formatting for partitions self._fmtChapter = nwHeadFmt.TITLE # Formatting for numbered chapters self._fmtUnNum = nwHeadFmt.TITLE # Formatting for unnumbered chapters self._fmtScene = nwHeadFmt.TITLE # Formatting for scenes self._fmtHScene = nwHeadFmt.TITLE # Formatting for hard scenes self._fmtSection = nwHeadFmt.TITLE # Formatting for sections self._hidePart = False # Do not include partition headings self._hideChapter = False # Do not include chapter headings self._hideUnNum = False # Do not include unnumbered headings self._hideScene = False # Do not include scene headings self._hideHScene = False # Do not include hard scene headings self._hideSection = False # Do not include section headings self._linkHeadings = False # Add an anchor before headings self._titleStyle = BlockFmt.CENTRE | BlockFmt.PBB self._partStyle = BlockFmt.CENTRE | BlockFmt.PBB self._chapterStyle = BlockFmt.PBB self._sceneStyle = BlockFmt.NONE # Instance Variables self._hFormatter = HeadingFormatter(self._project) self._noSep = True # Flag to indicate that we don't want a scene separator self._noIndent = False # Flag to disable text indent on next paragraph self._breakNext = False # Add a page break on next token # This File self._isNovel = False # Document is a novel document self._isFirst = True # Document is the first in a set # Error Handling self._errData = [] # Function Mapping self._localLookup = self._project.localLookup # Format RegEx self._rxMarkdown = [ (REGEX_PATTERNS.markdownItalic, [0, TextFmt.I_B, 0, TextFmt.I_E]), (REGEX_PATTERNS.markdownBold, [0, TextFmt.B_B, 0, TextFmt.B_E]), (REGEX_PATTERNS.markdownStrike, [0, TextFmt.D_B, 0, TextFmt.D_E]), (REGEX_PATTERNS.markdownMark, [0, TextFmt.M_B, 0, TextFmt.M_E]), ] self._shortCodeFmt = { nwShortcode.ITALIC_O: TextFmt.I_B, nwShortcode.ITALIC_C: TextFmt.I_E, nwShortcode.BOLD_O: TextFmt.B_B, nwShortcode.BOLD_C: TextFmt.B_E, nwShortcode.STRIKE_O: TextFmt.D_B, nwShortcode.STRIKE_C: TextFmt.D_E, nwShortcode.ULINE_O: TextFmt.U_B, nwShortcode.ULINE_C: TextFmt.U_E, nwShortcode.MARK_O: TextFmt.M_B, nwShortcode.MARK_C: TextFmt.M_E, nwShortcode.SUP_O: TextFmt.SUP_B, nwShortcode.SUP_C: TextFmt.SUP_E, nwShortcode.SUB_O: TextFmt.SUB_B, nwShortcode.SUB_C: TextFmt.SUB_E, } self._shortCodeVals = { nwShortcode.FOOTNOTE_B: TextFmt.FNOTE, nwShortcode.FIELD_B: TextFmt.FIELD, } # Dialogue self._hlightDialog = False self._rxAltDialog = REGEX_PATTERNS.altDialogStyle self._dialogParser = DialogParser() self._dialogParser.initParser() return ## # Properties ## @property def textStats(self) -> dict[str, int]: """The collected stats about the text.""" return self._counts @property def textOutline(self) -> dict[str, str]: """The generated outline of the text.""" return self._outline @property def errData(self) -> list[str]: """The error data.""" return self._errData ## # Setters ## def setLanguage(self, language: str | None) -> None: """Set language for the document.""" if language: self._dLocale = QLocale(language) return def setTheme(self, theme: TextDocumentTheme) -> None: """Set the document colour theme.""" self._theme = theme return def setPartitionFormat(self, hFormat: str, hide: bool = False) -> None: """Set the partition format pattern.""" self._fmtPart = hFormat.strip() self._hidePart = hide return def setChapterFormat(self, hFormat: str, hide: bool = False) -> None: """Set the chapter format pattern.""" self._fmtChapter = hFormat.strip() self._hideChapter = hide return def setUnNumberedFormat(self, hFormat: str, hide: bool = False) -> None: """Set the unnumbered format pattern.""" self._fmtUnNum = hFormat.strip() self._hideUnNum = hide return def setSceneFormat(self, hFormat: str, hide: bool = False) -> None: """Set the scene format pattern and hidden status.""" self._fmtScene = hFormat.strip() self._hideScene = hide return def setHardSceneFormat(self, hFormat: str, hide: bool = False) -> None: """Set the hard scene format pattern and hidden status.""" self._fmtHScene = hFormat.strip() self._hideHScene = hide return def setSectionFormat(self, hFormat: str, hide: bool = False) -> None: """Set the section format pattern and hidden status.""" self._fmtSection = hFormat.strip() self._hideSection = hide return def setTitleStyle(self, center: bool, pageBreak: bool) -> None: """Set the title heading style.""" self._titleStyle = BlockFmt.CENTRE if center else BlockFmt.NONE self._titleStyle |= BlockFmt.PBB if pageBreak else BlockFmt.NONE return def setPartitionStyle(self, center: bool, pageBreak: bool) -> None: """Set the partition heading style.""" self._partStyle = BlockFmt.CENTRE if center else BlockFmt.NONE self._partStyle |= BlockFmt.PBB if pageBreak else BlockFmt.NONE return def setChapterStyle(self, center: bool, pageBreak: bool) -> None: """Set the chapter heading style.""" self._chapterStyle = BlockFmt.CENTRE if center else BlockFmt.NONE self._chapterStyle |= BlockFmt.PBB if pageBreak else BlockFmt.NONE return def setSceneStyle(self, center: bool, pageBreak: bool) -> None: """Set the scene heading style.""" self._sceneStyle = BlockFmt.CENTRE if center else BlockFmt.NONE self._sceneStyle |= BlockFmt.PBB if pageBreak else BlockFmt.NONE return def setTextFont(self, font: QFont) -> None: """Set the build font.""" self._textFont = fontMatcher(font) return def setLineHeight(self, height: float) -> None: """Set the line height between 0.5 and 5.0.""" self._lineHeight = min(max(float(height), 0.5), 5.0) return def setHeadingStyles(self, color: bool, scale: bool, bold: bool) -> None: """Set text style for headings.""" self._colorHeads = color self._scaleHeads = scale self._boldHeads = bold return def setBlockIndent(self, indent: float) -> None: """Set the block indent between 0.0 and 10.0.""" self._blockIndent = min(max(float(indent), 0.0), 10.0) return def setFirstLineIndent(self, state: bool, indent: float, first: bool) -> None: """Set first line indent and whether to also indent first paragraph after a heading. """ self._firstIndent = state self._firstWidth = indent self._indentFirst = first return def setJustify(self, state: bool) -> None: """Enable or disable text justification.""" self._doJustify = state return def setDialogHighlight(self, state: bool) -> None: """Enable or disable dialogue highlighting.""" self._hlightDialog = state return def setTitleMargins(self, upper: float, lower: float) -> None: """Set the upper and lower title margin.""" self._marginTitle = (float(upper), float(lower)) return def setHead1Margins(self, upper: float, lower: float) -> None: """Set the upper and lower heading 1 margin.""" self._marginHead1 = (float(upper), float(lower)) return def setHead2Margins(self, upper: float, lower: float) -> None: """Set the upper and lower heading 2 margin.""" self._marginHead2 = (float(upper), float(lower)) return def setHead3Margins(self, upper: float, lower: float) -> None: """Set the upper and lower heading 3 margin.""" self._marginHead3 = (float(upper), float(lower)) return def setHead4Margins(self, upper: float, lower: float) -> None: """Set the upper and lower heading 4 margin.""" self._marginHead4 = (float(upper), float(lower)) return def setTextMargins(self, upper: float, lower: float) -> None: """Set the upper and lower text margin.""" self._marginText = (float(upper), float(lower)) return def setMetaMargins(self, upper: float, lower: float) -> None: """Set the upper and lower meta text margin.""" self._marginMeta = (float(upper), float(lower)) return def setSeparatorMargins(self, upper: float, lower: float) -> None: """Set the upper and lower meta text margin.""" self._marginSep = (float(upper), float(lower)) return def setLinkHeadings(self, state: bool) -> None: """Enable or disable adding an anchor before headings.""" self._linkHeadings = state return def setBodyText(self, state: bool) -> None: """Include body text in build.""" self._doBodyText = state return def setCommentType(self, comment: nwComment, state: bool) -> None: """Toggle the inclusion og certain comment types.""" if state: self._doComments.add(comment) else: self._doComments.discard(comment) return def setKeywords(self, state: bool) -> None: """Include keywords in build.""" self._doKeywords = state return def setIgnoredKeywords(self, keywords: str) -> None: """Comma separated string of keywords to ignore.""" self._skipKeywords = set(x.lower().strip() for x in keywords.split(",")) return def setKeepLineBreaks(self, state: bool) -> None: """Keep line breaks in paragraphs.""" self._keepBreaks = state return ## # Class Methods ## @abstractmethod def doConvert(self) -> None: raise NotImplementedError @abstractmethod def closeDocument(self) -> None: raise NotImplementedError @abstractmethod def saveDocument(self, path: Path) -> None: raise NotImplementedError def initDocument(self) -> None: """Initialise data after settings.""" self._classes["modifier"] = self._theme.modifier self._classes["note"] = self._theme.note self._classes["comment"] = self._theme.comment self._classes["dialog"] = self._theme.dialog self._classes["altdialog"] = self._theme.altdialog self._classes["tag"] = self._theme.tag self._classes["keyword"] = self._theme.keyword self._classes["optional"] = self._theme.optional return def setBreakNext(self) -> None: """Set a page break for next block.""" self._breakNext = True return def addRootHeading(self, tHandle: str) -> None: """Add a heading at the start of a new root folder.""" self._text = "" self._handle = None if (item := self._project.tree[tHandle]) and item.isRootType(): self._handle = tHandle style = BlockFmt.CENTRE if self._isFirst: self._isFirst = False else: style |= BlockFmt.PBB title = item.itemName if not item.isNovelLike(): notes = self._localLookup("Notes") title = f"{notes}: {title}" self._blocks = [( BlockTyp.TITLE, f"{self._handle}:T0001", title, [], style )] if self._keepRaw: self._raw.append(f"#! {title}\n\n") return def setText(self, tHandle: str, text: str | None = None) -> None: """Set the text for the tokenizer from a handle. If text is not set, it's is loaded from the file. """ self._text = "" self._handle = None if nwItem := self._project.tree[tHandle]: self._text = text or self._project.storage.getDocumentText(tHandle) self._handle = tHandle self._isNovel = nwItem.itemLayout == nwItemLayout.DOCUMENT return def doPreProcessing(self) -> None: """Run pre-processing jobs before the text is tokenized.""" # Process the user's auto-replace dictionary if entry := self._project.data.autoReplace: replace = {f"<{k}>": v for k, v in entry.items()} rxRep = re.compile("|".join([re.escape(k) for k in replace]), flags=re.DOTALL) self._text = rxRep.sub(lambda x: replace[x.group(0)], self._text) return def tokenizeText(self) -> None: """Scan the text for either lines starting with specific characters that indicate headings, comments, commands etc, or just contain plain text. In the case of plain text, apply the same RegExes that the syntax highlighter uses and save the locations of these formatting tags into the blocks list. The format of the blocs list is an entry with a five-tuple for each line in the file. The tuple is as follows: 1: The type of the block, BlockType.* 2: The heading number under which the text is placed 3: The text content of the block, without leading tags 4: The internal formatting map of the text, TxtFmt.* 5: The formats of the block, BlockFmt.* """ if self._keepRaw: self._raw.append(f"{self._text.rstrip()}\n\n") if self._noTokens: return if self._isNovel: self._hFormatter.setHandle(self._handle) # Cache Flags isNovel = self._isNovel doJustify = self._doJustify keepBreaks = self._keepBreaks indentFirst = self._indentFirst firstIndent = self._firstIndent # Replace all instances of [br] with a placeholder character text = REGEX_PATTERNS.lineBreak.sub(nwUnicode.U_NAC2, self._text) # Translation Maps transMapA = str.maketrans({ nwUnicode.U_NAC2: "", # Used when [br] is ignored nwUnicode.U_MAPOS: nwUnicode.U_RSQUO, nwUnicode.U_HBAR: nwUnicode.U_EMDASH, }) transMapB = str.maketrans({ nwUnicode.U_NAC2: "\n", # Used when [br] is not ignored nwUnicode.U_MAPOS: nwUnicode.U_RSQUO, nwUnicode.U_HBAR: nwUnicode.U_EMDASH, }) nHead = 0 tHandle = self._handle or "" tBlocks: list[T_Block] = [B_EMPTY] for bLine in text.splitlines(): aLine = bLine.translate(transMapA) sLine = aLine.strip().lower() # Check for blank lines if not sLine: tBlocks.append(B_EMPTY) continue if self._breakNext: tStyle = BlockFmt.PBB self._breakNext = False else: tStyle = BlockFmt.NONE # Check Line Format # ================= if aLine.startswith("["): # Special Formats # =============== # Parse special formatting line. This must be a separate if # statement, as it may not reach a continue statement and must # therefore proceed to check other formats. if sLine in ("[newpage]", "[new page]"): self._breakNext = True continue elif sLine == "[vspace]": tBlocks.append( (BlockTyp.SKIP, "", "", [], tStyle) ) continue elif sLine.startswith("[vspace:") and sLine.endswith("]"): nSkip = checkInt(sLine[8:-1], 0) if nSkip >= 1: tBlocks.append( (BlockTyp.SKIP, "", "", [], tStyle) ) if nSkip > 1: tBlocks += (nSkip - 1) * [ (BlockTyp.SKIP, "", "", [], BlockFmt.NONE) ] continue if aLine.startswith("%"): # Comments # ======== # All style comments are processed and the exact type exact # style extracted. Ignored comments on the '%~' format are # skipped completely. if aLine.startswith("%~"): continue cStyle, cKey, cText, _, _ = processComment(aLine) if cStyle not in self._doComments: continue if doJustify and not tStyle & BlockFmt.ALIGNED: tStyle |= BlockFmt.JUSTIFY if cStyle in ( nwComment.SYNOPSIS, nwComment.SHORT, nwComment.PLAIN, nwComment.STORY, nwComment.NOTE, ): bStyle = COMMENT_STYLE[cStyle] tLine, tFmt = self._formatComment(bStyle, cKey, cText) tBlocks.append(( BlockTyp.COMMENT, "", tLine, tFmt, tStyle )) elif cStyle == nwComment.FOOTNOTE: tLine, tFmt = self._extractFormats(cText, skip=TextFmt.FNOTE) self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt) elif aLine.startswith("@"): # Keywords # ======== # Only valid keyword lines are parsed, and any ignored keywords # are automatically skipped. if self._doKeywords: tTag, tLine, tFmt = self._formatMeta(aLine) if tLine: tBlocks.append(( BlockTyp.KEYWORD, tTag[1:], tLine, tFmt, tStyle )) elif aLine.startswith(("# ", "#! ")): # Title or Partition Headings # =========================== # Main titles are allowed in any document, and they are always # centred and start on a new page. For novel documents, we also # reset all counters when such a title is encountered. # Partition headings are only formatted in novel documents, and # otherwise unchanged. Scene separators are disabled # immediately after partitions, and scene numbers are reset. isPlain = aLine.startswith("# ") nHead += 1 tText = aLine[2:].strip() tType = BlockTyp.HEAD1 if isPlain else BlockTyp.TITLE sHide = self._hidePart if isPlain else False if not (isPlain or (isNovel and sHide)): tStyle |= self._titleStyle if isNovel: tType = BlockTyp.PART if isPlain else BlockTyp.TITLE if sHide: tText = "" tType = BlockTyp.EMPTY elif isPlain: tText = self._hFormatter.apply(self._fmtPart, tText, nHead) tStyle |= self._partStyle if isPlain: self._hFormatter.resetScene() else: self._hFormatter.resetAll() self._noSep = True tBlocks.append(( tType, f"{tHandle}:T{nHead:04d}", tText, [], tStyle )) elif aLine.startswith(("## ", "##! ")): # (Unnumbered) Chapter Headings # ============================= # Chapter headings are only formatted in novel documents, and # otherwise unchanged. Chapter numbers are bumped before the # heading is formatted. Scene separators are disabled # immediately after chapter headings, and scene numbers are # reset. Unnumbered chapters are only meaningful in Novel docs, # so if we're in a note, we keep them as level 2 headings. isPlain = aLine.startswith("## ") nHead += 1 tText = aLine[3:].strip() tType = BlockTyp.HEAD2 sHide = self._hideChapter if isPlain else self._hideUnNum tFormat = self._fmtChapter if isPlain else self._fmtUnNum if isNovel: tType = BlockTyp.HEAD1 # Promote if isPlain: self._hFormatter.incChapter() if sHide: tText = "" tType = BlockTyp.EMPTY else: tText = self._hFormatter.apply(tFormat, tText, nHead) tStyle |= self._chapterStyle self._hFormatter.resetScene() self._noSep = True tBlocks.append(( tType, f"{tHandle}:T{nHead:04d}", tText, [], tStyle )) elif aLine.startswith(("### ", "###! ")): # (Alternative) Scene Headings # ============================ # Scene headings in novel documents are treated as centred # separators if the formatting does not change the text. If the # format is empty, the scene can be hidden or a blank paragraph # (skip). When the scene title has static text or no text, it # is always ignored if the noSep flag is set. This prevents # separators immediately after other titles. Scene numbers are # always incremented before formatting. For notes, the heading # is unchanged. isPlain = aLine.startswith("### ") nHead += 1 tText = aLine[4:].strip() tType = BlockTyp.HEAD3 sHide = self._hideScene if isPlain else self._hideHScene tFormat = self._fmtScene if isPlain else self._fmtHScene if isNovel: tType = BlockTyp.HEAD2 # Promote self._hFormatter.incScene() if sHide: tText = "" tType = BlockTyp.EMPTY else: tText = self._hFormatter.apply(tFormat, tText, nHead) tStyle |= self._sceneStyle if tText == "": # Empty Format tType = BlockTyp.EMPTY if self._noSep else BlockTyp.SKIP elif tText == tFormat: # Static Format tText = "" if self._noSep else tText tType = BlockTyp.EMPTY if self._noSep else BlockTyp.SEP tStyle |= BlockFmt.NONE if self._noSep else BlockFmt.CENTRE self._noSep = False tBlocks.append(( tType, f"{tHandle}:T{nHead:04d}", tText, [], tStyle )) elif aLine.startswith("#### "): # Section Headings # ================= # Section headings in novel docs are treated as centred # separators if the formatting does not change the text. If the # format is empty, the section can be hidden or a blank # paragraph (skip). For notes, the heading is unchanged. nHead += 1 tText = aLine[5:].strip() tType = BlockTyp.HEAD4 if isNovel: tType = BlockTyp.HEAD3 # Promote if self._hideSection: tText = "" tType = BlockTyp.EMPTY else: tText = self._hFormatter.apply(self._fmtSection, tText, nHead) if tText == "": # Empty Format tType = BlockTyp.SKIP elif tText == self._fmtSection: # Static Format tType = BlockTyp.SEP tStyle |= BlockFmt.CENTRE tBlocks.append(( tType, f"{tHandle}:T{nHead:04d}", tText, [], tStyle )) else: # Text Lines # ========== # Anything remaining at this point is body text. If body text # is not disabled, we proceed to process text formatting. if not self._doBodyText: # Skip all body text continue # Check Alignment and Indentation alnLeft = False alnRight = False indLeft = False indRight = False if bLine.startswith(">>"): alnRight = True bLine = bLine[2:].lstrip(" ") elif bLine.startswith(">"): indLeft = True bLine = bLine[1:].lstrip(" ") if bLine.endswith("<<"): alnLeft = True bLine = bLine[:-2].rstrip(" ") elif bLine.endswith("<"): indRight = True bLine = bLine[:-1].rstrip(" ") if alnLeft and alnRight: tStyle |= BlockFmt.CENTRE elif alnLeft: tStyle |= BlockFmt.LEFT elif alnRight: tStyle |= BlockFmt.RIGHT if indLeft: tStyle |= BlockFmt.IND_L if indRight: tStyle |= BlockFmt.IND_R # Process formats tLine, tFmt = self._extractFormats(bLine, hDialog=isNovel) tBlocks.append(( BlockTyp.TEXT, "", tLine, tFmt, tStyle )) # If we have content, turn off the first page flag if self._isFirst and len(tBlocks) > 1: self._isFirst = False # First document has been processed # Make sure the blocks array doesn't start with a page break # on the very first block, adding a blank first page. for n, cBlock in enumerate(tBlocks): if cBlock[0] != BlockTyp.EMPTY: if cBlock[4] & BlockFmt.PBB: tBlocks[n] = ( cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB ) break # Always add an empty line at the end of the file tBlocks.append(B_EMPTY) # Second Pass # =========== # This second pass strips away consecutive blank lines, and # combines consecutive text lines into the same paragraph. # It also ensures that there isn't paragraph spacing between # meta data lines for formats that have spacing. lineSep = "\n" if keepBreaks else " " pLines: list[T_Block] = [] sBlocks: list[T_Block] = [] for n, cBlock in enumerate(tBlocks[1:-1], 1): pBlock = tBlocks[n-1] # Look behind nBlock = tBlocks[n+1] # Look ahead if cBlock[0] in SKIP_INDENT and not indentFirst: # Unless the indentFirst flag is set, we set up the next # paragraph to not be indented if we see a block of a # specific type self._noIndent = True if cBlock[0] == BlockTyp.EMPTY: # We don't need to keep the empty lines after this pass pass elif cBlock[0] == BlockTyp.KEYWORD: # Adjust margins for lines in a list of keyword lines aStyle = cBlock[4] if pBlock[0] == BlockTyp.KEYWORD: aStyle |= BlockFmt.Z_TOP if nBlock[0] == BlockTyp.KEYWORD: aStyle |= BlockFmt.Z_BTM sBlocks.append(( cBlock[0], cBlock[1], cBlock[2], cBlock[3], aStyle )) elif cBlock[0] == BlockTyp.TEXT: # Combine lines from the same paragraph pLines.append(cBlock) if nBlock[0] != BlockTyp.TEXT: # Next block is not text, so we add the buffer to blocks nLines = len(pLines) cStyle = pLines[0][4] if firstIndent and not (self._noIndent or cStyle & BlockFmt.ALIGNED): # If paragraph indentation is enabled, not temporarily # turned off, and the block is not aligned, we add the # text indentation flag cStyle |= BlockFmt.IND_T if nLines == 1: # The paragraph contains a single line, so we just save # that directly to the blocks list. If justify is # enabled, and there is no alignment, we apply it. if doJustify and not cStyle & BlockFmt.ALIGNED: cStyle |= BlockFmt.JUSTIFY pTxt = pLines[0][2].translate(transMapB) sBlocks.append(( BlockTyp.TEXT, pLines[0][1], pTxt, pLines[0][3], cStyle )) elif nLines > 1: # The paragraph contains multiple lines, so we need to # join them according to the line break policy, and # recompute all the formatting markers tTxt = "" tFmt: T_Formats = [] for aBlock in pLines: tLen = len(tTxt) tTxt += f"{aBlock[2]}{lineSep}" tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3]) cStyle |= aBlock[4] pTxt = tTxt[:-1].translate(transMapB) sBlocks.append(( BlockTyp.TEXT, pLines[0][1], pTxt, tFmt, cStyle )) # Reset buffer and make sure text indent is on for next pass pLines = [] self._noIndent = False else: sBlocks.append(cBlock) self._blocks = sBlocks return def buildOutline(self) -> None: """Build an outline of the text up to level 3 headings.""" isNovel = self._isNovel for tType, tKey, tText, _, _ in self._blocks: if tType == BlockTyp.TITLE: prefix = "TT" elif tType == BlockTyp.PART: prefix = "PT" elif tType == BlockTyp.HEAD1: prefix = "CH" if isNovel else "H1" elif tType == BlockTyp.HEAD2: prefix = "SC" if isNovel else "H2" elif tType == BlockTyp.HEAD3 and not isNovel: prefix = "H3" else: continue text = tText.replace(nwHeadFmt.BR, " ").replace("&", "&") self._outline[tKey] = f"{prefix}|{text}" return def countStats(self) -> None: """Count stats on the tokenized text.""" titleCount = self._counts.get(nwStats.TITLES, 0) paragraphCount = self._counts.get(nwStats.PARAGRAPHS, 0) allWords = self._counts.get(nwStats.WORDS, 0) textWords = self._counts.get(nwStats.WORDS_TEXT, 0) titleWords = self._counts.get(nwStats.WORDS_TITLE, 0) allChars = self._counts.get(nwStats.CHARS, 0) textChars = self._counts.get(nwStats.CHARS_TEXT, 0) titleChars = self._counts.get(nwStats.CHARS_TITLE, 0) allWordChars = self._counts.get(nwStats.WCHARS_ALL, 0) textWordChars = self._counts.get(nwStats.WCHARS_TEXT, 0) titleWordChars = self._counts.get(nwStats.WCHARS_TITLE, 0) for tType, _, tText, _, _ in self._blocks: tText = tText.replace(nwUnicode.U_ENDASH, " ") tText = tText.replace(nwUnicode.U_EMDASH, " ") tWords = tText.split() nWords = len(tWords) nChars = len(tText) nWChars = len("".join(tWords)) if tType == BlockTyp.TEXT: tPWords = tText.split() nPWords = len(tPWords) nPChars = len(tText) nPWChars = len("".join(tPWords)) paragraphCount += 1 allWords += nPWords textWords += nPWords allChars += nPChars textChars += nPChars allWordChars += nPWChars textWordChars += nPWChars elif tType in HEADINGS: titleCount += 1 allWords += nWords titleWords += nWords allChars += nChars allWordChars += nWChars titleChars += nChars titleWordChars += nWChars elif tType == BlockTyp.SEP: allWords += nWords allChars += nChars allWordChars += nWChars elif tType in (BlockTyp.COMMENT, BlockTyp.KEYWORD): words = tText.split() allWords += len(words) allChars += len(tText) allWordChars += len("".join(words)) self._counts[nwStats.TITLES] = titleCount self._counts[nwStats.PARAGRAPHS] = paragraphCount self._counts[nwStats.WORDS] = allWords self._counts[nwStats.WORDS_TEXT] = textWords self._counts[nwStats.WORDS_TITLE] = titleWords self._counts[nwStats.CHARS] = allChars self._counts[nwStats.CHARS_TEXT] = textChars self._counts[nwStats.CHARS_TITLE] = titleChars self._counts[nwStats.WCHARS_ALL] = allWordChars self._counts[nwStats.WCHARS_TEXT] = textWordChars self._counts[nwStats.WCHARS_TITLE] = titleWordChars return ## # Internal Functions ## def _formatInt(self, value: int) -> str: """Return a localised integer.""" return self._dLocale.toString(value) def _formatComment(self, style: ComStyle, key: str, text: str) -> tuple[str, T_Formats]: """Apply formatting to comments and notes.""" rFmt = [] tTxt, tFmt = self._extractFormats(text) tFmt.insert(0, (0, TextFmt.COL_B, style.textClass)) tFmt.append((len(tTxt), TextFmt.COL_E, "")) term = f" ({key.title()})" if key else "" if label := f"{self._localLookup(style.label)}{term}".strip(): shift = len(label) + 2 tTxt = f"{label}: {tTxt}" rFmt = [(0, TextFmt.B_B, ""), (shift - 1, TextFmt.B_E, "")] if style.labelClass: rFmt.insert(1, (0, TextFmt.COL_B, style.labelClass)) rFmt.insert(2, (shift - 1, TextFmt.COL_E, "")) rFmt.extend((p + shift, f, d) for p, f, d in tFmt) return tTxt, rFmt def _formatMeta(self, text: str) -> tuple[str, str, T_Formats]: """Apply formatting to a meta data line.""" tag = "" txt = [] fmt = [] valid, bits, _ = self._project.index.scanThis(text) if valid and bits and bits[0] in nwLabels.KEY_NAME and bits[0] not in self._skipKeywords: tag = bits[0] pos = 0 lbl = f"{self._localLookup(nwLabels.KEY_NAME[tag])}:" end = len(lbl) fmt = [ (pos, TextFmt.B_B, ""), (pos, TextFmt.COL_B, "keyword"), (end, TextFmt.COL_E, ""), (end, TextFmt.B_E, ""), ] txt = [lbl, " "] pos = end + 1 if (num := len(bits)) > 1: if bits[0] == nwKeyWords.TAG_KEY: one, two = self._project.index.parseValue(bits[1]) end = pos + len(one) fmt.append((pos, TextFmt.COL_B, "tag")) fmt.append((pos, TextFmt.ANM_B, f"tag_{one}".lower())) fmt.append((end, TextFmt.ANM_E, "")) fmt.append((end, TextFmt.COL_E, "")) txt.append(one) pos = end if two: txt.append(" | ") pos += 3 end = pos + len(two) fmt.append((pos, TextFmt.COL_B, "optional")) fmt.append((end, TextFmt.COL_E, "")) txt.append(two) pos = end else: for n, bit in enumerate(bits[1:], 2): end = pos + len(bit) fmt.append((pos, TextFmt.COL_B, "tag")) fmt.append((pos, TextFmt.ARF_B, f"#tag_{bit}".lower())) fmt.append((end, TextFmt.ARF_E, "")) fmt.append((end, TextFmt.COL_E, "")) txt.append(bit) pos = end if n < num: txt.append(", ") pos += 2 return tag, "".join(txt), fmt def _extractFormats( self, text: str, skip: int = 0, hDialog: bool = False ) -> tuple[str, T_Formats]: """Extract format markers from a text paragraph. In order to also process dialogue highlighting, the hDialog flag must be set to True. See issues #2011 and #2013. """ temp: list[tuple[int, int, int, str]] = [] # Match Markdown for regEx, fmts in self._rxMarkdown: for res in regEx.finditer(text): temp.extend( (res.start(n), res.end(n), fmt, "") for n, fmt in enumerate(fmts) if fmt > 0 ) # Match URLs for res in REGEX_PATTERNS.url.finditer(text): temp.append((res.start(0), 0, TextFmt.HRF_B, res.group(0))) temp.append((res.end(0), 0, TextFmt.HRF_E, "")) # Match Shortcodes temp.extend( (res.start(1), res.end(1), self._shortCodeFmt.get(res.group(1).lower(), 0), "") for res in REGEX_PATTERNS.shortcodePlain.finditer(text) ) # Match Shortcode w/Values tHandle = self._handle or "" for res in REGEX_PATTERNS.shortcodeValue.finditer(text): kind = self._shortCodeVals.get(res.group(1).lower(), 0) temp.append(( res.start(0), res.end(0), TextFmt.STRIP if kind == skip else kind, f"{tHandle}:{res.group(2)}", )) # Match Dialogue if self._hlightDialog and hDialog: if self._dialogParser.enabled: for pos, end in self._dialogParser(text): temp.append((pos, 0, TextFmt.COL_B, "dialog")) temp.append((end, 0, TextFmt.COL_E, "")) if self._rxAltDialog: for res in self._rxAltDialog.finditer(text): temp.append((res.start(0), 0, TextFmt.COL_B, "altdialog")) temp.append((res.end(0), 0, TextFmt.COL_E, "")) # Post-process text and format result = text formats = [] for pos, end, fmt, meta in reversed(sorted(temp, key=lambda x: x[0])): if fmt > 0: if end > pos: result = result[:pos] + result[end:] formats = [(p+pos-end if p > pos else p, f, m) for p, f, m in formats] formats.insert(0, (pos, fmt, meta)) return result, formats class HeadingFormatter: def __init__( self, project: NWProject, chapter: int = 0, scene: int = 0, absolute: int = 0, ) -> None: self._project = project self._handle = None self._chapter = chapter self._scene = scene self._absolute = absolute return def setHandle(self, tHandle: str | None) -> None: """Set the handle currently being processed.""" self._handle = tHandle return def incChapter(self) -> None: """Increment the chapter counter.""" self._chapter += 1 return def incScene(self) -> None: """Increment the scene counters.""" self._scene += 1 self._absolute += 1 return def resetAll(self) -> None: """Reset all counters.""" self._chapter = 0 self._scene = 0 self._absolute = 0 return def resetScene(self) -> None: """Reset the chapter scene counter.""" self._scene = 0 return def apply(self, hFormat: str, text: str, nHead: int) -> str: """Apply formatting to a specific heading.""" hFormat = hFormat.replace(nwHeadFmt.TITLE, text) hFormat = hFormat.replace(nwHeadFmt.BR, "\n") hFormat = hFormat.replace(nwHeadFmt.CH_NUM, str(self._chapter)) hFormat = hFormat.replace(nwHeadFmt.SC_NUM, str(self._scene)) hFormat = hFormat.replace(nwHeadFmt.SC_ABS, str(self._absolute)) if nwHeadFmt.CH_WORD in hFormat: chWord = self._project.localLookup(self._chapter) hFormat = hFormat.replace(nwHeadFmt.CH_WORD, chWord) if nwHeadFmt.CH_ROML in hFormat: chRom = numberToRoman(self._chapter, toLower=True) hFormat = hFormat.replace(nwHeadFmt.CH_ROML, chRom) if nwHeadFmt.CH_ROMU in hFormat: chRom = numberToRoman(self._chapter, toLower=False) hFormat = hFormat.replace(nwHeadFmt.CH_ROMU, chRom) if nwHeadFmt.CHAR_POV in hFormat or nwHeadFmt.CHAR_FOCUS in hFormat: if self._handle and nHead > 0: index = self._project.index pList = index.getReferenceForHeader(self._handle, nHead, nwKeyWords.POV_KEY) fList = index.getReferenceForHeader(self._handle, nHead, nwKeyWords.FOCUS_KEY) pText = pList[0] if pList else nwUnicode.U_ENDASH fText = fList[0] if fList else nwUnicode.U_ENDASH else: pText = trConst(nwLabels.KEY_NAME[nwKeyWords.POV_KEY]) fText = trConst(nwLabels.KEY_NAME[nwKeyWords.FOCUS_KEY]) hFormat = hFormat.replace(nwHeadFmt.CHAR_POV, pText) hFormat = hFormat.replace(nwHeadFmt.CHAR_FOCUS, fText) return hFormat