diff --git a/novelwriter/formats/shared.py b/novelwriter/formats/shared.py new file mode 100644 index 00000000..82a7c2c4 --- /dev/null +++ b/novelwriter/formats/shared.py @@ -0,0 +1,129 @@ +""" +novelWriter – Formats Shared +============================ + +File History: +Created: 2024-10-21 [2.6b1] TextFmt +Created: 2024-10-21 [2.6b1] BlockTyp +Created: 2024-10-21 [2.6b1] BlockFmt + +This file is a part of novelWriter +Copyright 2018–2024, Veronica Berglyd Olsen + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" +from __future__ import annotations + +import re + +from enum import Flag, IntEnum + +ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""} +RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL) + + +def stripEscape(text: str) -> str: + """Strip escaped Markdown characters from paragraph text.""" + if "\\" in text: + return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text) + return text + + +# Enums +# ===== + +class TextFmt(IntEnum): + """Text Format. + + An enum indicating the beginning or end of a text format region. + They must be paired with a position, and apply to locations in a + text block. + """ + + B_B = 1 # Begin bold + B_E = 2 # End bold + I_B = 3 # Begin italics + I_E = 4 # End italics + D_B = 5 # Begin strikeout + D_E = 6 # End strikeout + U_B = 7 # Begin underline + U_E = 8 # End underline + M_B = 9 # Begin mark + M_E = 10 # End mark + SUP_B = 11 # Begin superscript + SUP_E = 12 # End superscript + SUB_B = 13 # Begin subscript + SUB_E = 14 # End subscript + DL_B = 15 # Begin dialogue + DL_E = 16 # End dialogue + ADL_B = 17 # Begin alt dialogue + ADL_E = 18 # End alt dialogue + FNOTE = 19 # Footnote marker + STRIP = 20 # Strip the format code + + +class BlockTyp(IntEnum): + """Text Block Type. + + An enum indicating the type of a text block. + """ + + EMPTY = 1 # Empty line (new paragraph) + SYNOPSIS = 2 # Synopsis comment + SHORT = 3 # Short description comment + COMMENT = 4 # Comment line + KEYWORD = 5 # Command line + TITLE = 6 # Title + HEAD1 = 7 # Heading 1 + HEAD2 = 8 # Heading 2 + HEAD3 = 9 # Heading 3 + HEAD4 = 10 # Heading 4 + TEXT = 11 # Text line + SEP = 12 # Scene separator + SKIP = 13 # Paragraph break + + +class BlockFmt(Flag): + """Text Block Format. + + An enum of flags that can be combined to format a text block. + """ + + NONE = 0x0000 # No special style + LEFT = 0x0001 # Left aligned + RIGHT = 0x0002 # Right aligned + CENTRE = 0x0004 # Centred + JUSTIFY = 0x0008 # Justified + PBB = 0x0010 # Page break before + PBA = 0x0020 # Page break after + Z_TOPMRG = 0x0040 # Zero top margin + Z_BTMMRG = 0x0080 # Zero bottom margin + IND_L = 0x0100 # Left indentation + IND_R = 0x0200 # Right indentation + IND_T = 0x0400 # Text indentation + + +# Types +# ===== + +# A list of formats for a single text string, consisting of: +# text position, text format, and meta data +T_Formats = list[tuple[int, TextFmt, str]] + +# A note or comment with text and associated text formats +T_Note = tuple[str, T_Formats] + +# A tokenized text block, consisting of: +# type, header number, text, text formats, and block format +T_Block = tuple[BlockTyp, int, str, T_Formats, BlockFmt] diff --git a/novelwriter/formats/todocx.py b/novelwriter/formats/todocx.py index 8af138a4..1b5a0c80 100644 --- a/novelwriter/formats/todocx.py +++ b/novelwriter/formats/todocx.py @@ -39,7 +39,8 @@ from novelwriter import __version__ from novelwriter.common import firstFloat, xmlSubElem from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer +from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt +from novelwriter.formats.tokenizer import Tokenizer logger = logging.getLogger(__name__) @@ -219,7 +220,7 @@ class ToDocX(Tokenizer): bIndent = self._fontSize * self._blockIndent - for tType, _, tText, tFormat, tStyle in self._tokens: + for tType, _, tText, tFormat, tStyle in self._blocks: # Create Paragraph par = DocXParagraph() diff --git a/novelwriter/formats/tohtml.py b/novelwriter/formats/tohtml.py index 7755befb..41449fc3 100644 --- a/novelwriter/formats/tohtml.py +++ b/novelwriter/formats/tohtml.py @@ -32,9 +32,8 @@ from time import time from novelwriter.common import formatTimeStamp from novelwriter.constants import nwHeadFmt, nwHtmlUnicode, nwKeyWords, nwLabels from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import ( - BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape -) +from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape +from novelwriter.formats.tokenizer import Tokenizer from novelwriter.types import FONT_STYLE, FONT_WEIGHTS logger = logging.getLogger(__name__) @@ -158,7 +157,7 @@ class ToHtml(Tokenizer): lines = [] tHandle = self._handle - for tType, nHead, tText, tFormat, tStyle in self._tokens: + for tType, nHead, tText, tFormat, tStyle in self._blocks: # Replace < and > with HTML entities if tFormat: diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py index dc78d220..ae316dae 100644 --- a/novelwriter/formats/tokenizer.py +++ b/novelwriter/formats/tokenizer.py @@ -29,7 +29,6 @@ import logging import re from abc import ABC, abstractmethod -from enum import Flag, IntEnum from functools import partial from pathlib import Path from time import time @@ -45,82 +44,11 @@ from novelwriter.constants import ( from novelwriter.core.index import processComment from novelwriter.core.project import NWProject from novelwriter.enum import nwComment, nwItemLayout +from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Block, T_Formats, T_Note, TextFmt from novelwriter.text.patterns import REGEX_PATTERNS logger = logging.getLogger(__name__) -ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""} -RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL) - - -def stripEscape(text: str) -> str: - """Strip escaped Markdown characters from paragraph text.""" - if "\\" in text: - return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text) - return text - - -class TextFmt(IntEnum): - - B_B = 1 # Begin bold - B_E = 2 # End bold - I_B = 3 # Begin italics - I_E = 4 # End italics - D_B = 5 # Begin strikeout - D_E = 6 # End strikeout - U_B = 7 # Begin underline - U_E = 8 # End underline - M_B = 9 # Begin mark - M_E = 10 # End mark - SUP_B = 11 # Begin superscript - SUP_E = 12 # End superscript - SUB_B = 13 # Begin subscript - SUB_E = 14 # End subscript - DL_B = 15 # Begin dialogue - DL_E = 16 # End dialogue - ADL_B = 17 # Begin alt dialogue - ADL_E = 18 # End alt dialogue - FNOTE = 19 # Footnote marker - STRIP = 20 # Strip the format code - - -class BlockTyp(IntEnum): - - EMPTY = 1 # Empty line (new paragraph) - SYNOPSIS = 2 # Synopsis comment - SHORT = 3 # Short description comment - COMMENT = 4 # Comment line - KEYWORD = 5 # Command line - TITLE = 6 # Title - HEAD1 = 7 # Heading 1 - HEAD2 = 8 # Heading 2 - HEAD3 = 9 # Heading 3 - HEAD4 = 10 # Heading 4 - TEXT = 11 # Text line - SEP = 12 # Scene separator - SKIP = 13 # Paragraph break - - -class BlockFmt(Flag): - - NONE = 0x0000 # No special style - LEFT = 0x0001 # Left aligned - RIGHT = 0x0002 # Right aligned - CENTRE = 0x0004 # Centred - JUSTIFY = 0x0008 # Justified - PBB = 0x0010 # Page break before - PBA = 0x0020 # Page break after - Z_TOPMRG = 0x0040 # Zero top margin - Z_BTMMRG = 0x0080 # Zero bottom margin - IND_L = 0x0100 # Left indentation - IND_R = 0x0200 # Right indentation - IND_T = 0x0400 # Text indentation - - -T_Formats = list[tuple[int, TextFmt, str]] -T_Comment = tuple[str, T_Formats] -T_Token = tuple[BlockTyp, int, str, T_Formats, BlockFmt] - class Tokenizer(ABC): """Core: Text Tokenizer Abstract Base Class @@ -154,11 +82,11 @@ class Tokenizer(ABC): self._result = "" # The result of the last document self._keepRaw = False # Whether to keep the raw text, used by ToRaw - # Tokens and Meta Data (Per Document) - self._tokens: list[T_Token] = [] - self._footnotes: dict[str, T_Comment] = {} + # Blocks and Meta Data (Per Document) + self._blocks: list[T_Block] = [] + self._footnotes: dict[str, T_Note] = {} - # Tokens and Meta Data (Per Instance) + # Blocks and Meta Data (Per Instance) self._counts: dict[str, int] = {} self._outline: dict[str, str] = {} self._markdown: list[str] = [] @@ -518,8 +446,8 @@ class Tokenizer(ABC): trNotes = self._localLookup("Notes") title = f"{trNotes}: {tItem.itemName}" - self._tokens = [] - self._tokens.append(( + self._blocks = [] + self._blocks.append(( BlockTyp.TITLE, 1, title, [], textAlign )) if self._keepRaw: @@ -561,15 +489,15 @@ class Tokenizer(ABC): characters that indicate headings, comments, commands etc, or just contain plain text. In the case of plain text, apply the same RegExes that the syntax highlighter uses and save the - locations of these formatting tags into the token array. + locations of these formatting tags into the blocks list. - The format of the token list is an entry with a five-tuple for + The format of the blocs list is an entry with a five-tuple for each line in the file. The tuple is as follows: 1: The type of the block, BlockType.* 2: The heading number under which the text is placed 3: The text content of the block, without leading tags 4: The internal formatting map of the text, TxtFmt.* - 5: The style of the block, BlockFmt.* + 5: The formats of the block, BlockFmt.* """ if self._isNovel: self._hFormatter.setHandle(self._handle) @@ -578,13 +506,13 @@ class Tokenizer(ABC): breakNext = False tmpMarkdown = [] tHandle = self._handle or "" - tokens: list[T_Token] = [] + blocks: list[T_Block] = [] for aLine in self._text.splitlines(): sLine = aLine.strip().lower() # Check for blank lines if len(sLine) == 0: - tokens.append(( + blocks.append(( BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE )) if self._keepRaw: @@ -613,7 +541,7 @@ class Tokenizer(ABC): continue elif sLine == "[vspace]": - tokens.append( + blocks.append( (BlockTyp.SKIP, nHead, "", [], sAlign) ) continue @@ -621,11 +549,11 @@ class Tokenizer(ABC): elif sLine.startswith("[vspace:") and sLine.endswith("]"): nSkip = checkInt(sLine[8:-1], 0) if nSkip >= 1: - tokens.append( + blocks.append( (BlockTyp.SKIP, nHead, "", [], sAlign) ) if nSkip > 1: - tokens += (nSkip - 1) * [ + blocks += (nSkip - 1) * [ (BlockTyp.SKIP, nHead, "", [], BlockFmt.NONE) ] continue @@ -645,14 +573,14 @@ class Tokenizer(ABC): cStyle, cKey, cText, _, _ = processComment(aLine) if cStyle == nwComment.SYNOPSIS: tLine, tFmt = self._extractFormats(cText) - tokens.append(( + blocks.append(( BlockTyp.SYNOPSIS, nHead, tLine, tFmt, sAlign )) if self._doSynopsis and self._keepRaw: tmpMarkdown.append(f"{aLine}\n") elif cStyle == nwComment.SHORT: tLine, tFmt = self._extractFormats(cText) - tokens.append(( + blocks.append(( BlockTyp.SHORT, nHead, tLine, tFmt, sAlign )) if self._doSynopsis and self._keepRaw: @@ -664,7 +592,7 @@ class Tokenizer(ABC): tmpMarkdown.append(f"{aLine}\n") else: tLine, tFmt = self._extractFormats(cText) - tokens.append(( + blocks.append(( BlockTyp.COMMENT, nHead, tLine, tFmt, sAlign )) if self._doComments and self._keepRaw: @@ -681,7 +609,7 @@ class Tokenizer(ABC): valid and bits and bits[0] in nwLabels.KEY_NAME and bits[0] not in self._skipKeywords ): - tokens.append(( + blocks.append(( BlockTyp.KEYWORD, nHead, aLine[1:].strip(), [], sAlign )) if self._doKeywords and self._keepRaw: @@ -717,7 +645,7 @@ class Tokenizer(ABC): self._hFormatter.resetAll() self._noSep = True - tokens.append(( + blocks.append(( tType, nHead, tText, [], tStyle )) if self._keepRaw: @@ -752,7 +680,7 @@ class Tokenizer(ABC): self._hFormatter.resetScene() self._noSep = True - tokens.append(( + blocks.append(( tType, nHead, tText, [], tStyle )) if self._keepRaw: @@ -793,7 +721,7 @@ class Tokenizer(ABC): tStyle = BlockFmt.NONE if self._noSep else BlockFmt.CENTRE self._noSep = False - tokens.append(( + blocks.append(( tType, nHead, tText, [], tStyle )) if self._keepRaw: @@ -823,7 +751,7 @@ class Tokenizer(ABC): tType = BlockTyp.SEP tStyle = BlockFmt.CENTRE - tokens.append(( + blocks.append(( tType, nHead, tText, [], tStyle )) if self._keepRaw: @@ -871,26 +799,26 @@ class Tokenizer(ABC): # Process formats tLine, tFmt = self._extractFormats(aLine, hDialog=self._isNovel) - tokens.append(( + blocks.append(( BlockTyp.TEXT, nHead, tLine, tFmt, sAlign )) if self._keepRaw: tmpMarkdown.append(f"{aLine}\n") # If we have content, turn off the first page flag - if self._isFirst and tokens: + if self._isFirst and blocks: self._isFirst = False # First document has been processed - # Make sure the token array doesn't start with a page break + # Make sure the blocks array doesn't start with a page break # on the very first page, adding a blank first page. - if tokens[0][4] & BlockFmt.PBB: - cToken = tokens[0] - tokens[0] = ( - cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~BlockFmt.PBB + if blocks[0][4] & BlockFmt.PBB: + cBlock = blocks[0] + blocks[0] = ( + cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB ) # Always add an empty line at the end of the file - tokens.append(( + blocks.append(( BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE )) if self._keepRaw: @@ -904,48 +832,48 @@ class Tokenizer(ABC): # It also ensures that there isn't paragraph spacing between # meta data lines for formats that has spacing. - self._tokens = [] - pToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE) - nToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE) + self._blocks = [] + pBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE) + nBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE) lineSep = "\n" if self._keepBreaks else " " - pLines: list[T_Token] = [] + pLines: list[T_Block] = [] - tCount = len(tokens) - for n, cToken in enumerate(tokens): + tCount = len(blocks) + for n, cBlock in enumerate(blocks): if n > 0: - pToken = tokens[n-1] # Look behind + pBlock = blocks[n-1] # Look behind if n < tCount - 1: - nToken = tokens[n+1] # Look ahead + nBlock = blocks[n+1] # Look ahead - if cToken[0] in self.L_SKIP_INDENT and not self._indentFirst: + if cBlock[0] in self.L_SKIP_INDENT and not self._indentFirst: # Unless the indentFirst flag is set, we set up the next # paragraph to not be indented if we see a block of a # specific type self._noIndent = True - if cToken[0] == BlockTyp.EMPTY: + if cBlock[0] == BlockTyp.EMPTY: # We don't need to keep the empty lines after this pass pass - elif cToken[0] == BlockTyp.KEYWORD: + elif cBlock[0] == BlockTyp.KEYWORD: # Adjust margins for lines in a list of keyword lines - aStyle = cToken[4] - if pToken[0] == BlockTyp.KEYWORD: + aStyle = cBlock[4] + if pBlock[0] == BlockTyp.KEYWORD: aStyle |= BlockFmt.Z_TOPMRG - if nToken[0] == BlockTyp.KEYWORD: + if nBlock[0] == BlockTyp.KEYWORD: aStyle |= BlockFmt.Z_BTMMRG - self._tokens.append(( - cToken[0], cToken[1], cToken[2], cToken[3], aStyle + self._blocks.append(( + cBlock[0], cBlock[1], cBlock[2], cBlock[3], aStyle )) - elif cToken[0] == BlockTyp.TEXT: + elif cBlock[0] == BlockTyp.TEXT: # Combine lines from the same paragraph - pLines.append(cToken) + pLines.append(cBlock) - if nToken[0] != BlockTyp.TEXT: - # Next token is not text, so we add the buffer to tokens + if nBlock[0] != BlockTyp.TEXT: + # Next block is not text, so we add the buffer to blocks nLines = len(pLines) cStyle = pLines[0][4] if self._firstIndent and not (self._noIndent or cStyle & self.M_ALIGNED): @@ -956,11 +884,11 @@ class Tokenizer(ABC): if nLines == 1: # The paragraph contains a single line, so we just save - # that directly to the token list. If justify is + # that directly to the blocks list. If justify is # enabled, and there is no alignment, we apply it. if self._doJustify and not cStyle & self.M_ALIGNED: cStyle |= BlockFmt.JUSTIFY - self._tokens.append(( + self._blocks.append(( BlockTyp.TEXT, pLines[0][1], pLines[0][2], pLines[0][3], cStyle )) elif nLines > 1: @@ -969,11 +897,11 @@ class Tokenizer(ABC): # recompute all the formatting markers tTxt = "" tFmt: T_Formats = [] - for aToken in pLines: + for aBlock in pLines: tLen = len(tTxt) - tTxt += f"{aToken[2]}{lineSep}" - tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3]) - self._tokens.append(( + tTxt += f"{aBlock[2]}{lineSep}" + tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3]) + self._blocks.append(( BlockTyp.TEXT, pLines[0][1], tTxt[:-1], tFmt, cStyle )) @@ -982,7 +910,7 @@ class Tokenizer(ABC): self._noIndent = False else: - self._tokens.append(cToken) + self._blocks.append(cBlock) return @@ -990,7 +918,7 @@ class Tokenizer(ABC): """Build an outline of the text up to level 3 headings.""" tHandle = self._handle or "" isNovel = self._isNovel - for tType, nHead, tText, _, _ in self._tokens: + for tType, nHead, tText, _, _ in self._blocks: if tType == BlockTyp.TITLE: prefix = "TT" elif tType == BlockTyp.HEAD1: @@ -1025,7 +953,7 @@ class Tokenizer(ABC): textWordChars = self._counts.get("textWordChars", 0) titleWordChars = self._counts.get("titleWordChars", 0) - for tType, _, tText, _, _ in self._tokens: + for tType, _, tText, _, _ in self._blocks: tText = tText.replace(nwUnicode.U_ENDASH, " ") tText = tText.replace(nwUnicode.U_EMDASH, " ") diff --git a/novelwriter/formats/tomarkdown.py b/novelwriter/formats/tomarkdown.py index dcd83a9b..2f9ccc93 100644 --- a/novelwriter/formats/tomarkdown.py +++ b/novelwriter/formats/tomarkdown.py @@ -29,7 +29,8 @@ from pathlib import Path from novelwriter.constants import nwHeadFmt, nwLabels, nwUnicode from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer +from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt +from novelwriter.formats.tokenizer import Tokenizer logger = logging.getLogger(__name__) @@ -117,7 +118,7 @@ class ToMarkdown(Tokenizer): cSkip = "" lines = [] - for tType, _, tText, tFormat, tStyle in self._tokens: + for tType, _, tText, tFormat, tStyle in self._blocks: if tType == BlockTyp.TEXT: tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n") @@ -180,7 +181,7 @@ class ToMarkdown(Tokenizer): for key, index in self._usedNotes.items(): if content := self._footnotes.get(key): marker = f"{index}. " - text = self._formatText(*content, tags) + text = self._formatText(content[0], content[1], tags) lines.append(f"{marker}{text}\n") lines.append("\n") diff --git a/novelwriter/formats/toodt.py b/novelwriter/formats/toodt.py index 7af7f66d..d5bde589 100644 --- a/novelwriter/formats/toodt.py +++ b/novelwriter/formats/toodt.py @@ -41,9 +41,8 @@ from novelwriter import __version__ from novelwriter.common import xmlIndent, xmlSubElem from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import ( - BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape -) +from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape +from novelwriter.formats.tokenizer import Tokenizer from novelwriter.types import FONT_STYLE, FONT_WEIGHTS logger = logging.getLogger(__name__) @@ -426,7 +425,7 @@ class ToOdt(Tokenizer): self._result = "" # Not used, but cleared just in case xText = self._xText - for tType, _, tText, tFormat, tStyle in self._tokens: + for tType, _, tText, tFormat, tStyle in self._blocks: # Styles oStyle = ODTParagraphStyle("New") diff --git a/novelwriter/formats/toqdoc.py b/novelwriter/formats/toqdoc.py index dbde490a..243059ed 100644 --- a/novelwriter/formats/toqdoc.py +++ b/novelwriter/formats/toqdoc.py @@ -36,7 +36,8 @@ from PyQt5.QtPrintSupport import QPrinter from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles, nwUnicode from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer +from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt +from novelwriter.formats.tokenizer import Tokenizer from novelwriter.types import ( QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight, QtPageBreakAfter, QtPageBreakBefore, QtTransparent, QtVAlignNormal, @@ -221,7 +222,7 @@ class ToQTextDocument(Tokenizer): cursor = QTextCursor(self._document) cursor.movePosition(QTextCursor.MoveOperation.End) - for tType, nHead, tText, tFormat, tStyle in self._tokens: + for tType, nHead, tText, tFormat, tStyle in self._blocks: # Styles bFmt = QTextBlockFormat(self._blockFmt) diff --git a/tests/test_formats/test_fmt_todocx.py b/tests/test_formats/test_fmt_todocx.py index ffc07c7c..cd64b55e 100644 --- a/tests/test_formats/test_fmt_todocx.py +++ b/tests/test_formats/test_fmt_todocx.py @@ -31,11 +31,11 @@ from novelwriter.core.buildsettings import BuildSettings from novelwriter.core.docbuild import NWBuildDocument from novelwriter.core.project import NWProject from novelwriter.enum import nwBuildFmt +from novelwriter.formats.shared import BlockFmt, BlockTyp from novelwriter.formats.todocx import ( S_FNOTE, S_HEAD1, S_HEAD2, S_HEAD3, S_HEAD4, S_META, S_NORM, S_SEP, S_TITLE, ToDocX, _mkTag, _wTag ) -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp from tests.tools import DOCX_IGNORE, cmpFiles @@ -71,7 +71,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Normal Text xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -81,7 +81,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Title xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TITLE, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.TITLE, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -91,7 +91,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Heading Level 1 xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.HEAD1, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.HEAD1, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -101,7 +101,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Heading Level 2 xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.HEAD2, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.HEAD2, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -111,7 +111,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Heading Level 3 xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.HEAD3, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.HEAD3, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -121,7 +121,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Heading Level 4 xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.HEAD4, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.HEAD4, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -131,7 +131,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Separator xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.SEP, 0, "* * *", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.SEP, 0, "* * *", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -141,7 +141,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Empty Paragraph xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.SKIP, 0, "* * *", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.SKIP, 0, "* * *", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -150,7 +150,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Synopsis xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.SYNOPSIS, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.SYNOPSIS, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -162,7 +162,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Short xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.SHORT, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.SHORT, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -174,7 +174,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Comment xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.COMMENT, 0, "Hello World", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.COMMENT, 0, "Hello World", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -186,7 +186,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Tags and References (Single) xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.KEYWORD, 0, "tag: Stuff", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.KEYWORD, 0, "tag: Stuff", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -198,7 +198,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Tags and References (Multiple) xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.KEYWORD, 0, "char: Jane, John", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.KEYWORD, 0, "char: Jane, John", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -210,7 +210,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI): # Tags and References (Invalid) xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.KEYWORD, 0, "stuff: Stuff", [], BlockFmt.NONE)] + doc._blocks = [(BlockTyp.KEYWORD, 0, "stuff: Stuff", [], BlockFmt.NONE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -230,7 +230,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Left Align xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.LEFT)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.LEFT)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -240,7 +240,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Right Align xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.RIGHT)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.RIGHT)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -250,7 +250,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Center Align xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.CENTRE)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.CENTRE)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -260,7 +260,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Justify xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.JUSTIFY)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.JUSTIFY)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -270,7 +270,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Page Break Before xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBB)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBB)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -282,7 +282,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Page Break After xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBA)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBA)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -294,7 +294,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Zero Margins xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -305,7 +305,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # Indent xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_L | BlockFmt.IND_R)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_L | BlockFmt.IND_R)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( @@ -316,7 +316,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI): # First Line Indent xTest = ET.Element(_wTag("body")) - doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_T)] + doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_T)] doc.doConvert() doc._pars[-1].toXml(xTest) assert xmlToText(xTest) == ( diff --git a/tests/test_formats/test_fmt_tohtml.py b/tests/test_formats/test_fmt_tohtml.py index f27aef7a..71863338 100644 --- a/tests/test_formats/test_fmt_tohtml.py +++ b/tests/test_formats/test_fmt_tohtml.py @@ -26,8 +26,8 @@ import pytest from novelwriter import CONFIG from novelwriter.core.project import NWProject +from novelwriter.formats.shared import BlockFmt, BlockTyp from novelwriter.formats.tohtml import ToHtml -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp @pytest.mark.core @@ -359,7 +359,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): # ============== # Title - html._tokens = [ + html._blocks = [ (BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE), ] html.doConvert() @@ -369,7 +369,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): ) # Unnumbered - html._tokens = [ + html._blocks = [ (BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB), ] html.doConvert() @@ -382,14 +382,14 @@ def testFmtToHtml_ConvertDirect(mockGUI): # ========== # Separator - html._tokens = [ + html._blocks = [ (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), ] html.doConvert() assert html.result == "

* * *

\n" # Skip - html._tokens = [ + html._blocks = [ (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), ] html.doConvert() @@ -402,7 +402,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): # Align Left html.setStyles(False) - html._tokens = [ + html._blocks = [ (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT), ] html.doConvert() @@ -413,7 +413,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): html.setStyles(True) # Align Left - html._tokens = [ + html._blocks = [ (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT), ] html.doConvert() @@ -422,7 +422,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): ) # Align Right - html._tokens = [ + html._blocks = [ (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.RIGHT), ] html.doConvert() @@ -431,7 +431,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): ) # Align Centre - html._tokens = [ + html._blocks = [ (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.CENTRE), ] html.doConvert() @@ -440,7 +440,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): ) # Align Justify - html._tokens = [ + html._blocks = [ (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.JUSTIFY), ] html.doConvert() @@ -452,7 +452,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): # ========== # Page Break Always - html._tokens = [ + html._blocks = [ (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.PBB | BlockFmt.PBA), ] html.doConvert() @@ -465,7 +465,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): # ====== # Indent Left - html._tokens = [ + html._blocks = [ (BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_L), ] html.doConvert() @@ -474,7 +474,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): ) # Indent Right - html._tokens = [ + html._blocks = [ (BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_R), ] html.doConvert() @@ -483,7 +483,7 @@ def testFmtToHtml_ConvertDirect(mockGUI): ) # Text Indent - html._tokens = [ + html._blocks = [ (BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_T), ] html.doConvert() diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py index 6a621fa3..10809b67 100644 --- a/tests/test_formats/test_fmt_tokenizer.py +++ b/tests/test_formats/test_fmt_tokenizer.py @@ -29,9 +29,8 @@ from PyQt5.QtGui import QFont from novelwriter import CONFIG from novelwriter.constants import nwHeadFmt, nwStyles from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import ( - BlockFmt, BlockTyp, HeadingFormatter, TextFmt, Tokenizer, stripEscape -) +from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt, stripEscape +from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer from novelwriter.formats.tomarkdown import ToMarkdown from novelwriter.formats.toraw import ToRaw @@ -200,22 +199,22 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath): assert project.saveProject() # Root Heading - assert len(tokens._tokens) == 0 + assert len(tokens._blocks) == 0 tokens.addRootHeading("stuff") tokens.addRootHeading(C.hSceneDoc) - assert len(tokens._tokens) == 0 + assert len(tokens._blocks) == 0 # First Page tokens.addRootHeading(C.hPlotRoot) assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n" - assert tokens._tokens[-1] == ( + assert tokens._blocks[-1] == ( BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE ) # Not First Page tokens.addRootHeading(C.hPlotRoot) assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n" - assert tokens._tokens[-1] == ( + assert tokens._blocks[-1] == ( BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE | BlockFmt.PBB ) @@ -275,7 +274,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "#! Novel Title\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TITLE, 1, "Novel Title", [], BlockFmt.CENTRE), ] assert tokens.allMarkdown[-1] == "#! Novel Title\n\n" @@ -286,7 +285,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "#! Note Title\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TITLE, 1, "Note Title", [], BlockFmt.CENTRE), ] assert tokens.allMarkdown[-1] == "#! Note Title\n\n" @@ -300,7 +299,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "# Novel Title\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Novel Title", [], BlockFmt.CENTRE), ] assert tokens.allMarkdown[-1] == "# Novel Title\n\n" @@ -311,7 +310,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "# Note Title\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Note Title", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "# Note Title\n\n" @@ -324,7 +323,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "## Chapter One\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB), ] assert tokens.allMarkdown[-1] == "## Chapter One\n\n" @@ -334,7 +333,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "## Heading 2\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "Heading 2", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "## Heading 2\n\n" @@ -347,7 +346,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "### Scene One\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD3, 1, "Scene One", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "### Scene One\n\n" @@ -357,7 +356,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "### Heading 3\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD3, 1, "Heading 3", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "### Heading 3\n\n" @@ -370,7 +369,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "#### A Section\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD4, 1, "A Section", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "#### A Section\n\n" @@ -380,7 +379,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "#### Heading 4\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD4, 1, "Heading 4", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "#### Heading 4\n\n" @@ -394,7 +393,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "#! Title\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE), ] assert tokens.allMarkdown[-1] == "#! Title\n\n" @@ -405,7 +404,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "#! Title\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE), ] assert tokens.allMarkdown[-1] == "#! Title\n\n" @@ -418,7 +417,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "##! Prologue\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB), ] assert tokens.allMarkdown[-1] == "##! Prologue\n\n" @@ -428,7 +427,7 @@ def testFmtToken_HeaderFormat(mockGUI): tokens._text = "##! Prologue\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "##! Prologue\n\n" @@ -444,7 +443,7 @@ def testFmtToken_HeaderStyle(mockGUI): tokens._text = text tokens._isFirst = first tokens.tokenizeText() - return tokens._tokens[0][4] + return tokens._blocks[0][4] # No Styles # ========= @@ -704,7 +703,7 @@ def testFmtToken_MetaFormat(mockGUI): # Comment tokens._text = "% A comment\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.COMMENT, 0, "A comment", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -716,18 +715,18 @@ def testFmtToken_MetaFormat(mockGUI): # Ignore Text tokens._text = "%~ Some text\n" tokens.tokenizeText() - assert tokens._tokens == [] + assert tokens._blocks == [] assert tokens.allMarkdown[-1] == "\n" # Synopsis tokens._text = "%synopsis: The synopsis\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE), ] tokens._text = "% synopsis: The synopsis\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -740,7 +739,7 @@ def testFmtToken_MetaFormat(mockGUI): tokens.setSynopsis(False) tokens._text = "% short: A short description\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.SHORT, 0, "A short description", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -752,7 +751,7 @@ def testFmtToken_MetaFormat(mockGUI): # Keyword tokens._text = "@char: Bod\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "\n" @@ -766,7 +765,7 @@ def testFmtToken_MetaFormat(mockGUI): styTop = BlockFmt.NONE | BlockFmt.Z_BTMMRG styMid = BlockFmt.NONE | BlockFmt.Z_BTMMRG | BlockFmt.Z_TOPMRG styBtm = BlockFmt.NONE | BlockFmt.Z_TOPMRG - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.KEYWORD, 0, "pov: Bod", [], styTop), (BlockTyp.KEYWORD, 0, "plot: Main", [], styMid), (BlockTyp.KEYWORD, 0, "location: Europe", [], styBtm), @@ -777,7 +776,7 @@ def testFmtToken_MetaFormat(mockGUI): tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n" tokens.setIgnoredKeywords("@plot, @location") tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.KEYWORD, 0, "pov: Bod", [], BlockFmt.NONE), ] @@ -802,7 +801,7 @@ def testFmtToken_MarginFormat(mockGUI): ">> Right-indent, right-aligned <\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TEXT, 0, "Some regular text", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "Some left-aligned text", [], BlockFmt.LEFT), (BlockTyp.TEXT, 0, "Some right-aligned text", [], BlockFmt.RIGHT), @@ -939,7 +938,7 @@ def testFmtToken_Paragraphs(mockGUI): # Collapse empty lines tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TEXT, 0, "First paragraph", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "Second paragraph", [], BlockFmt.NONE), ] @@ -948,7 +947,7 @@ def testFmtToken_Paragraphs(mockGUI): tokens._text = "This is text\nspanning multiple\nlines" tokens.setKeepLineBreaks(True) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TEXT, 0, "This is text\nspanning multiple\nlines", [], BlockFmt.NONE), ] @@ -956,7 +955,7 @@ def testFmtToken_Paragraphs(mockGUI): tokens._text = "This is text\nspanning multiple\nlines" tokens.setKeepLineBreaks(False) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TEXT, 0, "This is text spanning multiple lines", [], BlockFmt.NONE), ] @@ -964,7 +963,7 @@ def testFmtToken_Paragraphs(mockGUI): tokens._text = "This **is text**\nspanning _multiple_\nlines" tokens.setKeepLineBreaks(False) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ ( BlockTyp.TEXT, 0, "This is text spanning multiple lines", @@ -982,7 +981,7 @@ def testFmtToken_Paragraphs(mockGUI): tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n" tokens.setKeepLineBreaks(False) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title", [], BlockFmt.NONE), ( BlockTyp.TEXT, 1, "Text on two lines.", [ @@ -1011,21 +1010,21 @@ def testFmtToken_TextFormat(mockGUI): # Text tokens._text = "Some plain text\non two lines\n\n\n" tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TEXT, 0, "Some plain text\non two lines", [], BlockFmt.NONE), ] assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n" tokens.setBodyText(False) tokens.tokenizeText() - assert tokens._tokens == [] + assert tokens._blocks == [] assert tokens.allMarkdown[-1] == "\n\n\n" tokens.setBodyText(True) # Text Emphasis tokens._text = "Some **bolded text** on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Some bolded text on this lines", [ (5, TextFmt.B_B, ""), @@ -1037,7 +1036,7 @@ def testFmtToken_TextFormat(mockGUI): tokens._text = "Some _italic text_ on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Some italic text on this lines", [ (5, TextFmt.I_B, ""), @@ -1049,7 +1048,7 @@ def testFmtToken_TextFormat(mockGUI): tokens._text = "Some **_bold italic text_** on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Some bold italic text on this lines", [ (5, TextFmt.B_B, ""), @@ -1063,7 +1062,7 @@ def testFmtToken_TextFormat(mockGUI): tokens._text = "Some ~~strikethrough text~~ on this lines\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Some strikethrough text on this lines", [ (5, TextFmt.D_B, ""), @@ -1075,7 +1074,7 @@ def testFmtToken_TextFormat(mockGUI): tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Some nested bold and italic and strikethrough text here", [ (5, TextFmt.B_B, ""), @@ -1113,7 +1112,7 @@ def testFmtToken_Dialogue(mockGUI): # Single quotes tokens._text = "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019", [ @@ -1128,7 +1127,7 @@ def testFmtToken_Dialogue(mockGUI): # Double quotes tokens._text = "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d", [ @@ -1143,7 +1142,7 @@ def testFmtToken_Dialogue(mockGUI): # Alt quotes tokens._text = "Text with ::dialogue one,:: and ::dialogue two.::\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "Text with ::dialogue one,:: and ::dialogue two.::", [ @@ -1158,7 +1157,7 @@ def testFmtToken_Dialogue(mockGUI): # Dialogue line with narrator break tokens._text = "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?", [ @@ -1176,7 +1175,7 @@ def testFmtToken_Dialogue(mockGUI): # Dialogue + formatting on same index (Issue #2012) tokens._text = "[i]\u201cDialogue text.\u201d[/i]\n" tokens.tokenizeText() - assert tokens._tokens == [( + assert tokens._blocks == [( BlockTyp.TEXT, 0, "\u201cDialogue text.\u201d", [ @@ -1213,7 +1212,7 @@ def testFmtToken_SpecialFormat(mockGUI): "# Title Two\n\n" ) tokens.tokenizeText() - assert tokens._tokens == correctResp + assert tokens._blocks == correctResp # Command w/Space tokens._isFirst = True @@ -1223,7 +1222,7 @@ def testFmtToken_SpecialFormat(mockGUI): "# Title Two\n\n" ) tokens.tokenizeText() - assert tokens._tokens == correctResp + assert tokens._blocks == correctResp # Trailing Spaces tokens._isFirst = True @@ -1233,7 +1232,7 @@ def testFmtToken_SpecialFormat(mockGUI): "# Title Two\n\n" ) tokens.tokenizeText() - assert tokens._tokens == correctResp + assert tokens._blocks == correctResp # Single Empty Paragraph # ====================== @@ -1244,7 +1243,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), @@ -1260,7 +1259,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), @@ -1273,7 +1272,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), @@ -1288,7 +1287,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), ] @@ -1300,7 +1299,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), ] @@ -1312,7 +1311,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), ] @@ -1328,7 +1327,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.SKIP, 1, "", [], BlockFmt.PBB), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), @@ -1342,7 +1341,7 @@ def testFmtToken_SpecialFormat(mockGUI): "Some text to go here ...\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.SKIP, 1, "", [], BlockFmt.PBB), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), @@ -1374,7 +1373,7 @@ def testFmtToken_TextIndent(mockGUI): "Second paragraph.\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE), (BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE), (BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.NONE), @@ -1389,7 +1388,7 @@ def testFmtToken_TextIndent(mockGUI): "%Synopsis: Stuff happens.\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD3, 1, "Scene Two", [], BlockFmt.NONE), (BlockTyp.SYNOPSIS, 1, "Stuff happens.", [], BlockFmt.NONE), ] @@ -1402,7 +1401,7 @@ def testFmtToken_TextIndent(mockGUI): "Second paragraph.\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.TEXT, 0, "First paragraph.", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "Second paragraph.", [], BlockFmt.IND_T), ] @@ -1425,7 +1424,7 @@ def testFmtToken_TextIndent(mockGUI): "Second paragraph.\n\n" ) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE), (BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE), (BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.IND_T), @@ -1451,7 +1450,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "# Part One\n" tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}") tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.CENTRE), ] @@ -1460,7 +1459,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "# Part One\n" tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}") tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.PBB | BlockFmt.CENTRE), ] @@ -1471,7 +1470,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "## Chapter One\n" tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}") tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "C: Chapter One", [], BlockFmt.PBB), ] @@ -1479,7 +1478,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "##! Prologue\n" tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}") tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "U: Prologue", [], BlockFmt.PBB), ] @@ -1488,7 +1487,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}") tokens._hFormatter._chCount = 0 tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB), ] @@ -1496,7 +1495,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "## Chapter\n" tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}") tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "Chapter II", [], BlockFmt.PBB), ] @@ -1504,7 +1503,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "## Chapter\n" tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}") tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD2, 1, "Chapter iii", [], BlockFmt.PBB), ] @@ -1515,7 +1514,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "### Scene One\n" tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD3, 1, "S: Scene One", [], BlockFmt.NONE), ] @@ -1523,21 +1522,21 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "### Scene One\n" tokens.setSceneFormat("", True) tokens.tokenizeText() - assert tokens._tokens == [] + assert tokens._blocks == [] # H3: Scene wo/Format, first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) tokens._noSep = True tokens.tokenizeText() - assert tokens._tokens == [] + assert tokens._blocks == [] # H3: Scene wo/Format, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("", False) tokens._noSep = False tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), ] @@ -1546,14 +1545,14 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens.setSceneFormat("* * *", False) tokens._noSep = True tokens.tokenizeText() - assert tokens._tokens == [] + assert tokens._blocks == [] # H3: Scene Separator, not first tokens._text = "### Scene One\n" tokens.setSceneFormat("* * *", False) tokens._noSep = False tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), ] @@ -1563,7 +1562,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._hFormatter._scAbsCount = 0 tokens._hFormatter._scChCount = 0 tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD3, 1, "Scene 1", [], BlockFmt.NONE), ] @@ -1573,7 +1572,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._hFormatter._scAbsCount = 0 tokens._hFormatter._scChCount = 1 tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD3, 1, "Scene 3.2", [], BlockFmt.NONE), ] @@ -1584,13 +1583,13 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat("", True) tokens.tokenizeText() - assert tokens._tokens == [] + assert tokens._blocks == [] # H4: Section Visible wo/Format tokens._text = "#### A Section\n" tokens.setSectionFormat("", False) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), ] @@ -1598,7 +1597,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.HEAD4, 1, "X: A Section", [], BlockFmt.NONE), ] @@ -1606,7 +1605,7 @@ def testFmtToken_ProcessHeaders(mockGUI): tokens._text = "#### A Section\n" tokens.setSectionFormat("* * *", False) tokens.tokenizeText() - assert tokens._tokens == [ + assert tokens._blocks == [ (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), ] @@ -1704,7 +1703,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText): tokens._counts = {} tokens.tokenizeText() tokens.countStats() - assert tokens._tokens[0][2] == "A Chapter Title" + assert tokens._blocks[0][2] == "A Chapter Title" assert tokens.textStats == { "titleCount": 1, "paragraphCount": 0, "allWords": 3, "textWords": 0, "titleWords": 3, @@ -1719,7 +1718,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText): tokens._hFormatter.resetAll() tokens.tokenizeText() tokens.countStats() - assert tokens._tokens[0][2] == "C 1: A Chapter Title" + assert tokens._blocks[0][2] == "C 1: A Chapter Title" assert tokens.textStats == { "titleCount": 1, "paragraphCount": 0, "allWords": 5, "textWords": 0, "titleWords": 5, @@ -1747,7 +1746,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText): tokens.setSceneFormat("* * *", False) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == ["Chapter", "Text", "* * *", "Text"] + assert [t[2] for t in tokens._blocks] == ["Chapter", "Text", "* * *", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 2, "allWords": 6, "textWords": 2, "titleWords": 1, @@ -1764,7 +1763,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText): tokens.setSynopsis(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] + assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 4, "textWords": 1, "titleWords": 1, @@ -1781,7 +1780,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText): tokens.setSynopsis(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] + assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 5, "textWords": 1, "titleWords": 1, @@ -1798,7 +1797,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText): tokens.setComments(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] + assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 4, "textWords": 1, "titleWords": 1, @@ -1815,7 +1814,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText): tokens.setKeywords(True) tokens.tokenizeText() tokens.countStats() - assert [t[2] for t in tokens._tokens] == ["Chapter", "pov: Jane", "Text"] + assert [t[2] for t in tokens._blocks] == ["Chapter", "pov: Jane", "Text"] assert tokens.textStats == { "titleCount": 1, "paragraphCount": 1, "allWords": 6, "textWords": 1, "titleWords": 1, diff --git a/tests/test_formats/test_fmt_tomarkdown.py b/tests/test_formats/test_fmt_tomarkdown.py index abef4f73..fffcd3c2 100644 --- a/tests/test_formats/test_fmt_tomarkdown.py +++ b/tests/test_formats/test_fmt_tomarkdown.py @@ -23,7 +23,7 @@ from __future__ import annotations import pytest from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp +from novelwriter.formats.shared import BlockFmt, BlockTyp from novelwriter.formats.tomarkdown import ToMarkdown @@ -220,7 +220,7 @@ def testFmtToMarkdown_ConvertDirect(mockGUI): # ============== # Title - toMD._tokens = [ + toMD._blocks = [ (BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE), ] toMD.doConvert() @@ -230,14 +230,14 @@ def testFmtToMarkdown_ConvertDirect(mockGUI): # ========== # Separator - toMD._tokens = [ + toMD._blocks = [ (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), ] toMD.doConvert() assert toMD.result == "* * *\n\n" # Skip - toMD._tokens = [ + toMD._blocks = [ (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), ] toMD.doConvert() diff --git a/tests/test_formats/test_fmt_toodt.py b/tests/test_formats/test_fmt_toodt.py index bd17ff77..7f75b935 100644 --- a/tests/test_formats/test_fmt_toodt.py +++ b/tests/test_formats/test_fmt_toodt.py @@ -30,7 +30,7 @@ import pytest from novelwriter.common import xmlIndent from novelwriter.constants import nwHeadFmt from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, TextFmt +from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt from novelwriter.formats.toodt import ODTParagraphStyle, ODTTextStyle, ToOdt, XMLParagraph, _mkTag from tests.tools import ODT_IGNORE, cmpFiles @@ -719,7 +719,7 @@ def testFmtToOdt_ConvertDirect(mockGUI): # Justified doc = ToOdt(project, isFlat=True) - doc._tokens = [ + doc._blocks = [ (BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.JUSTIFY), ] doc.initDocument() @@ -739,7 +739,7 @@ def testFmtToOdt_ConvertDirect(mockGUI): # Page Break After doc = ToOdt(project, isFlat=True) - doc._tokens = [ + doc._blocks = [ (BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.PBA), ] doc.initDocument() diff --git a/tests/test_formats/test_fmt_toqdoc.py b/tests/test_formats/test_fmt_toqdoc.py index ab65b3b9..ee04e757 100644 --- a/tests/test_formats/test_fmt_toqdoc.py +++ b/tests/test_formats/test_fmt_toqdoc.py @@ -27,7 +27,7 @@ from PyQt5.QtGui import QTextBlock, QTextCharFormat, QTextCursor from novelwriter import CONFIG from novelwriter.constants import nwUnicode from novelwriter.core.project import NWProject -from novelwriter.formats.tokenizer import BlockFmt, BlockTyp +from novelwriter.formats.shared import BlockFmt, BlockTyp from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument from novelwriter.types import ( QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight, @@ -410,7 +410,7 @@ def testFmtToQTextDocument_TextBlockFormats(mockGUI): # Some formatting markers are currently not reachable qdoc.document.clear() - qdoc._tokens = [ + qdoc._blocks = [ (BlockTyp.TEXT, 1, "This is justified", [], BlockFmt.JUSTIFY), (BlockTyp.TEXT, 1, "This has a page break", [], BlockFmt.PBA), ]