From 16076ebfb6f8c90d377516982bb8266f78e6a23e Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Tue, 22 Oct 2024 00:25:35 +0200
Subject: [PATCH] Move shared formats code to separate file, and rename some
variables
---
novelwriter/formats/shared.py | 129 +++++++++++++++
novelwriter/formats/todocx.py | 5 +-
novelwriter/formats/tohtml.py | 7 +-
novelwriter/formats/tokenizer.py | 192 +++++++---------------
novelwriter/formats/tomarkdown.py | 7 +-
novelwriter/formats/toodt.py | 7 +-
novelwriter/formats/toqdoc.py | 5 +-
tests/test_formats/test_fmt_todocx.py | 48 +++---
tests/test_formats/test_fmt_tohtml.py | 28 ++--
tests/test_formats/test_fmt_tokenizer.py | 177 ++++++++++----------
tests/test_formats/test_fmt_tomarkdown.py | 8 +-
tests/test_formats/test_fmt_toodt.py | 6 +-
tests/test_formats/test_fmt_toqdoc.py | 4 +-
13 files changed, 340 insertions(+), 283 deletions(-)
create mode 100644 novelwriter/formats/shared.py
diff --git a/novelwriter/formats/shared.py b/novelwriter/formats/shared.py
new file mode 100644
index 00000000..82a7c2c4
--- /dev/null
+++ b/novelwriter/formats/shared.py
@@ -0,0 +1,129 @@
+"""
+novelWriter – Formats Shared
+============================
+
+File History:
+Created: 2024-10-21 [2.6b1] TextFmt
+Created: 2024-10-21 [2.6b1] BlockTyp
+Created: 2024-10-21 [2.6b1] BlockFmt
+
+This file is a part of novelWriter
+Copyright 2018–2024, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+"""
+from __future__ import annotations
+
+import re
+
+from enum import Flag, IntEnum
+
+ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
+RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
+
+
+def stripEscape(text: str) -> str:
+ """Strip escaped Markdown characters from paragraph text."""
+ if "\\" in text:
+ return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text)
+ return text
+
+
+# Enums
+# =====
+
+class TextFmt(IntEnum):
+ """Text Format.
+
+ An enum indicating the beginning or end of a text format region.
+ They must be paired with a position, and apply to locations in a
+ text block.
+ """
+
+ B_B = 1 # Begin bold
+ B_E = 2 # End bold
+ I_B = 3 # Begin italics
+ I_E = 4 # End italics
+ D_B = 5 # Begin strikeout
+ D_E = 6 # End strikeout
+ U_B = 7 # Begin underline
+ U_E = 8 # End underline
+ M_B = 9 # Begin mark
+ M_E = 10 # End mark
+ SUP_B = 11 # Begin superscript
+ SUP_E = 12 # End superscript
+ SUB_B = 13 # Begin subscript
+ SUB_E = 14 # End subscript
+ DL_B = 15 # Begin dialogue
+ DL_E = 16 # End dialogue
+ ADL_B = 17 # Begin alt dialogue
+ ADL_E = 18 # End alt dialogue
+ FNOTE = 19 # Footnote marker
+ STRIP = 20 # Strip the format code
+
+
+class BlockTyp(IntEnum):
+ """Text Block Type.
+
+ An enum indicating the type of a text block.
+ """
+
+ EMPTY = 1 # Empty line (new paragraph)
+ SYNOPSIS = 2 # Synopsis comment
+ SHORT = 3 # Short description comment
+ COMMENT = 4 # Comment line
+ KEYWORD = 5 # Command line
+ TITLE = 6 # Title
+ HEAD1 = 7 # Heading 1
+ HEAD2 = 8 # Heading 2
+ HEAD3 = 9 # Heading 3
+ HEAD4 = 10 # Heading 4
+ TEXT = 11 # Text line
+ SEP = 12 # Scene separator
+ SKIP = 13 # Paragraph break
+
+
+class BlockFmt(Flag):
+ """Text Block Format.
+
+ An enum of flags that can be combined to format a text block.
+ """
+
+ NONE = 0x0000 # No special style
+ LEFT = 0x0001 # Left aligned
+ RIGHT = 0x0002 # Right aligned
+ CENTRE = 0x0004 # Centred
+ JUSTIFY = 0x0008 # Justified
+ PBB = 0x0010 # Page break before
+ PBA = 0x0020 # Page break after
+ Z_TOPMRG = 0x0040 # Zero top margin
+ Z_BTMMRG = 0x0080 # Zero bottom margin
+ IND_L = 0x0100 # Left indentation
+ IND_R = 0x0200 # Right indentation
+ IND_T = 0x0400 # Text indentation
+
+
+# Types
+# =====
+
+# A list of formats for a single text string, consisting of:
+# text position, text format, and meta data
+T_Formats = list[tuple[int, TextFmt, str]]
+
+# A note or comment with text and associated text formats
+T_Note = tuple[str, T_Formats]
+
+# A tokenized text block, consisting of:
+# type, header number, text, text formats, and block format
+T_Block = tuple[BlockTyp, int, str, T_Formats, BlockFmt]
diff --git a/novelwriter/formats/todocx.py b/novelwriter/formats/todocx.py
index 8af138a4..1b5a0c80 100644
--- a/novelwriter/formats/todocx.py
+++ b/novelwriter/formats/todocx.py
@@ -39,7 +39,8 @@ from novelwriter import __version__
from novelwriter.common import firstFloat, xmlSubElem
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer
+from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
+from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__)
@@ -219,7 +220,7 @@ class ToDocX(Tokenizer):
bIndent = self._fontSize * self._blockIndent
- for tType, _, tText, tFormat, tStyle in self._tokens:
+ for tType, _, tText, tFormat, tStyle in self._blocks:
# Create Paragraph
par = DocXParagraph()
diff --git a/novelwriter/formats/tohtml.py b/novelwriter/formats/tohtml.py
index 7755befb..41449fc3 100644
--- a/novelwriter/formats/tohtml.py
+++ b/novelwriter/formats/tohtml.py
@@ -32,9 +32,8 @@ from time import time
from novelwriter.common import formatTimeStamp
from novelwriter.constants import nwHeadFmt, nwHtmlUnicode, nwKeyWords, nwLabels
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import (
- BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape
-)
+from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape
+from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.types import FONT_STYLE, FONT_WEIGHTS
logger = logging.getLogger(__name__)
@@ -158,7 +157,7 @@ class ToHtml(Tokenizer):
lines = []
tHandle = self._handle
- for tType, nHead, tText, tFormat, tStyle in self._tokens:
+ for tType, nHead, tText, tFormat, tStyle in self._blocks:
# Replace < and > with HTML entities
if tFormat:
diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py
index dc78d220..ae316dae 100644
--- a/novelwriter/formats/tokenizer.py
+++ b/novelwriter/formats/tokenizer.py
@@ -29,7 +29,6 @@ import logging
import re
from abc import ABC, abstractmethod
-from enum import Flag, IntEnum
from functools import partial
from pathlib import Path
from time import time
@@ -45,82 +44,11 @@ from novelwriter.constants import (
from novelwriter.core.index import processComment
from novelwriter.core.project import NWProject
from novelwriter.enum import nwComment, nwItemLayout
+from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Block, T_Formats, T_Note, TextFmt
from novelwriter.text.patterns import REGEX_PATTERNS
logger = logging.getLogger(__name__)
-ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
-RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
-
-
-def stripEscape(text: str) -> str:
- """Strip escaped Markdown characters from paragraph text."""
- if "\\" in text:
- return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text)
- return text
-
-
-class TextFmt(IntEnum):
-
- B_B = 1 # Begin bold
- B_E = 2 # End bold
- I_B = 3 # Begin italics
- I_E = 4 # End italics
- D_B = 5 # Begin strikeout
- D_E = 6 # End strikeout
- U_B = 7 # Begin underline
- U_E = 8 # End underline
- M_B = 9 # Begin mark
- M_E = 10 # End mark
- SUP_B = 11 # Begin superscript
- SUP_E = 12 # End superscript
- SUB_B = 13 # Begin subscript
- SUB_E = 14 # End subscript
- DL_B = 15 # Begin dialogue
- DL_E = 16 # End dialogue
- ADL_B = 17 # Begin alt dialogue
- ADL_E = 18 # End alt dialogue
- FNOTE = 19 # Footnote marker
- STRIP = 20 # Strip the format code
-
-
-class BlockTyp(IntEnum):
-
- EMPTY = 1 # Empty line (new paragraph)
- SYNOPSIS = 2 # Synopsis comment
- SHORT = 3 # Short description comment
- COMMENT = 4 # Comment line
- KEYWORD = 5 # Command line
- TITLE = 6 # Title
- HEAD1 = 7 # Heading 1
- HEAD2 = 8 # Heading 2
- HEAD3 = 9 # Heading 3
- HEAD4 = 10 # Heading 4
- TEXT = 11 # Text line
- SEP = 12 # Scene separator
- SKIP = 13 # Paragraph break
-
-
-class BlockFmt(Flag):
-
- NONE = 0x0000 # No special style
- LEFT = 0x0001 # Left aligned
- RIGHT = 0x0002 # Right aligned
- CENTRE = 0x0004 # Centred
- JUSTIFY = 0x0008 # Justified
- PBB = 0x0010 # Page break before
- PBA = 0x0020 # Page break after
- Z_TOPMRG = 0x0040 # Zero top margin
- Z_BTMMRG = 0x0080 # Zero bottom margin
- IND_L = 0x0100 # Left indentation
- IND_R = 0x0200 # Right indentation
- IND_T = 0x0400 # Text indentation
-
-
-T_Formats = list[tuple[int, TextFmt, str]]
-T_Comment = tuple[str, T_Formats]
-T_Token = tuple[BlockTyp, int, str, T_Formats, BlockFmt]
-
class Tokenizer(ABC):
"""Core: Text Tokenizer Abstract Base Class
@@ -154,11 +82,11 @@ class Tokenizer(ABC):
self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw
- # Tokens and Meta Data (Per Document)
- self._tokens: list[T_Token] = []
- self._footnotes: dict[str, T_Comment] = {}
+ # Blocks and Meta Data (Per Document)
+ self._blocks: list[T_Block] = []
+ self._footnotes: dict[str, T_Note] = {}
- # Tokens and Meta Data (Per Instance)
+ # Blocks and Meta Data (Per Instance)
self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {}
self._markdown: list[str] = []
@@ -518,8 +446,8 @@ class Tokenizer(ABC):
trNotes = self._localLookup("Notes")
title = f"{trNotes}: {tItem.itemName}"
- self._tokens = []
- self._tokens.append((
+ self._blocks = []
+ self._blocks.append((
BlockTyp.TITLE, 1, title, [], textAlign
))
if self._keepRaw:
@@ -561,15 +489,15 @@ class Tokenizer(ABC):
characters that indicate headings, comments, commands etc, or
just contain plain text. In the case of plain text, apply the
same RegExes that the syntax highlighter uses and save the
- locations of these formatting tags into the token array.
+ locations of these formatting tags into the blocks list.
- The format of the token list is an entry with a five-tuple for
+ The format of the blocs list is an entry with a five-tuple for
each line in the file. The tuple is as follows:
1: The type of the block, BlockType.*
2: The heading number under which the text is placed
3: The text content of the block, without leading tags
4: The internal formatting map of the text, TxtFmt.*
- 5: The style of the block, BlockFmt.*
+ 5: The formats of the block, BlockFmt.*
"""
if self._isNovel:
self._hFormatter.setHandle(self._handle)
@@ -578,13 +506,13 @@ class Tokenizer(ABC):
breakNext = False
tmpMarkdown = []
tHandle = self._handle or ""
- tokens: list[T_Token] = []
+ blocks: list[T_Block] = []
for aLine in self._text.splitlines():
sLine = aLine.strip().lower()
# Check for blank lines
if len(sLine) == 0:
- tokens.append((
+ blocks.append((
BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE
))
if self._keepRaw:
@@ -613,7 +541,7 @@ class Tokenizer(ABC):
continue
elif sLine == "[vspace]":
- tokens.append(
+ blocks.append(
(BlockTyp.SKIP, nHead, "", [], sAlign)
)
continue
@@ -621,11 +549,11 @@ class Tokenizer(ABC):
elif sLine.startswith("[vspace:") and sLine.endswith("]"):
nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1:
- tokens.append(
+ blocks.append(
(BlockTyp.SKIP, nHead, "", [], sAlign)
)
if nSkip > 1:
- tokens += (nSkip - 1) * [
+ blocks += (nSkip - 1) * [
(BlockTyp.SKIP, nHead, "", [], BlockFmt.NONE)
]
continue
@@ -645,14 +573,14 @@ class Tokenizer(ABC):
cStyle, cKey, cText, _, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS:
tLine, tFmt = self._extractFormats(cText)
- tokens.append((
+ blocks.append((
BlockTyp.SYNOPSIS, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText)
- tokens.append((
+ blocks.append((
BlockTyp.SHORT, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepRaw:
@@ -664,7 +592,7 @@ class Tokenizer(ABC):
tmpMarkdown.append(f"{aLine}\n")
else:
tLine, tFmt = self._extractFormats(cText)
- tokens.append((
+ blocks.append((
BlockTyp.COMMENT, nHead, tLine, tFmt, sAlign
))
if self._doComments and self._keepRaw:
@@ -681,7 +609,7 @@ class Tokenizer(ABC):
valid and bits and bits[0] in nwLabels.KEY_NAME
and bits[0] not in self._skipKeywords
):
- tokens.append((
+ blocks.append((
BlockTyp.KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepRaw:
@@ -717,7 +645,7 @@ class Tokenizer(ABC):
self._hFormatter.resetAll()
self._noSep = True
- tokens.append((
+ blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -752,7 +680,7 @@ class Tokenizer(ABC):
self._hFormatter.resetScene()
self._noSep = True
- tokens.append((
+ blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -793,7 +721,7 @@ class Tokenizer(ABC):
tStyle = BlockFmt.NONE if self._noSep else BlockFmt.CENTRE
self._noSep = False
- tokens.append((
+ blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -823,7 +751,7 @@ class Tokenizer(ABC):
tType = BlockTyp.SEP
tStyle = BlockFmt.CENTRE
- tokens.append((
+ blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -871,26 +799,26 @@ class Tokenizer(ABC):
# Process formats
tLine, tFmt = self._extractFormats(aLine, hDialog=self._isNovel)
- tokens.append((
+ blocks.append((
BlockTyp.TEXT, nHead, tLine, tFmt, sAlign
))
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag
- if self._isFirst and tokens:
+ if self._isFirst and blocks:
self._isFirst = False # First document has been processed
- # Make sure the token array doesn't start with a page break
+ # Make sure the blocks array doesn't start with a page break
# on the very first page, adding a blank first page.
- if tokens[0][4] & BlockFmt.PBB:
- cToken = tokens[0]
- tokens[0] = (
- cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~BlockFmt.PBB
+ if blocks[0][4] & BlockFmt.PBB:
+ cBlock = blocks[0]
+ blocks[0] = (
+ cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB
)
# Always add an empty line at the end of the file
- tokens.append((
+ blocks.append((
BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE
))
if self._keepRaw:
@@ -904,48 +832,48 @@ class Tokenizer(ABC):
# It also ensures that there isn't paragraph spacing between
# meta data lines for formats that has spacing.
- self._tokens = []
- pToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
- nToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
+ self._blocks = []
+ pBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
+ nBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
lineSep = "\n" if self._keepBreaks else " "
- pLines: list[T_Token] = []
+ pLines: list[T_Block] = []
- tCount = len(tokens)
- for n, cToken in enumerate(tokens):
+ tCount = len(blocks)
+ for n, cBlock in enumerate(blocks):
if n > 0:
- pToken = tokens[n-1] # Look behind
+ pBlock = blocks[n-1] # Look behind
if n < tCount - 1:
- nToken = tokens[n+1] # Look ahead
+ nBlock = blocks[n+1] # Look ahead
- if cToken[0] in self.L_SKIP_INDENT and not self._indentFirst:
+ if cBlock[0] in self.L_SKIP_INDENT and not self._indentFirst:
# Unless the indentFirst flag is set, we set up the next
# paragraph to not be indented if we see a block of a
# specific type
self._noIndent = True
- if cToken[0] == BlockTyp.EMPTY:
+ if cBlock[0] == BlockTyp.EMPTY:
# We don't need to keep the empty lines after this pass
pass
- elif cToken[0] == BlockTyp.KEYWORD:
+ elif cBlock[0] == BlockTyp.KEYWORD:
# Adjust margins for lines in a list of keyword lines
- aStyle = cToken[4]
- if pToken[0] == BlockTyp.KEYWORD:
+ aStyle = cBlock[4]
+ if pBlock[0] == BlockTyp.KEYWORD:
aStyle |= BlockFmt.Z_TOPMRG
- if nToken[0] == BlockTyp.KEYWORD:
+ if nBlock[0] == BlockTyp.KEYWORD:
aStyle |= BlockFmt.Z_BTMMRG
- self._tokens.append((
- cToken[0], cToken[1], cToken[2], cToken[3], aStyle
+ self._blocks.append((
+ cBlock[0], cBlock[1], cBlock[2], cBlock[3], aStyle
))
- elif cToken[0] == BlockTyp.TEXT:
+ elif cBlock[0] == BlockTyp.TEXT:
# Combine lines from the same paragraph
- pLines.append(cToken)
+ pLines.append(cBlock)
- if nToken[0] != BlockTyp.TEXT:
- # Next token is not text, so we add the buffer to tokens
+ if nBlock[0] != BlockTyp.TEXT:
+ # Next block is not text, so we add the buffer to blocks
nLines = len(pLines)
cStyle = pLines[0][4]
if self._firstIndent and not (self._noIndent or cStyle & self.M_ALIGNED):
@@ -956,11 +884,11 @@ class Tokenizer(ABC):
if nLines == 1:
# The paragraph contains a single line, so we just save
- # that directly to the token list. If justify is
+ # that directly to the blocks list. If justify is
# enabled, and there is no alignment, we apply it.
if self._doJustify and not cStyle & self.M_ALIGNED:
cStyle |= BlockFmt.JUSTIFY
- self._tokens.append((
+ self._blocks.append((
BlockTyp.TEXT, pLines[0][1], pLines[0][2], pLines[0][3], cStyle
))
elif nLines > 1:
@@ -969,11 +897,11 @@ class Tokenizer(ABC):
# recompute all the formatting markers
tTxt = ""
tFmt: T_Formats = []
- for aToken in pLines:
+ for aBlock in pLines:
tLen = len(tTxt)
- tTxt += f"{aToken[2]}{lineSep}"
- tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3])
- self._tokens.append((
+ tTxt += f"{aBlock[2]}{lineSep}"
+ tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3])
+ self._blocks.append((
BlockTyp.TEXT, pLines[0][1], tTxt[:-1], tFmt, cStyle
))
@@ -982,7 +910,7 @@ class Tokenizer(ABC):
self._noIndent = False
else:
- self._tokens.append(cToken)
+ self._blocks.append(cBlock)
return
@@ -990,7 +918,7 @@ class Tokenizer(ABC):
"""Build an outline of the text up to level 3 headings."""
tHandle = self._handle or ""
isNovel = self._isNovel
- for tType, nHead, tText, _, _ in self._tokens:
+ for tType, nHead, tText, _, _ in self._blocks:
if tType == BlockTyp.TITLE:
prefix = "TT"
elif tType == BlockTyp.HEAD1:
@@ -1025,7 +953,7 @@ class Tokenizer(ABC):
textWordChars = self._counts.get("textWordChars", 0)
titleWordChars = self._counts.get("titleWordChars", 0)
- for tType, _, tText, _, _ in self._tokens:
+ for tType, _, tText, _, _ in self._blocks:
tText = tText.replace(nwUnicode.U_ENDASH, " ")
tText = tText.replace(nwUnicode.U_EMDASH, " ")
diff --git a/novelwriter/formats/tomarkdown.py b/novelwriter/formats/tomarkdown.py
index dcd83a9b..2f9ccc93 100644
--- a/novelwriter/formats/tomarkdown.py
+++ b/novelwriter/formats/tomarkdown.py
@@ -29,7 +29,8 @@ from pathlib import Path
from novelwriter.constants import nwHeadFmt, nwLabels, nwUnicode
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer
+from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
+from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__)
@@ -117,7 +118,7 @@ class ToMarkdown(Tokenizer):
cSkip = ""
lines = []
- for tType, _, tText, tFormat, tStyle in self._tokens:
+ for tType, _, tText, tFormat, tStyle in self._blocks:
if tType == BlockTyp.TEXT:
tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n")
@@ -180,7 +181,7 @@ class ToMarkdown(Tokenizer):
for key, index in self._usedNotes.items():
if content := self._footnotes.get(key):
marker = f"{index}. "
- text = self._formatText(*content, tags)
+ text = self._formatText(content[0], content[1], tags)
lines.append(f"{marker}{text}\n")
lines.append("\n")
diff --git a/novelwriter/formats/toodt.py b/novelwriter/formats/toodt.py
index 7af7f66d..d5bde589 100644
--- a/novelwriter/formats/toodt.py
+++ b/novelwriter/formats/toodt.py
@@ -41,9 +41,8 @@ from novelwriter import __version__
from novelwriter.common import xmlIndent, xmlSubElem
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import (
- BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape
-)
+from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape
+from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.types import FONT_STYLE, FONT_WEIGHTS
logger = logging.getLogger(__name__)
@@ -426,7 +425,7 @@ class ToOdt(Tokenizer):
self._result = "" # Not used, but cleared just in case
xText = self._xText
- for tType, _, tText, tFormat, tStyle in self._tokens:
+ for tType, _, tText, tFormat, tStyle in self._blocks:
# Styles
oStyle = ODTParagraphStyle("New")
diff --git a/novelwriter/formats/toqdoc.py b/novelwriter/formats/toqdoc.py
index dbde490a..243059ed 100644
--- a/novelwriter/formats/toqdoc.py
+++ b/novelwriter/formats/toqdoc.py
@@ -36,7 +36,8 @@ from PyQt5.QtPrintSupport import QPrinter
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles, nwUnicode
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer
+from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
+from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.types import (
QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight,
QtPageBreakAfter, QtPageBreakBefore, QtTransparent, QtVAlignNormal,
@@ -221,7 +222,7 @@ class ToQTextDocument(Tokenizer):
cursor = QTextCursor(self._document)
cursor.movePosition(QTextCursor.MoveOperation.End)
- for tType, nHead, tText, tFormat, tStyle in self._tokens:
+ for tType, nHead, tText, tFormat, tStyle in self._blocks:
# Styles
bFmt = QTextBlockFormat(self._blockFmt)
diff --git a/tests/test_formats/test_fmt_todocx.py b/tests/test_formats/test_fmt_todocx.py
index ffc07c7c..cd64b55e 100644
--- a/tests/test_formats/test_fmt_todocx.py
+++ b/tests/test_formats/test_fmt_todocx.py
@@ -31,11 +31,11 @@ from novelwriter.core.buildsettings import BuildSettings
from novelwriter.core.docbuild import NWBuildDocument
from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt
+from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.todocx import (
S_FNOTE, S_HEAD1, S_HEAD2, S_HEAD3, S_HEAD4, S_META, S_NORM, S_SEP,
S_TITLE, ToDocX, _mkTag, _wTag
)
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp
from tests.tools import DOCX_IGNORE, cmpFiles
@@ -71,7 +71,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Normal Text
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -81,7 +81,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Title
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TITLE, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.TITLE, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -91,7 +91,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 1
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.HEAD1, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.HEAD1, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -101,7 +101,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 2
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.HEAD2, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.HEAD2, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -111,7 +111,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 3
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.HEAD3, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.HEAD3, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -121,7 +121,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 4
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.HEAD4, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.HEAD4, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -131,7 +131,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Separator
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.SEP, 0, "* * *", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.SEP, 0, "* * *", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -141,7 +141,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Empty Paragraph
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.SKIP, 0, "* * *", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.SKIP, 0, "* * *", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -150,7 +150,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Synopsis
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.SYNOPSIS, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.SYNOPSIS, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -162,7 +162,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Short
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.SHORT, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.SHORT, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -174,7 +174,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Comment
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.COMMENT, 0, "Hello World", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.COMMENT, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -186,7 +186,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Tags and References (Single)
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.KEYWORD, 0, "tag: Stuff", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.KEYWORD, 0, "tag: Stuff", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -198,7 +198,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Tags and References (Multiple)
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.KEYWORD, 0, "char: Jane, John", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.KEYWORD, 0, "char: Jane, John", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -210,7 +210,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Tags and References (Invalid)
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.KEYWORD, 0, "stuff: Stuff", [], BlockFmt.NONE)]
+ doc._blocks = [(BlockTyp.KEYWORD, 0, "stuff: Stuff", [], BlockFmt.NONE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -230,7 +230,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Left Align
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.LEFT)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.LEFT)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -240,7 +240,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Right Align
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.RIGHT)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.RIGHT)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -250,7 +250,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Center Align
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.CENTRE)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.CENTRE)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -260,7 +260,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Justify
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.JUSTIFY)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.JUSTIFY)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -270,7 +270,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Page Break Before
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBB)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBB)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -282,7 +282,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Page Break After
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBA)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBA)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -294,7 +294,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Zero Margins
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -305,7 +305,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Indent
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_L | BlockFmt.IND_R)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_L | BlockFmt.IND_R)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
@@ -316,7 +316,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# First Line Indent
xTest = ET.Element(_wTag("body"))
- doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_T)]
+ doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_T)]
doc.doConvert()
doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == (
diff --git a/tests/test_formats/test_fmt_tohtml.py b/tests/test_formats/test_fmt_tohtml.py
index f27aef7a..71863338 100644
--- a/tests/test_formats/test_fmt_tohtml.py
+++ b/tests/test_formats/test_fmt_tohtml.py
@@ -26,8 +26,8 @@ import pytest
from novelwriter import CONFIG
from novelwriter.core.project import NWProject
+from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.tohtml import ToHtml
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp
@pytest.mark.core
@@ -359,7 +359,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ==============
# Title
- html._tokens = [
+ html._blocks = [
(BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
html.doConvert()
@@ -369,7 +369,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
)
# Unnumbered
- html._tokens = [
+ html._blocks = [
(BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB),
]
html.doConvert()
@@ -382,14 +382,14 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ==========
# Separator
- html._tokens = [
+ html._blocks = [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
]
html.doConvert()
assert html.result == "* * *
\n"
# Skip
- html._tokens = [
+ html._blocks = [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
]
html.doConvert()
@@ -402,7 +402,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# Align Left
html.setStyles(False)
- html._tokens = [
+ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT),
]
html.doConvert()
@@ -413,7 +413,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
html.setStyles(True)
# Align Left
- html._tokens = [
+ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT),
]
html.doConvert()
@@ -422,7 +422,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
)
# Align Right
- html._tokens = [
+ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.RIGHT),
]
html.doConvert()
@@ -431,7 +431,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
)
# Align Centre
- html._tokens = [
+ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.CENTRE),
]
html.doConvert()
@@ -440,7 +440,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
)
# Align Justify
- html._tokens = [
+ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.JUSTIFY),
]
html.doConvert()
@@ -452,7 +452,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ==========
# Page Break Always
- html._tokens = [
+ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.PBB | BlockFmt.PBA),
]
html.doConvert()
@@ -465,7 +465,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ======
# Indent Left
- html._tokens = [
+ html._blocks = [
(BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_L),
]
html.doConvert()
@@ -474,7 +474,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
)
# Indent Right
- html._tokens = [
+ html._blocks = [
(BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_R),
]
html.doConvert()
@@ -483,7 +483,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
)
# Text Indent
- html._tokens = [
+ html._blocks = [
(BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_T),
]
html.doConvert()
diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py
index 6a621fa3..10809b67 100644
--- a/tests/test_formats/test_fmt_tokenizer.py
+++ b/tests/test_formats/test_fmt_tokenizer.py
@@ -29,9 +29,8 @@ from PyQt5.QtGui import QFont
from novelwriter import CONFIG
from novelwriter.constants import nwHeadFmt, nwStyles
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import (
- BlockFmt, BlockTyp, HeadingFormatter, TextFmt, Tokenizer, stripEscape
-)
+from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt, stripEscape
+from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer
from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toraw import ToRaw
@@ -200,22 +199,22 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
assert project.saveProject()
# Root Heading
- assert len(tokens._tokens) == 0
+ assert len(tokens._blocks) == 0
tokens.addRootHeading("stuff")
tokens.addRootHeading(C.hSceneDoc)
- assert len(tokens._tokens) == 0
+ assert len(tokens._blocks) == 0
# First Page
tokens.addRootHeading(C.hPlotRoot)
assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n"
- assert tokens._tokens[-1] == (
+ assert tokens._blocks[-1] == (
BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE
)
# Not First Page
tokens.addRootHeading(C.hPlotRoot)
assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n"
- assert tokens._tokens[-1] == (
+ assert tokens._blocks[-1] == (
BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE | BlockFmt.PBB
)
@@ -275,7 +274,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Novel Title\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Novel Title", [], BlockFmt.CENTRE),
]
assert tokens.allMarkdown[-1] == "#! Novel Title\n\n"
@@ -286,7 +285,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Note Title\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Note Title", [], BlockFmt.CENTRE),
]
assert tokens.allMarkdown[-1] == "#! Note Title\n\n"
@@ -300,7 +299,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "# Novel Title\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Novel Title", [], BlockFmt.CENTRE),
]
assert tokens.allMarkdown[-1] == "# Novel Title\n\n"
@@ -311,7 +310,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "# Note Title\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Note Title", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "# Note Title\n\n"
@@ -324,7 +323,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "## Chapter One\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB),
]
assert tokens.allMarkdown[-1] == "## Chapter One\n\n"
@@ -334,7 +333,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "## Heading 2\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Heading 2", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "## Heading 2\n\n"
@@ -347,7 +346,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "### Scene One\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene One", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "### Scene One\n\n"
@@ -357,7 +356,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "### Heading 3\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Heading 3", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "### Heading 3\n\n"
@@ -370,7 +369,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#### A Section\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD4, 1, "A Section", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "#### A Section\n\n"
@@ -380,7 +379,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#### Heading 4\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD4, 1, "Heading 4", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "#### Heading 4\n\n"
@@ -394,7 +393,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Title\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
assert tokens.allMarkdown[-1] == "#! Title\n\n"
@@ -405,7 +404,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Title\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
assert tokens.allMarkdown[-1] == "#! Title\n\n"
@@ -418,7 +417,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "##! Prologue\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB),
]
assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
@@ -428,7 +427,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "##! Prologue\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
@@ -444,7 +443,7 @@ def testFmtToken_HeaderStyle(mockGUI):
tokens._text = text
tokens._isFirst = first
tokens.tokenizeText()
- return tokens._tokens[0][4]
+ return tokens._blocks[0][4]
# No Styles
# =========
@@ -704,7 +703,7 @@ def testFmtToken_MetaFormat(mockGUI):
# Comment
tokens._text = "% A comment\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.COMMENT, 0, "A comment", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -716,18 +715,18 @@ def testFmtToken_MetaFormat(mockGUI):
# Ignore Text
tokens._text = "%~ Some text\n"
tokens.tokenizeText()
- assert tokens._tokens == []
+ assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n"
# Synopsis
tokens._text = "%synopsis: The synopsis\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE),
]
tokens._text = "% synopsis: The synopsis\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -740,7 +739,7 @@ def testFmtToken_MetaFormat(mockGUI):
tokens.setSynopsis(False)
tokens._text = "% short: A short description\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.SHORT, 0, "A short description", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -752,7 +751,7 @@ def testFmtToken_MetaFormat(mockGUI):
# Keyword
tokens._text = "@char: Bod\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "\n"
@@ -766,7 +765,7 @@ def testFmtToken_MetaFormat(mockGUI):
styTop = BlockFmt.NONE | BlockFmt.Z_BTMMRG
styMid = BlockFmt.NONE | BlockFmt.Z_BTMMRG | BlockFmt.Z_TOPMRG
styBtm = BlockFmt.NONE | BlockFmt.Z_TOPMRG
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "pov: Bod", [], styTop),
(BlockTyp.KEYWORD, 0, "plot: Main", [], styMid),
(BlockTyp.KEYWORD, 0, "location: Europe", [], styBtm),
@@ -777,7 +776,7 @@ def testFmtToken_MetaFormat(mockGUI):
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
tokens.setIgnoredKeywords("@plot, @location")
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "pov: Bod", [], BlockFmt.NONE),
]
@@ -802,7 +801,7 @@ def testFmtToken_MarginFormat(mockGUI):
">> Right-indent, right-aligned <\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "Some regular text", [], BlockFmt.NONE),
(BlockTyp.TEXT, 0, "Some left-aligned text", [], BlockFmt.LEFT),
(BlockTyp.TEXT, 0, "Some right-aligned text", [], BlockFmt.RIGHT),
@@ -939,7 +938,7 @@ def testFmtToken_Paragraphs(mockGUI):
# Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "First paragraph", [], BlockFmt.NONE),
(BlockTyp.TEXT, 0, "Second paragraph", [], BlockFmt.NONE),
]
@@ -948,7 +947,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "This is text\nspanning multiple\nlines"
tokens.setKeepLineBreaks(True)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "This is text\nspanning multiple\nlines", [], BlockFmt.NONE),
]
@@ -956,7 +955,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "This is text\nspanning multiple\nlines"
tokens.setKeepLineBreaks(False)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "This is text spanning multiple lines", [], BlockFmt.NONE),
]
@@ -964,7 +963,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "This **is text**\nspanning _multiple_\nlines"
tokens.setKeepLineBreaks(False)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(
BlockTyp.TEXT, 0,
"This is text spanning multiple lines",
@@ -982,7 +981,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n"
tokens.setKeepLineBreaks(False)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title", [], BlockFmt.NONE),
(
BlockTyp.TEXT, 1, "Text on two lines.", [
@@ -1011,21 +1010,21 @@ def testFmtToken_TextFormat(mockGUI):
# Text
tokens._text = "Some plain text\non two lines\n\n\n"
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "Some plain text\non two lines", [], BlockFmt.NONE),
]
assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"
tokens.setBodyText(False)
tokens.tokenizeText()
- assert tokens._tokens == []
+ assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n\n\n"
tokens.setBodyText(True)
# Text Emphasis
tokens._text = "Some **bolded text** on this lines\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some bolded text on this lines",
[
(5, TextFmt.B_B, ""),
@@ -1037,7 +1036,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some _italic text_ on this lines\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some italic text on this lines",
[
(5, TextFmt.I_B, ""),
@@ -1049,7 +1048,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some **_bold italic text_** on this lines\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some bold italic text on this lines",
[
(5, TextFmt.B_B, ""),
@@ -1063,7 +1062,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some ~~strikethrough text~~ on this lines\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some strikethrough text on this lines",
[
(5, TextFmt.D_B, ""),
@@ -1075,7 +1074,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some nested bold and italic and strikethrough text here",
[
(5, TextFmt.B_B, ""),
@@ -1113,7 +1112,7 @@ def testFmtToken_Dialogue(mockGUI):
# Single quotes
tokens._text = "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0,
"Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019",
[
@@ -1128,7 +1127,7 @@ def testFmtToken_Dialogue(mockGUI):
# Double quotes
tokens._text = "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0,
"Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d",
[
@@ -1143,7 +1142,7 @@ def testFmtToken_Dialogue(mockGUI):
# Alt quotes
tokens._text = "Text with ::dialogue one,:: and ::dialogue two.::\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0,
"Text with ::dialogue one,:: and ::dialogue two.::",
[
@@ -1158,7 +1157,7 @@ def testFmtToken_Dialogue(mockGUI):
# Dialogue line with narrator break
tokens._text = "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0,
"\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?",
[
@@ -1176,7 +1175,7 @@ def testFmtToken_Dialogue(mockGUI):
# Dialogue + formatting on same index (Issue #2012)
tokens._text = "[i]\u201cDialogue text.\u201d[/i]\n"
tokens.tokenizeText()
- assert tokens._tokens == [(
+ assert tokens._blocks == [(
BlockTyp.TEXT, 0,
"\u201cDialogue text.\u201d",
[
@@ -1213,7 +1212,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"# Title Two\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == correctResp
+ assert tokens._blocks == correctResp
# Command w/Space
tokens._isFirst = True
@@ -1223,7 +1222,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"# Title Two\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == correctResp
+ assert tokens._blocks == correctResp
# Trailing Spaces
tokens._isFirst = True
@@ -1233,7 +1232,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"# Title Two\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == correctResp
+ assert tokens._blocks == correctResp
# Single Empty Paragraph
# ======================
@@ -1244,7 +1243,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
@@ -1260,7 +1259,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
@@ -1273,7 +1272,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
@@ -1288,7 +1287,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
]
@@ -1300,7 +1299,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
]
@@ -1312,7 +1311,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
]
@@ -1328,7 +1327,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.PBB),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
@@ -1342,7 +1341,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.PBB),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
@@ -1374,7 +1373,7 @@ def testFmtToken_TextIndent(mockGUI):
"Second paragraph.\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE),
(BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE),
(BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.NONE),
@@ -1389,7 +1388,7 @@ def testFmtToken_TextIndent(mockGUI):
"%Synopsis: Stuff happens.\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene Two", [], BlockFmt.NONE),
(BlockTyp.SYNOPSIS, 1, "Stuff happens.", [], BlockFmt.NONE),
]
@@ -1402,7 +1401,7 @@ def testFmtToken_TextIndent(mockGUI):
"Second paragraph.\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "First paragraph.", [], BlockFmt.NONE),
(BlockTyp.TEXT, 0, "Second paragraph.", [], BlockFmt.IND_T),
]
@@ -1425,7 +1424,7 @@ def testFmtToken_TextIndent(mockGUI):
"Second paragraph.\n\n"
)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE),
(BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE),
(BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.IND_T),
@@ -1451,7 +1450,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "# Part One\n"
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.CENTRE),
]
@@ -1460,7 +1459,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "# Part One\n"
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
@@ -1471,7 +1470,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter One\n"
tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "C: Chapter One", [], BlockFmt.PBB),
]
@@ -1479,7 +1478,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "##! Prologue\n"
tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "U: Prologue", [], BlockFmt.PBB),
]
@@ -1488,7 +1487,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}")
tokens._hFormatter._chCount = 0
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB),
]
@@ -1496,7 +1495,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter II", [], BlockFmt.PBB),
]
@@ -1504,7 +1503,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter iii", [], BlockFmt.PBB),
]
@@ -1515,7 +1514,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "### Scene One\n"
tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "S: Scene One", [], BlockFmt.NONE),
]
@@ -1523,21 +1522,21 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", True)
tokens.tokenizeText()
- assert tokens._tokens == []
+ assert tokens._blocks == []
# H3: Scene wo/Format, first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._noSep = True
tokens.tokenizeText()
- assert tokens._tokens == []
+ assert tokens._blocks == []
# H3: Scene wo/Format, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False)
tokens._noSep = False
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
]
@@ -1546,14 +1545,14 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setSceneFormat("* * *", False)
tokens._noSep = True
tokens.tokenizeText()
- assert tokens._tokens == []
+ assert tokens._blocks == []
# H3: Scene Separator, not first
tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False)
tokens._noSep = False
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
]
@@ -1563,7 +1562,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 0
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene 1", [], BlockFmt.NONE),
]
@@ -1573,7 +1572,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 1
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene 3.2", [], BlockFmt.NONE),
]
@@ -1584,13 +1583,13 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n"
tokens.setSectionFormat("", True)
tokens.tokenizeText()
- assert tokens._tokens == []
+ assert tokens._blocks == []
# H4: Section Visible wo/Format
tokens._text = "#### A Section\n"
tokens.setSectionFormat("", False)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
]
@@ -1598,7 +1597,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n"
tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.HEAD4, 1, "X: A Section", [], BlockFmt.NONE),
]
@@ -1606,7 +1605,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n"
tokens.setSectionFormat("* * *", False)
tokens.tokenizeText()
- assert tokens._tokens == [
+ assert tokens._blocks == [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
]
@@ -1704,7 +1703,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens._counts = {}
tokens.tokenizeText()
tokens.countStats()
- assert tokens._tokens[0][2] == "A Chapter Title"
+ assert tokens._blocks[0][2] == "A Chapter Title"
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 0,
"allWords": 3, "textWords": 0, "titleWords": 3,
@@ -1719,7 +1718,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens._hFormatter.resetAll()
tokens.tokenizeText()
tokens.countStats()
- assert tokens._tokens[0][2] == "C 1: A Chapter Title"
+ assert tokens._blocks[0][2] == "C 1: A Chapter Title"
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 0,
"allWords": 5, "textWords": 0, "titleWords": 5,
@@ -1747,7 +1746,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSceneFormat("* * *", False)
tokens.tokenizeText()
tokens.countStats()
- assert [t[2] for t in tokens._tokens] == ["Chapter", "Text", "* * *", "Text"]
+ assert [t[2] for t in tokens._blocks] == ["Chapter", "Text", "* * *", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 2,
"allWords": 6, "textWords": 2, "titleWords": 1,
@@ -1764,7 +1763,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.countStats()
- assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"]
+ assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
@@ -1781,7 +1780,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSynopsis(True)
tokens.tokenizeText()
tokens.countStats()
- assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"]
+ assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 5, "textWords": 1, "titleWords": 1,
@@ -1798,7 +1797,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setComments(True)
tokens.tokenizeText()
tokens.countStats()
- assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"]
+ assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1,
@@ -1815,7 +1814,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setKeywords(True)
tokens.tokenizeText()
tokens.countStats()
- assert [t[2] for t in tokens._tokens] == ["Chapter", "pov: Jane", "Text"]
+ assert [t[2] for t in tokens._blocks] == ["Chapter", "pov: Jane", "Text"]
assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1,
"allWords": 6, "textWords": 1, "titleWords": 1,
diff --git a/tests/test_formats/test_fmt_tomarkdown.py b/tests/test_formats/test_fmt_tomarkdown.py
index abef4f73..fffcd3c2 100644
--- a/tests/test_formats/test_fmt_tomarkdown.py
+++ b/tests/test_formats/test_fmt_tomarkdown.py
@@ -23,7 +23,7 @@ from __future__ import annotations
import pytest
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp
+from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.tomarkdown import ToMarkdown
@@ -220,7 +220,7 @@ def testFmtToMarkdown_ConvertDirect(mockGUI):
# ==============
# Title
- toMD._tokens = [
+ toMD._blocks = [
(BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
]
toMD.doConvert()
@@ -230,14 +230,14 @@ def testFmtToMarkdown_ConvertDirect(mockGUI):
# ==========
# Separator
- toMD._tokens = [
+ toMD._blocks = [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
]
toMD.doConvert()
assert toMD.result == "* * *\n\n"
# Skip
- toMD._tokens = [
+ toMD._blocks = [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
]
toMD.doConvert()
diff --git a/tests/test_formats/test_fmt_toodt.py b/tests/test_formats/test_fmt_toodt.py
index bd17ff77..7f75b935 100644
--- a/tests/test_formats/test_fmt_toodt.py
+++ b/tests/test_formats/test_fmt_toodt.py
@@ -30,7 +30,7 @@ import pytest
from novelwriter.common import xmlIndent
from novelwriter.constants import nwHeadFmt
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, TextFmt
+from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt
from novelwriter.formats.toodt import ODTParagraphStyle, ODTTextStyle, ToOdt, XMLParagraph, _mkTag
from tests.tools import ODT_IGNORE, cmpFiles
@@ -719,7 +719,7 @@ def testFmtToOdt_ConvertDirect(mockGUI):
# Justified
doc = ToOdt(project, isFlat=True)
- doc._tokens = [
+ doc._blocks = [
(BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.JUSTIFY),
]
doc.initDocument()
@@ -739,7 +739,7 @@ def testFmtToOdt_ConvertDirect(mockGUI):
# Page Break After
doc = ToOdt(project, isFlat=True)
- doc._tokens = [
+ doc._blocks = [
(BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.PBA),
]
doc.initDocument()
diff --git a/tests/test_formats/test_fmt_toqdoc.py b/tests/test_formats/test_fmt_toqdoc.py
index ab65b3b9..ee04e757 100644
--- a/tests/test_formats/test_fmt_toqdoc.py
+++ b/tests/test_formats/test_fmt_toqdoc.py
@@ -27,7 +27,7 @@ from PyQt5.QtGui import QTextBlock, QTextCharFormat, QTextCursor
from novelwriter import CONFIG
from novelwriter.constants import nwUnicode
from novelwriter.core.project import NWProject
-from novelwriter.formats.tokenizer import BlockFmt, BlockTyp
+from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
from novelwriter.types import (
QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight,
@@ -410,7 +410,7 @@ def testFmtToQTextDocument_TextBlockFormats(mockGUI):
# Some formatting markers are currently not reachable
qdoc.document.clear()
- qdoc._tokens = [
+ qdoc._blocks = [
(BlockTyp.TEXT, 1, "This is justified", [], BlockFmt.JUSTIFY),
(BlockTyp.TEXT, 1, "This has a page break", [], BlockFmt.PBA),
]