Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move shared formats code to separate file, and rename some variables

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-22 00:25:35 +02:00
parent 38ca447c8a
commit 16076ebfb6
13 changed files with 340 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/formats/shared.py
+129
View File
@@ -0,0 +1,129 @@
"""
novelWriter – Formats Shared
============================
File History:
Created: 2024-10-21 [2.6b1] TextFmt
Created: 2024-10-21 [2.6b1] BlockTyp
Created: 2024-10-21 [2.6b1] BlockFmt
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import re
from enum import Flag, IntEnum
ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
def stripEscape(text: str) -> str:
"""Strip escaped Markdown characters from paragraph text."""
if "\\" in text:
return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text)
return text
# Enums
# =====
class TextFmt(IntEnum):
"""Text Format.
An enum indicating the beginning or end of a text format region.
They must be paired with a position, and apply to locations in a
text block.
"""
B_B = 1 # Begin bold
B_E = 2 # End bold
I_B = 3 # Begin italics
I_E = 4 # End italics
D_B = 5 # Begin strikeout
D_E = 6 # End strikeout
U_B = 7 # Begin underline
U_E = 8 # End underline
M_B = 9 # Begin mark
M_E = 10 # End mark
SUP_B = 11 # Begin superscript
SUP_E = 12 # End superscript
SUB_B = 13 # Begin subscript
SUB_E = 14 # End subscript
DL_B = 15 # Begin dialogue
DL_E = 16 # End dialogue
ADL_B = 17 # Begin alt dialogue
ADL_E = 18 # End alt dialogue
FNOTE = 19 # Footnote marker
STRIP = 20 # Strip the format code
class BlockTyp(IntEnum):
"""Text Block Type.
An enum indicating the type of a text block.
"""
EMPTY = 1 # Empty line (new paragraph)
SYNOPSIS = 2 # Synopsis comment
SHORT = 3 # Short description comment
COMMENT = 4 # Comment line
KEYWORD = 5 # Command line
TITLE = 6 # Title
HEAD1 = 7 # Heading 1
HEAD2 = 8 # Heading 2
HEAD3 = 9 # Heading 3
HEAD4 = 10 # Heading 4
TEXT = 11 # Text line
SEP = 12 # Scene separator
SKIP = 13 # Paragraph break
class BlockFmt(Flag):
"""Text Block Format.
An enum of flags that can be combined to format a text block.
"""
NONE = 0x0000 # No special style
LEFT = 0x0001 # Left aligned
RIGHT = 0x0002 # Right aligned
CENTRE = 0x0004 # Centred
JUSTIFY = 0x0008 # Justified
PBB = 0x0010 # Page break before
PBA = 0x0020 # Page break after
Z_TOPMRG = 0x0040 # Zero top margin
Z_BTMMRG = 0x0080 # Zero bottom margin
IND_L = 0x0100 # Left indentation
IND_R = 0x0200 # Right indentation
IND_T = 0x0400 # Text indentation
# Types
# =====
# A list of formats for a single text string, consisting of:
# text position, text format, and meta data
T_Formats = list[tuple[int, TextFmt, str]]
# A note or comment with text and associated text formats
T_Note = tuple[str, T_Formats]
# A tokenized text block, consisting of:
# type, header number, text, text formats, and block format
T_Block = tuple[BlockTyp, int, str, T_Formats, BlockFmt]
novelwriter/formats/todocx.py
+3 -2
View File
@@ -39,7 +39,8 @@ from novelwriter import __version__
from novelwriter.common import firstFloat, xmlSubElem from novelwriter.common import firstFloat, xmlSubElem
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -219,7 +220,7 @@ class ToDocX(Tokenizer):
bIndent = self._fontSize * self._blockIndent bIndent = self._fontSize * self._blockIndent
for tType, _, tText, tFormat, tStyle in self._tokens: for tType, _, tText, tFormat, tStyle in self._blocks:
# Create Paragraph # Create Paragraph
par = DocXParagraph() par = DocXParagraph()
novelwriter/formats/tohtml.py
+3 -4
View File
@@ -32,9 +32,8 @@ from time import time
from novelwriter.common import formatTimeStamp from novelwriter.common import formatTimeStamp
from novelwriter.constants import nwHeadFmt, nwHtmlUnicode, nwKeyWords, nwLabels from novelwriter.constants import nwHeadFmt, nwHtmlUnicode, nwKeyWords, nwLabels
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import ( from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape
BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape from novelwriter.formats.tokenizer import Tokenizer
)
from novelwriter.types import FONT_STYLE, FONT_WEIGHTS from novelwriter.types import FONT_STYLE, FONT_WEIGHTS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -158,7 +157,7 @@ class ToHtml(Tokenizer):
lines = [] lines = []
tHandle = self._handle tHandle = self._handle
for tType, nHead, tText, tFormat, tStyle in self._tokens: for tType, nHead, tText, tFormat, tStyle in self._blocks:
# Replace < and > with HTML entities # Replace < and > with HTML entities
if tFormat: if tFormat:
novelwriter/formats/tokenizer.py
+60 -132
View File
@@ -29,7 +29,6 @@ import logging
import re import re
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from enum import Flag, IntEnum
from functools import partial from functools import partial
from pathlib import Path from pathlib import Path
from time import time from time import time
@@ -45,82 +44,11 @@ from novelwriter.constants import (
from novelwriter.core.index import processComment from novelwriter.core.index import processComment
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.enum import nwComment, nwItemLayout from novelwriter.enum import nwComment, nwItemLayout
from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Block, T_Formats, T_Note, TextFmt
from novelwriter.text.patterns import REGEX_PATTERNS from novelwriter.text.patterns import REGEX_PATTERNS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
def stripEscape(text: str) -> str:
"""Strip escaped Markdown characters from paragraph text."""
if "\\" in text:
return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text)
return text
class TextFmt(IntEnum):
B_B = 1 # Begin bold
B_E = 2 # End bold
I_B = 3 # Begin italics
I_E = 4 # End italics
D_B = 5 # Begin strikeout
D_E = 6 # End strikeout
U_B = 7 # Begin underline
U_E = 8 # End underline
M_B = 9 # Begin mark
M_E = 10 # End mark
SUP_B = 11 # Begin superscript
SUP_E = 12 # End superscript
SUB_B = 13 # Begin subscript
SUB_E = 14 # End subscript
DL_B = 15 # Begin dialogue
DL_E = 16 # End dialogue
ADL_B = 17 # Begin alt dialogue
ADL_E = 18 # End alt dialogue
FNOTE = 19 # Footnote marker
STRIP = 20 # Strip the format code
class BlockTyp(IntEnum):
EMPTY = 1 # Empty line (new paragraph)
SYNOPSIS = 2 # Synopsis comment
SHORT = 3 # Short description comment
COMMENT = 4 # Comment line
KEYWORD = 5 # Command line
TITLE = 6 # Title
HEAD1 = 7 # Heading 1
HEAD2 = 8 # Heading 2
HEAD3 = 9 # Heading 3
HEAD4 = 10 # Heading 4
TEXT = 11 # Text line
SEP = 12 # Scene separator
SKIP = 13 # Paragraph break
class BlockFmt(Flag):
NONE = 0x0000 # No special style
LEFT = 0x0001 # Left aligned
RIGHT = 0x0002 # Right aligned
CENTRE = 0x0004 # Centred
JUSTIFY = 0x0008 # Justified
PBB = 0x0010 # Page break before
PBA = 0x0020 # Page break after
Z_TOPMRG = 0x0040 # Zero top margin
Z_BTMMRG = 0x0080 # Zero bottom margin
IND_L = 0x0100 # Left indentation
IND_R = 0x0200 # Right indentation
IND_T = 0x0400 # Text indentation
T_Formats = list[tuple[int, TextFmt, str]]
T_Comment = tuple[str, T_Formats]
T_Token = tuple[BlockTyp, int, str, T_Formats, BlockFmt]
class Tokenizer(ABC): class Tokenizer(ABC):
"""Core: Text Tokenizer Abstract Base Class """Core: Text Tokenizer Abstract Base Class
@@ -154,11 +82,11 @@ class Tokenizer(ABC):
self._result = "" # The result of the last document self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Tokens and Meta Data (Per Document) # Blocks and Meta Data (Per Document)
self._tokens: list[T_Token] = [] self._blocks: list[T_Block] = []
self._footnotes: dict[str, T_Comment] = {} self._footnotes: dict[str, T_Note] = {}
# Tokens and Meta Data (Per Instance) # Blocks and Meta Data (Per Instance)
self._counts: dict[str, int] = {} self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {} self._outline: dict[str, str] = {}
self._markdown: list[str] = [] self._markdown: list[str] = []
@@ -518,8 +446,8 @@ class Tokenizer(ABC):
trNotes = self._localLookup("Notes") trNotes = self._localLookup("Notes")
title = f"{trNotes}: {tItem.itemName}" title = f"{trNotes}: {tItem.itemName}"
self._tokens = [] self._blocks = []
self._tokens.append(( self._blocks.append((
BlockTyp.TITLE, 1, title, [], textAlign BlockTyp.TITLE, 1, title, [], textAlign
)) ))
if self._keepRaw: if self._keepRaw:
@@ -561,15 +489,15 @@ class Tokenizer(ABC):
characters that indicate headings, comments, commands etc, or characters that indicate headings, comments, commands etc, or
just contain plain text. In the case of plain text, apply the just contain plain text. In the case of plain text, apply the
same RegExes that the syntax highlighter uses and save the same RegExes that the syntax highlighter uses and save the
locations of these formatting tags into the token array. locations of these formatting tags into the blocks list.
The format of the token list is an entry with a five-tuple for The format of the blocs list is an entry with a five-tuple for
each line in the file. The tuple is as follows: each line in the file. The tuple is as follows:
1: The type of the block, BlockType.* 1: The type of the block, BlockType.*
2: The heading number under which the text is placed 2: The heading number under which the text is placed
3: The text content of the block, without leading tags 3: The text content of the block, without leading tags
4: The internal formatting map of the text, TxtFmt.* 4: The internal formatting map of the text, TxtFmt.*
5: The style of the block, BlockFmt.* 5: The formats of the block, BlockFmt.*
""" """
if self._isNovel: if self._isNovel:
self._hFormatter.setHandle(self._handle) self._hFormatter.setHandle(self._handle)
@@ -578,13 +506,13 @@ class Tokenizer(ABC):
breakNext = False breakNext = False
tmpMarkdown = [] tmpMarkdown = []
tHandle = self._handle or "" tHandle = self._handle or ""
tokens: list[T_Token] = [] blocks: list[T_Block] = []
for aLine in self._text.splitlines(): for aLine in self._text.splitlines():
sLine = aLine.strip().lower() sLine = aLine.strip().lower()
# Check for blank lines # Check for blank lines
if len(sLine) == 0: if len(sLine) == 0:
tokens.append(( blocks.append((
BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE
)) ))
if self._keepRaw: if self._keepRaw:
@@ -613,7 +541,7 @@ class Tokenizer(ABC):
continue continue
elif sLine == "[vspace]": elif sLine == "[vspace]":
tokens.append( blocks.append(
(BlockTyp.SKIP, nHead, "", [], sAlign) (BlockTyp.SKIP, nHead, "", [], sAlign)
) )
continue continue
@@ -621,11 +549,11 @@ class Tokenizer(ABC):
elif sLine.startswith("[vspace:") and sLine.endswith("]"): elif sLine.startswith("[vspace:") and sLine.endswith("]"):
nSkip = checkInt(sLine[8:-1], 0) nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1: if nSkip >= 1:
tokens.append( blocks.append(
(BlockTyp.SKIP, nHead, "", [], sAlign) (BlockTyp.SKIP, nHead, "", [], sAlign)
) )
if nSkip > 1: if nSkip > 1:
tokens += (nSkip - 1) * [ blocks += (nSkip - 1) * [
(BlockTyp.SKIP, nHead, "", [], BlockFmt.NONE) (BlockTyp.SKIP, nHead, "", [], BlockFmt.NONE)
] ]
continue continue
@@ -645,14 +573,14 @@ class Tokenizer(ABC):
cStyle, cKey, cText, _, _ = processComment(aLine) cStyle, cKey, cText, _, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS: if cStyle == nwComment.SYNOPSIS:
tLine, tFmt = self._extractFormats(cText) tLine, tFmt = self._extractFormats(cText)
tokens.append(( blocks.append((
BlockTyp.SYNOPSIS, nHead, tLine, tFmt, sAlign BlockTyp.SYNOPSIS, nHead, tLine, tFmt, sAlign
)) ))
if self._doSynopsis and self._keepRaw: if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT: elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText) tLine, tFmt = self._extractFormats(cText)
tokens.append(( blocks.append((
BlockTyp.SHORT, nHead, tLine, tFmt, sAlign BlockTyp.SHORT, nHead, tLine, tFmt, sAlign
)) ))
if self._doSynopsis and self._keepRaw: if self._doSynopsis and self._keepRaw:
@@ -664,7 +592,7 @@ class Tokenizer(ABC):
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
else: else:
tLine, tFmt = self._extractFormats(cText) tLine, tFmt = self._extractFormats(cText)
tokens.append(( blocks.append((
BlockTyp.COMMENT, nHead, tLine, tFmt, sAlign BlockTyp.COMMENT, nHead, tLine, tFmt, sAlign
)) ))
if self._doComments and self._keepRaw: if self._doComments and self._keepRaw:
@@ -681,7 +609,7 @@ class Tokenizer(ABC):
valid and bits and bits[0] in nwLabels.KEY_NAME valid and bits and bits[0] in nwLabels.KEY_NAME
and bits[0] not in self._skipKeywords and bits[0] not in self._skipKeywords
): ):
tokens.append(( blocks.append((
BlockTyp.KEYWORD, nHead, aLine[1:].strip(), [], sAlign BlockTyp.KEYWORD, nHead, aLine[1:].strip(), [], sAlign
)) ))
if self._doKeywords and self._keepRaw: if self._doKeywords and self._keepRaw:
@@ -717,7 +645,7 @@ class Tokenizer(ABC):
self._hFormatter.resetAll() self._hFormatter.resetAll()
self._noSep = True self._noSep = True
tokens.append(( blocks.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepRaw: if self._keepRaw:
@@ -752,7 +680,7 @@ class Tokenizer(ABC):
self._hFormatter.resetScene() self._hFormatter.resetScene()
self._noSep = True self._noSep = True
tokens.append(( blocks.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepRaw: if self._keepRaw:
@@ -793,7 +721,7 @@ class Tokenizer(ABC):
tStyle = BlockFmt.NONE if self._noSep else BlockFmt.CENTRE tStyle = BlockFmt.NONE if self._noSep else BlockFmt.CENTRE
self._noSep = False self._noSep = False
tokens.append(( blocks.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepRaw: if self._keepRaw:
@@ -823,7 +751,7 @@ class Tokenizer(ABC):
tType = BlockTyp.SEP tType = BlockTyp.SEP
tStyle = BlockFmt.CENTRE tStyle = BlockFmt.CENTRE
tokens.append(( blocks.append((
tType, nHead, tText, [], tStyle tType, nHead, tText, [], tStyle
)) ))
if self._keepRaw: if self._keepRaw:
@@ -871,26 +799,26 @@ class Tokenizer(ABC):
# Process formats # Process formats
tLine, tFmt = self._extractFormats(aLine, hDialog=self._isNovel) tLine, tFmt = self._extractFormats(aLine, hDialog=self._isNovel)
tokens.append(( blocks.append((
BlockTyp.TEXT, nHead, tLine, tFmt, sAlign BlockTyp.TEXT, nHead, tLine, tFmt, sAlign
)) ))
if self._keepRaw: if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n") tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag # If we have content, turn off the first page flag
if self._isFirst and tokens: if self._isFirst and blocks:
self._isFirst = False # First document has been processed self._isFirst = False # First document has been processed
# Make sure the token array doesn't start with a page break # Make sure the blocks array doesn't start with a page break
# on the very first page, adding a blank first page. # on the very first page, adding a blank first page.
if tokens[0][4] & BlockFmt.PBB: if blocks[0][4] & BlockFmt.PBB:
cToken = tokens[0] cBlock = blocks[0]
tokens[0] = ( blocks[0] = (
cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~BlockFmt.PBB cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB
) )
# Always add an empty line at the end of the file # Always add an empty line at the end of the file
tokens.append(( blocks.append((
BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE
)) ))
if self._keepRaw: if self._keepRaw:
@@ -904,48 +832,48 @@ class Tokenizer(ABC):
# It also ensures that there isn't paragraph spacing between # It also ensures that there isn't paragraph spacing between
# meta data lines for formats that has spacing. # meta data lines for formats that has spacing.
self._tokens = [] self._blocks = []
pToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE) pBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
nToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE) nBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
lineSep = "\n" if self._keepBreaks else " " lineSep = "\n" if self._keepBreaks else " "
pLines: list[T_Token] = [] pLines: list[T_Block] = []
tCount = len(tokens) tCount = len(blocks)
for n, cToken in enumerate(tokens): for n, cBlock in enumerate(blocks):
if n > 0: if n > 0:
pToken = tokens[n-1] # Look behind pBlock = blocks[n-1] # Look behind
if n < tCount - 1: if n < tCount - 1:
nToken = tokens[n+1] # Look ahead nBlock = blocks[n+1] # Look ahead
if cToken[0] in self.L_SKIP_INDENT and not self._indentFirst: if cBlock[0] in self.L_SKIP_INDENT and not self._indentFirst:
# Unless the indentFirst flag is set, we set up the next # Unless the indentFirst flag is set, we set up the next
# paragraph to not be indented if we see a block of a # paragraph to not be indented if we see a block of a
# specific type # specific type
self._noIndent = True self._noIndent = True
if cToken[0] == BlockTyp.EMPTY: if cBlock[0] == BlockTyp.EMPTY:
# We don't need to keep the empty lines after this pass # We don't need to keep the empty lines after this pass
pass pass
elif cToken[0] == BlockTyp.KEYWORD: elif cBlock[0] == BlockTyp.KEYWORD:
# Adjust margins for lines in a list of keyword lines # Adjust margins for lines in a list of keyword lines
aStyle = cToken[4] aStyle = cBlock[4]
if pToken[0] == BlockTyp.KEYWORD: if pBlock[0] == BlockTyp.KEYWORD:
aStyle |= BlockFmt.Z_TOPMRG aStyle |= BlockFmt.Z_TOPMRG
if nToken[0] == BlockTyp.KEYWORD: if nBlock[0] == BlockTyp.KEYWORD:
aStyle |= BlockFmt.Z_BTMMRG aStyle |= BlockFmt.Z_BTMMRG
self._tokens.append(( self._blocks.append((
cToken[0], cToken[1], cToken[2], cToken[3], aStyle cBlock[0], cBlock[1], cBlock[2], cBlock[3], aStyle
)) ))
elif cToken[0] == BlockTyp.TEXT: elif cBlock[0] == BlockTyp.TEXT:
# Combine lines from the same paragraph # Combine lines from the same paragraph
pLines.append(cToken) pLines.append(cBlock)
if nToken[0] != BlockTyp.TEXT: if nBlock[0] != BlockTyp.TEXT:
# Next token is not text, so we add the buffer to tokens # Next block is not text, so we add the buffer to blocks
nLines = len(pLines) nLines = len(pLines)
cStyle = pLines[0][4] cStyle = pLines[0][4]
if self._firstIndent and not (self._noIndent or cStyle & self.M_ALIGNED): if self._firstIndent and not (self._noIndent or cStyle & self.M_ALIGNED):
@@ -956,11 +884,11 @@ class Tokenizer(ABC):
if nLines == 1: if nLines == 1:
# The paragraph contains a single line, so we just save # The paragraph contains a single line, so we just save
# that directly to the token list. If justify is # that directly to the blocks list. If justify is
# enabled, and there is no alignment, we apply it. # enabled, and there is no alignment, we apply it.
if self._doJustify and not cStyle & self.M_ALIGNED: if self._doJustify and not cStyle & self.M_ALIGNED:
cStyle |= BlockFmt.JUSTIFY cStyle |= BlockFmt.JUSTIFY
self._tokens.append(( self._blocks.append((
BlockTyp.TEXT, pLines[0][1], pLines[0][2], pLines[0][3], cStyle BlockTyp.TEXT, pLines[0][1], pLines[0][2], pLines[0][3], cStyle
)) ))
elif nLines > 1: elif nLines > 1:
@@ -969,11 +897,11 @@ class Tokenizer(ABC):
# recompute all the formatting markers # recompute all the formatting markers
tTxt = "" tTxt = ""
tFmt: T_Formats = [] tFmt: T_Formats = []
for aToken in pLines: for aBlock in pLines:
tLen = len(tTxt) tLen = len(tTxt)
tTxt += f"{aToken[2]}{lineSep}" tTxt += f"{aBlock[2]}{lineSep}"
tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3]) tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3])
self._tokens.append(( self._blocks.append((
BlockTyp.TEXT, pLines[0][1], tTxt[:-1], tFmt, cStyle BlockTyp.TEXT, pLines[0][1], tTxt[:-1], tFmt, cStyle
)) ))
@@ -982,7 +910,7 @@ class Tokenizer(ABC):
self._noIndent = False self._noIndent = False
else: else:
self._tokens.append(cToken) self._blocks.append(cBlock)
return return
@@ -990,7 +918,7 @@ class Tokenizer(ABC):
"""Build an outline of the text up to level 3 headings.""" """Build an outline of the text up to level 3 headings."""
tHandle = self._handle or "" tHandle = self._handle or ""
isNovel = self._isNovel isNovel = self._isNovel
for tType, nHead, tText, _, _ in self._tokens: for tType, nHead, tText, _, _ in self._blocks:
if tType == BlockTyp.TITLE: if tType == BlockTyp.TITLE:
prefix = "TT" prefix = "TT"
elif tType == BlockTyp.HEAD1: elif tType == BlockTyp.HEAD1:
@@ -1025,7 +953,7 @@ class Tokenizer(ABC):
textWordChars = self._counts.get("textWordChars", 0) textWordChars = self._counts.get("textWordChars", 0)
titleWordChars = self._counts.get("titleWordChars", 0) titleWordChars = self._counts.get("titleWordChars", 0)
for tType, _, tText, _, _ in self._tokens: for tType, _, tText, _, _ in self._blocks:
tText = tText.replace(nwUnicode.U_ENDASH, " ") tText = tText.replace(nwUnicode.U_ENDASH, " ")
tText = tText.replace(nwUnicode.U_EMDASH, " ") tText = tText.replace(nwUnicode.U_EMDASH, " ")
novelwriter/formats/tomarkdown.py
+4 -3
View File
@@ -29,7 +29,8 @@ from pathlib import Path
from novelwriter.constants import nwHeadFmt, nwLabels, nwUnicode from novelwriter.constants import nwHeadFmt, nwLabels, nwUnicode
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -117,7 +118,7 @@ class ToMarkdown(Tokenizer):
cSkip = "" cSkip = ""
lines = [] lines = []
for tType, _, tText, tFormat, tStyle in self._tokens: for tType, _, tText, tFormat, tStyle in self._blocks:
if tType == BlockTyp.TEXT: if tType == BlockTyp.TEXT:
tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n") tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n")
@@ -180,7 +181,7 @@ class ToMarkdown(Tokenizer):
for key, index in self._usedNotes.items(): for key, index in self._usedNotes.items():
if content := self._footnotes.get(key): if content := self._footnotes.get(key):
marker = f"{index}. " marker = f"{index}. "
text = self._formatText(*content, tags) text = self._formatText(content[0], content[1], tags)
lines.append(f"{marker}{text}\n") lines.append(f"{marker}{text}\n")
lines.append("\n") lines.append("\n")
novelwriter/formats/toodt.py
+3 -4
View File
@@ -41,9 +41,8 @@ from novelwriter import __version__
from novelwriter.common import xmlIndent, xmlSubElem from novelwriter.common import xmlIndent, xmlSubElem
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import ( from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape
BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape from novelwriter.formats.tokenizer import Tokenizer
)
from novelwriter.types import FONT_STYLE, FONT_WEIGHTS from novelwriter.types import FONT_STYLE, FONT_WEIGHTS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -426,7 +425,7 @@ class ToOdt(Tokenizer):
self._result = "" # Not used, but cleared just in case self._result = "" # Not used, but cleared just in case
xText = self._xText xText = self._xText
for tType, _, tText, tFormat, tStyle in self._tokens: for tType, _, tText, tFormat, tStyle in self._blocks:
# Styles # Styles
oStyle = ODTParagraphStyle("New") oStyle = ODTParagraphStyle("New")
novelwriter/formats/toqdoc.py
+3 -2
View File
@@ -36,7 +36,8 @@ from PyQt5.QtPrintSupport import QPrinter
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles, nwUnicode from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles, nwUnicode
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.types import ( from novelwriter.types import (
QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight, QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight,
QtPageBreakAfter, QtPageBreakBefore, QtTransparent, QtVAlignNormal, QtPageBreakAfter, QtPageBreakBefore, QtTransparent, QtVAlignNormal,
@@ -221,7 +222,7 @@ class ToQTextDocument(Tokenizer):
cursor = QTextCursor(self._document) cursor = QTextCursor(self._document)
cursor.movePosition(QTextCursor.MoveOperation.End) cursor.movePosition(QTextCursor.MoveOperation.End)
for tType, nHead, tText, tFormat, tStyle in self._tokens: for tType, nHead, tText, tFormat, tStyle in self._blocks:
# Styles # Styles
bFmt = QTextBlockFormat(self._blockFmt) bFmt = QTextBlockFormat(self._blockFmt)
tests/test_formats/test_fmt_todocx.py
+24 -24
View File
@@ -31,11 +31,11 @@ from novelwriter.core.buildsettings import BuildSettings
from novelwriter.core.docbuild import NWBuildDocument from novelwriter.core.docbuild import NWBuildDocument
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.enum import nwBuildFmt from novelwriter.enum import nwBuildFmt
from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.todocx import ( from novelwriter.formats.todocx import (
S_FNOTE, S_HEAD1, S_HEAD2, S_HEAD3, S_HEAD4, S_META, S_NORM, S_SEP, S_FNOTE, S_HEAD1, S_HEAD2, S_HEAD3, S_HEAD4, S_META, S_NORM, S_SEP,
S_TITLE, ToDocX, _mkTag, _wTag S_TITLE, ToDocX, _mkTag, _wTag
) )
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp
from tests.tools import DOCX_IGNORE, cmpFiles from tests.tools import DOCX_IGNORE, cmpFiles
@@ -71,7 +71,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Normal Text # Normal Text
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -81,7 +81,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Title # Title
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TITLE, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.TITLE, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -91,7 +91,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 1 # Heading Level 1
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.HEAD1, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.HEAD1, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -101,7 +101,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 2 # Heading Level 2
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.HEAD2, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.HEAD2, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -111,7 +111,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 3 # Heading Level 3
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.HEAD3, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.HEAD3, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -121,7 +121,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Heading Level 4 # Heading Level 4
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.HEAD4, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.HEAD4, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -131,7 +131,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Separator # Separator
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.SEP, 0, "* * *", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.SEP, 0, "* * *", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -141,7 +141,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Empty Paragraph # Empty Paragraph
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.SKIP, 0, "* * *", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.SKIP, 0, "* * *", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -150,7 +150,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Synopsis # Synopsis
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.SYNOPSIS, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.SYNOPSIS, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -162,7 +162,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Short # Short
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.SHORT, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.SHORT, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -174,7 +174,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Comment # Comment
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.COMMENT, 0, "Hello World", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.COMMENT, 0, "Hello World", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -186,7 +186,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Tags and References (Single) # Tags and References (Single)
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.KEYWORD, 0, "tag: Stuff", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.KEYWORD, 0, "tag: Stuff", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -198,7 +198,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Tags and References (Multiple) # Tags and References (Multiple)
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.KEYWORD, 0, "char: Jane, John", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.KEYWORD, 0, "char: Jane, John", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -210,7 +210,7 @@ def testFmtToDocX_ParagraphStyles(mockGUI):
# Tags and References (Invalid) # Tags and References (Invalid)
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.KEYWORD, 0, "stuff: Stuff", [], BlockFmt.NONE)] doc._blocks = [(BlockTyp.KEYWORD, 0, "stuff: Stuff", [], BlockFmt.NONE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -230,7 +230,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Left Align # Left Align
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.LEFT)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.LEFT)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -240,7 +240,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Right Align # Right Align
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.RIGHT)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.RIGHT)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -250,7 +250,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Center Align # Center Align
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.CENTRE)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.CENTRE)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -260,7 +260,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Justify # Justify
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.JUSTIFY)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.JUSTIFY)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -270,7 +270,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Page Break Before # Page Break Before
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBB)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBB)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -282,7 +282,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Page Break After # Page Break After
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBA)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.PBA)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -294,7 +294,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Zero Margins # Zero Margins
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -305,7 +305,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# Indent # Indent
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_L | BlockFmt.IND_R)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_L | BlockFmt.IND_R)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
@@ -316,7 +316,7 @@ def testFmtToDocX_ParagraphFormatting(mockGUI):
# First Line Indent # First Line Indent
xTest = ET.Element(_wTag("body")) xTest = ET.Element(_wTag("body"))
doc._tokens = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_T)] doc._blocks = [(BlockTyp.TEXT, 0, "Hello World", [], BlockFmt.IND_T)]
doc.doConvert() doc.doConvert()
doc._pars[-1].toXml(xTest) doc._pars[-1].toXml(xTest)
assert xmlToText(xTest) == ( assert xmlToText(xTest) == (
tests/test_formats/test_fmt_tohtml.py
+14 -14
View File
@@ -26,8 +26,8 @@ import pytest
from novelwriter import CONFIG from novelwriter import CONFIG
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.tohtml import ToHtml from novelwriter.formats.tohtml import ToHtml
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp
@pytest.mark.core @pytest.mark.core
@@ -359,7 +359,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ============== # ==============
# Title # Title
html._tokens = [ html._blocks = [
(BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
] ]
html.doConvert() html.doConvert()
@@ -369,7 +369,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
) )
# Unnumbered # Unnumbered
html._tokens = [ html._blocks = [
(BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB),
] ]
html.doConvert() html.doConvert()
@@ -382,14 +382,14 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ========== # ==========
# Separator # Separator
html._tokens = [ html._blocks = [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
] ]
html.doConvert() html.doConvert()
assert html.result == "<p class='sep' style='text-align: center;'>* * *</p>\n" assert html.result == "<p class='sep' style='text-align: center;'>* * *</p>\n"
# Skip # Skip
html._tokens = [ html._blocks = [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
] ]
html.doConvert() html.doConvert()
@@ -402,7 +402,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# Align Left # Align Left
html.setStyles(False) html.setStyles(False)
html._tokens = [ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT), (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT),
] ]
html.doConvert() html.doConvert()
@@ -413,7 +413,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
html.setStyles(True) html.setStyles(True)
# Align Left # Align Left
html._tokens = [ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT), (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.LEFT),
] ]
html.doConvert() html.doConvert()
@@ -422,7 +422,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
) )
# Align Right # Align Right
html._tokens = [ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.RIGHT), (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.RIGHT),
] ]
html.doConvert() html.doConvert()
@@ -431,7 +431,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
) )
# Align Centre # Align Centre
html._tokens = [ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.CENTRE),
] ]
html.doConvert() html.doConvert()
@@ -440,7 +440,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
) )
# Align Justify # Align Justify
html._tokens = [ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.JUSTIFY), (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.JUSTIFY),
] ]
html.doConvert() html.doConvert()
@@ -452,7 +452,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ========== # ==========
# Page Break Always # Page Break Always
html._tokens = [ html._blocks = [
(BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.PBB | BlockFmt.PBA), (BlockTyp.HEAD1, 1, "A Title", [], BlockFmt.PBB | BlockFmt.PBA),
] ]
html.doConvert() html.doConvert()
@@ -465,7 +465,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
# ====== # ======
# Indent Left # Indent Left
html._tokens = [ html._blocks = [
(BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_L), (BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_L),
] ]
html.doConvert() html.doConvert()
@@ -474,7 +474,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
) )
# Indent Right # Indent Right
html._tokens = [ html._blocks = [
(BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_R), (BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_R),
] ]
html.doConvert() html.doConvert()
@@ -483,7 +483,7 @@ def testFmtToHtml_ConvertDirect(mockGUI):
) )
# Text Indent # Text Indent
html._tokens = [ html._blocks = [
(BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_T), (BlockTyp.TEXT, 1, "Some text ...", [], BlockFmt.IND_T),
] ]
html.doConvert() html.doConvert()
tests/test_formats/test_fmt_tokenizer.py
+88 -89
View File
@@ -29,9 +29,8 @@ from PyQt5.QtGui import QFont
from novelwriter import CONFIG from novelwriter import CONFIG
from novelwriter.constants import nwHeadFmt, nwStyles from novelwriter.constants import nwHeadFmt, nwStyles
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import ( from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt, stripEscape
BlockFmt, BlockTyp, HeadingFormatter, TextFmt, Tokenizer, stripEscape from novelwriter.formats.tokenizer import HeadingFormatter, Tokenizer
)
from novelwriter.formats.tomarkdown import ToMarkdown from novelwriter.formats.tomarkdown import ToMarkdown
from novelwriter.formats.toraw import ToRaw from novelwriter.formats.toraw import ToRaw
@@ -200,22 +199,22 @@ def testFmtToken_TextOps(monkeypatch, mockGUI, mockRnd, fncPath):
assert project.saveProject() assert project.saveProject()
# Root Heading # Root Heading
assert len(tokens._tokens) == 0 assert len(tokens._blocks) == 0
tokens.addRootHeading("stuff") tokens.addRootHeading("stuff")
tokens.addRootHeading(C.hSceneDoc) tokens.addRootHeading(C.hSceneDoc)
assert len(tokens._tokens) == 0 assert len(tokens._blocks) == 0
# First Page # First Page
tokens.addRootHeading(C.hPlotRoot) tokens.addRootHeading(C.hPlotRoot)
assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n" assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n"
assert tokens._tokens[-1] == ( assert tokens._blocks[-1] == (
BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE
) )
# Not First Page # Not First Page
tokens.addRootHeading(C.hPlotRoot) tokens.addRootHeading(C.hPlotRoot)
assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n" assert tokens.allMarkdown[-1] == "#! Notes: Plot\n\n"
assert tokens._tokens[-1] == ( assert tokens._blocks[-1] == (
BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE | BlockFmt.PBB BlockTyp.TITLE, 1, "Notes: Plot", [], BlockFmt.CENTRE | BlockFmt.PBB
) )
@@ -275,7 +274,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Novel Title\n" tokens._text = "#! Novel Title\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Novel Title", [], BlockFmt.CENTRE), (BlockTyp.TITLE, 1, "Novel Title", [], BlockFmt.CENTRE),
] ]
assert tokens.allMarkdown[-1] == "#! Novel Title\n\n" assert tokens.allMarkdown[-1] == "#! Novel Title\n\n"
@@ -286,7 +285,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Note Title\n" tokens._text = "#! Note Title\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Note Title", [], BlockFmt.CENTRE), (BlockTyp.TITLE, 1, "Note Title", [], BlockFmt.CENTRE),
] ]
assert tokens.allMarkdown[-1] == "#! Note Title\n\n" assert tokens.allMarkdown[-1] == "#! Note Title\n\n"
@@ -300,7 +299,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "# Novel Title\n" tokens._text = "# Novel Title\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Novel Title", [], BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Novel Title", [], BlockFmt.CENTRE),
] ]
assert tokens.allMarkdown[-1] == "# Novel Title\n\n" assert tokens.allMarkdown[-1] == "# Novel Title\n\n"
@@ -311,7 +310,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "# Note Title\n" tokens._text = "# Note Title\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Note Title", [], BlockFmt.NONE), (BlockTyp.HEAD1, 1, "Note Title", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "# Note Title\n\n" assert tokens.allMarkdown[-1] == "# Note Title\n\n"
@@ -324,7 +323,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "## Chapter One\n" tokens._text = "## Chapter One\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB),
] ]
assert tokens.allMarkdown[-1] == "## Chapter One\n\n" assert tokens.allMarkdown[-1] == "## Chapter One\n\n"
@@ -334,7 +333,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "## Heading 2\n" tokens._text = "## Heading 2\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Heading 2", [], BlockFmt.NONE), (BlockTyp.HEAD2, 1, "Heading 2", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "## Heading 2\n\n" assert tokens.allMarkdown[-1] == "## Heading 2\n\n"
@@ -347,7 +346,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene One", [], BlockFmt.NONE), (BlockTyp.HEAD3, 1, "Scene One", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "### Scene One\n\n" assert tokens.allMarkdown[-1] == "### Scene One\n\n"
@@ -357,7 +356,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "### Heading 3\n" tokens._text = "### Heading 3\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Heading 3", [], BlockFmt.NONE), (BlockTyp.HEAD3, 1, "Heading 3", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "### Heading 3\n\n" assert tokens.allMarkdown[-1] == "### Heading 3\n\n"
@@ -370,7 +369,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#### A Section\n" tokens._text = "#### A Section\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD4, 1, "A Section", [], BlockFmt.NONE), (BlockTyp.HEAD4, 1, "A Section", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "#### A Section\n\n" assert tokens.allMarkdown[-1] == "#### A Section\n\n"
@@ -380,7 +379,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#### Heading 4\n" tokens._text = "#### Heading 4\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD4, 1, "Heading 4", [], BlockFmt.NONE), (BlockTyp.HEAD4, 1, "Heading 4", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "#### Heading 4\n\n" assert tokens.allMarkdown[-1] == "#### Heading 4\n\n"
@@ -394,7 +393,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Title\n" tokens._text = "#! Title\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
] ]
assert tokens.allMarkdown[-1] == "#! Title\n\n" assert tokens.allMarkdown[-1] == "#! Title\n\n"
@@ -405,7 +404,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "#! Title\n" tokens._text = "#! Title\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.TITLE, 1, "Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
] ]
assert tokens.allMarkdown[-1] == "#! Title\n\n" assert tokens.allMarkdown[-1] == "#! Title\n\n"
@@ -418,7 +417,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "##! Prologue\n" tokens._text = "##! Prologue\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.PBB),
] ]
assert tokens.allMarkdown[-1] == "##! Prologue\n\n" assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
@@ -428,7 +427,7 @@ def testFmtToken_HeaderFormat(mockGUI):
tokens._text = "##! Prologue\n" tokens._text = "##! Prologue\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.NONE), (BlockTyp.HEAD2, 1, "Prologue", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "##! Prologue\n\n" assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
@@ -444,7 +443,7 @@ def testFmtToken_HeaderStyle(mockGUI):
tokens._text = text tokens._text = text
tokens._isFirst = first tokens._isFirst = first
tokens.tokenizeText() tokens.tokenizeText()
return tokens._tokens[0][4] return tokens._blocks[0][4]
# No Styles # No Styles
# ========= # =========
@@ -704,7 +703,7 @@ def testFmtToken_MetaFormat(mockGUI):
# Comment # Comment
tokens._text = "% A comment\n" tokens._text = "% A comment\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.COMMENT, 0, "A comment", [], BlockFmt.NONE), (BlockTyp.COMMENT, 0, "A comment", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "\n" assert tokens.allMarkdown[-1] == "\n"
@@ -716,18 +715,18 @@ def testFmtToken_MetaFormat(mockGUI):
# Ignore Text # Ignore Text
tokens._text = "%~ Some text\n" tokens._text = "%~ Some text\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [] assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n" assert tokens.allMarkdown[-1] == "\n"
# Synopsis # Synopsis
tokens._text = "%synopsis: The synopsis\n" tokens._text = "%synopsis: The synopsis\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE), (BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE),
] ]
tokens._text = "% synopsis: The synopsis\n" tokens._text = "% synopsis: The synopsis\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE), (BlockTyp.SYNOPSIS, 0, "The synopsis", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "\n" assert tokens.allMarkdown[-1] == "\n"
@@ -740,7 +739,7 @@ def testFmtToken_MetaFormat(mockGUI):
tokens.setSynopsis(False) tokens.setSynopsis(False)
tokens._text = "% short: A short description\n" tokens._text = "% short: A short description\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.SHORT, 0, "A short description", [], BlockFmt.NONE), (BlockTyp.SHORT, 0, "A short description", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "\n" assert tokens.allMarkdown[-1] == "\n"
@@ -752,7 +751,7 @@ def testFmtToken_MetaFormat(mockGUI):
# Keyword # Keyword
tokens._text = "@char: Bod\n" tokens._text = "@char: Bod\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE), (BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "\n" assert tokens.allMarkdown[-1] == "\n"
@@ -766,7 +765,7 @@ def testFmtToken_MetaFormat(mockGUI):
styTop = BlockFmt.NONE | BlockFmt.Z_BTMMRG styTop = BlockFmt.NONE | BlockFmt.Z_BTMMRG
styMid = BlockFmt.NONE | BlockFmt.Z_BTMMRG | BlockFmt.Z_TOPMRG styMid = BlockFmt.NONE | BlockFmt.Z_BTMMRG | BlockFmt.Z_TOPMRG
styBtm = BlockFmt.NONE | BlockFmt.Z_TOPMRG styBtm = BlockFmt.NONE | BlockFmt.Z_TOPMRG
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "pov: Bod", [], styTop), (BlockTyp.KEYWORD, 0, "pov: Bod", [], styTop),
(BlockTyp.KEYWORD, 0, "plot: Main", [], styMid), (BlockTyp.KEYWORD, 0, "plot: Main", [], styMid),
(BlockTyp.KEYWORD, 0, "location: Europe", [], styBtm), (BlockTyp.KEYWORD, 0, "location: Europe", [], styBtm),
@@ -777,7 +776,7 @@ def testFmtToken_MetaFormat(mockGUI):
tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n" tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
tokens.setIgnoredKeywords("@plot, @location") tokens.setIgnoredKeywords("@plot, @location")
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.KEYWORD, 0, "pov: Bod", [], BlockFmt.NONE), (BlockTyp.KEYWORD, 0, "pov: Bod", [], BlockFmt.NONE),
] ]
@@ -802,7 +801,7 @@ def testFmtToken_MarginFormat(mockGUI):
">> Right-indent, right-aligned <\n\n" ">> Right-indent, right-aligned <\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "Some regular text", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "Some regular text", [], BlockFmt.NONE),
(BlockTyp.TEXT, 0, "Some left-aligned text", [], BlockFmt.LEFT), (BlockTyp.TEXT, 0, "Some left-aligned text", [], BlockFmt.LEFT),
(BlockTyp.TEXT, 0, "Some right-aligned text", [], BlockFmt.RIGHT), (BlockTyp.TEXT, 0, "Some right-aligned text", [], BlockFmt.RIGHT),
@@ -939,7 +938,7 @@ def testFmtToken_Paragraphs(mockGUI):
# Collapse empty lines # Collapse empty lines
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n" tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "First paragraph", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "First paragraph", [], BlockFmt.NONE),
(BlockTyp.TEXT, 0, "Second paragraph", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "Second paragraph", [], BlockFmt.NONE),
] ]
@@ -948,7 +947,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "This is text\nspanning multiple\nlines" tokens._text = "This is text\nspanning multiple\nlines"
tokens.setKeepLineBreaks(True) tokens.setKeepLineBreaks(True)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "This is text\nspanning multiple\nlines", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "This is text\nspanning multiple\nlines", [], BlockFmt.NONE),
] ]
@@ -956,7 +955,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "This is text\nspanning multiple\nlines" tokens._text = "This is text\nspanning multiple\nlines"
tokens.setKeepLineBreaks(False) tokens.setKeepLineBreaks(False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "This is text spanning multiple lines", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "This is text spanning multiple lines", [], BlockFmt.NONE),
] ]
@@ -964,7 +963,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "This **is text**\nspanning _multiple_\nlines" tokens._text = "This **is text**\nspanning _multiple_\nlines"
tokens.setKeepLineBreaks(False) tokens.setKeepLineBreaks(False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
( (
BlockTyp.TEXT, 0, BlockTyp.TEXT, 0,
"This is text spanning multiple lines", "This is text spanning multiple lines",
@@ -982,7 +981,7 @@ def testFmtToken_Paragraphs(mockGUI):
tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n" tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n"
tokens.setKeepLineBreaks(False) tokens.setKeepLineBreaks(False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title", [], BlockFmt.NONE), (BlockTyp.HEAD1, 1, "Title", [], BlockFmt.NONE),
( (
BlockTyp.TEXT, 1, "Text on two lines.", [ BlockTyp.TEXT, 1, "Text on two lines.", [
@@ -1011,21 +1010,21 @@ def testFmtToken_TextFormat(mockGUI):
# Text # Text
tokens._text = "Some plain text\non two lines\n\n\n" tokens._text = "Some plain text\non two lines\n\n\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "Some plain text\non two lines", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "Some plain text\non two lines", [], BlockFmt.NONE),
] ]
assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n" assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"
tokens.setBodyText(False) tokens.setBodyText(False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [] assert tokens._blocks == []
assert tokens.allMarkdown[-1] == "\n\n\n" assert tokens.allMarkdown[-1] == "\n\n\n"
tokens.setBodyText(True) tokens.setBodyText(True)
# Text Emphasis # Text Emphasis
tokens._text = "Some **bolded text** on this lines\n" tokens._text = "Some **bolded text** on this lines\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some bolded text on this lines", BlockTyp.TEXT, 0, "Some bolded text on this lines",
[ [
(5, TextFmt.B_B, ""), (5, TextFmt.B_B, ""),
@@ -1037,7 +1036,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some _italic text_ on this lines\n" tokens._text = "Some _italic text_ on this lines\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some italic text on this lines", BlockTyp.TEXT, 0, "Some italic text on this lines",
[ [
(5, TextFmt.I_B, ""), (5, TextFmt.I_B, ""),
@@ -1049,7 +1048,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some **_bold italic text_** on this lines\n" tokens._text = "Some **_bold italic text_** on this lines\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some bold italic text on this lines", BlockTyp.TEXT, 0, "Some bold italic text on this lines",
[ [
(5, TextFmt.B_B, ""), (5, TextFmt.B_B, ""),
@@ -1063,7 +1062,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some ~~strikethrough text~~ on this lines\n" tokens._text = "Some ~~strikethrough text~~ on this lines\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some strikethrough text on this lines", BlockTyp.TEXT, 0, "Some strikethrough text on this lines",
[ [
(5, TextFmt.D_B, ""), (5, TextFmt.D_B, ""),
@@ -1075,7 +1074,7 @@ def testFmtToken_TextFormat(mockGUI):
tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n" tokens._text = "Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, "Some nested bold and italic and strikethrough text here", BlockTyp.TEXT, 0, "Some nested bold and italic and strikethrough text here",
[ [
(5, TextFmt.B_B, ""), (5, TextFmt.B_B, ""),
@@ -1113,7 +1112,7 @@ def testFmtToken_Dialogue(mockGUI):
# Single quotes # Single quotes
tokens._text = "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019\n" tokens._text = "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, BlockTyp.TEXT, 0,
"Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019", "Text with \u2018dialogue one,\u2019 and \u2018dialogue two.\u2019",
[ [
@@ -1128,7 +1127,7 @@ def testFmtToken_Dialogue(mockGUI):
# Double quotes # Double quotes
tokens._text = "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d\n" tokens._text = "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, BlockTyp.TEXT, 0,
"Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d", "Text with \u201cdialogue one,\u201d and \u201cdialogue two.\u201d",
[ [
@@ -1143,7 +1142,7 @@ def testFmtToken_Dialogue(mockGUI):
# Alt quotes # Alt quotes
tokens._text = "Text with ::dialogue one,:: and ::dialogue two.::\n" tokens._text = "Text with ::dialogue one,:: and ::dialogue two.::\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, BlockTyp.TEXT, 0,
"Text with ::dialogue one,:: and ::dialogue two.::", "Text with ::dialogue one,:: and ::dialogue two.::",
[ [
@@ -1158,7 +1157,7 @@ def testFmtToken_Dialogue(mockGUI):
# Dialogue line with narrator break # Dialogue line with narrator break
tokens._text = "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?\n" tokens._text = "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, BlockTyp.TEXT, 0,
"\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?", "\u2013 Dialogue with a narrator break, \u2013he said,\u2013 see?",
[ [
@@ -1176,7 +1175,7 @@ def testFmtToken_Dialogue(mockGUI):
# Dialogue + formatting on same index (Issue #2012) # Dialogue + formatting on same index (Issue #2012)
tokens._text = "[i]\u201cDialogue text.\u201d[/i]\n" tokens._text = "[i]\u201cDialogue text.\u201d[/i]\n"
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [( assert tokens._blocks == [(
BlockTyp.TEXT, 0, BlockTyp.TEXT, 0,
"\u201cDialogue text.\u201d", "\u201cDialogue text.\u201d",
[ [
@@ -1213,7 +1212,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"# Title Two\n\n" "# Title Two\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == correctResp assert tokens._blocks == correctResp
# Command w/Space # Command w/Space
tokens._isFirst = True tokens._isFirst = True
@@ -1223,7 +1222,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"# Title Two\n\n" "# Title Two\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == correctResp assert tokens._blocks == correctResp
# Trailing Spaces # Trailing Spaces
tokens._isFirst = True tokens._isFirst = True
@@ -1233,7 +1232,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"# Title Two\n\n" "# Title Two\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == correctResp assert tokens._blocks == correctResp
# Single Empty Paragraph # Single Empty Paragraph
# ====================== # ======================
@@ -1244,7 +1243,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
@@ -1260,7 +1259,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
@@ -1273,7 +1272,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
@@ -1288,7 +1287,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
] ]
@@ -1300,7 +1299,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
] ]
@@ -1312,7 +1311,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
] ]
@@ -1328,7 +1327,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.PBB), (BlockTyp.SKIP, 1, "", [], BlockFmt.PBB),
(BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE), (BlockTyp.TEXT, 1, "Some text to go here ...", [], BlockFmt.NONE),
@@ -1342,7 +1341,7 @@ def testFmtToken_SpecialFormat(mockGUI):
"Some text to go here ...\n\n" "Some text to go here ...\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.PBB | BlockFmt.CENTRE),
(BlockTyp.SKIP, 1, "", [], BlockFmt.PBB), (BlockTyp.SKIP, 1, "", [], BlockFmt.PBB),
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
@@ -1374,7 +1373,7 @@ def testFmtToken_TextIndent(mockGUI):
"Second paragraph.\n\n" "Second paragraph.\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE),
(BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE), (BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE),
(BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.NONE), (BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.NONE),
@@ -1389,7 +1388,7 @@ def testFmtToken_TextIndent(mockGUI):
"%Synopsis: Stuff happens.\n\n" "%Synopsis: Stuff happens.\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene Two", [], BlockFmt.NONE), (BlockTyp.HEAD3, 1, "Scene Two", [], BlockFmt.NONE),
(BlockTyp.SYNOPSIS, 1, "Stuff happens.", [], BlockFmt.NONE), (BlockTyp.SYNOPSIS, 1, "Stuff happens.", [], BlockFmt.NONE),
] ]
@@ -1402,7 +1401,7 @@ def testFmtToken_TextIndent(mockGUI):
"Second paragraph.\n\n" "Second paragraph.\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.TEXT, 0, "First paragraph.", [], BlockFmt.NONE), (BlockTyp.TEXT, 0, "First paragraph.", [], BlockFmt.NONE),
(BlockTyp.TEXT, 0, "Second paragraph.", [], BlockFmt.IND_T), (BlockTyp.TEXT, 0, "Second paragraph.", [], BlockFmt.IND_T),
] ]
@@ -1425,7 +1424,7 @@ def testFmtToken_TextIndent(mockGUI):
"Second paragraph.\n\n" "Second paragraph.\n\n"
) )
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE), (BlockTyp.HEAD1, 1, "Title One", [], BlockFmt.NONE),
(BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE), (BlockTyp.HEAD3, 2, "Scene One", [], BlockFmt.NONE),
(BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.IND_T), (BlockTyp.TEXT, 2, "First paragraph.", [], BlockFmt.IND_T),
@@ -1451,7 +1450,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "# Part One\n" tokens._text = "# Part One\n"
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}") tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.CENTRE),
] ]
@@ -1460,7 +1459,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "# Part One\n" tokens._text = "# Part One\n"
tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}") tokens.setPartitionFormat(f"T: {nwHeadFmt.TITLE}")
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.HEAD1, 1, "T: Part One", [], BlockFmt.PBB | BlockFmt.CENTRE),
] ]
@@ -1471,7 +1470,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter One\n" tokens._text = "## Chapter One\n"
tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}") tokens.setChapterFormat(f"C: {nwHeadFmt.TITLE}")
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "C: Chapter One", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "C: Chapter One", [], BlockFmt.PBB),
] ]
@@ -1479,7 +1478,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "##! Prologue\n" tokens._text = "##! Prologue\n"
tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}") tokens.setUnNumberedFormat(f"U: {nwHeadFmt.TITLE}")
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "U: Prologue", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "U: Prologue", [], BlockFmt.PBB),
] ]
@@ -1488,7 +1487,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}") tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_WORD}")
tokens._hFormatter._chCount = 0 tokens._hFormatter._chCount = 0
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "Chapter One", [], BlockFmt.PBB),
] ]
@@ -1496,7 +1495,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter\n" tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}") tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROMU}")
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter II", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "Chapter II", [], BlockFmt.PBB),
] ]
@@ -1504,7 +1503,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "## Chapter\n" tokens._text = "## Chapter\n"
tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}") tokens.setChapterFormat(f"Chapter {nwHeadFmt.CH_ROML}")
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD2, 1, "Chapter iii", [], BlockFmt.PBB), (BlockTyp.HEAD2, 1, "Chapter iii", [], BlockFmt.PBB),
] ]
@@ -1515,7 +1514,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False) tokens.setSceneFormat(f"S: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "S: Scene One", [], BlockFmt.NONE), (BlockTyp.HEAD3, 1, "S: Scene One", [], BlockFmt.NONE),
] ]
@@ -1523,21 +1522,21 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.setSceneFormat("", True) tokens.setSceneFormat("", True)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [] assert tokens._blocks == []
# H3: Scene wo/Format, first # H3: Scene wo/Format, first
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False) tokens.setSceneFormat("", False)
tokens._noSep = True tokens._noSep = True
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [] assert tokens._blocks == []
# H3: Scene wo/Format, not first # H3: Scene wo/Format, not first
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.setSceneFormat("", False) tokens.setSceneFormat("", False)
tokens._noSep = False tokens._noSep = False
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
] ]
@@ -1546,14 +1545,14 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens.setSceneFormat("* * *", False) tokens.setSceneFormat("* * *", False)
tokens._noSep = True tokens._noSep = True
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [] assert tokens._blocks == []
# H3: Scene Separator, not first # H3: Scene Separator, not first
tokens._text = "### Scene One\n" tokens._text = "### Scene One\n"
tokens.setSceneFormat("* * *", False) tokens.setSceneFormat("* * *", False)
tokens._noSep = False tokens._noSep = False
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
] ]
@@ -1563,7 +1562,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scAbsCount = 0 tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 0 tokens._hFormatter._scChCount = 0
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene 1", [], BlockFmt.NONE), (BlockTyp.HEAD3, 1, "Scene 1", [], BlockFmt.NONE),
] ]
@@ -1573,7 +1572,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._hFormatter._scAbsCount = 0 tokens._hFormatter._scAbsCount = 0
tokens._hFormatter._scChCount = 1 tokens._hFormatter._scChCount = 1
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD3, 1, "Scene 3.2", [], BlockFmt.NONE), (BlockTyp.HEAD3, 1, "Scene 3.2", [], BlockFmt.NONE),
] ]
@@ -1584,13 +1583,13 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n" tokens._text = "#### A Section\n"
tokens.setSectionFormat("", True) tokens.setSectionFormat("", True)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [] assert tokens._blocks == []
# H4: Section Visible wo/Format # H4: Section Visible wo/Format
tokens._text = "#### A Section\n" tokens._text = "#### A Section\n"
tokens.setSectionFormat("", False) tokens.setSectionFormat("", False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
] ]
@@ -1598,7 +1597,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n" tokens._text = "#### A Section\n"
tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False) tokens.setSectionFormat(f"X: {nwHeadFmt.TITLE}", False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.HEAD4, 1, "X: A Section", [], BlockFmt.NONE), (BlockTyp.HEAD4, 1, "X: A Section", [], BlockFmt.NONE),
] ]
@@ -1606,7 +1605,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
tokens._text = "#### A Section\n" tokens._text = "#### A Section\n"
tokens.setSectionFormat("* * *", False) tokens.setSectionFormat("* * *", False)
tokens.tokenizeText() tokens.tokenizeText()
assert tokens._tokens == [ assert tokens._blocks == [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
] ]
@@ -1704,7 +1703,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens._counts = {} tokens._counts = {}
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert tokens._tokens[0][2] == "A Chapter Title" assert tokens._blocks[0][2] == "A Chapter Title"
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 0, "titleCount": 1, "paragraphCount": 0,
"allWords": 3, "textWords": 0, "titleWords": 3, "allWords": 3, "textWords": 0, "titleWords": 3,
@@ -1719,7 +1718,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens._hFormatter.resetAll() tokens._hFormatter.resetAll()
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert tokens._tokens[0][2] == "C 1: A Chapter Title" assert tokens._blocks[0][2] == "C 1: A Chapter Title"
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 0, "titleCount": 1, "paragraphCount": 0,
"allWords": 5, "textWords": 0, "titleWords": 5, "allWords": 5, "textWords": 0, "titleWords": 5,
@@ -1747,7 +1746,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSceneFormat("* * *", False) tokens.setSceneFormat("* * *", False)
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == ["Chapter", "Text", "* * *", "Text"] assert [t[2] for t in tokens._blocks] == ["Chapter", "Text", "* * *", "Text"]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 2, "titleCount": 1, "paragraphCount": 2,
"allWords": 6, "textWords": 2, "titleWords": 1, "allWords": 6, "textWords": 2, "titleWords": 1,
@@ -1764,7 +1763,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSynopsis(True) tokens.setSynopsis(True)
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1, "allWords": 4, "textWords": 1, "titleWords": 1,
@@ -1781,7 +1780,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setSynopsis(True) tokens.setSynopsis(True)
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 5, "textWords": 1, "titleWords": 1, "allWords": 5, "textWords": 1, "titleWords": 1,
@@ -1798,7 +1797,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setComments(True) tokens.setComments(True)
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"] assert [t[2] for t in tokens._blocks] == ["Chapter", "Stuff", "Text"]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 4, "textWords": 1, "titleWords": 1, "allWords": 4, "textWords": 1, "titleWords": 1,
@@ -1815,7 +1814,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
tokens.setKeywords(True) tokens.setKeywords(True)
tokens.tokenizeText() tokens.tokenizeText()
tokens.countStats() tokens.countStats()
assert [t[2] for t in tokens._tokens] == ["Chapter", "pov: Jane", "Text"] assert [t[2] for t in tokens._blocks] == ["Chapter", "pov: Jane", "Text"]
assert tokens.textStats == { assert tokens.textStats == {
"titleCount": 1, "paragraphCount": 1, "titleCount": 1, "paragraphCount": 1,
"allWords": 6, "textWords": 1, "titleWords": 1, "allWords": 6, "textWords": 1, "titleWords": 1,
tests/test_formats/test_fmt_tomarkdown.py
+4 -4
View File
@@ -23,7 +23,7 @@ from __future__ import annotations
import pytest import pytest
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.tomarkdown import ToMarkdown from novelwriter.formats.tomarkdown import ToMarkdown
@@ -220,7 +220,7 @@ def testFmtToMarkdown_ConvertDirect(mockGUI):
# ============== # ==============
# Title # Title
toMD._tokens = [ toMD._blocks = [
(BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE), (BlockTyp.TITLE, 1, "A Title", [], BlockFmt.PBB | BlockFmt.CENTRE),
] ]
toMD.doConvert() toMD.doConvert()
@@ -230,14 +230,14 @@ def testFmtToMarkdown_ConvertDirect(mockGUI):
# ========== # ==========
# Separator # Separator
toMD._tokens = [ toMD._blocks = [
(BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE), (BlockTyp.SEP, 1, "* * *", [], BlockFmt.CENTRE),
] ]
toMD.doConvert() toMD.doConvert()
assert toMD.result == "* * *\n\n" assert toMD.result == "* * *\n\n"
# Skip # Skip
toMD._tokens = [ toMD._blocks = [
(BlockTyp.SKIP, 1, "", [], BlockFmt.NONE), (BlockTyp.SKIP, 1, "", [], BlockFmt.NONE),
] ]
toMD.doConvert() toMD.doConvert()
tests/test_formats/test_fmt_toodt.py
+3 -3
View File
@@ -30,7 +30,7 @@ import pytest
from novelwriter.common import xmlIndent from novelwriter.common import xmlIndent
from novelwriter.constants import nwHeadFmt from novelwriter.constants import nwHeadFmt
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, TextFmt from novelwriter.formats.shared import BlockFmt, BlockTyp, TextFmt
from novelwriter.formats.toodt import ODTParagraphStyle, ODTTextStyle, ToOdt, XMLParagraph, _mkTag from novelwriter.formats.toodt import ODTParagraphStyle, ODTTextStyle, ToOdt, XMLParagraph, _mkTag
from tests.tools import ODT_IGNORE, cmpFiles from tests.tools import ODT_IGNORE, cmpFiles
@@ -719,7 +719,7 @@ def testFmtToOdt_ConvertDirect(mockGUI):
# Justified # Justified
doc = ToOdt(project, isFlat=True) doc = ToOdt(project, isFlat=True)
doc._tokens = [ doc._blocks = [
(BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.JUSTIFY), (BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.JUSTIFY),
] ]
doc.initDocument() doc.initDocument()
@@ -739,7 +739,7 @@ def testFmtToOdt_ConvertDirect(mockGUI):
# Page Break After # Page Break After
doc = ToOdt(project, isFlat=True) doc = ToOdt(project, isFlat=True)
doc._tokens = [ doc._blocks = [
(BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.PBA), (BlockTyp.TEXT, 1, "This is a paragraph", [], BlockFmt.PBA),
] ]
doc.initDocument() doc.initDocument()
tests/test_formats/test_fmt_toqdoc.py
+2 -2
View File
@@ -27,7 +27,7 @@ from PyQt5.QtGui import QTextBlock, QTextCharFormat, QTextCursor
from novelwriter import CONFIG from novelwriter import CONFIG
from novelwriter.constants import nwUnicode from novelwriter.constants import nwUnicode
from novelwriter.core.project import NWProject from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp from novelwriter.formats.shared import BlockFmt, BlockTyp
from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument from novelwriter.formats.toqdoc import TextDocumentTheme, ToQTextDocument
from novelwriter.types import ( from novelwriter.types import (
QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight, QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight,
@@ -410,7 +410,7 @@ def testFmtToQTextDocument_TextBlockFormats(mockGUI):
# Some formatting markers are currently not reachable # Some formatting markers are currently not reachable
qdoc.document.clear() qdoc.document.clear()
qdoc._tokens = [ qdoc._blocks = [
(BlockTyp.TEXT, 1, "This is justified", [], BlockFmt.JUSTIFY), (BlockTyp.TEXT, 1, "This is justified", [], BlockFmt.JUSTIFY),
(BlockTyp.TEXT, 1, "This has a page break", [], BlockFmt.PBA), (BlockTyp.TEXT, 1, "This has a page break", [], BlockFmt.PBA),
] ]