Improve the text tokenizer (#1885)
This commit is contained in:
@@ -99,6 +99,7 @@ class nwHeaders:
|
||||
|
||||
H_VALID = ("H0", "H1", "H2", "H3", "H4")
|
||||
H_LEVEL = {"H0": 0, "H1": 1, "H2": 2, "H3": 3, "H4": 4}
|
||||
H_SIZES = {0: 1.00, 1: 2.00, 2: 1.75, 3: 1.50, 4: 1.25}
|
||||
|
||||
|
||||
class nwFiles:
|
||||
|
||||
@@ -37,7 +37,7 @@ from novelwriter.core.item import NWItem
|
||||
from novelwriter.core.project import NWProject
|
||||
from novelwriter.core.tohtml import ToHtml
|
||||
from novelwriter.core.tokenizer import Tokenizer
|
||||
from novelwriter.core.tomd import ToMarkdown
|
||||
from novelwriter.core.tomarkdown import ToMarkdown
|
||||
from novelwriter.core.toodt import ToOdt
|
||||
from novelwriter.enum import nwBuildFmt
|
||||
from novelwriter.error import formatException, logException
|
||||
|
||||
@@ -171,9 +171,7 @@ class ToHtml(Tokenizer):
|
||||
h3 = "h3"
|
||||
h4 = "h4"
|
||||
|
||||
para = []
|
||||
lines = []
|
||||
pStyle = None
|
||||
tHandle = self._handle
|
||||
|
||||
for tType, nHead, tText, tFormat, tStyle in self._tokens:
|
||||
@@ -240,37 +238,27 @@ class ToHtml(Tokenizer):
|
||||
aNm = ""
|
||||
|
||||
# Process Text Type
|
||||
if tType == self.T_EMPTY:
|
||||
if pStyle is None:
|
||||
pStyle = ""
|
||||
if len(para) > 1 and self._cssStyles:
|
||||
pClass = " class='break'"
|
||||
else:
|
||||
pClass = ""
|
||||
if len(para) > 0:
|
||||
tTemp = "<br/>".join(para)
|
||||
lines.append(f"<p{pClass+pStyle}>{tTemp.rstrip()}</p>\n")
|
||||
para = []
|
||||
pStyle = None
|
||||
if tType == self.T_TEXT:
|
||||
lines.append(f"<p{hStyle}>{self._formatText(tText, tFormat, hTags)}</p>\n")
|
||||
|
||||
elif tType == self.T_TITLE:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br>")
|
||||
lines.append(f"<h1 class='title'{hStyle}>{aNm}{tHead}</h1>\n")
|
||||
|
||||
elif tType == self.T_HEAD1:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br>")
|
||||
lines.append(f"<{h1}{h1Cl}{hStyle}>{aNm}{tHead}</{h1}>\n")
|
||||
|
||||
elif tType == self.T_HEAD2:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br>")
|
||||
lines.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
|
||||
|
||||
elif tType == self.T_HEAD3:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br>")
|
||||
lines.append(f"<{h3}{hStyle}>{aNm}{tHead}</{h3}>\n")
|
||||
|
||||
elif tType == self.T_HEAD4:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br/>")
|
||||
tHead = tText.replace(nwHeadFmt.BR, "<br>")
|
||||
lines.append(f"<{h4}{hStyle}>{aNm}{tHead}</{h4}>\n")
|
||||
|
||||
elif tType == self.T_SEP:
|
||||
@@ -279,11 +267,6 @@ class ToHtml(Tokenizer):
|
||||
elif tType == self.T_SKIP:
|
||||
lines.append(f"<p class='skip'{hStyle}> </p>\n")
|
||||
|
||||
elif tType == self.T_TEXT:
|
||||
if pStyle is None:
|
||||
pStyle = hStyle
|
||||
para.append(self._formatText(tText, tFormat, hTags).rstrip())
|
||||
|
||||
elif tType == self.T_SYNOPSIS and self._doSynopsis:
|
||||
lines.append(self._formatSynopsis(self._formatText(tText, tFormat, hTags), True))
|
||||
|
||||
@@ -491,6 +474,7 @@ class ToHtml(Tokenizer):
|
||||
else:
|
||||
html = tags.get(fmt, "ERR")
|
||||
temp = f"{temp[:pos]}{html}{temp[pos:]}"
|
||||
temp = temp.replace("\n", "<br>")
|
||||
return stripEscape(temp)
|
||||
|
||||
def _formatSynopsis(self, text: str, synopsis: bool) -> str:
|
||||
|
||||
@@ -50,6 +50,7 @@ RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.D
|
||||
|
||||
T_Formats = list[tuple[int, int, str]]
|
||||
T_Comment = tuple[str, T_Formats]
|
||||
T_Token = tuple[int, int, str, T_Formats, int]
|
||||
|
||||
|
||||
def stripEscape(text: str) -> str:
|
||||
@@ -129,7 +130,7 @@ class Tokenizer(ABC):
|
||||
self._keepMD = False # Whether to keep the markdown text
|
||||
|
||||
# Tokens and Meta Data (Per Document)
|
||||
self._tokens: list[tuple[int, int, str, T_Formats, int]] = []
|
||||
self._tokens: list[T_Token] = []
|
||||
self._footnotes: dict[str, T_Comment] = {}
|
||||
|
||||
# Tokens and Meta Data (Per Instance)
|
||||
@@ -152,6 +153,7 @@ class Tokenizer(ABC):
|
||||
self._doComments = False # Also process comments
|
||||
self._doKeywords = False # Also process keywords like tags and references
|
||||
self._skipKeywords = set() # Keywords to ignore
|
||||
self._keepBreaks = True # Keep line breaks in paragraphs
|
||||
|
||||
# Margins
|
||||
self._marginTitle = (1.000, 0.500)
|
||||
@@ -409,6 +411,11 @@ class Tokenizer(ABC):
|
||||
self._skipKeywords = set(x.lower().strip() for x in keywords.split(","))
|
||||
return
|
||||
|
||||
def setKeepLineBreaks(self, state: bool) -> None:
|
||||
"""Keep line breaks in paragraphs."""
|
||||
self._keepBreaks = state
|
||||
return
|
||||
|
||||
def setKeepMarkdown(self, state: bool) -> None:
|
||||
"""Keep original markdown during build."""
|
||||
self._keepMD = state
|
||||
@@ -490,7 +497,6 @@ class Tokenizer(ABC):
|
||||
4: The internal formatting map of the text, self.FMT_*
|
||||
5: The style of the block, self.A_*
|
||||
"""
|
||||
self._tokens = []
|
||||
if self._isNovel:
|
||||
self._hFormatter.setHandle(self._handle)
|
||||
|
||||
@@ -498,12 +504,13 @@ class Tokenizer(ABC):
|
||||
breakNext = False
|
||||
tmpMarkdown = []
|
||||
tHandle = self._handle or ""
|
||||
tokens: list[T_Token] = []
|
||||
for aLine in self._text.splitlines():
|
||||
sLine = aLine.strip().lower()
|
||||
|
||||
# Check for blank lines
|
||||
if len(sLine) == 0:
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
self.T_EMPTY, nHead, "", [], self.A_NONE
|
||||
))
|
||||
if self._keepMD:
|
||||
@@ -532,7 +539,7 @@ class Tokenizer(ABC):
|
||||
continue
|
||||
|
||||
elif sLine == "[vspace]":
|
||||
self._tokens.append(
|
||||
tokens.append(
|
||||
(self.T_SKIP, nHead, "", [], sAlign)
|
||||
)
|
||||
continue
|
||||
@@ -540,11 +547,11 @@ class Tokenizer(ABC):
|
||||
elif sLine.startswith("[vspace:") and sLine.endswith("]"):
|
||||
nSkip = checkInt(sLine[8:-1], 0)
|
||||
if nSkip >= 1:
|
||||
self._tokens.append(
|
||||
tokens.append(
|
||||
(self.T_SKIP, nHead, "", [], sAlign)
|
||||
)
|
||||
if nSkip > 1:
|
||||
self._tokens += (nSkip - 1) * [
|
||||
tokens += (nSkip - 1) * [
|
||||
(self.T_SKIP, nHead, "", [], self.A_NONE)
|
||||
]
|
||||
continue
|
||||
@@ -561,14 +568,14 @@ class Tokenizer(ABC):
|
||||
cStyle, cKey, cText, _, _ = processComment(aLine)
|
||||
if cStyle == nwComment.SYNOPSIS:
|
||||
tLine, tFmt = self._extractFormats(cText)
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
|
||||
))
|
||||
if self._doSynopsis and self._keepMD:
|
||||
tmpMarkdown.append(f"{aLine}\n")
|
||||
elif cStyle == nwComment.SHORT:
|
||||
tLine, tFmt = self._extractFormats(cText)
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
self.T_SHORT, nHead, tLine, tFmt, sAlign
|
||||
))
|
||||
if self._doSynopsis and self._keepMD:
|
||||
@@ -580,7 +587,7 @@ class Tokenizer(ABC):
|
||||
tmpMarkdown.append(f"{aLine}\n")
|
||||
else:
|
||||
tLine, tFmt = self._extractFormats(cText)
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
self.T_COMMENT, nHead, tLine, tFmt, sAlign
|
||||
))
|
||||
if self._doComments and self._keepMD:
|
||||
@@ -594,7 +601,7 @@ class Tokenizer(ABC):
|
||||
|
||||
valid, bits, _ = self._project.index.scanThis(aLine)
|
||||
if valid and bits and bits[0] not in self._skipKeywords:
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
|
||||
))
|
||||
if self._doKeywords and self._keepMD:
|
||||
@@ -630,7 +637,7 @@ class Tokenizer(ABC):
|
||||
self._hFormatter.resetAll()
|
||||
self._noSep = True
|
||||
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
tType, nHead, tText, [], tStyle
|
||||
))
|
||||
if self._keepMD:
|
||||
@@ -665,7 +672,7 @@ class Tokenizer(ABC):
|
||||
self._hFormatter.resetScene()
|
||||
self._noSep = True
|
||||
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
tType, nHead, tText, [], tStyle
|
||||
))
|
||||
if self._keepMD:
|
||||
@@ -706,7 +713,7 @@ class Tokenizer(ABC):
|
||||
tStyle = self.A_NONE if self._noSep else self.A_CENTRE
|
||||
self._noSep = False
|
||||
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
tType, nHead, tText, [], tStyle
|
||||
))
|
||||
if self._keepMD:
|
||||
@@ -736,7 +743,7 @@ class Tokenizer(ABC):
|
||||
tType = self.T_SEP
|
||||
tStyle = self.A_CENTRE
|
||||
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
tType, nHead, tText, [], tStyle
|
||||
))
|
||||
if self._keepMD:
|
||||
@@ -784,26 +791,26 @@ class Tokenizer(ABC):
|
||||
|
||||
# Process formats
|
||||
tLine, tFmt = self._extractFormats(aLine)
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
self.T_TEXT, nHead, tLine, tFmt, sAlign
|
||||
))
|
||||
if self._keepMD:
|
||||
tmpMarkdown.append(f"{aLine}\n")
|
||||
|
||||
# If we have content, turn off the first page flag
|
||||
if self._isFirst and self._tokens:
|
||||
if self._isFirst and tokens:
|
||||
self._isFirst = False # First document has been processed
|
||||
|
||||
# Make sure the token array doesn't start with a page break
|
||||
# on the very first page, adding a blank first page.
|
||||
if self._tokens[0][4] & self.A_PBB:
|
||||
token = self._tokens[0]
|
||||
self._tokens[0] = (
|
||||
token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
|
||||
if tokens[0][4] & self.A_PBB:
|
||||
cToken = tokens[0]
|
||||
tokens[0] = (
|
||||
cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~self.A_PBB
|
||||
)
|
||||
|
||||
# Always add an empty line at the end of the file
|
||||
self._tokens.append((
|
||||
tokens.append((
|
||||
self.T_EMPTY, nHead, "", [], self.A_NONE
|
||||
))
|
||||
if self._keepMD:
|
||||
@@ -812,27 +819,62 @@ class Tokenizer(ABC):
|
||||
|
||||
# Second Pass
|
||||
# ===========
|
||||
# Some items need a second pass
|
||||
# This second pass strips away consecutive blank lines, and
|
||||
# combines consecutive text lines into the same paragraph.
|
||||
# It also ensures that there isn't paragraph spacing between
|
||||
# meta data lines for formats that has spacing.
|
||||
|
||||
pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
|
||||
nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
|
||||
tCount = len(self._tokens)
|
||||
for n, token in enumerate(self._tokens):
|
||||
self._tokens = []
|
||||
pToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE)
|
||||
nToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE)
|
||||
|
||||
lineSep = "\n" if self._keepBreaks else " "
|
||||
pLines: list[T_Token] = []
|
||||
|
||||
tCount = len(tokens)
|
||||
for n, cToken in enumerate(tokens):
|
||||
|
||||
if n > 0:
|
||||
pToken = self._tokens[n-1]
|
||||
pToken = tokens[n-1] # Look behind
|
||||
if n < tCount - 1:
|
||||
nToken = self._tokens[n+1]
|
||||
nToken = tokens[n+1] # Look ahead
|
||||
|
||||
if token[0] == self.T_KEYWORD:
|
||||
aStyle = token[4]
|
||||
if cToken[0] == self.T_EMPTY:
|
||||
# We don't need to keep the empty lines after this pass
|
||||
pass
|
||||
|
||||
elif cToken[0] == self.T_KEYWORD:
|
||||
# Adjust margins for lines in a list of keyword lines
|
||||
aStyle = cToken[4]
|
||||
if pToken[0] == self.T_KEYWORD:
|
||||
aStyle |= self.A_Z_TOPMRG
|
||||
if nToken[0] == self.T_KEYWORD:
|
||||
aStyle |= self.A_Z_BTMMRG
|
||||
self._tokens[n] = (
|
||||
token[0], token[1], token[2], token[3], aStyle
|
||||
)
|
||||
self._tokens.append((
|
||||
cToken[0], cToken[1], cToken[2], cToken[3], aStyle
|
||||
))
|
||||
|
||||
elif cToken[0] == self.T_TEXT:
|
||||
# Combine lines from the same paragraph
|
||||
pLines.append(cToken)
|
||||
if nToken[0] != self.T_TEXT:
|
||||
nLines = len(pLines)
|
||||
if nLines == 1:
|
||||
self._tokens.append(pLines[0])
|
||||
elif nLines > 1:
|
||||
tTxt = ""
|
||||
tFmt: T_Formats = []
|
||||
for aToken in pLines:
|
||||
tLen = len(tTxt)
|
||||
tTxt += f"{aToken[2]}{lineSep}"
|
||||
tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3])
|
||||
self._tokens.append((
|
||||
self.T_TEXT, pLines[0][1], tTxt[:-1], tFmt, pLines[0][4]
|
||||
))
|
||||
pLines = []
|
||||
|
||||
else:
|
||||
self._tokens.append(cToken)
|
||||
|
||||
return
|
||||
|
||||
@@ -875,7 +917,6 @@ class Tokenizer(ABC):
|
||||
textWordChars = self._counts.get("textWordChars", 0)
|
||||
titleWordChars = self._counts.get("titleWordChars", 0)
|
||||
|
||||
para = []
|
||||
for tType, _, tText, _, _ in self._tokens:
|
||||
tText = tText.replace(nwUnicode.U_ENDASH, " ")
|
||||
tText = tText.replace(nwUnicode.U_EMDASH, " ")
|
||||
@@ -885,22 +926,19 @@ class Tokenizer(ABC):
|
||||
nChars = len(tText)
|
||||
nWChars = len("".join(tWords))
|
||||
|
||||
if tType == self.T_EMPTY:
|
||||
if len(para) > 0:
|
||||
tTemp = "\n".join(para)
|
||||
tPWords = tTemp.split()
|
||||
nPWords = len(tPWords)
|
||||
nPChars = len(tTemp)
|
||||
nPWChars = len("".join(tPWords))
|
||||
if tType == self.T_TEXT:
|
||||
tPWords = tText.split()
|
||||
nPWords = len(tPWords)
|
||||
nPChars = len(tText)
|
||||
nPWChars = len("".join(tPWords))
|
||||
|
||||
paragraphCount += 1
|
||||
allWords += nPWords
|
||||
textWords += nPWords
|
||||
allChars += nPChars
|
||||
textChars += nPChars
|
||||
allWordChars += nPWChars
|
||||
textWordChars += nPWChars
|
||||
para = []
|
||||
paragraphCount += 1
|
||||
allWords += nPWords
|
||||
textWords += nPWords
|
||||
allChars += nPChars
|
||||
textChars += nPChars
|
||||
allWordChars += nPWChars
|
||||
textWordChars += nPWChars
|
||||
|
||||
elif tType in self.L_HEADINGS:
|
||||
titleCount += 1
|
||||
@@ -916,9 +954,6 @@ class Tokenizer(ABC):
|
||||
allChars += nChars
|
||||
allWordChars += nWChars
|
||||
|
||||
elif tType == self.T_TEXT:
|
||||
para.append(tText.rstrip())
|
||||
|
||||
elif tType == self.T_SYNOPSIS and self._doSynopsis:
|
||||
text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
|
||||
words = text.split()
|
||||
|
||||
@@ -139,17 +139,12 @@ class ToMarkdown(Tokenizer):
|
||||
mTags = EXT_MD
|
||||
cSkip = nwUnicode.U_MMSP
|
||||
|
||||
para = []
|
||||
lines = []
|
||||
lineSep = " \n" if self._preserveBreaks else " "
|
||||
|
||||
for tType, _, tText, tFormat, tStyle in self._tokens:
|
||||
|
||||
if tType == self.T_EMPTY:
|
||||
if para:
|
||||
tTemp = (lineSep.join(para)).rstrip(" ")
|
||||
lines.append(f"{tTemp}\n\n")
|
||||
para = []
|
||||
if tType == self.T_TEXT:
|
||||
tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n")
|
||||
lines.append(f"{tTemp}\n\n")
|
||||
|
||||
elif tType == self.T_TITLE:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "\n")
|
||||
@@ -177,9 +172,6 @@ class ToMarkdown(Tokenizer):
|
||||
elif tType == self.T_SKIP:
|
||||
lines.append(f"{cSkip}\n\n")
|
||||
|
||||
elif tType == self.T_TEXT:
|
||||
para.append(self._formatText(tText, tFormat, mTags).rstrip())
|
||||
|
||||
elif tType == self.T_SYNOPSIS and self._doSynopsis:
|
||||
label = self._localLookup("Synopsis")
|
||||
lines.append(f"**{label}:** {self._formatText(tText, tFormat, mTags)}\n\n")
|
||||
+49
-64
@@ -96,6 +96,18 @@ M_MRK = ~X_MRK
|
||||
M_SUP = ~X_SUP
|
||||
M_SUB = ~X_SUB
|
||||
|
||||
# ODT Styles
|
||||
S_TITLE = "Title"
|
||||
S_HEAD1 = "Heading_20_1"
|
||||
S_HEAD2 = "Heading_20_2"
|
||||
S_HEAD3 = "Heading_20_3"
|
||||
S_HEAD4 = "Heading_20_4"
|
||||
S_SEP = "Separator"
|
||||
S_FIND = "First_20_line_20_indent"
|
||||
S_TEXT = "Text_20_body"
|
||||
S_META = "Text_20_Meta"
|
||||
S_HNF = "Header_20_and_20_Footer"
|
||||
|
||||
|
||||
class ToOdt(Tokenizer):
|
||||
"""Core: Open Document Writer
|
||||
@@ -406,11 +418,8 @@ class ToOdt(Tokenizer):
|
||||
"""Convert the list of text tokens into XML elements."""
|
||||
self._result = "" # Not used, but cleared just in case
|
||||
|
||||
pFmt: list[T_Formats] = []
|
||||
pText = []
|
||||
pStyle = None
|
||||
pIndent = True
|
||||
xText = self._xText
|
||||
pIndent = True
|
||||
for tType, _, tText, tFormat, tStyle in self._tokens:
|
||||
|
||||
# Styles
|
||||
@@ -445,79 +454,55 @@ class ToOdt(Tokenizer):
|
||||
pIndent = False
|
||||
|
||||
# Process Text Types
|
||||
if tType == self.T_EMPTY:
|
||||
if len(pText) > 1 and pStyle is not None:
|
||||
if self._doJustify:
|
||||
pStyle.setTextAlign("left")
|
||||
|
||||
if len(pText) > 0 and pStyle is not None:
|
||||
tTxt = ""
|
||||
tFmt: T_Formats = []
|
||||
for nText, nFmt in zip(pText, pFmt):
|
||||
tLen = len(tTxt)
|
||||
tTxt += f"{nText}\n"
|
||||
tFmt.extend((p+tLen, fmt, key) for p, fmt, key in nFmt)
|
||||
|
||||
# Don't indent a paragraph if it has alignment set
|
||||
tIndent = self._firstIndent and pIndent and pStyle.isUnaligned()
|
||||
self._addTextPar(
|
||||
xText, "First_20_line_20_indent" if tIndent else "Text_20_body",
|
||||
pStyle, tTxt.rstrip(), tFmt=tFmt
|
||||
)
|
||||
pIndent = True
|
||||
|
||||
pFmt = []
|
||||
pText = []
|
||||
pStyle = None
|
||||
if tType == self.T_TEXT:
|
||||
if self._firstIndent and pIndent and oStyle.isUnaligned():
|
||||
self._addTextPar(xText, S_FIND, oStyle, tText, tFmt=tFormat)
|
||||
else:
|
||||
self._addTextPar(xText, S_TEXT, oStyle, tText, tFmt=tFormat)
|
||||
pIndent = True
|
||||
|
||||
elif tType == self.T_TITLE:
|
||||
# Title must be text:p
|
||||
tHead = tText.replace(nwHeadFmt.BR, "\n")
|
||||
self._addTextPar(xText, "Title", oStyle, tHead, isHead=False)
|
||||
self._addTextPar(xText, S_TITLE, oStyle, tHead, isHead=False)
|
||||
|
||||
elif tType == self.T_HEAD1:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "\n")
|
||||
self._addTextPar(xText, "Heading_20_1", oStyle, tHead, isHead=True, oLevel="1")
|
||||
self._addTextPar(xText, S_HEAD1, oStyle, tHead, isHead=True, oLevel="1")
|
||||
|
||||
elif tType == self.T_HEAD2:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "\n")
|
||||
self._addTextPar(xText, "Heading_20_2", oStyle, tHead, isHead=True, oLevel="2")
|
||||
self._addTextPar(xText, S_HEAD2, oStyle, tHead, isHead=True, oLevel="2")
|
||||
|
||||
elif tType == self.T_HEAD3:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "\n")
|
||||
self._addTextPar(xText, "Heading_20_3", oStyle, tHead, isHead=True, oLevel="3")
|
||||
self._addTextPar(xText, S_HEAD3, oStyle, tHead, isHead=True, oLevel="3")
|
||||
|
||||
elif tType == self.T_HEAD4:
|
||||
tHead = tText.replace(nwHeadFmt.BR, "\n")
|
||||
self._addTextPar(xText, "Heading_20_4", oStyle, tHead, isHead=True, oLevel="4")
|
||||
self._addTextPar(xText, S_HEAD4, oStyle, tHead, isHead=True, oLevel="4")
|
||||
|
||||
elif tType == self.T_SEP:
|
||||
self._addTextPar(xText, "Separator", oStyle, tText)
|
||||
self._addTextPar(xText, S_SEP, oStyle, tText)
|
||||
|
||||
elif tType == self.T_SKIP:
|
||||
self._addTextPar(xText, "Separator", oStyle, "")
|
||||
|
||||
elif tType == self.T_TEXT:
|
||||
if pStyle is None:
|
||||
pStyle = oStyle
|
||||
pText.append(tText)
|
||||
pFmt.append(tFormat)
|
||||
self._addTextPar(xText, S_SEP, oStyle, "")
|
||||
|
||||
elif tType == self.T_SYNOPSIS and self._doSynopsis:
|
||||
tTemp, tFmt = self._formatSynopsis(tText, tFormat, True)
|
||||
self._addTextPar(xText, "Text_20_Meta", oStyle, tTemp, tFmt=tFmt)
|
||||
self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt)
|
||||
|
||||
elif tType == self.T_SHORT and self._doSynopsis:
|
||||
tTemp, tFmt = self._formatSynopsis(tText, tFormat, False)
|
||||
self._addTextPar(xText, "Text_20_Meta", oStyle, tTemp, tFmt=tFmt)
|
||||
self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt)
|
||||
|
||||
elif tType == self.T_COMMENT and self._doComments:
|
||||
tTemp, tFmt = self._formatComments(tText, tFormat)
|
||||
self._addTextPar(xText, "Text_20_Meta", oStyle, tTemp, tFmt=tFmt)
|
||||
self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt)
|
||||
|
||||
elif tType == self.T_KEYWORD and self._doKeywords:
|
||||
tTemp, tFmt = self._formatKeywords(tText)
|
||||
self._addTextPar(xText, "Text_20_Meta", oStyle, tTemp, tFmt=tFmt)
|
||||
self._addTextPar(xText, S_META, oStyle, tTemp, tFmt=tFmt)
|
||||
|
||||
return
|
||||
|
||||
@@ -847,7 +832,7 @@ class ToOdt(Tokenizer):
|
||||
_mkTag("style", "name"): "Heading",
|
||||
_mkTag("style", "family"): "paragraph",
|
||||
_mkTag("style", "parent-style-name"): "Standard",
|
||||
_mkTag("style", "next-style-name"): "Text_20_body",
|
||||
_mkTag("style", "next-style-name"): S_TEXT,
|
||||
_mkTag("style", "class"): "text",
|
||||
})
|
||||
ET.SubElement(xStyl, _mkTag("style", "paragraph-properties"), attrib={
|
||||
@@ -863,7 +848,7 @@ class ToOdt(Tokenizer):
|
||||
|
||||
# Add Header and Footer Styles
|
||||
ET.SubElement(self._xStyl, _mkTag("style", "style"), attrib={
|
||||
_mkTag("style", "name"): "Header_20_and_20_Footer",
|
||||
_mkTag("style", "name"): S_HNF,
|
||||
_mkTag("style", "display-name"): "Header and Footer",
|
||||
_mkTag("style", "family"): "paragraph",
|
||||
_mkTag("style", "parent-style-name"): "Standard",
|
||||
@@ -875,7 +860,7 @@ class ToOdt(Tokenizer):
|
||||
def _useableStyles(self) -> None:
|
||||
"""Set the usable styles."""
|
||||
# Add Text Body Style
|
||||
style = ODTParagraphStyle("Text_20_body")
|
||||
style = ODTParagraphStyle(S_TEXT)
|
||||
style.setDisplayName("Text body")
|
||||
style.setParentStyleName("Standard")
|
||||
style.setClass("text")
|
||||
@@ -890,16 +875,16 @@ class ToOdt(Tokenizer):
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add First Line Indent Style
|
||||
style = ODTParagraphStyle("First_20_line_20_indent")
|
||||
style = ODTParagraphStyle(S_FIND)
|
||||
style.setDisplayName("First line indent")
|
||||
style.setParentStyleName("Text_20_body")
|
||||
style.setParentStyleName(S_TEXT)
|
||||
style.setClass("text")
|
||||
style.setTextIndent(self._fTextIndent)
|
||||
style.packXML(self._xStyl)
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add Text Meta Style
|
||||
style = ODTParagraphStyle("Text_20_Meta")
|
||||
style = ODTParagraphStyle(S_META)
|
||||
style.setDisplayName("Text Meta")
|
||||
style.setParentStyleName("Standard")
|
||||
style.setClass("text")
|
||||
@@ -915,10 +900,10 @@ class ToOdt(Tokenizer):
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add Title Style
|
||||
style = ODTParagraphStyle("Title")
|
||||
style = ODTParagraphStyle(S_TITLE)
|
||||
style.setDisplayName("Title")
|
||||
style.setParentStyleName("Heading")
|
||||
style.setNextStyleName("Text_20_body")
|
||||
style.setNextStyleName(S_TEXT)
|
||||
style.setClass("chapter")
|
||||
style.setMarginTop(self._mTopTitle)
|
||||
style.setMarginBottom(self._mBotTitle)
|
||||
@@ -931,10 +916,10 @@ class ToOdt(Tokenizer):
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add Separator Style
|
||||
style = ODTParagraphStyle("Separator")
|
||||
style = ODTParagraphStyle(S_SEP)
|
||||
style.setDisplayName("Separator")
|
||||
style.setParentStyleName("Standard")
|
||||
style.setNextStyleName("Text_20_body")
|
||||
style.setNextStyleName(S_TEXT)
|
||||
style.setClass("text")
|
||||
style.setMarginTop(self._mTopText)
|
||||
style.setMarginBottom(self._mBotText)
|
||||
@@ -947,10 +932,10 @@ class ToOdt(Tokenizer):
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add Heading 1 Style
|
||||
style = ODTParagraphStyle("Heading_20_1")
|
||||
style = ODTParagraphStyle(S_HEAD1)
|
||||
style.setDisplayName("Heading 1")
|
||||
style.setParentStyleName("Heading")
|
||||
style.setNextStyleName("Text_20_body")
|
||||
style.setNextStyleName(S_TEXT)
|
||||
style.setOutlineLevel("1")
|
||||
style.setClass("text")
|
||||
style.setMarginTop(self._mTopHead1)
|
||||
@@ -965,10 +950,10 @@ class ToOdt(Tokenizer):
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add Heading 2 Style
|
||||
style = ODTParagraphStyle("Heading_20_2")
|
||||
style = ODTParagraphStyle(S_HEAD2)
|
||||
style.setDisplayName("Heading 2")
|
||||
style.setParentStyleName("Heading")
|
||||
style.setNextStyleName("Text_20_body")
|
||||
style.setNextStyleName(S_TEXT)
|
||||
style.setOutlineLevel("2")
|
||||
style.setClass("text")
|
||||
style.setMarginTop(self._mTopHead2)
|
||||
@@ -983,10 +968,10 @@ class ToOdt(Tokenizer):
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add Heading 3 Style
|
||||
style = ODTParagraphStyle("Heading_20_3")
|
||||
style = ODTParagraphStyle(S_HEAD3)
|
||||
style.setDisplayName("Heading 3")
|
||||
style.setParentStyleName("Heading")
|
||||
style.setNextStyleName("Text_20_body")
|
||||
style.setNextStyleName(S_TEXT)
|
||||
style.setOutlineLevel("3")
|
||||
style.setClass("text")
|
||||
style.setMarginTop(self._mTopHead3)
|
||||
@@ -1001,10 +986,10 @@ class ToOdt(Tokenizer):
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
# Add Heading 4 Style
|
||||
style = ODTParagraphStyle("Heading_20_4")
|
||||
style = ODTParagraphStyle(S_HEAD4)
|
||||
style.setDisplayName("Heading 4")
|
||||
style.setParentStyleName("Heading")
|
||||
style.setNextStyleName("Text_20_body")
|
||||
style.setNextStyleName(S_TEXT)
|
||||
style.setOutlineLevel("4")
|
||||
style.setClass("text")
|
||||
style.setMarginTop(self._mTopHead4)
|
||||
@@ -1021,7 +1006,7 @@ class ToOdt(Tokenizer):
|
||||
# Add Header Style
|
||||
style = ODTParagraphStyle("Header")
|
||||
style.setDisplayName("Header")
|
||||
style.setParentStyleName("Header_20_and_20_Footer")
|
||||
style.setParentStyleName(S_HNF)
|
||||
style.setTextAlign("right")
|
||||
style.packXML(self._xStyl)
|
||||
self._mainPara[style.name] = style
|
||||
|
||||
@@ -36,7 +36,7 @@ from PyQt5.QtGui import (
|
||||
|
||||
from novelwriter import CONFIG, SHARED
|
||||
from novelwriter.common import checkInt
|
||||
from novelwriter.constants import nwRegEx, nwUnicode
|
||||
from novelwriter.constants import nwHeaders, nwRegEx, nwUnicode
|
||||
from novelwriter.core.index import processComment
|
||||
from novelwriter.enum import nwComment
|
||||
from novelwriter.types import QRegExUnicode
|
||||
@@ -95,14 +95,14 @@ class GuiDocHighlighter(QSyntaxHighlighter):
|
||||
|
||||
# Create Character Formats
|
||||
self._addCharFormat("text", SHARED.theme.colText)
|
||||
self._addCharFormat("header1", SHARED.theme.colHead, "b", 1.8)
|
||||
self._addCharFormat("header2", SHARED.theme.colHead, "b", 1.6)
|
||||
self._addCharFormat("header3", SHARED.theme.colHead, "b", 1.4)
|
||||
self._addCharFormat("header4", SHARED.theme.colHead, "b", 1.2)
|
||||
self._addCharFormat("head1h", SHARED.theme.colHeadH, "b", 1.8)
|
||||
self._addCharFormat("head2h", SHARED.theme.colHeadH, "b", 1.6)
|
||||
self._addCharFormat("head3h", SHARED.theme.colHeadH, "b", 1.4)
|
||||
self._addCharFormat("head4h", SHARED.theme.colHeadH, "b", 1.2)
|
||||
self._addCharFormat("header1", SHARED.theme.colHead, "b", nwHeaders.H_SIZES[1])
|
||||
self._addCharFormat("header2", SHARED.theme.colHead, "b", nwHeaders.H_SIZES[2])
|
||||
self._addCharFormat("header3", SHARED.theme.colHead, "b", nwHeaders.H_SIZES[3])
|
||||
self._addCharFormat("header4", SHARED.theme.colHead, "b", nwHeaders.H_SIZES[4])
|
||||
self._addCharFormat("head1h", SHARED.theme.colHeadH, "b", nwHeaders.H_SIZES[1])
|
||||
self._addCharFormat("head2h", SHARED.theme.colHeadH, "b", nwHeaders.H_SIZES[2])
|
||||
self._addCharFormat("head3h", SHARED.theme.colHeadH, "b", nwHeaders.H_SIZES[3])
|
||||
self._addCharFormat("head4h", SHARED.theme.colHeadH, "b", nwHeaders.H_SIZES[4])
|
||||
self._addCharFormat("bold", colEmph, "b")
|
||||
self._addCharFormat("italic", colEmph, "i")
|
||||
self._addCharFormat("strike", SHARED.theme.colHidden, "s")
|
||||
|
||||
@@ -24,7 +24,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from __future__ import annotations
|
||||
|
||||
from PyQt5.QtCore import QRegularExpression, Qt
|
||||
from PyQt5.QtGui import QColor, QPainter, QTextCursor
|
||||
from PyQt5.QtGui import QColor, QPainter, QTextCursor, QTextFormat
|
||||
from PyQt5.QtWidgets import QDialogButtonBox, QSizePolicy, QStyle
|
||||
|
||||
# Qt Alignment Flags
|
||||
@@ -44,6 +44,11 @@ QtAlignRightMiddle = Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter
|
||||
QtAlignRightTop = Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignTop
|
||||
QtAlignTop = Qt.AlignmentFlag.AlignTop
|
||||
|
||||
# Qt Page Break
|
||||
|
||||
QtPageBreakBefore = QTextFormat.PageBreakFlag.PageBreak_AlwaysBefore
|
||||
QtPageBreakAfter = QTextFormat.PageBreakFlag.PageBreak_AlwaysAfter
|
||||
|
||||
# Qt Painter Types
|
||||
|
||||
QtTransparent = QColor(0, 0, 0, 0)
|
||||
|
||||
@@ -31,7 +31,7 @@ from novelwriter.core.buildsettings import BuildSettings
|
||||
from novelwriter.core.docbuild import NWBuildDocument
|
||||
from novelwriter.core.project import NWProject
|
||||
from novelwriter.core.tohtml import ToHtml
|
||||
from novelwriter.core.tomd import ToMarkdown
|
||||
from novelwriter.core.tomarkdown import ToMarkdown
|
||||
from novelwriter.core.toodt import ToOdt
|
||||
from novelwriter.enum import nwBuildFmt
|
||||
|
||||
|
||||
@@ -165,11 +165,11 @@ def testCoreToHtml_ConvertParagraphs(mockGUI):
|
||||
)
|
||||
|
||||
# Text w/Hard Break
|
||||
html._text = "Line one \nLine two \nLine three\n"
|
||||
html._text = "Line one\nLine two\nLine three\n"
|
||||
html.tokenizeText()
|
||||
html.doConvert()
|
||||
assert html.result == (
|
||||
"<p class='break'>Line one<br/>Line two<br/>Line three</p>\n"
|
||||
"<p>Line one<br>Line two<br>Line three</p>\n"
|
||||
)
|
||||
|
||||
# Synopsis, Short
|
||||
|
||||
@@ -27,7 +27,7 @@ import pytest
|
||||
from novelwriter.constants import nwHeadFmt
|
||||
from novelwriter.core.project import NWProject
|
||||
from novelwriter.core.tokenizer import HeadingFormatter, Tokenizer, stripEscape
|
||||
from novelwriter.core.tomd import ToMarkdown
|
||||
from novelwriter.core.tomarkdown import ToMarkdown
|
||||
|
||||
from tests.tools import C, buildTestProject, readFile
|
||||
|
||||
@@ -262,7 +262,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TITLE, 1, "Novel Title", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "#! Novel Title\n\n"
|
||||
|
||||
@@ -274,7 +273,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TITLE, 1, "Note Title", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "#! Note Title\n\n"
|
||||
|
||||
@@ -289,7 +287,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Novel Title", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "# Novel Title\n\n"
|
||||
|
||||
@@ -301,7 +298,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Note Title", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "# Note Title\n\n"
|
||||
|
||||
@@ -315,7 +311,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "## Chapter One\n\n"
|
||||
|
||||
@@ -326,7 +321,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "Heading 2", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "## Heading 2\n\n"
|
||||
|
||||
@@ -340,7 +334,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD3, 1, "Scene One", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "### Scene One\n\n"
|
||||
|
||||
@@ -351,7 +344,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD3, 1, "Heading 3", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "### Heading 3\n\n"
|
||||
|
||||
@@ -365,7 +357,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD4, 1, "A Section", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "#### A Section\n\n"
|
||||
|
||||
@@ -376,7 +367,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD4, 1, "Heading 4", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "#### Heading 4\n\n"
|
||||
|
||||
@@ -391,7 +381,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "#! Title\n\n"
|
||||
|
||||
@@ -403,7 +392,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TITLE, 1, "Title", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "#! Title\n\n"
|
||||
|
||||
@@ -417,7 +405,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
|
||||
|
||||
@@ -428,7 +415,6 @@ def testCoreToken_HeaderFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "Prologue", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "##! Prologue\n\n"
|
||||
|
||||
@@ -706,7 +692,6 @@ def testCoreToken_MetaFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_COMMENT, 0, "A comment", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "\n"
|
||||
|
||||
@@ -717,9 +702,7 @@ def testCoreToken_MetaFormat(mockGUI):
|
||||
# Ignore Text
|
||||
tokens._text = "%~ Some text\n"
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens._tokens == []
|
||||
assert tokens.allMarkdown[-1] == "\n"
|
||||
|
||||
# Synopsis
|
||||
@@ -727,13 +710,11 @@ def testCoreToken_MetaFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
tokens._text = "% synopsis: The synopsis\n"
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_SYNOPSIS, 0, "The synopsis", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "\n"
|
||||
|
||||
@@ -747,7 +728,6 @@ def testCoreToken_MetaFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_SHORT, 0, "A short description", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "\n"
|
||||
|
||||
@@ -760,7 +740,6 @@ def testCoreToken_MetaFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_KEYWORD, 0, "char: Bod", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "\n"
|
||||
|
||||
@@ -777,7 +756,6 @@ def testCoreToken_MetaFormat(mockGUI):
|
||||
(Tokenizer.T_KEYWORD, 0, "pov: Bod", [], styTop),
|
||||
(Tokenizer.T_KEYWORD, 0, "plot: Main", [], styMid),
|
||||
(Tokenizer.T_KEYWORD, 0, "location: Europe", [], styBtm),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n"
|
||||
|
||||
@@ -787,7 +765,6 @@ def testCoreToken_MetaFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_KEYWORD, 0, "pov: Bod", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
|
||||
@@ -814,22 +791,13 @@ def testCoreToken_MarginFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TEXT, 0, "Some regular text", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Some left-aligned text", [], Tokenizer.A_LEFT),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Some right-aligned text", [], Tokenizer.A_RIGHT),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Some centered text", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Left-indented block", [], Tokenizer.A_IND_L),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Right-indented block", [], Tokenizer.A_IND_R),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Double-indented block", [], dblIndent),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Right-indent, right-aligned", [], rIndAlign),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == (
|
||||
"Some regular text\n\n"
|
||||
@@ -950,6 +918,79 @@ def testCoreToken_ExtractFormats(mockGUI):
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core
|
||||
def testCoreToken_Paragraphs(mockGUI):
|
||||
"""Test the splitting of paragraphs."""
|
||||
project = NWProject()
|
||||
tokens = BareTokenizer(project)
|
||||
tokens.setKeepMarkdown(True)
|
||||
|
||||
# Collapse empty lines
|
||||
tokens._text = "First paragraph\n\n\nSecond paragraph\n\n\n"
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TEXT, 0, "First paragraph", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Second paragraph", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Combine multi-line paragraphs, keep breaks
|
||||
tokens._text = "This is text\nspanning multiple\nlines"
|
||||
tokens.setKeepLineBreaks(True)
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TEXT, 0, "This is text\nspanning multiple\nlines", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Combine multi-line paragraphs, remove breaks
|
||||
tokens._text = "This is text\nspanning multiple\nlines"
|
||||
tokens.setKeepLineBreaks(False)
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TEXT, 0, "This is text spanning multiple lines", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Combine multi-line paragraphs, remove breaks, with formatting
|
||||
tokens._text = "This **is text**\nspanning _multiple_\nlines"
|
||||
tokens.setKeepLineBreaks(False)
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(
|
||||
Tokenizer.T_TEXT, 0,
|
||||
"This is text spanning multiple lines",
|
||||
[
|
||||
(5, Tokenizer.FMT_B_B, ""),
|
||||
(12, Tokenizer.FMT_B_E, ""),
|
||||
(22, Tokenizer.FMT_I_B, ""),
|
||||
(30, Tokenizer.FMT_I_E, ""),
|
||||
],
|
||||
Tokenizer.A_NONE
|
||||
),
|
||||
]
|
||||
|
||||
# Make sure titles break a paragraph
|
||||
tokens._text = "# Title\nText _on_\ntwo lines.\n## Chapter\nMore **text**\n_here_.\n\n\n"
|
||||
tokens.setKeepLineBreaks(False)
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title", [], Tokenizer.A_NONE),
|
||||
(
|
||||
Tokenizer.T_TEXT, 1, "Text on two lines.", [
|
||||
(5, Tokenizer.FMT_I_B, ""),
|
||||
(7, Tokenizer.FMT_I_E, ""),
|
||||
], Tokenizer.A_NONE
|
||||
),
|
||||
(Tokenizer.T_HEAD2, 2, "Chapter", [], Tokenizer.A_NONE),
|
||||
(
|
||||
Tokenizer.T_TEXT, 2, "More text here.", [
|
||||
(5, Tokenizer.FMT_B_B, ""),
|
||||
(9, Tokenizer.FMT_B_E, ""),
|
||||
(10, Tokenizer.FMT_I_B, ""),
|
||||
(14, Tokenizer.FMT_I_E, ""),
|
||||
], Tokenizer.A_NONE
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core
|
||||
def testCoreToken_TextFormat(mockGUI):
|
||||
"""Test the tokenization of text formats in the Tokenizer class."""
|
||||
@@ -961,21 +1002,13 @@ def testCoreToken_TextFormat(mockGUI):
|
||||
tokens._text = "Some plain text\non two lines\n\n\n"
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_TEXT, 0, "Some plain text", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "on two lines", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 0, "Some plain text\non two lines", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "Some plain text\non two lines\n\n\n\n"
|
||||
|
||||
tokens.setBodyText(False)
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens._tokens == []
|
||||
assert tokens.allMarkdown[-1] == "\n\n\n"
|
||||
tokens.setBodyText(True)
|
||||
|
||||
@@ -992,7 +1025,6 @@ def testCoreToken_TextFormat(mockGUI):
|
||||
],
|
||||
Tokenizer.A_NONE
|
||||
),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "Some **bolded text** on this lines\n\n"
|
||||
|
||||
@@ -1008,7 +1040,6 @@ def testCoreToken_TextFormat(mockGUI):
|
||||
],
|
||||
Tokenizer.A_NONE
|
||||
),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "Some _italic text_ on this lines\n\n"
|
||||
|
||||
@@ -1026,7 +1057,6 @@ def testCoreToken_TextFormat(mockGUI):
|
||||
],
|
||||
Tokenizer.A_NONE
|
||||
),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "Some **_bold italic text_** on this lines\n\n"
|
||||
|
||||
@@ -1042,7 +1072,6 @@ def testCoreToken_TextFormat(mockGUI):
|
||||
],
|
||||
Tokenizer.A_NONE
|
||||
),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == "Some ~~strikethrough text~~ on this lines\n\n"
|
||||
|
||||
@@ -1062,7 +1091,6 @@ def testCoreToken_TextFormat(mockGUI):
|
||||
],
|
||||
Tokenizer.A_NONE
|
||||
),
|
||||
(Tokenizer.T_EMPTY, 0, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens.allMarkdown[-1] == (
|
||||
"Some **nested bold and _italic_ and ~~strikethrough~~ text** here\n\n"
|
||||
@@ -1082,11 +1110,7 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
|
||||
correctResp = [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_HEAD1, 2, "Title Two", [], Tokenizer.A_CENTRE | Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 2, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Command wo/Space
|
||||
@@ -1130,12 +1154,8 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Multiple Empty Paragraphs
|
||||
@@ -1150,12 +1170,8 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Three Skips
|
||||
@@ -1167,14 +1183,10 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Malformed Command, Case 1
|
||||
@@ -1186,11 +1198,7 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Malformed Command, Case 2
|
||||
@@ -1202,11 +1210,7 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Malformed Command, Case 3
|
||||
@@ -1218,11 +1222,7 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Empty Paragraph and Page Break
|
||||
@@ -1238,13 +1238,8 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Multiple Skip
|
||||
@@ -1257,15 +1252,10 @@ def testCoreToken_SpecialFormat(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "Title One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_TEXT, 1, "Some text to go here ...", [], 0),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
|
||||
@@ -1293,7 +1283,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H1: Title, Not First Page
|
||||
@@ -1303,7 +1292,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD1, 1, "T: Part One", [], Tokenizer.A_PBB | Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Chapters
|
||||
@@ -1315,7 +1303,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "C: Chapter One", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H2: Unnumbered Chapter
|
||||
@@ -1324,7 +1311,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "U: Prologue", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H2: Chapter Word Number
|
||||
@@ -1334,7 +1320,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "Chapter One", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H2: Chapter Roman Number Upper Case
|
||||
@@ -1343,7 +1328,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "Chapter II", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H2: Chapter Roman Number Lower Case
|
||||
@@ -1352,7 +1336,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD2, 1, "Chapter iii", [], Tokenizer.A_PBB),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Scenes
|
||||
@@ -1364,27 +1347,20 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD3, 1, "S: Scene One", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H3: Scene Hidden wo/Format
|
||||
tokens._text = "### Scene One\n"
|
||||
tokens.setSceneFormat("", True)
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens._tokens == []
|
||||
|
||||
# H3: Scene wo/Format, first
|
||||
tokens._text = "### Scene One\n"
|
||||
tokens.setSceneFormat("", False)
|
||||
tokens._noSep = True
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens._tokens == []
|
||||
|
||||
# H3: Scene wo/Format, not first
|
||||
tokens._text = "### Scene One\n"
|
||||
@@ -1393,7 +1369,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H3: Scene Separator, first
|
||||
@@ -1401,10 +1376,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.setSceneFormat("* * *", False)
|
||||
tokens._noSep = True
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens._tokens == []
|
||||
|
||||
# H3: Scene Separator, not first
|
||||
tokens._text = "### Scene One\n"
|
||||
@@ -1413,7 +1385,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H3: Scene w/Absolute Number
|
||||
@@ -1424,7 +1395,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD3, 1, "Scene 1", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H3: Scene w/Chapter Number
|
||||
@@ -1435,7 +1405,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD3, 1, "Scene 3.2", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Sections
|
||||
@@ -1445,10 +1414,7 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens._text = "#### A Section\n"
|
||||
tokens.setSectionFormat("", True)
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
assert tokens._tokens == []
|
||||
|
||||
# H4: Section Visible wo/Format
|
||||
tokens._text = "#### A Section\n"
|
||||
@@ -1456,7 +1422,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_SKIP, 1, "", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H4: Section w/Format
|
||||
@@ -1465,7 +1430,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_HEAD4, 1, "X: A Section", [], Tokenizer.A_NONE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# H4: Section Separator
|
||||
@@ -1474,7 +1438,6 @@ def testCoreToken_ProcessHeaders(mockGUI):
|
||||
tokens.tokenizeText()
|
||||
assert tokens._tokens == [
|
||||
(Tokenizer.T_SEP, 1, "* * *", [], Tokenizer.A_CENTRE),
|
||||
(Tokenizer.T_EMPTY, 1, "", [], Tokenizer.A_NONE),
|
||||
]
|
||||
|
||||
# Check the first scene detector, plain text
|
||||
@@ -1614,9 +1577,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
|
||||
tokens.setSceneFormat("* * *", False)
|
||||
tokens.tokenizeText()
|
||||
tokens.countStats()
|
||||
assert [t[2] for t in tokens._tokens] == [
|
||||
"Chapter", "", "", "", "Text", "", "* * *", "", "Text", ""
|
||||
]
|
||||
assert [t[2] for t in tokens._tokens] == ["Chapter", "Text", "* * *", "Text"]
|
||||
assert tokens.textStats == {
|
||||
"titleCount": 1, "paragraphCount": 2,
|
||||
"allWords": 6, "textWords": 2, "titleWords": 1,
|
||||
@@ -1633,9 +1594,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
|
||||
tokens.setSynopsis(True)
|
||||
tokens.tokenizeText()
|
||||
tokens.countStats()
|
||||
assert [t[2] for t in tokens._tokens] == [
|
||||
"Chapter", "", "", "", "Stuff", "", "Text", ""
|
||||
]
|
||||
assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"]
|
||||
assert tokens.textStats == {
|
||||
"titleCount": 1, "paragraphCount": 1,
|
||||
"allWords": 4, "textWords": 1, "titleWords": 1,
|
||||
@@ -1652,9 +1611,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
|
||||
tokens.setSynopsis(True)
|
||||
tokens.tokenizeText()
|
||||
tokens.countStats()
|
||||
assert [t[2] for t in tokens._tokens] == [
|
||||
"Chapter", "", "", "", "Stuff", "", "Text", ""
|
||||
]
|
||||
assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"]
|
||||
assert tokens.textStats == {
|
||||
"titleCount": 1, "paragraphCount": 1,
|
||||
"allWords": 5, "textWords": 1, "titleWords": 1,
|
||||
@@ -1671,9 +1628,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
|
||||
tokens.setComments(True)
|
||||
tokens.tokenizeText()
|
||||
tokens.countStats()
|
||||
assert [t[2] for t in tokens._tokens] == [
|
||||
"Chapter", "", "", "", "Stuff", "", "Text", ""
|
||||
]
|
||||
assert [t[2] for t in tokens._tokens] == ["Chapter", "Stuff", "Text"]
|
||||
assert tokens.textStats == {
|
||||
"titleCount": 1, "paragraphCount": 1,
|
||||
"allWords": 4, "textWords": 1, "titleWords": 1,
|
||||
@@ -1690,9 +1645,7 @@ def testCoreToken_CountStats(mockGUI, ipsumText):
|
||||
tokens.setKeywords(True)
|
||||
tokens.tokenizeText()
|
||||
tokens.countStats()
|
||||
assert [t[2] for t in tokens._tokens] == [
|
||||
"Chapter", "", "", "", "pov: Jane", "", "Text", ""
|
||||
]
|
||||
assert [t[2] for t in tokens._tokens] == ["Chapter", "pov: Jane", "Text"]
|
||||
assert tokens.textStats == {
|
||||
"titleCount": 1, "paragraphCount": 1,
|
||||
"allWords": 6, "textWords": 1, "titleWords": 1,
|
||||
|
||||
@@ -23,7 +23,7 @@ from __future__ import annotations
|
||||
import pytest
|
||||
|
||||
from novelwriter.core.project import NWProject
|
||||
from novelwriter.core.tomd import ToMarkdown
|
||||
from novelwriter.core.tomarkdown import ToMarkdown
|
||||
|
||||
|
||||
@pytest.mark.core
|
||||
@@ -125,14 +125,14 @@ def testCoreToMarkdown_ConvertParagraphs(mockGUI):
|
||||
)
|
||||
|
||||
# Text w/Hard Break
|
||||
toMD._text = "Line one \nLine two \nLine three\n"
|
||||
toMD._text = "Line one\nLine two\nLine three\n"
|
||||
toMD.tokenizeText()
|
||||
toMD.doConvert()
|
||||
assert toMD.result == "Line one \nLine two \nLine three\n\n"
|
||||
|
||||
# Text wo/Hard Break
|
||||
toMD._text = "Line one \nLine two \nLine three\n"
|
||||
toMD.setPreserveBreaks(False)
|
||||
toMD._text = "Line one\nLine two\nLine three\n"
|
||||
toMD.setKeepLineBreaks(False)
|
||||
toMD.tokenizeText()
|
||||
toMD.doConvert()
|
||||
assert toMD.result == "Line one Line two Line three\n\n"
|
||||
@@ -587,7 +587,7 @@ def testCoreToOdt_ConvertParagraphs(mockGUI):
|
||||
'<office:text>'
|
||||
'<text:h text:style-name="Heading_20_3" text:outline-level="3">Scene</text:h>'
|
||||
'<text:p text:style-name="Text_20_body">Regular paragraph</text:p>'
|
||||
'<text:p text:style-name="P7">with<text:line-break />break</text:p>'
|
||||
'<text:p text:style-name="Text_20_body">with<text:line-break />break</text:p>'
|
||||
'<text:p text:style-name="P7">Left Align</text:p>'
|
||||
'</office:text>'
|
||||
)
|
||||
|
||||
@@ -29,13 +29,12 @@ from novelwriter import CONFIG, SHARED
|
||||
from novelwriter.constants import nwKeyWords, nwUnicode
|
||||
from novelwriter.enum import nwDocAction, nwDocInsert
|
||||
from novelwriter.gui.doceditor import GuiDocEditor
|
||||
from novelwriter.types import QtMouseLeft
|
||||
|
||||
from tests.tools import C, buildTestProject, writeFile
|
||||
|
||||
|
||||
@pytest.mark.gui
|
||||
def testGuiMenu_EditFormat(qtbot, monkeypatch, nwGUI, prjLipsum):
|
||||
def testGuiMainMenu_EditFormat(qtbot, monkeypatch, nwGUI, prjLipsum):
|
||||
"""Test the main menu Edit and Format entries."""
|
||||
monkeypatch.setattr(GuiDocEditor, "hasFocus", lambda *a: True)
|
||||
|
||||
@@ -345,84 +344,7 @@ def testGuiMenu_EditFormat(qtbot, monkeypatch, nwGUI, prjLipsum):
|
||||
|
||||
|
||||
@pytest.mark.gui
|
||||
def testGuiMenu_ContextMenus(qtbot, nwGUI, prjLipsum):
|
||||
"""Test the context menus."""
|
||||
assert nwGUI.openProject(prjLipsum)
|
||||
assert nwGUI.openDocument("4c4f28287af27")
|
||||
|
||||
# Editor Context Menu
|
||||
cursor = nwGUI.docEditor.textCursor()
|
||||
cursor.setPosition(127)
|
||||
nwGUI.docEditor.setTextCursor(cursor)
|
||||
rect = nwGUI.docEditor.cursorRect()
|
||||
|
||||
nwGUI.docEditor._openContextMenu(rect.bottomRight())
|
||||
qtbot.mouseClick(nwGUI.docEditor, QtMouseLeft, pos=rect.topLeft())
|
||||
|
||||
nwGUI.docEditor._makePosSelection(QTextCursor.WordUnderCursor, rect.center())
|
||||
cursor = nwGUI.docEditor.textCursor()
|
||||
assert cursor.selectedText() == "imperdiet"
|
||||
|
||||
nwGUI.docEditor._makePosSelection(QTextCursor.BlockUnderCursor, rect.center())
|
||||
cursor = nwGUI.docEditor.textCursor()
|
||||
assert cursor.selectedText() == (
|
||||
"Pellentesque nec erat ut nulla posuere commodo. Curabitur nisi augue, imperdiet et porta "
|
||||
"imperdiet, efficitur id leo. Cras finibus arcu at nibh commodo congue. Proin suscipit "
|
||||
"placerat condimentum. Aenean ante enim, cursus id lorem a, blandit venenatis nibh. "
|
||||
"Maecenas suscipit porta elit, sit amet porta felis porttitor eu. Sed a dui nibh. "
|
||||
"Phasellus sed faucibus dui. Pellentesque felis nulla, ultrices non efficitur quis, "
|
||||
"rutrum id mi. Mauris tempus auctor nisl, in bibendum enim pellentesque sit amet. Proin "
|
||||
"nunc lacus, imperdiet nec posuere ac, interdum non lectus."
|
||||
)
|
||||
|
||||
# Viewer Context Menu
|
||||
assert nwGUI.viewDocument("4c4f28287af27")
|
||||
|
||||
cursor = nwGUI.docViewer.textCursor()
|
||||
cursor.setPosition(127)
|
||||
nwGUI.docViewer.setTextCursor(cursor)
|
||||
rect = nwGUI.docViewer.cursorRect()
|
||||
|
||||
nwGUI.docViewer._openContextMenu(rect.bottomRight())
|
||||
qtbot.mouseClick(nwGUI.docViewer, QtMouseLeft, pos=rect.topLeft())
|
||||
|
||||
nwGUI.docViewer._makePosSelection(QTextCursor.WordUnderCursor, rect.center())
|
||||
cursor = nwGUI.docViewer.textCursor()
|
||||
assert cursor.selectedText() == "imperdiet"
|
||||
|
||||
nwGUI.docEditor._makePosSelection(QTextCursor.BlockUnderCursor, rect.center())
|
||||
cursor = nwGUI.docEditor.textCursor()
|
||||
assert cursor.selectedText() == (
|
||||
"Pellentesque nec erat ut nulla posuere commodo. Curabitur nisi augue, imperdiet et porta "
|
||||
"imperdiet, efficitur id leo. Cras finibus arcu at nibh commodo congue. Proin suscipit "
|
||||
"placerat condimentum. Aenean ante enim, cursus id lorem a, blandit venenatis nibh. "
|
||||
"Maecenas suscipit porta elit, sit amet porta felis porttitor eu. Sed a dui nibh. "
|
||||
"Phasellus sed faucibus dui. Pellentesque felis nulla, ultrices non efficitur quis, "
|
||||
"rutrum id mi. Mauris tempus auctor nisl, in bibendum enim pellentesque sit amet. Proin "
|
||||
"nunc lacus, imperdiet nec posuere ac, interdum non lectus."
|
||||
)
|
||||
|
||||
# Navigation History
|
||||
assert nwGUI.viewDocument("04468803b92e1")
|
||||
assert nwGUI.docViewer.docHandle == "04468803b92e1"
|
||||
assert nwGUI.docViewer.docHeader.backButton.isEnabled()
|
||||
assert not nwGUI.docViewer.docHeader.forwardButton.isEnabled()
|
||||
|
||||
qtbot.mouseClick(nwGUI.docViewer.docHeader.backButton, QtMouseLeft)
|
||||
assert nwGUI.docViewer.docHandle == "4c4f28287af27"
|
||||
assert not nwGUI.docViewer.docHeader.backButton.isEnabled()
|
||||
assert nwGUI.docViewer.docHeader.forwardButton.isEnabled()
|
||||
|
||||
qtbot.mouseClick(nwGUI.docViewer.docHeader.forwardButton, QtMouseLeft)
|
||||
assert nwGUI.docViewer.docHandle == "04468803b92e1"
|
||||
assert nwGUI.docViewer.docHeader.backButton.isEnabled()
|
||||
assert not nwGUI.docViewer.docHeader.forwardButton.isEnabled()
|
||||
|
||||
# qtbot.stop()
|
||||
|
||||
|
||||
@pytest.mark.gui
|
||||
def testGuiMenu_Insert(qtbot, monkeypatch, nwGUI, fncPath, projPath, mockRnd):
|
||||
def testGuiMainMenu_Insert(qtbot, monkeypatch, nwGUI, fncPath, projPath, mockRnd):
|
||||
"""Test the Insert menu."""
|
||||
buildTestProject(nwGUI, projPath)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user