Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move shared formats code to separate file, and rename some variables

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2024-10-22 00:25:35 +02:00
parent 38ca447c8a
commit 16076ebfb6
13 changed files with 340 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/formats/shared.py
+129
View File
@@ -0,0 +1,129 @@
"""
novelWriter – Formats Shared
============================
File History:
Created: 2024-10-21 [2.6b1] TextFmt
Created: 2024-10-21 [2.6b1] BlockTyp
Created: 2024-10-21 [2.6b1] BlockFmt
This file is a part of novelWriter
Copyright 2018–2024, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import re
from enum import Flag, IntEnum
ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
def stripEscape(text: str) -> str:
"""Strip escaped Markdown characters from paragraph text."""
if "\\" in text:
return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text)
return text
# Enums
# =====
class TextFmt(IntEnum):
"""Text Format.
An enum indicating the beginning or end of a text format region.
They must be paired with a position, and apply to locations in a
text block.
"""
B_B = 1 # Begin bold
B_E = 2 # End bold
I_B = 3 # Begin italics
I_E = 4 # End italics
D_B = 5 # Begin strikeout
D_E = 6 # End strikeout
U_B = 7 # Begin underline
U_E = 8 # End underline
M_B = 9 # Begin mark
M_E = 10 # End mark
SUP_B = 11 # Begin superscript
SUP_E = 12 # End superscript
SUB_B = 13 # Begin subscript
SUB_E = 14 # End subscript
DL_B = 15 # Begin dialogue
DL_E = 16 # End dialogue
ADL_B = 17 # Begin alt dialogue
ADL_E = 18 # End alt dialogue
FNOTE = 19 # Footnote marker
STRIP = 20 # Strip the format code
class BlockTyp(IntEnum):
"""Text Block Type.
An enum indicating the type of a text block.
"""
EMPTY = 1 # Empty line (new paragraph)
SYNOPSIS = 2 # Synopsis comment
SHORT = 3 # Short description comment
COMMENT = 4 # Comment line
KEYWORD = 5 # Command line
TITLE = 6 # Title
HEAD1 = 7 # Heading 1
HEAD2 = 8 # Heading 2
HEAD3 = 9 # Heading 3
HEAD4 = 10 # Heading 4
TEXT = 11 # Text line
SEP = 12 # Scene separator
SKIP = 13 # Paragraph break
class BlockFmt(Flag):
"""Text Block Format.
An enum of flags that can be combined to format a text block.
"""
NONE = 0x0000 # No special style
LEFT = 0x0001 # Left aligned
RIGHT = 0x0002 # Right aligned
CENTRE = 0x0004 # Centred
JUSTIFY = 0x0008 # Justified
PBB = 0x0010 # Page break before
PBA = 0x0020 # Page break after
Z_TOPMRG = 0x0040 # Zero top margin
Z_BTMMRG = 0x0080 # Zero bottom margin
IND_L = 0x0100 # Left indentation
IND_R = 0x0200 # Right indentation
IND_T = 0x0400 # Text indentation
# Types
# =====
# A list of formats for a single text string, consisting of:
# text position, text format, and meta data
T_Formats = list[tuple[int, TextFmt, str]]
# A note or comment with text and associated text formats
T_Note = tuple[str, T_Formats]
# A tokenized text block, consisting of:
# type, header number, text, text formats, and block format
T_Block = tuple[BlockTyp, int, str, T_Formats, BlockFmt]
novelwriter/formats/todocx.py
+3 -2
View File
@@ -39,7 +39,8 @@ from novelwriter import __version__
from novelwriter.common import firstFloat, xmlSubElem
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer
from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__)
@@ -219,7 +220,7 @@ class ToDocX(Tokenizer):
bIndent = self._fontSize * self._blockIndent
for tType, _, tText, tFormat, tStyle in self._tokens:
for tType, _, tText, tFormat, tStyle in self._blocks:
# Create Paragraph
par = DocXParagraph()
novelwriter/formats/tohtml.py
+3 -4
View File
@@ -32,9 +32,8 @@ from time import time
from novelwriter.common import formatTimeStamp
from novelwriter.constants import nwHeadFmt, nwHtmlUnicode, nwKeyWords, nwLabels
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import (
BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape
)
from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape
from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.types import FONT_STYLE, FONT_WEIGHTS
logger = logging.getLogger(__name__)
@@ -158,7 +157,7 @@ class ToHtml(Tokenizer):
lines = []
tHandle = self._handle
for tType, nHead, tText, tFormat, tStyle in self._tokens:
for tType, nHead, tText, tFormat, tStyle in self._blocks:
# Replace < and > with HTML entities
if tFormat:
novelwriter/formats/tokenizer.py
+60 -132
View File
@@ -29,7 +29,6 @@ import logging
import re
from abc import ABC, abstractmethod
from enum import Flag, IntEnum
from functools import partial
from pathlib import Path
from time import time
@@ -45,82 +44,11 @@ from novelwriter.constants import (
from novelwriter.core.index import processComment
from novelwriter.core.project import NWProject
from novelwriter.enum import nwComment, nwItemLayout
from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Block, T_Formats, T_Note, TextFmt
from novelwriter.text.patterns import REGEX_PATTERNS
logger = logging.getLogger(__name__)
ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
def stripEscape(text: str) -> str:
"""Strip escaped Markdown characters from paragraph text."""
if "\\" in text:
return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text)
return text
class TextFmt(IntEnum):
B_B = 1 # Begin bold
B_E = 2 # End bold
I_B = 3 # Begin italics
I_E = 4 # End italics
D_B = 5 # Begin strikeout
D_E = 6 # End strikeout
U_B = 7 # Begin underline
U_E = 8 # End underline
M_B = 9 # Begin mark
M_E = 10 # End mark
SUP_B = 11 # Begin superscript
SUP_E = 12 # End superscript
SUB_B = 13 # Begin subscript
SUB_E = 14 # End subscript
DL_B = 15 # Begin dialogue
DL_E = 16 # End dialogue
ADL_B = 17 # Begin alt dialogue
ADL_E = 18 # End alt dialogue
FNOTE = 19 # Footnote marker
STRIP = 20 # Strip the format code
class BlockTyp(IntEnum):
EMPTY = 1 # Empty line (new paragraph)
SYNOPSIS = 2 # Synopsis comment
SHORT = 3 # Short description comment
COMMENT = 4 # Comment line
KEYWORD = 5 # Command line
TITLE = 6 # Title
HEAD1 = 7 # Heading 1
HEAD2 = 8 # Heading 2
HEAD3 = 9 # Heading 3
HEAD4 = 10 # Heading 4
TEXT = 11 # Text line
SEP = 12 # Scene separator
SKIP = 13 # Paragraph break
class BlockFmt(Flag):
NONE = 0x0000 # No special style
LEFT = 0x0001 # Left aligned
RIGHT = 0x0002 # Right aligned
CENTRE = 0x0004 # Centred
JUSTIFY = 0x0008 # Justified
PBB = 0x0010 # Page break before
PBA = 0x0020 # Page break after
Z_TOPMRG = 0x0040 # Zero top margin
Z_BTMMRG = 0x0080 # Zero bottom margin
IND_L = 0x0100 # Left indentation
IND_R = 0x0200 # Right indentation
IND_T = 0x0400 # Text indentation
T_Formats = list[tuple[int, TextFmt, str]]
T_Comment = tuple[str, T_Formats]
T_Token = tuple[BlockTyp, int, str, T_Formats, BlockFmt]
class Tokenizer(ABC):
"""Core: Text Tokenizer Abstract Base Class
@@ -154,11 +82,11 @@ class Tokenizer(ABC):
self._result = "" # The result of the last document
self._keepRaw = False # Whether to keep the raw text, used by ToRaw
# Tokens and Meta Data (Per Document)
self._tokens: list[T_Token] = []
self._footnotes: dict[str, T_Comment] = {}
# Blocks and Meta Data (Per Document)
self._blocks: list[T_Block] = []
self._footnotes: dict[str, T_Note] = {}
# Tokens and Meta Data (Per Instance)
# Blocks and Meta Data (Per Instance)
self._counts: dict[str, int] = {}
self._outline: dict[str, str] = {}
self._markdown: list[str] = []
@@ -518,8 +446,8 @@ class Tokenizer(ABC):
trNotes = self._localLookup("Notes")
title = f"{trNotes}: {tItem.itemName}"
self._tokens = []
self._tokens.append((
self._blocks = []
self._blocks.append((
BlockTyp.TITLE, 1, title, [], textAlign
))
if self._keepRaw:
@@ -561,15 +489,15 @@ class Tokenizer(ABC):
characters that indicate headings, comments, commands etc, or
just contain plain text. In the case of plain text, apply the
same RegExes that the syntax highlighter uses and save the
locations of these formatting tags into the token array.
locations of these formatting tags into the blocks list.
The format of the token list is an entry with a five-tuple for
The format of the blocs list is an entry with a five-tuple for
each line in the file. The tuple is as follows:
1: The type of the block, BlockType.*
2: The heading number under which the text is placed
3: The text content of the block, without leading tags
4: The internal formatting map of the text, TxtFmt.*
5: The style of the block, BlockFmt.*
5: The formats of the block, BlockFmt.*
"""
if self._isNovel:
self._hFormatter.setHandle(self._handle)
@@ -578,13 +506,13 @@ class Tokenizer(ABC):
breakNext = False
tmpMarkdown = []
tHandle = self._handle or ""
tokens: list[T_Token] = []
blocks: list[T_Block] = []
for aLine in self._text.splitlines():
sLine = aLine.strip().lower()
# Check for blank lines
if len(sLine) == 0:
tokens.append((
blocks.append((
BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE
))
if self._keepRaw:
@@ -613,7 +541,7 @@ class Tokenizer(ABC):
continue
elif sLine == "[vspace]":
tokens.append(
blocks.append(
(BlockTyp.SKIP, nHead, "", [], sAlign)
)
continue
@@ -621,11 +549,11 @@ class Tokenizer(ABC):
elif sLine.startswith("[vspace:") and sLine.endswith("]"):
nSkip = checkInt(sLine[8:-1], 0)
if nSkip >= 1:
tokens.append(
blocks.append(
(BlockTyp.SKIP, nHead, "", [], sAlign)
)
if nSkip > 1:
tokens += (nSkip - 1) * [
blocks += (nSkip - 1) * [
(BlockTyp.SKIP, nHead, "", [], BlockFmt.NONE)
]
continue
@@ -645,14 +573,14 @@ class Tokenizer(ABC):
cStyle, cKey, cText, _, _ = processComment(aLine)
if cStyle == nwComment.SYNOPSIS:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
blocks.append((
BlockTyp.SYNOPSIS, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
elif cStyle == nwComment.SHORT:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
blocks.append((
BlockTyp.SHORT, nHead, tLine, tFmt, sAlign
))
if self._doSynopsis and self._keepRaw:
@@ -664,7 +592,7 @@ class Tokenizer(ABC):
tmpMarkdown.append(f"{aLine}\n")
else:
tLine, tFmt = self._extractFormats(cText)
tokens.append((
blocks.append((
BlockTyp.COMMENT, nHead, tLine, tFmt, sAlign
))
if self._doComments and self._keepRaw:
@@ -681,7 +609,7 @@ class Tokenizer(ABC):
valid and bits and bits[0] in nwLabels.KEY_NAME
and bits[0] not in self._skipKeywords
):
tokens.append((
blocks.append((
BlockTyp.KEYWORD, nHead, aLine[1:].strip(), [], sAlign
))
if self._doKeywords and self._keepRaw:
@@ -717,7 +645,7 @@ class Tokenizer(ABC):
self._hFormatter.resetAll()
self._noSep = True
tokens.append((
blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -752,7 +680,7 @@ class Tokenizer(ABC):
self._hFormatter.resetScene()
self._noSep = True
tokens.append((
blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -793,7 +721,7 @@ class Tokenizer(ABC):
tStyle = BlockFmt.NONE if self._noSep else BlockFmt.CENTRE
self._noSep = False
tokens.append((
blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -823,7 +751,7 @@ class Tokenizer(ABC):
tType = BlockTyp.SEP
tStyle = BlockFmt.CENTRE
tokens.append((
blocks.append((
tType, nHead, tText, [], tStyle
))
if self._keepRaw:
@@ -871,26 +799,26 @@ class Tokenizer(ABC):
# Process formats
tLine, tFmt = self._extractFormats(aLine, hDialog=self._isNovel)
tokens.append((
blocks.append((
BlockTyp.TEXT, nHead, tLine, tFmt, sAlign
))
if self._keepRaw:
tmpMarkdown.append(f"{aLine}\n")
# If we have content, turn off the first page flag
if self._isFirst and tokens:
if self._isFirst and blocks:
self._isFirst = False # First document has been processed
# Make sure the token array doesn't start with a page break
# Make sure the blocks array doesn't start with a page break
# on the very first page, adding a blank first page.
if tokens[0][4] & BlockFmt.PBB:
cToken = tokens[0]
tokens[0] = (
cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~BlockFmt.PBB
if blocks[0][4] & BlockFmt.PBB:
cBlock = blocks[0]
blocks[0] = (
cBlock[0], cBlock[1], cBlock[2], cBlock[3], cBlock[4] & ~BlockFmt.PBB
)
# Always add an empty line at the end of the file
tokens.append((
blocks.append((
BlockTyp.EMPTY, nHead, "", [], BlockFmt.NONE
))
if self._keepRaw:
@@ -904,48 +832,48 @@ class Tokenizer(ABC):
# It also ensures that there isn't paragraph spacing between
# meta data lines for formats that has spacing.
self._tokens = []
pToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
nToken: T_Token = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
self._blocks = []
pBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
nBlock: T_Block = (BlockTyp.EMPTY, 0, "", [], BlockFmt.NONE)
lineSep = "\n" if self._keepBreaks else " "
pLines: list[T_Token] = []
pLines: list[T_Block] = []
tCount = len(tokens)
for n, cToken in enumerate(tokens):
tCount = len(blocks)
for n, cBlock in enumerate(blocks):
if n > 0:
pToken = tokens[n-1] # Look behind
pBlock = blocks[n-1] # Look behind
if n < tCount - 1:
nToken = tokens[n+1] # Look ahead
nBlock = blocks[n+1] # Look ahead
if cToken[0] in self.L_SKIP_INDENT and not self._indentFirst:
if cBlock[0] in self.L_SKIP_INDENT and not self._indentFirst:
# Unless the indentFirst flag is set, we set up the next
# paragraph to not be indented if we see a block of a
# specific type
self._noIndent = True
if cToken[0] == BlockTyp.EMPTY:
if cBlock[0] == BlockTyp.EMPTY:
# We don't need to keep the empty lines after this pass
pass
elif cToken[0] == BlockTyp.KEYWORD:
elif cBlock[0] == BlockTyp.KEYWORD:
# Adjust margins for lines in a list of keyword lines
aStyle = cToken[4]
if pToken[0] == BlockTyp.KEYWORD:
aStyle = cBlock[4]
if pBlock[0] == BlockTyp.KEYWORD:
aStyle |= BlockFmt.Z_TOPMRG
if nToken[0] == BlockTyp.KEYWORD:
if nBlock[0] == BlockTyp.KEYWORD:
aStyle |= BlockFmt.Z_BTMMRG
self._tokens.append((
cToken[0], cToken[1], cToken[2], cToken[3], aStyle
self._blocks.append((
cBlock[0], cBlock[1], cBlock[2], cBlock[3], aStyle
))
elif cToken[0] == BlockTyp.TEXT:
elif cBlock[0] == BlockTyp.TEXT:
# Combine lines from the same paragraph
pLines.append(cToken)
pLines.append(cBlock)
if nToken[0] != BlockTyp.TEXT:
# Next token is not text, so we add the buffer to tokens
if nBlock[0] != BlockTyp.TEXT:
# Next block is not text, so we add the buffer to blocks
nLines = len(pLines)
cStyle = pLines[0][4]
if self._firstIndent and not (self._noIndent or cStyle & self.M_ALIGNED):
@@ -956,11 +884,11 @@ class Tokenizer(ABC):
if nLines == 1:
# The paragraph contains a single line, so we just save
# that directly to the token list. If justify is
# that directly to the blocks list. If justify is
# enabled, and there is no alignment, we apply it.
if self._doJustify and not cStyle & self.M_ALIGNED:
cStyle |= BlockFmt.JUSTIFY
self._tokens.append((
self._blocks.append((
BlockTyp.TEXT, pLines[0][1], pLines[0][2], pLines[0][3], cStyle
))
elif nLines > 1:
@@ -969,11 +897,11 @@ class Tokenizer(ABC):
# recompute all the formatting markers
tTxt = ""
tFmt: T_Formats = []
for aToken in pLines:
for aBlock in pLines:
tLen = len(tTxt)
tTxt += f"{aToken[2]}{lineSep}"
tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3])
self._tokens.append((
tTxt += f"{aBlock[2]}{lineSep}"
tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aBlock[3])
self._blocks.append((
BlockTyp.TEXT, pLines[0][1], tTxt[:-1], tFmt, cStyle
))
@@ -982,7 +910,7 @@ class Tokenizer(ABC):
self._noIndent = False
else:
self._tokens.append(cToken)
self._blocks.append(cBlock)
return
@@ -990,7 +918,7 @@ class Tokenizer(ABC):
"""Build an outline of the text up to level 3 headings."""
tHandle = self._handle or ""
isNovel = self._isNovel
for tType, nHead, tText, _, _ in self._tokens:
for tType, nHead, tText, _, _ in self._blocks:
if tType == BlockTyp.TITLE:
prefix = "TT"
elif tType == BlockTyp.HEAD1:
@@ -1025,7 +953,7 @@ class Tokenizer(ABC):
textWordChars = self._counts.get("textWordChars", 0)
titleWordChars = self._counts.get("titleWordChars", 0)
for tType, _, tText, _, _ in self._tokens:
for tType, _, tText, _, _ in self._blocks:
tText = tText.replace(nwUnicode.U_ENDASH, " ")
tText = tText.replace(nwUnicode.U_EMDASH, " ")
novelwriter/formats/tomarkdown.py
+4 -3
View File
@@ -29,7 +29,8 @@ from pathlib import Path
from novelwriter.constants import nwHeadFmt, nwLabels, nwUnicode
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer
from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
from novelwriter.formats.tokenizer import Tokenizer
logger = logging.getLogger(__name__)
@@ -117,7 +118,7 @@ class ToMarkdown(Tokenizer):
cSkip = ""
lines = []
for tType, _, tText, tFormat, tStyle in self._tokens:
for tType, _, tText, tFormat, tStyle in self._blocks:
if tType == BlockTyp.TEXT:
tTemp = self._formatText(tText, tFormat, mTags).replace("\n", " \n")
@@ -180,7 +181,7 @@ class ToMarkdown(Tokenizer):
for key, index in self._usedNotes.items():
if content := self._footnotes.get(key):
marker = f"{index}. "
text = self._formatText(*content, tags)
text = self._formatText(content[0], content[1], tags)
lines.append(f"{marker}{text}\n")
lines.append("\n")
novelwriter/formats/toodt.py
+3 -4
View File
@@ -41,9 +41,8 @@ from novelwriter import __version__
from novelwriter.common import xmlIndent, xmlSubElem
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import (
BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer, stripEscape
)
from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt, stripEscape
from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.types import FONT_STYLE, FONT_WEIGHTS
logger = logging.getLogger(__name__)
@@ -426,7 +425,7 @@ class ToOdt(Tokenizer):
self._result = "" # Not used, but cleared just in case
xText = self._xText
for tType, _, tText, tFormat, tStyle in self._tokens:
for tType, _, tText, tFormat, tStyle in self._blocks:
# Styles
oStyle = ODTParagraphStyle("New")
novelwriter/formats/toqdoc.py
+3 -2
View File
@@ -36,7 +36,8 @@ from PyQt5.QtPrintSupport import QPrinter
from novelwriter.constants import nwHeadFmt, nwKeyWords, nwLabels, nwStyles, nwUnicode
from novelwriter.core.project import NWProject
from novelwriter.formats.tokenizer import BlockFmt, BlockTyp, T_Formats, TextFmt, Tokenizer
from novelwriter.formats.shared import BlockFmt, BlockTyp, T_Formats, TextFmt
from novelwriter.formats.tokenizer import Tokenizer
from novelwriter.types import (
QtAlignAbsolute, QtAlignCenter, QtAlignJustify, QtAlignLeft, QtAlignRight,
QtPageBreakAfter, QtPageBreakBefore, QtTransparent, QtVAlignNormal,
@@ -221,7 +222,7 @@ class ToQTextDocument(Tokenizer):
cursor = QTextCursor(self._document)
cursor.movePosition(QTextCursor.MoveOperation.End)
for tType, nHead, tText, tFormat, tStyle in self._tokens:
for tType, nHead, tText, tFormat, tStyle in self._blocks:
# Styles
bFmt = QTextBlockFormat(self._blockFmt)