diff --git a/.gitignore b/.gitignore index a7e67336..a69e4f00 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ ToC.txt # Coverage /.coverage /coverage.* + +# Other +/test.py diff --git a/docs/source/index.rst b/docs/source/index.rst index 1495d713..34daef4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -66,7 +66,6 @@ with pip. See :ref:`a_started` for more details. usage_format usage_shortcuts usage_typography - usage_projectformat .. toctree:: :maxdepth: 1 @@ -80,7 +79,15 @@ with pip. See :ref:`a_started` for more details. .. toctree:: :maxdepth: 1 - :caption: Additional Topics + :caption: Additional Details + :hidden: + + more_projectformat + more_counting + +.. toctree:: + :maxdepth: 1 + :caption: Technical Topics :hidden: tech_locations diff --git a/docs/source/more_counting.rst b/docs/source/more_counting.rst new file mode 100644 index 00000000..d312fd76 --- /dev/null +++ b/docs/source/more_counting.rst @@ -0,0 +1,98 @@ +.. _a_counting: + +******************** +Word and Text Counts +******************** + +This is an overview of how words and other counts of your text are performed. The counting rules +should be relatively standard, and are compared to LibreOffice Writer rules. + +The counts provided in the app on the raw text is meant to be approximate. For more accurate +counts, you need to build your manuscript in the :guilabel:`Manuscript Tool` and check the counts +on the generated preview. + + +Text Word Counts and Stats +========================== + +These are the rules for the main counts available for for each document in a project. + +For all counts, the following rules apply. + +#. Short (–) and long (—) dashes are considered word separators. +#. Any line starting with ``%`` or ``@`` is ignored. +#. Trailing white spaces are ignored, including line breaks. +#. Leading ``>`` and trailing ``<`` are ignored with any spaces next to them. +#. Valid shortcodes and other commands wrapped in brackets ``[]`` are ignored. +#. In-line Markdown syntax in text paragraphs is treated as part of the text. + +After the above preparation of the text, the following counts are available. + +**Character Count** + The character count is the sum of characters per line, including leading and in-text white space + characters, but excluding trailing white space characters. Shortcodes in the text are not + included, but Markdown codes are. Only headers and text are counted. + +**Word Count** + The words count is the sum of blocks of continuous character per line separated by any number of + white space characters or dashes. Only headers and text are counted. + +**Paragraph Count** + The paragraph count is the number of text blocks separated by one or more empty line. A line + consisting only of white spaces is considered empty. + + +Manuscript Counts +================= + +These are the rules for the counts available for a manuscript in the :guilabel:`Manuscript Tool`. +The rules have been tuned to agree with LibreOffice Writer, but will vary slightly depending on the +content of your text. LibreOffice Writer also counts the text in the page header, which the +Manuscript Tool does not. + +The content of each line is counted after all formatting has been processed, so the result will be +more accurate than the counts for text documents elsewhere in the app. The following rules apply: + +#. Short (–) and long (—) dashes are considered word separators. +#. Leading and trailing white spaces are generally included, but paragraph breaks are not. +#. Hard line breaks within paragraph are considered white space characters. +#. All formatting codes are ignored, including shortcodes, commands and Markdown. +#. Scene and section separators are counted. +#. Comments and meta data lines are counted after they are formatted. +#. Headers are counted after they are formatted with custom formats. + +The following counts are available: + +**Header Count** + The number of headers in the manuscript. + +**Paragraph Count** + The number of body text paragraphs in the manuscript. + +**Total Word Count** + The number of words in the manuscript, including any comments and meta data text. + +**Text Word Count** + The number of words in body text paragraphs, excluding all other text. + +**Header Word Count** + The number of words in headers, including inserted formatting like chapter numbers, etc. + +**Total Character Count** + The number of characters on all lines, including any comments and meta data text. Paragraph + breaks are not counted, but in-paragraph hard line breaks are. + +**Text Character Count** + The number of characters in body text paragraphs. Paragraph breaks are not counted, but + in-paragraph hard line breaks are. + +**Header Character Count** + The number of characters in headings. + +**Text Words Character Count** + The number of characters in body text paragraphs considered part of a word or punctuation. That + is, white space characters are not counted. + +**Header Words Character Count** + The number of characters in headers considered part of a word or punctuation. That is, white + space characters are not counted. diff --git a/docs/source/usage_projectformat.rst b/docs/source/more_projectformat.rst similarity index 100% rename from docs/source/usage_projectformat.rst rename to docs/source/more_projectformat.rst diff --git a/novelwriter/assets/icons/typicons_dark/icons.conf b/novelwriter/assets/icons/typicons_dark/icons.conf index d5725121..e183b1a0 100644 --- a/novelwriter/assets/icons/typicons_dark/icons.conf +++ b/novelwriter/assets/icons/typicons_dark/icons.conf @@ -100,6 +100,8 @@ status_time = typ_stopwatch-grey.svg sticky-off = typ_pin-outline.svg sticky-on = typ_pin.svg unchecked = mixed_input-unchecked.svg +unfold-hide = typ_arrow-right.svg +unfold-show = typ_arrow-down.svg up = typ_chevron-up.svg view = typ_eye.svg view_build = typ_export-grey.svg diff --git a/novelwriter/assets/icons/typicons_dark/typ_arrow-down.svg b/novelwriter/assets/icons/typicons_dark/typ_arrow-down.svg new file mode 100644 index 00000000..356a1edd --- /dev/null +++ b/novelwriter/assets/icons/typicons_dark/typ_arrow-down.svg @@ -0,0 +1,4 @@ + + + + diff --git a/novelwriter/assets/icons/typicons_dark/typ_arrow-right.svg b/novelwriter/assets/icons/typicons_dark/typ_arrow-right.svg new file mode 100644 index 00000000..288465d0 --- /dev/null +++ b/novelwriter/assets/icons/typicons_dark/typ_arrow-right.svg @@ -0,0 +1,4 @@ + + + + diff --git a/novelwriter/assets/icons/typicons_light/icons.conf b/novelwriter/assets/icons/typicons_light/icons.conf index 5720ba58..9763734e 100644 --- a/novelwriter/assets/icons/typicons_light/icons.conf +++ b/novelwriter/assets/icons/typicons_light/icons.conf @@ -100,6 +100,8 @@ status_time = typ_stopwatch-grey.svg sticky-off = typ_pin-outline.svg sticky-on = typ_pin.svg unchecked = mixed_input-unchecked.svg +unfold-hide = typ_arrow-right.svg +unfold-show = typ_arrow-down.svg up = typ_chevron-up.svg view = typ_eye.svg view_build = typ_export-grey.svg diff --git a/novelwriter/assets/icons/typicons_light/typ_arrow-down.svg b/novelwriter/assets/icons/typicons_light/typ_arrow-down.svg new file mode 100644 index 00000000..f3c1a221 --- /dev/null +++ b/novelwriter/assets/icons/typicons_light/typ_arrow-down.svg @@ -0,0 +1,4 @@ + + + + diff --git a/novelwriter/assets/icons/typicons_light/typ_arrow-right.svg b/novelwriter/assets/icons/typicons_light/typ_arrow-right.svg new file mode 100644 index 00000000..c7df8491 --- /dev/null +++ b/novelwriter/assets/icons/typicons_light/typ_arrow-right.svg @@ -0,0 +1,4 @@ + + + + diff --git a/novelwriter/constants.py b/novelwriter/constants.py index 0ba85a57..130378bd 100644 --- a/novelwriter/constants.py +++ b/novelwriter/constants.py @@ -23,8 +23,6 @@ along with this program. If not, see . """ from __future__ import annotations -import re - from PyQt5.QtCore import QCoreApplication, QT_TRANSLATE_NOOP from novelwriter.enum import nwBuildFmt, nwItemClass, nwItemLayout, nwOutline @@ -70,9 +68,6 @@ class nwRegEx: FMT_SC = r"(?i)(? None: + def __init__(self, project: NWProject, build: BuildSettings, doCount: bool = False) -> None: self._project = project self._build = build self._queue = [] self._error = None self._cache = None + self._count = doCount return ## @@ -314,11 +315,15 @@ class NWBuildDocument: bldObj.addRootHeading(tHandle) if convert: bldObj.doConvert() + if self._count: + bldObj.countStats() elif tItem.isFileType(): bldObj.setText(tHandle) bldObj.doPreProcessing() bldObj.tokenizeText() bldObj.doHeaders() + if self._count: + bldObj.countStats() if convert: bldObj.doConvert() else: diff --git a/novelwriter/core/index.py b/novelwriter/core/index.py index d5f035aa..6f275c77 100644 --- a/novelwriter/core/index.py +++ b/novelwriter/core/index.py @@ -3,7 +3,6 @@ novelWriter – Project Index =========================== File History: -Created: 2019-04-22 [0.0.1] countWords Created: 2019-05-27 [0.1.4] NWIndex Created: 2022-05-28 [2.0rc1] IndexItem Created: 2022-05-28 [2.0rc1] IndexHeading @@ -40,7 +39,8 @@ from novelwriter import SHARED from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout from novelwriter.error import logException from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode -from novelwriter.constants import nwFiles, nwKeyWords, nwRegEx, nwUnicode, nwHeaders +from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders +from novelwriter.text.counting import standardCounter if TYPE_CHECKING: # pragma: no cover from novelwriter.core.item import NWItem @@ -266,7 +266,7 @@ class NWIndex: self._itemIndex.add(tHandle, tItem) # Run word counter for the whole text - cC, wC, pC = countWords(text) + cC, wC, pC = standardCounter(text) tItem.setCharCount(cC) tItem.setWordCount(wC) tItem.setParaCount(pC) @@ -400,7 +400,7 @@ class NWIndex: def _indexWordCounts(self, tHandle: str, text: str, sTitle: str) -> None: """Count text stats and save the counts to the index.""" - cC, wC, pC = countWords(text) + cC, wC, pC = standardCounter(text) self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC) return @@ -1315,86 +1315,3 @@ def processComment(text: str) -> tuple[nwComment, str, int]: if content and (clean := classifier.strip().lower()) in CLASSIFIERS: return CLASSIFIERS[clean], content.strip(), text.find(":") + 1 return nwComment.PLAIN, check, 0 - - -def countWords(text: str) -> tuple[int, int, int]: - """Count words in a piece of text, skipping special syntax and - comments. - """ - charCount = 0 - wordCount = 0 - paraCount = 0 - prevEmpty = True - - if not isinstance(text, str): - return charCount, wordCount, paraCount - - # We need to treat dashes as word separators for counting words. - # The check+replace approach is much faster than direct replace for - # large texts, and a bit slower for small texts, but in the latter - # case it doesn't really matter. - if nwUnicode.U_ENDASH in text: - text = text.replace(nwUnicode.U_ENDASH, " ") - if nwUnicode.U_EMDASH in text: - text = text.replace(nwUnicode.U_EMDASH, " ") - - # Strip shortcodes - if "[" in text: - text = nwRegEx.RX_SC.sub("", text) - - for line in text.splitlines(): - - countPara = True - - if not line: - prevEmpty = True - continue - - if line[0] == "@" or line[0] == "%": - continue - - if line[0] == "[": - check = line.lower() - if check.startswith(("[newpage]", "[new page]", "[vspace]")): - continue - elif check.startswith("[vspace:") and line.endswith("]"): - continue - - elif line[0] == "#": - if line[:5] == "#### ": - line = line[5:] - countPara = False - elif line[:4] == "### ": - line = line[4:] - countPara = False - elif line[:3] == "## ": - line = line[3:] - countPara = False - elif line[:2] == "# ": - line = line[2:] - countPara = False - elif line[:3] == "#! ": - line = line[3:] - countPara = False - elif line[:4] == "##! ": - line = line[4:] - countPara = False - - elif line[0] == ">" or line[-1] == "<": - if line[:2] == ">>": - line = line[2:].lstrip(" ") - elif line[:1] == ">": - line = line[1:].lstrip(" ") - if line[-2:] == "<<": - line = line[:-2].rstrip(" ") - elif line[-1:] == "<": - line = line[:-1].rstrip(" ") - - wordCount += len(line.split()) - charCount += len(line) - if countPara and prevEmpty: - paraCount += 1 - - prevEmpty = not countPara - - return charCount, wordCount, paraCount diff --git a/novelwriter/core/tohtml.py b/novelwriter/core/tohtml.py index fad2bc3b..8e1a67ca 100644 --- a/novelwriter/core/tohtml.py +++ b/novelwriter/core/tohtml.py @@ -368,13 +368,12 @@ class ToHtml(Tokenizer): def getStyleSheet(self) -> list[str]: """Generate a stylesheet for the current settings.""" - styles = [] if not self._cssStyles: - return styles + return [] mScale = self._lineHeight/1.15 - textAlign = "justify" if self._doJustify else "left" + styles = [] styles.append("body {{font-family: '{0:s}'; font-size: {1:d}pt;}}".format( self._textFont, self._textSize )) @@ -384,7 +383,7 @@ class ToHtml(Tokenizer): "margin-top: {2:.2f}em; margin-bottom: {3:.2f}em;" "}}" ).format( - textAlign, + "justify" if self._doJustify else "left", round(100 * self._lineHeight), mScale * self._marginText[0], mScale * self._marginText[1], @@ -449,6 +448,7 @@ class ToHtml(Tokenizer): )) styles.append("a {color: rgb(66, 113, 174);}") + styles.append("mark {background: rgb(255, 255, 166);}") styles.append(".tags {color: rgb(245, 135, 31); font-weight: bold;}") styles.append(".break {text-align: left;}") styles.append(".synopsis {font-style: italic;}") diff --git a/novelwriter/core/tokenizer.py b/novelwriter/core/tokenizer.py index dd9ff0a6..1bd3d794 100644 --- a/novelwriter/core/tokenizer.py +++ b/novelwriter/core/tokenizer.py @@ -110,6 +110,9 @@ class Tokenizer(ABC): A_IND_L = 0x0100 # Left indentation A_IND_R = 0x0200 # Right indentation + # Lookups + L_HEADINGS = [T_TITLE, T_UNNUM, T_HEAD1, T_HEAD2, T_HEAD3, T_HEAD4] + def __init__(self, project: NWProject) -> None: self._project = project @@ -118,6 +121,7 @@ class Tokenizer(ABC): self._text = "" # The raw text to be tokenized self._nwItem = None # The NWItem currently being processed self._result = "" # The result of the last document + self._counts = {} # Counter data self._keepMarkdown = False # Whether to keep the markdown text self._allMarkdown = [] # The result novelWriter markdown of all documents @@ -210,6 +214,11 @@ class Tokenizer(ABC): """The combined novelWriter Markdown text.""" return self._allMarkdown + @property + def textStats(self) -> dict[str, int]: + """The collected stats about the text.""" + return self._counts + @property def errData(self) -> list: """The error data.""" @@ -751,6 +760,115 @@ class Tokenizer(ABC): return True + def countStats(self) -> dict[str, int]: + """Count stats on the tokenized text.""" + titleCount = self._counts.get("titleCount", 0) + paragraphCount = self._counts.get("paragraphCount", 0) + + allWords = self._counts.get("allWords", 0) + textWords = self._counts.get("textWords", 0) + titleWords = self._counts.get("titleWords", 0) + + allChars = self._counts.get("allChars", 0) + textChars = self._counts.get("textChars", 0) + titleChars = self._counts.get("titleChars", 0) + + allWordChars = self._counts.get("allWordChars", 0) + textWordChars = self._counts.get("textWordChars", 0) + titleWordChars = self._counts.get("titleWordChars", 0) + + para = [] + for tType, _, tText, _, _ in self._tokens: + tText = tText.replace(nwUnicode.U_ENDASH, " ") + tText = tText.replace(nwUnicode.U_EMDASH, " ") + + tWords = tText.split() + nWords = len(tWords) + nChars = len(tText) + nWChars = len("".join(tWords)) + + if tType == self.T_EMPTY: + if len(para) > 0: + tTemp = "\n".join(para) + tPWords = tTemp.split() + nPWords = len(tPWords) + nPChars = len(tTemp) + nPWChars = len("".join(tPWords)) + + paragraphCount += 1 + allWords += nPWords + textWords += nPWords + allChars += nPChars + textChars += nPChars + allWordChars += nPWChars + textWordChars += nPWChars + para = [] + + elif tType in self.L_HEADINGS: + titleCount += 1 + allWords += nWords + titleWords += nWords + allChars += nChars + allWordChars += nWChars + titleChars += nChars + titleWordChars += nWChars + + elif tType == self.T_SEP: + allWords += nWords + allChars += nChars + allWordChars += nWChars + + elif tType == self.T_TEXT: + para.append(tText.rstrip()) + + elif tType == self.T_SYNOPSIS and self._doSynopsis: + text = "{0}: {1}".format(self._localLookup("Synopsis"), tText) + words = text.split() + allWords += len(words) + allChars += len(text) + allWordChars += len("".join(words)) + + elif tType == self.T_SHORT and self._doSynopsis: + text = "{0}: {1}".format(self._localLookup("Short Description"), tText) + words = text.split() + allWords += len(words) + allChars += len(text) + allWordChars += len("".join(words)) + + elif tType == self.T_COMMENT and self._doComments: + text = "{0}: {1}".format(self._localLookup("Comment"), tText) + words = text.split() + allWords += len(words) + allChars += len(text) + allWordChars += len("".join(words)) + + elif tType == self.T_KEYWORD and self._doKeywords: + valid, bits, _ = self._project.index.scanThis("@"+tText) + if valid and bits: + key = self._localLookup(nwLabels.KEY_NAME[bits[0]]) + text = "{0}: {1}".format(key, ", ".join(bits[1:])) + words = text.split() + allWords += len(words) + allChars += len(text) + allWordChars += len("".join(words)) + + self._counts["titleCount"] = titleCount + self._counts["paragraphCount"] = paragraphCount + + self._counts["allWords"] = allWords + self._counts["textWords"] = textWords + self._counts["titleWords"] = titleWords + + self._counts["allChars"] = allChars + self._counts["textChars"] = textChars + self._counts["titleChars"] = titleChars + + self._counts["allWordChars"] = allWordChars + self._counts["textWordChars"] = textWordChars + self._counts["titleWordChars"] = titleWordChars + + return {} + def saveRawMarkdown(self, path: str | Path) -> None: """Save the raw text to a plain text file.""" with open(path, mode="w", encoding="utf-8") as outFile: diff --git a/novelwriter/gui/doceditor.py b/novelwriter/gui/doceditor.py index 33f6672b..e715ca8f 100644 --- a/novelwriter/gui/doceditor.py +++ b/novelwriter/gui/doceditor.py @@ -56,9 +56,9 @@ from novelwriter import CONFIG, SHARED from novelwriter.enum import nwDocAction, nwDocInsert, nwDocMode, nwItemClass, nwTrinary from novelwriter.common import minmax, transferCase from novelwriter.constants import nwKeyWords, nwLabels, nwShortcode, nwUnicode, trConst -from novelwriter.core.index import countWords from novelwriter.tools.lipsum import GuiLipsum from novelwriter.core.document import NWDocument +from novelwriter.text.counting import standardCounter from novelwriter.gui.dochighlight import GuiDocHighlighter from novelwriter.gui.editordocument import GuiTextDocument from novelwriter.extensions.eventfilters import WheelEventFilter @@ -462,7 +462,7 @@ class GuiDocEditor(QPlainTextEdit): return False docText = self.getText() - cC, wC, pC = countWords(docText) + cC, wC, pC = standardCounter(docText) self._updateDocCounts(cC, wC, pC) self.saveCursorPosition() @@ -2226,7 +2226,7 @@ class BackgroundWordCounter(QRunnable): else: text = self._docEditor.getText() - cC, wC, pC = countWords(text) + cC, wC, pC = standardCounter(text) self.signals.countsReady.emit(cC, wC, pC) self._isRunning = False diff --git a/novelwriter/gui/theme.py b/novelwriter/gui/theme.py index fb8ca63c..eb86ba7a 100644 --- a/novelwriter/gui/theme.py +++ b/novelwriter/gui/theme.py @@ -462,6 +462,7 @@ class GuiIcons: # Switches "sticky-on", "sticky-off", "bullet-on", "bullet-off", + "unfold-show", "unfold-hide", # Decorations "deco_doc_h0", "deco_doc_h1", "deco_doc_h2", "deco_doc_h3", "deco_doc_h4", "deco_doc_more", @@ -472,6 +473,7 @@ class GuiIcons: TOGGLE_ICON_KEYS: dict[str, tuple[str, str]] = { "sticky": ("sticky-on", "sticky-off"), "bullet": ("bullet-on", "bullet-off"), + "unfold": ("unfold-show", "unfold-hide"), } IMAGE_MAP: dict[str, tuple[str, str]] = { diff --git a/novelwriter/text/__init__.py b/novelwriter/text/__init__.py new file mode 100644 index 00000000..69bc7fa4 --- /dev/null +++ b/novelwriter/text/__init__.py @@ -0,0 +1,3 @@ +""" +novelWriter – Text Analysis Tools +""" diff --git a/novelwriter/text/counting.py b/novelwriter/text/counting.py new file mode 100644 index 00000000..b381adea --- /dev/null +++ b/novelwriter/text/counting.py @@ -0,0 +1,135 @@ +""" +novelWriter – Text Counting Functions +===================================== + +File History: +Created: 2019-04-22 [0.0.1] standardCounter +Rewritten: 2024-02-27 [2.4b1] preProcessText, standardCounter +Created: 2024-02-27 [2.4b1] bodyTextCounter + +This file is a part of novelWriter +Copyright 2018–2024, Veronica Berglyd Olsen + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" +from __future__ import annotations + +import re + +from novelwriter.constants import nwRegEx, nwUnicode + +RX_SC = re.compile(nwRegEx.FMT_SC) +RX_LO = re.compile(r"(?i)(?{1,2}\s*|\s*<{1,2}$") + + +def preProcessText(text: str, keepHeaders: bool = True) -> list[str]: + """Strip formatting codes from the text and split into lines.""" + if not isinstance(text, str): + return [] + + # We need to treat dashes as word separators for counting words. + # The check+replace approach is much faster than direct replace for + # large texts, and a bit slower for small texts, but in the latter + # case it doesn't really matter. + if nwUnicode.U_ENDASH in text: + text = text.replace(nwUnicode.U_ENDASH, " ") + if nwUnicode.U_EMDASH in text: + text = text.replace(nwUnicode.U_EMDASH, " ") + + ignore = "%@" if keepHeaders else "%@#" + + result = [] + for line in text.splitlines(): + line = line.rstrip() + if line: + if line[0] in ignore: + continue + if line[0] == ">": + line = line.lstrip(">").lstrip(" ") + if line[-1] == "<": + line = line.rstrip("<").rstrip(" ") + if "[" in line: + # Strip shortcodes and special formatting + # RegEx is slow, so we do this only when necessary + line = RX_SC.sub("", line) + line = RX_LO.sub("", line) + + result.append(line) + + return result + + +def standardCounter(text: str) -> tuple[int, int, int]: + """A counter that counts paragraphs, words and characters. + This is the standard counter that includes headers in the word and + character counts. + """ + cCount = 0 + wCount = 0 + pCount = 0 + prevEmpty = True + + for line in preProcessText(text): + + countPara = True + if not line: + prevEmpty = True + continue + + if line[0] == "#": + if line[:5] == "#### ": + line = line[5:] + countPara = False + elif line[:4] == "### ": + line = line[4:] + countPara = False + elif line[:3] == "## ": + line = line[3:] + countPara = False + elif line[:2] == "# ": + line = line[2:] + countPara = False + elif line[:3] == "#! ": + line = line[3:] + countPara = False + elif line[:4] == "##! ": + line = line[4:] + countPara = False + + wCount += len(line.split()) + cCount += len(line) + if countPara and prevEmpty: + pCount += 1 + + prevEmpty = not countPara + + return cCount, wCount, pCount + + +def bodyTextCounter(text: str | list[str]) -> tuple[int, int, int]: + """A counter that counts body text words, characters, and characters + without white spaces. + """ + wCount = 0 + cCount = 0 + sCount = 0 + + for line in preProcessText(text, keepHeaders=False): + words = line.split() + wCount += len(words) + cCount += len(line) + sCount += len("".join(words)) + + return wCount, cCount, sCount diff --git a/novelwriter/tools/manuscript.py b/novelwriter/tools/manuscript.py index 8d95da6a..65926c1d 100644 --- a/novelwriter/tools/manuscript.py +++ b/novelwriter/tools/manuscript.py @@ -33,9 +33,10 @@ from datetime import datetime from PyQt5.QtGui import QCloseEvent, QColor, QCursor, QFont, QPalette, QResizeEvent from PyQt5.QtCore import QSize, QTimer, Qt, pyqtSlot from PyQt5.QtWidgets import ( - QAbstractItemView, QDialog, QGridLayout, QHBoxLayout, QLabel, QListWidget, - QListWidgetItem, QPushButton, QSplitter, QTextBrowser, QToolButton, - QTreeWidget, QTreeWidgetItem, QVBoxLayout, QWidget, qApp + QAbstractItemView, QDialog, QFormLayout, QGridLayout, QHBoxLayout, QLabel, + QListWidget, QListWidgetItem, QPushButton, QSizePolicy, QSplitter, + QStackedWidget, QTextBrowser, QToolButton, QTreeWidget, QTreeWidgetItem, + QVBoxLayout, QWidget, qApp ) from PyQt5.QtPrintSupport import QPrintPreviewDialog, QPrinter @@ -185,6 +186,15 @@ class GuiManuscript(QDialog): # ============ self.docPreview = _PreviewWidget(self) + self.docStats = _StatsWidget(self) + + self.docBox = QVBoxLayout() + self.docBox.addWidget(self.docPreview, 1) + self.docBox.addWidget(self.docStats, 0) + self.docBox.setContentsMargins(0, 0, 0, 0) + + self.docWdiget = QWidget(self) + self.docWdiget.setLayout(self.docBox) self.controlBox = QVBoxLayout() self.controlBox.addLayout(self.listToolBox, 0) @@ -192,12 +202,12 @@ class GuiManuscript(QDialog): self.controlBox.addLayout(self.processBox, 0) self.controlBox.setContentsMargins(0, 0, 0, 0) - self.optsWidget = QWidget() + self.optsWidget = QWidget(self) self.optsWidget.setLayout(self.controlBox) self.mainSplit = QSplitter() self.mainSplit.addWidget(self.optsWidget) - self.mainSplit.addWidget(self.docPreview) + self.mainSplit.addWidget(self.docWdiget) self.mainSplit.setCollapsible(0, False) self.mainSplit.setCollapsible(1, False) self.mainSplit.setStretchFactor(0, 0) @@ -245,7 +255,7 @@ class GuiManuscript(QDialog): if isinstance(build, BuildSettings): self._updatePreview(data, build) except Exception: - logger.error("Failed to save build cache") + logger.error("Failed to load build cache") logException() return @@ -327,7 +337,7 @@ class GuiManuscript(QDialog): if build is None: return - docBuild = NWBuildDocument(SHARED.project, build) + docBuild = NWBuildDocument(SHARED.project, build, doCount=True) docBuild.queueAll() self.docPreview.beginNewBuild(len(docBuild)) @@ -340,6 +350,7 @@ class GuiManuscript(QDialog): result = { "uuid": build.buildID, "time": int(time()), + "stats": buildObj.textStats, "styles": buildObj.getStyleSheet(), "html": buildObj.fullHTML, } @@ -395,6 +406,7 @@ class GuiManuscript(QDialog): self.docPreview.setJustify( build.getBool("format.justifyText") ) + self.docStats.updateStats(data.get("stats", {})) return def _getSelectedBuild(self) -> BuildSettings | None: @@ -756,20 +768,13 @@ class _PreviewWidget(QTextBrowser): self.buildProgress.setCentreText(self.tr("Processing ...")) qApp.processEvents() - styles = "\n".join(data.get("styles", [ - "h1, h2 {color: rgb(66, 113, 174);}", - "h3, h4 {color: rgb(50, 50, 50);}", - "a {color: rgb(66, 113, 174);}", - ".tags {color: rgb(245, 135, 31); font-weight: bold;}", - ])) + styles = "\n".join(data.get("styles", [])) self.document().setDefaultStyleSheet(styles) html = "".join(data.get("html", [])) html = html.replace("\t", "!!tab!!") html = html.replace("", "") html = html.replace("", "") - html = html.replace("", "") - html = html.replace("", "") self.setHtml(html) qApp.processEvents() while self.find("!!tab!!"): @@ -861,3 +866,195 @@ class _PreviewWidget(QTextBrowser): return # END Class _PreviewWidget + + +class _StatsWidget(QWidget): + + def __init__(self, parent: QWidget) -> None: + super().__init__(parent=parent) + + font = self.font() + font.setPointSizeF(0.9*SHARED.theme.fontPointSize) + self.setFont(font) + + self.minWidget = QWidget(self) + self.maxWidget = QWidget(self) + + fPx = int(0.6*SHARED.theme.fontPixelSize) + toggleIcon = SHARED.theme.getToggleIcon("unfold", (fPx, fPx)) + + fadeCol = self.palette().text().color() + buttonStyle = ( + "QToolButton {{padding: 0; border: none; background: transparent;}} " + "QToolButton:hover {{border: none; background: rgba({0},{1},{2},0.2);}}" + ).format(fadeCol.red(), fadeCol.green(), fadeCol.blue()) + + self.toggleButton = QToolButton(self) + self.toggleButton.setCheckable(True) + self.toggleButton.setIcon(toggleIcon) + self.toggleButton.setStyleSheet(buttonStyle) + self.toggleButton.toggled.connect(self._toggleView) + + self._buildMinimal() + self._buildMaximal() + + self.mainStack = QStackedWidget(self) + self.mainStack.addWidget(self.minWidget) + self.mainStack.addWidget(self.maxWidget) + + self.outerBox = QHBoxLayout() + self.outerBox.addWidget(self.toggleButton, 0, Qt.AlignmentFlag.AlignTop) + self.outerBox.addWidget(self.mainStack, 1, Qt.AlignmentFlag.AlignTop) + self.outerBox.setContentsMargins(0, 0, 0, 0) + + self.setLayout(self.outerBox) + + self._toggleView(False) + + return + + def updateStats(self, data: dict[str, int]) -> None: + """Update the stats values from a Tokenizer stats dict.""" + # Minimal + self.minWordCount.setText("{0:n}".format(data.get("allWords", 0))) + self.minCharCount.setText("{0:n}".format(data.get("allChars", 0))) + + # Maximal + self.maxTotalWords.setText("{0:n}".format(data.get("allWords", 0))) + self.maxHeaderWords.setText("{0:n}".format(data.get("titleWords", 0))) + self.maxTextWords.setText("{0:n}".format(data.get("textWords", 0))) + self.maxTitleCount.setText("{0:n}".format(data.get("titleCount", 0))) + self.maxParCount.setText("{0:n}".format(data.get("paragraphCount", 0))) + + self.maxTotalChars.setText("{0:n}".format(data.get("allChars", 0))) + self.maxHeaderChars.setText("{0:n}".format(data.get("titleChars", 0))) + self.maxTextChars.setText("{0:n}".format(data.get("textChars", 0))) + + self.maxTotalWordChars.setText("{0:n}".format(data.get("allWordChars", 0))) + self.maxHeaderWordChars.setText("{0:n}".format(data.get("titleWordChars", 0))) + self.maxTextWordChars.setText("{0:n}".format(data.get("textWordChars", 0))) + + return + + ## + # Private Slots + ## + + @pyqtSlot(bool) + def _toggleView(self, state: bool) -> None: + """Toggle minimal or maximal view.""" + ignored = QSizePolicy.Policy.Ignored + expanded = QSizePolicy.Policy.Expanding + if state: + self.mainStack.setCurrentWidget(self.maxWidget) + self.maxWidget.setSizePolicy(expanded, expanded) + self.minWidget.setSizePolicy(ignored, ignored) + else: + self.mainStack.setCurrentWidget(self.minWidget) + self.maxWidget.setSizePolicy(ignored, ignored) + self.minWidget.setSizePolicy(expanded, expanded) + self.maxWidget.adjustSize() + self.minWidget.adjustSize() + self.mainStack.adjustSize() + self.adjustSize() + return + + ## + # Internal Functions + ## + + def _buildMinimal(self) -> None: + """Build the minimal stats page.""" + mPx = CONFIG.pxInt(8) + + self.lblWordCount = QLabel(self.tr("Words:")) + self.minWordCount = QLabel(self) + + self.lblCharCount = QLabel(self.tr("Characters:")) + self.minCharCount = QLabel(self) + + # Assemble + self.minLayout = QHBoxLayout() + self.minLayout.addWidget(self.lblWordCount) + self.minLayout.addWidget(self.minWordCount) + self.minLayout.addSpacing(mPx) + self.minLayout.addWidget(self.lblCharCount) + self.minLayout.addWidget(self.minCharCount) + self.minLayout.addStretch(1) + self.minLayout.setSpacing(mPx) + self.minLayout.setContentsMargins(0, 0, 0, 0) + + self.minWidget.setLayout(self.minLayout) + + return + + def _buildMaximal(self) -> None: + """Build the maximal stats page.""" + hPx = CONFIG.pxInt(12) + vPx = CONFIG.pxInt(4) + + alignRight = Qt.AlignmentFlag.AlignRight + + # Left Column + self.maxTotalWords = QLabel(self) + self.maxHeaderWords = QLabel(self) + self.maxTextWords = QLabel(self) + self.maxTitleCount = QLabel(self) + self.maxParCount = QLabel(self) + + self.maxTotalWords.setAlignment(alignRight) + self.maxHeaderWords.setAlignment(alignRight) + self.maxTextWords.setAlignment(alignRight) + self.maxTitleCount.setAlignment(alignRight) + self.maxParCount.setAlignment(alignRight) + + self.leftForm = QFormLayout() + self.leftForm.addRow(self.tr("Words:"), self.maxTotalWords) + self.leftForm.addRow(self.tr("Header Words:"), self.maxHeaderWords) + self.leftForm.addRow(self.tr("Body Text Words:"), self.maxTextWords) + self.leftForm.addRow("", QLabel(self)) + self.leftForm.addRow(self.tr("Headers:"), self.maxTitleCount) + self.leftForm.addRow(self.tr("Paragraphs:"), self.maxParCount) + self.leftForm.setHorizontalSpacing(hPx) + self.leftForm.setVerticalSpacing(vPx) + + # Right Column + self.maxTotalChars = QLabel(self) + self.maxHeaderChars = QLabel(self) + self.maxTextChars = QLabel(self) + + self.maxTotalWordChars = QLabel(self) + self.maxHeaderWordChars = QLabel(self) + self.maxTextWordChars = QLabel(self) + + self.maxTotalChars.setAlignment(alignRight) + self.maxHeaderChars.setAlignment(alignRight) + self.maxTextChars.setAlignment(alignRight) + + self.maxTotalWordChars.setAlignment(alignRight) + self.maxHeaderWordChars.setAlignment(alignRight) + self.maxTextWordChars.setAlignment(alignRight) + + self.rightForm = QFormLayout() + self.rightForm.addRow(self.tr("Characters:"), self.maxTotalChars) + self.rightForm.addRow(self.tr("Header Characters:"), self.maxHeaderChars) + self.rightForm.addRow(self.tr("Body Text Characters:"), self.maxTextChars) + self.rightForm.addRow(self.tr("Characters, No Spaces:"), self.maxTotalWordChars) + self.rightForm.addRow(self.tr("Header Characters, No Spaces:"), self.maxHeaderWordChars) + self.rightForm.addRow(self.tr("Body Text Characters, No Spaces:"), self.maxTextWordChars) + self.rightForm.setHorizontalSpacing(hPx) + self.rightForm.setVerticalSpacing(vPx) + + # Assemble + self.maxLayout = QHBoxLayout() + self.maxLayout.addLayout(self.leftForm) + self.maxLayout.addLayout(self.rightForm) + self.maxLayout.addStretch(1) + self.maxLayout.setSpacing(CONFIG.pxInt(32)) + self.maxLayout.setContentsMargins(0, 0, 0, 0) + + self.maxWidget.setLayout(self.maxLayout) + + return + +# END Class _StatsWidget diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..65bec68e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +flake8 +flake8-pep585 diff --git a/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.htm b/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.htm index f3463792..f0aa4a92 100644 --- a/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.htm +++ b/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.htm @@ -14,6 +14,7 @@ h4 {color: rgb(50, 50, 50); page-break-after: avoid; margin-top: 0.76em; margin- .title {font-size: 2.5em; margin-top: 1.30em; margin-bottom: 0.65em;} .sep, .skip {text-align: center; margin-top: 1.30em; margin-bottom: 1.30em;} a {color: rgb(66, 113, 174);} +mark {background: rgb(255, 255, 166);} .tags {color: rgb(245, 135, 31); font-weight: bold;} .break {text-align: left;} .synopsis {font-style: italic;} diff --git a/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.json b/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.json index 50ae4237..6ff35e85 100644 --- a/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.json +++ b/tests/reference/mBuildDocBuild_HTML5_Lorem_Ipsum.json @@ -2,8 +2,8 @@ "meta": { "projectName": "Lorem Ipsum", "novelAuthor": "lipsum.com", - "buildTime": 1706648755, - "buildTimeStr": "2024-01-30 22:05:55" + "buildTime": 1709024252, + "buildTimeStr": "2024-02-27 09:57:32" }, "text": { "css": [ @@ -16,6 +16,7 @@ ".title {font-size: 2.5em; margin-top: 1.30em; margin-bottom: 0.65em;}", ".sep, .skip {text-align: center; margin-top: 1.30em; margin-bottom: 1.30em;}", "a {color: rgb(66, 113, 174);}", + "mark {background: rgb(255, 255, 166);}", ".tags {color: rgb(245, 135, 31); font-weight: bold;}", ".break {text-align: left;}", ".synopsis {font-style: italic;}", diff --git a/tests/test_core/test_core_docbuild.py b/tests/test_core/test_core_docbuild.py index b70269a8..991b378f 100644 --- a/tests/test_core/test_core_docbuild.py +++ b/tests/test_core/test_core_docbuild.py @@ -83,7 +83,7 @@ def testCoreDocBuild_OpenDocument(monkeypatch, mockGUI, prjLipsum, fncPath, tstP build = BuildSettings() build.unpack(BUILD_CONF) - docBuild = NWBuildDocument(project, build) + docBuild = NWBuildDocument(project, build, doCount=True) docBuild.queueAll() assert len(docBuild) == 21 diff --git a/tests/test_core/test_core_index.py b/tests/test_core/test_core_index.py index f225d0f2..b8daa6c6 100644 --- a/tests/test_core/test_core_index.py +++ b/tests/test_core/test_core_index.py @@ -31,7 +31,7 @@ from tools import C, buildTestProject, cmpFiles, writeFile from novelwriter.enum import nwComment, nwItemClass, nwItemLayout from novelwriter.constants import nwFiles -from novelwriter.core.index import IndexItem, NWIndex, countWords, TagsIndex, processComment +from novelwriter.core.index import IndexItem, NWIndex, TagsIndex, processComment from novelwriter.core.project import NWProject @@ -1319,105 +1319,3 @@ def testCoreIndex_processComment(): assert processComment("% \t SHORT : Hi:You") == (nwComment.SHORT, "Hi:You", 13) # END Test testCoreIndex_processComment - - -@pytest.mark.core -def testCoreIndex_countWords(): - """Test the word counter and the exclusion filers.""" - # Non-Text - assert countWords(None) == (0, 0, 0) # type: ignore - assert countWords(1234) == (0, 0, 0) # type: ignore - - # General Text - cC, wC, pC = countWords(( - "# Heading One\n" - "## Heading Two\n" - "### Heading Three\n" - "#### Heading Four\n\n" - "@tag: value\n\n" - "% A comment that should not be counted.\n\n" - "The first paragraph.\n\n" - "The second paragraph.\n\n\n" - "The third paragraph.\n\n" - "Dashes\u2013and even longer\u2014dashes." - )) - assert cC == 138 - assert wC == 22 - assert pC == 4 - - # Text Alignment - cC, wC, pC = countWords(( - "# Title\n\n" - "Left aligned<<\n\n" - "Left aligned <<\n\n" - "Right indent<\n\n" - "Right indent <\n\n" - )) - assert cC == 53 - assert wC == 9 - assert pC == 4 - - cC, wC, pC = countWords(( - "# Title\n\n" - ">>Right aligned\n\n" - ">> Right aligned\n\n" - ">Left indent\n\n" - "> Left indent\n\n" - )) - assert cC == 53 - assert wC == 9 - assert pC == 4 - - cC, wC, pC = countWords(( - "# Title\n\n" - ">>Centre aligned<<\n\n" - ">> Centre aligned <<\n\n" - ">Double indent<\n\n" - "> Double indent <\n\n" - )) - assert cC == 59 - assert wC == 9 - assert pC == 4 - - # Formatting Codes, Upper Case (Old Implementation) - cC, wC, pC = countWords(( - "Some text\n\n" - "[NEWPAGE]\n\n" - "more text\n\n" - "[NEW PAGE]]\n\n" - "even more text\n\n" - "[VSPACE]\n\n" - "and some final text\n\n" - "[VSPACE:4]\n\n" - "THE END\n\n" - )) - assert cC == 58 - assert wC == 13 - assert pC == 5 - - # Formatting Codes, Lower Case (Current Implementation) - cC, wC, pC = countWords(( - "Some text\n\n" - "[newpage]\n\n" - "more text\n\n" - "[new page]]\n\n" - "even more text\n\n" - "[vspace]\n\n" - "and some final text\n\n" - "[vspace:4]\n\n" - "THE END\n\n" - )) - assert cC == 58 - assert wC == 13 - assert pC == 5 - - # Check ShortCodes - cC, wC, pC = countWords(( - "Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n" - "Text with [b][i] nested [/i] emphasis [/b] in it.\n\n" - )) - assert cC == 78 - assert wC == 14 - assert pC == 2 - -# END Test testCoreIndex_countWords diff --git a/tests/test_core/test_core_tokenizer.py b/tests/test_core/test_core_tokenizer.py index 5c492a7f..f722c89f 100644 --- a/tests/test_core/test_core_tokenizer.py +++ b/tests/test_core/test_core_tokenizer.py @@ -1253,6 +1253,225 @@ def testCoreToken_ProcessHeaders(mockGUI): # END Test testCoreToken_ProcessHeaders +@pytest.mark.core +def testCoreToken_CountStats(mockGUI, ipsumText): + """Test stats counter of the Tokenizer class.""" + project = NWProject() + project.data.setLanguage("en") + project._loadProjectLocalisation() + tokens = BareTokenizer(project) + tokens._isNone = False + tokens._isNote = False + tokens._isNovel = True + + # Short Text + # ========== + + # Header wo/Format + tokens._text = "## A Chapter Title\n\n" + tokens._counts = {} + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert tokens._tokens[0][2] == "A Chapter Title" + assert tokens.textStats == { + "titleCount": 1, "paragraphCount": 0, + "allWords": 3, "textWords": 0, "titleWords": 3, + "allChars": 15, "textChars": 0, "titleChars": 15, + "allWordChars": 13, "textWordChars": 0, "titleWordChars": 13 + } + + # Header w/Format + tokens._text = "## A Chapter Title\n\n" + tokens._counts = {} + tokens.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}") + tokens._hFormatter.resetAll() + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert tokens._tokens[0][2] == "C 1: A Chapter Title" + assert tokens.textStats == { + "titleCount": 1, "paragraphCount": 0, + "allWords": 5, "textWords": 0, "titleWords": 5, + "allChars": 20, "textChars": 0, "titleChars": 20, + "allWordChars": 16, "textWordChars": 0, "titleWordChars": 16 + } + + # Two Paragraphs + # First break should be counted, the double breaks not. + tokens._text = "Some text\non two lines.\n\nWith a second paragraph.\n\n" + tokens._counts = {} + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert tokens.textStats == { + "titleCount": 0, "paragraphCount": 2, + "allWords": 9, "textWords": 9, "titleWords": 0, + "allChars": 47, "textChars": 47, "titleChars": 0, + "allWordChars": 40, "textWordChars": 40, "titleWordChars": 0 + } + + # Two Scenes w/Separator + tokens._text = "## Chapter\n\n### Scene\n\nText\n\n### Scene\n\nText" + tokens._counts = {} + tokens.setChapterFormat(nwHeadFmt.TITLE) + tokens.setSceneFormat("* * *", False) + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert [t[2] for t in tokens._tokens] == [ + "Chapter", "", "", "", "Text", "", "* * *", "", "Text", "" + ] + assert tokens.textStats == { + "titleCount": 1, "paragraphCount": 2, + "allWords": 6, "textWords": 2, "titleWords": 1, + "allChars": 20, "textChars": 8, "titleChars": 7, + "allWordChars": 18, "textWordChars": 8, "titleWordChars": 7 + } + + # Scene w/Synopsis + # Synopsis does not count as a paragraph, and counts as "Synopsis: Stuff" + tokens._text = "## Chapter\n\n### Scene\n\n%Synopsis: Stuff\n\nText" + tokens._counts = {} + tokens.setChapterFormat(nwHeadFmt.TITLE) + tokens.setSceneFormat("* * *", False) + tokens.setSynopsis(True) + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert [t[2] for t in tokens._tokens] == [ + "Chapter", "", "", "", "Stuff", "", "Text", "" + ] + assert tokens.textStats == { + "titleCount": 1, "paragraphCount": 1, + "allWords": 4, "textWords": 1, "titleWords": 1, + "allChars": 26, "textChars": 4, "titleChars": 7, + "allWordChars": 25, "textWordChars": 4, "titleWordChars": 7 + } + + # Scene w/Short + # Short does not count as a paragraph, and counts as "Short Description: Stuff" + tokens._text = "## Chapter\n\n### Scene\n\n%Short: Stuff\n\nText" + tokens._counts = {} + tokens.setChapterFormat(nwHeadFmt.TITLE) + tokens.setSceneFormat("* * *", False) + tokens.setSynopsis(True) + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert [t[2] for t in tokens._tokens] == [ + "Chapter", "", "", "", "Stuff", "", "Text", "" + ] + assert tokens.textStats == { + "titleCount": 1, "paragraphCount": 1, + "allWords": 5, "textWords": 1, "titleWords": 1, + "allChars": 35, "textChars": 4, "titleChars": 7, + "allWordChars": 33, "textWordChars": 4, "titleWordChars": 7 + } + + # Scene w/Comment + # Comment does not count as a paragraph, and counts as "Comment: Stuff" + tokens._text = "## Chapter\n\n### Scene\n\n% Stuff\n\nText" + tokens._counts = {} + tokens.setChapterFormat(nwHeadFmt.TITLE) + tokens.setSceneFormat("* * *", False) + tokens.setComments(True) + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert [t[2] for t in tokens._tokens] == [ + "Chapter", "", "", "", "Stuff", "", "Text", "" + ] + assert tokens.textStats == { + "titleCount": 1, "paragraphCount": 1, + "allWords": 4, "textWords": 1, "titleWords": 1, + "allChars": 25, "textChars": 4, "titleChars": 7, + "allWordChars": 24, "textWordChars": 4, "titleWordChars": 7 + } + + # Scene w/Keyword + # Keyword does not count as a paragraph, and counts as "Point of View: Jane" + tokens._text = "## Chapter\n\n### Scene\n\n@pov: Jane\n\nText" + tokens._counts = {} + tokens.setChapterFormat(nwHeadFmt.TITLE) + tokens.setSceneFormat("* * *", False) + tokens.setKeywords(True) + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert [t[2] for t in tokens._tokens] == [ + "Chapter", "", "", "", "pov: Jane", "", "Text", "" + ] + assert tokens.textStats == { + "titleCount": 1, "paragraphCount": 1, + "allWords": 6, "textWords": 1, "titleWords": 1, + "allChars": 30, "textChars": 4, "titleChars": 7, + "allWordChars": 27, "textWordChars": 4, "titleWordChars": 7 + } + + # Long Text + # ========= + tokens._text = ( + "# Act One\n\n" + + "## Chapter\n\n" + + "### Scene\n\n" + "@pov: Jane\n" + "@char: Jane, John\n\n" + "%Synopsis: A scene\n\n" + f"{ipsumText[0]}.\n\n" + + "### Scene\n\n" + "@pov: Jane\n" + "@char: Jane, John\n\n" + "%Synopsis: A scene\n\n" + f"{ipsumText[1]}.\n\n" + + "## Chapter\n\n" + + "### Scene\n\n" + "@pov: Jane\n" + "@char: Jane, John\n\n" + "%Synopsis: A scene\n\n" + f"{ipsumText[2]}.\n\n" + + "### Scene\n\n" + "@pov: Jane\n" + "@char: Jane, John\n\n" + "%Synopsis: A scene\n\n" + f"{ipsumText[3]}.\n\n" + + "## Chapter\n\n" + + "### Scene\n\n" + "@pov: Jane\n" + "@char: Jane, John\n\n" + "%Synopsis: A scene\n\n" + f"{ipsumText[4]}.\n\n" + ) + tokens._counts = {} + + tokens.setTitleFormat(f"T: {nwHeadFmt.TITLE}") + tokens.setChapterFormat(f"C {nwHeadFmt.CH_NUM}: {nwHeadFmt.TITLE}") + tokens.setSceneFormat("* * *", False) + tokens.setSynopsis(True) + tokens.setComments(True) + tokens.setKeywords(True) + + tokens.tokenizeText() + tokens.doHeaders() + tokens.countStats() + assert tokens.textStats == { + "titleCount": 4, "paragraphCount": 5, + "allWords": 596, "textWords": 528, "titleWords": 12, + "allChars": 3859, "textChars": 3513, "titleChars": 46, + "allWordChars": 3289, "textWordChars": 2990, "titleWordChars": 38 + } + +# END Test testCoreToken_CountStats + + @pytest.mark.core def testCoreToken_HeaderCounterAndVisibility(mockGUI): """Test the header counter and visibility of the Tokenizer class. diff --git a/tests/test_gui/test_gui_doceditor.py b/tests/test_gui/test_gui_doceditor.py index 8ce95710..419e4bf9 100644 --- a/tests/test_gui/test_gui_doceditor.py +++ b/tests/test_gui/test_gui_doceditor.py @@ -32,8 +32,8 @@ from PyQt5.QtWidgets import QAction, QMenu, qApp from novelwriter import CONFIG, SHARED from novelwriter.enum import nwDocAction, nwDocInsert, nwItemLayout, nwTrinary, nwWidget from novelwriter.constants import nwKeyWords, nwUnicode -from novelwriter.core.index import countWords from novelwriter.gui.doceditor import GuiDocEditor, GuiDocToolBar +from novelwriter.text.counting import standardCounter from novelwriter.dialogs.editlabel import GuiEditLabel KEY_DELAY = 1 @@ -1673,7 +1673,7 @@ def testGuiEditor_WordCounters(qtbot, monkeypatch, nwGUI, projPath, ipsumText, m assert nwGUI.openDocument(C.hSceneDoc) is True text = "\n\n".join(ipsumText) - cC, wC, pC = countWords(text) + cC, wC, pC = standardCounter(text) nwGUI.docEditor.replaceText(text) # Check that a busy counter is blocked diff --git a/tests/test_text/test_core_counting.py b/tests/test_text/test_core_counting.py new file mode 100644 index 00000000..bab0596e --- /dev/null +++ b/tests/test_text/test_core_counting.py @@ -0,0 +1,206 @@ +""" +novelWriter – Counter Module Tester +=================================== + +This file is a part of novelWriter +Copyright 2018–2024, Veronica Berglyd Olsen + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" +from __future__ import annotations + +import pytest + +from novelwriter.text.counting import bodyTextCounter, preProcessText, standardCounter + + +@pytest.mark.core +def testTextCounting_preProcessText(): + """Test the text preprocessor for counters.""" + # Not Text + assert preProcessText(None) == [] + + # No Text + assert preProcessText("") == [] + + text = ( + "#! Title\n" + "##! Prologue\n" + "# Heading One\n" + "## Heading Two\n" + "### Heading Three\n" + "#### Heading Four\n\n" + "@tag: value\n\n" + "% A comment.\n\n" + "A [b]paragraph[/b].\n\n" + "[vspace:3]\n\n" + "[New Page]\n\n" + "Dashes\u2013and even longer\u2014dashes.\n\n" + ) + + # Process Text, w/Headers + assert preProcessText(text) == [ + "#! Title", + "##! Prologue", + "# Heading One", + "## Heading Two", + "### Heading Three", + "#### Heading Four", + "", "", "", + "A paragraph.", "", + "", "", "", "", + "Dashes and even longer dashes.", "" + ] + + # Process Text, wo/Headers + assert preProcessText(text, keepHeaders=False) == [ + "", "", "", + "A paragraph.", "", + "", "", "", "", + "Dashes and even longer dashes.", "" + ] + +# END Test testTextCounting_preProcessText + + +@pytest.mark.core +def testTextCounting_standardCounter(): + """Test the standard counter.""" + # Non-Text + assert standardCounter(None) == (0, 0, 0) # type: ignore + assert standardCounter(1234) == (0, 0, 0) # type: ignore + + # General Text + cC, wC, pC = standardCounter(( + "#! Title\n\n" + "##! Prologue\n\n" + "# Heading One\n" + "## Heading Two\n" + "### Heading Three\n" + "#### Heading Four\n\n" + "@tag: value\n\n" + "% A comment that should not be counted.\n\n" + "The first paragraph.\n\n" + "The second paragraph.\n\n\n" + "The third paragraph.\n\n" + "Dashes\u2013and even longer\u2014dashes." + )) + assert cC == 151 + assert wC == 24 + assert pC == 4 + + # Text Alignment + cC, wC, pC = standardCounter(( + "# Title\n\n" + "Left aligned<<\n\n" + "Left aligned <<\n\n" + "Right indent<\n\n" + "Right indent <\n\n" + )) + assert cC == 53 + assert wC == 9 + assert pC == 4 + + cC, wC, pC = standardCounter(( + "# Title\n\n" + ">>Right aligned\n\n" + ">> Right aligned\n\n" + ">Left indent\n\n" + "> Left indent\n\n" + )) + assert cC == 53 + assert wC == 9 + assert pC == 4 + + cC, wC, pC = standardCounter(( + "# Title\n\n" + ">>Centre aligned<<\n\n" + ">> Centre aligned <<\n\n" + ">Double indent<\n\n" + "> Double indent <\n\n" + )) + assert cC == 59 + assert wC == 9 + assert pC == 4 + + # Formatting Codes, Upper Case (Old Implementation) + cC, wC, pC = standardCounter(( + "Some text\n\n" + "[NEWPAGE]\n\n" + "more text\n\n" + "[NEW PAGE]\n\n" + "even more text\n\n" + "[VSPACE]\n\n" + "and some final text\n\n" + "[VSPACE:4]\n\n" + "THE END\n\n" + )) + assert cC == 58 + assert wC == 13 + assert pC == 5 + + # Formatting Codes, Lower Case (Current Implementation) + cC, wC, pC = standardCounter(( + "Some text\n\n" + "[newpage]\n\n" + "more text\n\n" + "[new page]\n\n" + "even more text\n\n" + "[vspace]\n\n" + "and some final text\n\n" + "[vspace:4]\n\n" + "THE END\n\n" + )) + assert cC == 58 + assert wC == 13 + assert pC == 5 + + # Check ShortCodes + cC, wC, pC = standardCounter(( + "Text with [b]bold[/b] text and padded [b] bold [/b] text.\n\n" + "Text with [b][i] nested [/i] emphasis [/b] in it.\n\n" + )) + assert cC == 78 + assert wC == 14 + assert pC == 2 + +# END Test testTextCounting_standardCounter + + +@pytest.mark.core +def testTextCounting_bodyTextCounter(): + """Test the body text counter.""" + # Not Text + assert bodyTextCounter(None) == (0, 0, 0) + + # General Text + wC, cC, sC = bodyTextCounter(( + "#! Title\n\n" + "##! Prologue\n\n" + "# Heading One\n" + "## Heading Two\n" + "### Heading Three\n" + "#### Heading Four\n\n" + "@tag: value\n\n" + "% A comment that should not be counted.\n\n" + "The first paragraph.\n\n" + "The second paragraph.\n\n\n" + "The third paragraph.\n\n" + "Dashes\u2013and even longer\u2014dashes." + )) + assert wC == 14 + assert cC == 91 + assert sC == 81 + +# END Test testTextCounting_bodyTextCounter diff --git a/tests/test_tools/test_tools_manuscript.py b/tests/test_tools/test_tools_manuscript.py index 734642d4..b6c5df66 100644 --- a/tests/test_tools/test_tools_manuscript.py +++ b/tests/test_tools/test_tools_manuscript.py @@ -192,6 +192,16 @@ def testManuscript_Features(monkeypatch, qtbot: QtBot, nwGUI: GuiMain, projPath: assert manus.docPreview.toPlainText().strip() != "" assert cacheFile.exists() is True + # Check Preview Stats + assert manus.docStats.mainStack.currentWidget() == manus.docStats.minWidget + assert manus.docStats.minWordCount.text() == "7" + assert manus.docStats.minCharCount.text() == "31" + + manus.docStats.toggleButton.toggle() + assert manus.docStats.mainStack.currentWidget() == manus.docStats.maxWidget + assert manus.docStats.maxTotalWords.text() == "7" + assert manus.docStats.maxTotalChars.text() == "31" + # Toggle justify assert manus.docPreview.document().defaultTextOption().alignment() == Qt.AlignAbsolute manus.docPreview.setJustify(True)