From 041183f91420e5adaf7b2125b0f2a5923d8fcc1b Mon Sep 17 00:00:00 2001
From: Veronica Berglyd Olsen <1619840+vkbo@users.noreply.github.com>
Date: Mon, 28 Oct 2024 16:50:30 +0100
Subject: [PATCH] Add processing of fields in the Tokenizer

---
 novelwriter/constants.py                 |  1 +
 novelwriter/formats/shared.py            |  3 ++-
 novelwriter/formats/tokenizer.py         |  1 +
 sample/content/53b69b83cdafc.nwd         |  6 ++++--
 sample/nwProject.nwx                     | 12 +++++------
 tests/test_formats/test_fmt_tokenizer.py | 26 ++++++++++++++++++++++++
 6 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/novelwriter/constants.py b/novelwriter/constants.py
index 83c22ac6..af232328 100644
--- a/novelwriter/constants.py
+++ b/novelwriter/constants.py
@@ -89,6 +89,7 @@ class nwShortcode:
     BREAK    = "[br]"
 
     FOOTNOTE_B = "[footnote:"
+    FIELD_B    = "[field:"
 
     COMMENT_STYLES = {
         nwComment.FOOTNOTE: "[footnote:{0}]",
diff --git a/novelwriter/formats/shared.py b/novelwriter/formats/shared.py
index 170ee890..119eee9a 100644
--- a/novelwriter/formats/shared.py
+++ b/novelwriter/formats/shared.py
@@ -92,7 +92,8 @@ class TextFmt(IntEnum):
     HRF_B = 21  # Begin href link
     HRF_E = 22  # End href link
     FNOTE = 23  # Footnote marker
-    STRIP = 24  # Strip the format code
+    FIELD = 24  # Data field
+    STRIP = 25  # Strip the format code
 
 
 class BlockTyp(IntEnum):
diff --git a/novelwriter/formats/tokenizer.py b/novelwriter/formats/tokenizer.py
index 09e8e2ab..feb77ffd 100644
--- a/novelwriter/formats/tokenizer.py
+++ b/novelwriter/formats/tokenizer.py
@@ -193,6 +193,7 @@ class Tokenizer(ABC):
         }
         self._shortCodeVals = {
             nwShortcode.FOOTNOTE_B: TextFmt.FNOTE,
+            nwShortcode.FIELD_B:    TextFmt.FIELD,
         }
 
         # Dialogue
diff --git a/sample/content/53b69b83cdafc.nwd b/sample/content/53b69b83cdafc.nwd
index f793413a..66e5a513 100644
--- a/sample/content/53b69b83cdafc.nwd
+++ b/sample/content/53b69b83cdafc.nwd
@@ -1,8 +1,8 @@
 %%~name: Title Page
 %%~path: 7031beac91f75/53b69b83cdafc
 %%~kind: NOVEL/DOCUMENT
-%%~hash: 4072adb6d21ff877577f033f19714d9bd01396f3
-%%~date: Unknown/2024-10-24 16:25:44
+%%~hash: 45ec619c9fd1b1185bc35b38d28cda13b05f0a88
+%%~date: Unknown/2024-10-28 16:42:26
 
 Jane Smith[br]
 42 Main Street[br]
@@ -16,3 +16,5 @@ Jane Smith[br]
 
 >> This is the title page. <<
 >> It should be the first document of the project. <<
+
+>> Word Count: [field:textWords] <<
\ No newline at end of file
diff --git a/sample/nwProject.nwx b/sample/nwProject.nwx
index b8252bfc..1308d3b1 100644
--- a/sample/nwProject.nwx
+++ b/sample/nwProject.nwx
@@ -1,6 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<novelWriterXML appVersion="2.6a2" hexVersion="0x020600a2" fileVersion="1.5" fileRevision="4" timeStamp="2024-10-26 16:59:09">
-  <project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="2132" autoCount="279" editTime="95062">
+<novelWriterXML appVersion="2.6a3" hexVersion="0x020600a3" fileVersion="1.5" fileRevision="4" timeStamp="2024-10-28 16:42:35">
+  <project id="e2be99af-f9bf-4403-857a-c3d1ac25abea" saveCount="2134" autoCount="279" editTime="95103">
     <name>Sample Project</name>
     <author>Jane Smith</author>
   </project>
@@ -9,8 +9,8 @@
     <language>en_GB</language>
     <spellChecking auto="yes">None</spellChecking>
     <lastHandle>
-      <entry key="editor">636b6aa9b697b</entry>
-      <entry key="viewer">636b6aa9b697b</entry>
+      <entry key="editor">53b69b83cdafc</entry>
+      <entry key="viewer">53b69b83cdafc</entry>
       <entry key="novelTree">7031beac91f75</entry>
       <entry key="outline">7031beac91f75</entry>
     </lastHandle>
@@ -36,13 +36,13 @@
       <entry key="i56be10" count="1" red="220" green="138" blue="221" shape="BLOCK_4">Main</entry>
     </importance>
   </settings>
-  <content items="31" novelWords="1014" notesWords="416">
+  <content items="31" novelWords="1016" notesWords="416">
     <item handle="7031beac91f75" parent="None" root="7031beac91f75" order="0" type="ROOT" class="NOVEL">
       <meta expanded="yes" />
       <name status="sc24b8f" import="ia857f0">Novel</name>
     </item>
     <item handle="53b69b83cdafc" parent="7031beac91f75" root="7031beac91f75" order="0" type="FILE" class="NOVEL" layout="DOCUMENT">
-      <meta expanded="no" heading="H1" charCount="136" wordCount="27" paraCount="3" cursorPos="69" />
+      <meta expanded="no" heading="H1" charCount="148" wordCount="29" paraCount="4" cursorPos="227" />
       <name status="sc24b8f" import="ia857f0" active="yes">Title Page</name>
     </item>
     <item handle="974e400180a99" parent="7031beac91f75" root="7031beac91f75" order="1" type="FILE" class="NOVEL" layout="DOCUMENT">
diff --git a/tests/test_formats/test_fmt_tokenizer.py b/tests/test_formats/test_fmt_tokenizer.py
index 8d01445e..874f3211 100644
--- a/tests/test_formats/test_fmt_tokenizer.py
+++ b/tests/test_formats/test_fmt_tokenizer.py
@@ -1241,6 +1241,32 @@ def testFmtToken_LineBreak(mockGUI):
     ]
 
 
+@pytest.mark.core
+def testFmtToken_ShortcodeValue(mockGUI):
+    """Test processing of shortcodes with values."""
+    project = NWProject()
+    tokens = BareTokenizer(project)
+    tokens._handle = TMH
+
+    # Footnote
+    tokens._text = "Hello World[footnote:abcd] to you!"
+    tokens.tokenizeText()
+    assert tokens._blocks == [(
+        BlockTyp.TEXT, "", "Hello World to you!", [
+            (11, TextFmt.FNOTE, f"{TMH}:abcd"),
+        ], BlockFmt.NONE
+    )]
+
+    # Field
+    tokens._text = "Hello World: [field:abcd] times!"
+    tokens.tokenizeText()
+    assert tokens._blocks == [(
+        BlockTyp.TEXT, "", "Hello World:  times!", [
+            (13, TextFmt.FIELD, f"{TMH}:abcd"),
+        ], BlockFmt.NONE
+    )]
+
+
 @pytest.mark.core
 def testFmtToken_Dialogue(mockGUI):
     """Test the tokenization of dialogue in the Tokenizer class."""