Process tags and references in Tokenizer

tests/test_formats/test_fmt_tokenizer.py

@@ -770,30 +770,54 @@ def testFmtToken_MetaFormat(mockGUI):
 
     tokens.setKeywords(True)
     tokens.tokenizeText()
-    assert tokens._blocks == [
-        (BlockTyp.KEYWORD, 0, "char: Bod", [], BlockFmt.NONE),
-    ]
+    assert tokens._blocks == [(
+        BlockTyp.KEYWORD, 0, "Characters: Bod", [
+            (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
+            (11, TextFmt.B_E, ""), (11, TextFmt.COL_E, ""),
+            (12, TextFmt.COL_B, "tag"), (12, TextFmt.HRF_B, "#tag_bod"),
+            (15, TextFmt.HRF_E, ""), (15, TextFmt.COL_E, ""),
+        ], BlockFmt.NONE
+    )]
     assert tokens.allMarkdown[-1] == "@char: Bod\n\n"
 
     tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
     tokens.tokenizeText()
-    styTop = BlockFmt.NONE | BlockFmt.Z_BTMMRG
-    styMid = BlockFmt.NONE | BlockFmt.Z_BTMMRG | BlockFmt.Z_TOPMRG
-    styBtm = BlockFmt.NONE | BlockFmt.Z_TOPMRG
-    assert tokens._blocks == [
-        (BlockTyp.KEYWORD, 0, "pov: Bod", [], styTop),
-        (BlockTyp.KEYWORD, 0, "plot: Main", [], styMid),
-        (BlockTyp.KEYWORD, 0, "location: Europe", [], styBtm),
-    ]
+    assert tokens._blocks == [(
+        BlockTyp.KEYWORD, 0, "Point of View: Bod", [
+            (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
+            (14, TextFmt.B_E, ""), (14, TextFmt.COL_E, ""),
+            (15, TextFmt.COL_B, "tag"), (15, TextFmt.HRF_B, "#tag_bod"),
+            (18, TextFmt.HRF_E, ""), (18, TextFmt.COL_E, ""),
+        ], BlockFmt.Z_BTMMRG
+    ), (
+        BlockTyp.KEYWORD, 0, "Plot: Main", [
+            (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
+            (5, TextFmt.B_E, ""), (5, TextFmt.COL_E, ""),
+            (6, TextFmt.COL_B, "tag"), (6, TextFmt.HRF_B, "#tag_main"),
+            (10, TextFmt.HRF_E, ""), (10, TextFmt.COL_E, ""),
+        ], BlockFmt.Z_TOPMRG | BlockFmt.Z_BTMMRG
+    ), (
+        BlockTyp.KEYWORD, 0, "Locations: Europe", [
+            (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
+            (10, TextFmt.B_E, ""), (10, TextFmt.COL_E, ""),
+            (11, TextFmt.COL_B, "tag"), (11, TextFmt.HRF_B, "#tag_europe"),
+            (17, TextFmt.HRF_E, ""), (17, TextFmt.COL_E, ""),
+        ], BlockFmt.Z_TOPMRG
+    )]
     assert tokens.allMarkdown[-1] == "@pov: Bod\n@plot: Main\n@location: Europe\n\n"
 
     # Ignored keywords
     tokens._text = "@pov: Bod\n@plot: Main\n@location: Europe\n"
     tokens.setIgnoredKeywords("@plot, @location")
     tokens.tokenizeText()
-    assert tokens._blocks == [
-        (BlockTyp.KEYWORD, 0, "pov: Bod", [], BlockFmt.NONE),
-    ]
+    assert tokens._blocks == [(
+        BlockTyp.KEYWORD, 0, "Point of View: Bod", [
+            (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
+            (14, TextFmt.B_E, ""), (14, TextFmt.COL_E, ""),
+            (15, TextFmt.COL_B, "tag"), (15, TextFmt.HRF_B, "#tag_bod"),
+            (18, TextFmt.HRF_E, ""), (18, TextFmt.COL_E, ""),
+        ], BlockFmt.NONE
+    )]
 
 
 @pytest.mark.core
@@ -1643,7 +1667,7 @@ def testFmtToken_ProcessHeaders(mockGUI):
 
 
 @pytest.mark.core
-def testFmtToken_FormatNote(mockGUI, ipsumText):
+def testFmtToken_FormatComment(mockGUI):
     """Test note and comment formatting."""
     project = NWProject()
     project.data.setLanguage("en")
@@ -1671,6 +1695,36 @@ def testFmtToken_FormatNote(mockGUI, ipsumText):
     )
 
 
+@pytest.mark.core
+def testFmtToken_FormatMeta(mockGUI):
+    """Test meta formatting."""
+    project = NWProject()
+    project.data.setLanguage("en")
+    project._loadProjectLocalisation()
+    tokens = BareTokenizer(project)
+
+    assert tokens._formatMeta("@tag: Jane | Jane Smith") == (
+        "Tag: Jane | Jane Smith", [
+            (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
+            (4, TextFmt.B_E, ""), (4, TextFmt.COL_E, ""),
+            (5, TextFmt.COL_B, "tag"), (5, TextFmt.ANM_B, "tag_jane"),
+            (9, TextFmt.ANM_E, ""), (9, TextFmt.COL_E, ""),
+            (12, TextFmt.COL_B, "optional"), (22, TextFmt.COL_E, ""),
+        ]
+    )
+
+    assert tokens._formatMeta("@char: Jane, John") == (
+        "Characters: Jane, John", [
+            (0, TextFmt.B_B, ""), (0, TextFmt.COL_B, "keyword"),
+            (11, TextFmt.B_E, ""), (11, TextFmt.COL_E, ""),
+            (12, TextFmt.COL_B, "tag"), (12, TextFmt.HRF_B, "#tag_jane"),
+            (16, TextFmt.HRF_E, ""), (16, TextFmt.COL_E, ""),
+            (18, TextFmt.COL_B, "tag"), (18, TextFmt.HRF_B, "#tag_john"),
+            (22, TextFmt.HRF_E, ""), (22, TextFmt.COL_E, ""),
+        ]
+    )
+
+
 @pytest.mark.core
 def testFmtToken_BuildOutline(mockGUI, ipsumText):
     """Test stats counter of the Tokenizer class."""
@@ -1863,7 +1917,7 @@ def testFmtToken_CountStats(mockGUI, ipsumText):
     tokens.setKeywords(True)
     tokens.tokenizeText()
     tokens.countStats()
-    assert [t[2] for t in tokens._blocks] == ["Chapter", "pov: Jane", "Text"]
+    assert [t[2] for t in tokens._blocks] == ["Chapter", "Point of View: Jane", "Text"]
     assert tokens.textStats == {
         "titleCount": 1, "paragraphCount": 1,
         "allWords": 6, "textWords": 1, "titleWords": 1,