From e0936b5f33305fcfbf70907ed5ec14046ba938ba Mon Sep 17 00:00:00 2001
From: "Veronica K. B. Olsen" <code@vkbo.net>
Date: Tue, 7 May 2019 18:01:19 +0200
Subject: [PATCH] Text tokenizer done.

---
 nw/convert/tokenizer.py | 84 ++++++++++++++++++++++++++++++++++++++---
 nw/gui/dochighlight.py  |  8 ++--
 nw/gui/winmain.py       |  3 +-
 3 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/nw/convert/tokenizer.py b/nw/convert/tokenizer.py
index e6ca8c05..85596ebb 100644
--- a/nw/convert/tokenizer.py
+++ b/nw/convert/tokenizer.py
@@ -13,32 +13,104 @@
 import logging
 import nw
 
+from PyQt5.QtCore        import QRegularExpression
+
 from nw.project.document import NWDoc
 
 logger = logging.getLogger(__name__)
 
 class Tokenizer():
 
+    FMT_B_B = 1 # Begin Bold
+    FMT_B_E = 2 # End Bold
+    FMT_I_B = 3 # Begin Italics
+    FMT_I_E = 4 # End Italics
+    FMT_U_B = 5 # Begin Underline
+    FMT_U_E = 6 # End Underline
+
     def __init__(self, theProject, theParent):
 
         self.mainConf   = nw.CONFIG
         self.theProject = theProject
         self.theParent  = theParent
 
+        self.theText    = None
         self.theHandle  = None
         self.theItem    = None
         self.theTokens  = None
 
         return
 
-    def tokenizeText(self, tHandle):
+    def setText(self, theHandle, theText=None):
 
-        self.theItem = self.theProject.getItem(tHandle)
-        theDoc  = NWDoc(self.theProject, self.theParent)
-        theText = theDoc.openDocument(tHandle)
+        self.theHandle = theHandle
+        self.theItem   = self.theProject.getItem(theHandle)
 
-        for aLine in theText.splitlines():
-            print(aLine)
+        if theText is not None:
+            # If the text is set, just use that
+            self.theText = theText
+        else:
+            # Otherwise, load it from file
+            theDocument  = NWDoc(self.theProject, self.theParent)
+            self.theText = theDocument.openDocument(theHandle)
+
+        return
+
+    def tokenizeText(self):
+        """Scan the text for either lines starting with specific characters that indicate headers,
+        comments, commands etc, or just contains plain text. in the case of plain text, apply the
+        same RegExes that the syntax highlighter uses and save the locations of these formatting
+        tags into the token array.
+        """
+
+        # RegExes for adding formatting tags within text lines
+        # Keep in sync with the DocHighlighter class
+        rxFormats = [(
+            QRegularExpression(r"(?<![\w|\\])([\*]{2})(?!\s)(?m:(.+?))(?<![\s|\\])(\1)(?!\w)"),
+            [None, self.FMT_B_B, None, self.FMT_B_E]
+        ),(
+            QRegularExpression(r"(?<![\w|_|\\])([_])(?!\s|\1)(?m:(.+?))(?<![\s|\\])(\1)(?!\w)"),
+            [None, self.FMT_I_B, None, self.FMT_I_E]
+        ),(
+            QRegularExpression(r"(?<![\w|\\])([_]{2})(?!\s)(?m:(.+?))(?<![\s|\\])(\1)(?!\w)"),
+            [None, self.FMT_U_B, None, self.FMT_U_E]
+        )]
+
+        self.theTokens = []
+        for aLine in self.theText.splitlines():
+            aLine = aLine.strip()
+
+            # Tag lines starting with specific characters
+            if len(aLine) == 0:
+                self.theTokens.append(("empty","",None))
+            elif aLine[0] == "%":
+                self.theTokens.append(("comment",aLine[1:].strip(),None))
+            elif aLine[0] == "@":
+                self.theTokens.append(("command",aLine[1:].strip(),None))
+            elif aLine[:2] == "# ":
+                self.theTokens.append(("header1",aLine[2:].strip(),None))
+            elif aLine[:3] == "## ":
+                self.theTokens.append(("header2",aLine[3:].strip(),None))
+            elif aLine[:4] == "### ":
+                self.theTokens.append(("header3",aLine[4:].strip(),None))
+            elif aLine[:5] == "#### ":
+                self.theTokens.append(("header4",aLine[5:].strip(),None))
+            else:
+                # Otherwise we use RegEx to find formatting tags within a line of text
+                fmtPos = []
+                for theRX, theKeys in rxFormats:
+                    rxThis = theRX.globalMatch(aLine, 0)
+                    while rxThis.hasNext():
+                        rxMatch = rxThis.next()
+                        for n in range(1,len(theKeys)):
+                            if theKeys[n] is not None:
+                                xPos = rxMatch.capturedStart(n)
+                                xLen = rxMatch.capturedLength(n)
+                                fmtPos.append([xPos,xLen,theKeys[n]])
+                # Save the line as is, but append the array of formatting locations
+                self.theTokens.append(("text",aLine,fmtPos))
+
+        # print(self.theTokens)
 
         return
 
diff --git a/nw/gui/dochighlight.py b/nw/gui/dochighlight.py
index b1c35831..11d4a0d8 100644
--- a/nw/gui/dochighlight.py
+++ b/nw/gui/dochighlight.py
@@ -64,25 +64,25 @@ class GuiDocHighlighter(QSyntaxHighlighter):
 
         # Headers
         self.hRules.append((
-            r"^(#{1})[^#](.*)[^\n]", {
+            r"^(#{1}) (.*)[^\n]", {
                 0 : self.hStyles["header1"],
                 1 : self.hStyles["header1h"],
             }
         ))
         self.hRules.append((
-            r"^(#{2})[^#](.*)[^\n]", {
+            r"^(#{2}) (.*)[^\n]", {
                 0 : self.hStyles["header2"],
                 1 : self.hStyles["header2h"],
             }
         ))
         self.hRules.append((
-            r"^(#{3})[^#](.*)[^\n]", {
+            r"^(#{3}) (.*)[^\n]", {
                 0 : self.hStyles["header3"],
                 1 : self.hStyles["header3h"],
             }
         ))
         self.hRules.append((
-            r"^(#{4})[^#](.*)[^\n]", {
+            r"^(#{4}) (.*)[^\n]", {
                 0 : self.hStyles["header4"],
                 1 : self.hStyles["header4h"],
             }
diff --git a/nw/gui/winmain.py b/nw/gui/winmain.py
index e026db58..e5678d42 100644
--- a/nw/gui/winmain.py
+++ b/nw/gui/winmain.py
@@ -202,7 +202,8 @@ class GuiMain(QMainWindow):
             if tHandle not in theHandles:
                 continue
             aDoc = Tokenizer(self.theProject, self)
-            aDoc.tokenizeText(tHandle)
+            aDoc.setText(tHandle)
+            aDoc.tokenizeText()
 
         return