Update tokenizer and html converter

nw/convert/tohtml.py

@@ -31,6 +31,10 @@ class ToHtml(Tokenizer):
             "<"      : "&lt;",
             ">"      : "&gt;",
             "&"      : "&amp;",
+            "\u2013" : "&endash;",
+            "\u2014" : "$emdash;",
+            "\u2500" : "$emdash;",
+            "\u2026" : "&hellip;",
         }
         xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
         self.theText = xRep.sub(lambda x: repDict[x.group(0)], self.theText)

nw/convert/tokenizer.py

@@ -42,6 +42,7 @@ class Tokenizer():
     T_HEAD4   =  7 # Header 4
     T_TEXT    =  8 # Text line
     T_SEP     =  9 # Scene separator
+    T_PBREAK  = 10 # Page break
 
     A_LEFT    =  1 # Left aligned
     A_RIGHT   =  2 # Right aligned
@@ -262,6 +263,7 @@ class Tokenizer():
                         self.theTokens[n] = (tType,tTemp,None,self.A_LEFT)
 
         # For title page and partitions, we need to centre all text
+        # and for some formats, we need a page break
         if isTitle or isPart:
             for n in range(len(self.theTokens)):
                 tToken  = self.theTokens[n]
@@ -270,6 +272,8 @@ class Tokenizer():
                 tFormat = tToken[2]
                 self.theTokens[n] = (tType,tText,tFormat,self.A_CENTRE)
 
+            self.theTokens[n] = (self.T_PBREAK,"",None,self.A_LEFT)
+
         return
 
     def doConvert(self):