Merge pull request #80 from vkbo/latex_unicode_fix

LaTeX unicode fix

nw/convert/file/latex.py

@@ -24,6 +24,7 @@ class LaTeXFile(TextFile):
     def __init__(self, theProject, theParent):
         TextFile.__init__(self, theProject, theParent)
         self.theConv = ToLaTeX(self.theProject, self.theParent)
+        self.texCodecFail = False
         return
 
     ##
@@ -50,6 +51,8 @@ class LaTeXFile(TextFile):
             self.outFile.write("\\end{document}\n")
             self.outFile.close()
 
+        self.texCodecFail = self.theConv.texCodecFail
+
         return True
 
 # END Class LaTeXFile

nw/convert/text/tolatex.py

@@ -10,8 +10,8 @@
 
 """
 
-import textwrap
 import logging
+import codecs
 import re
 import nw
 
@@ -23,20 +23,7 @@ class ToLaTeX(Tokenizer):
 
     def __init__(self, theProject, theParent):
         Tokenizer.__init__(self, theProject, theParent)
-        return
+        self.texCodecFail = False
-
-    def doAutoReplace(self):
-        Tokenizer.doAutoReplace(self)
-
-        repDict = {
-            "\u2013" : "--",
-            "\u2014" : "---",
-            "\u2500" : "---",
-            "\u2026" : "...",
-        }
-        xRep = re.compile("|".join([re.escape(k) for k in repDict.keys()]), flags=re.DOTALL)
-        self.theText = xRep.sub(lambda x: repDict[x.group(0)], self.theText)
-
         return
 
     def doConvert(self):
@@ -50,34 +37,6 @@ class ToLaTeX(Tokenizer):
             self.FMT_U_E : r"}",
         }
 
-        if self.wordWrap > 0:
-            tWrap = textwrap.TextWrapper(
-                width                = self.wordWrap,
-                initial_indent       = "",
-                subsequent_indent    = "",
-                expand_tabs          = True,
-                replace_whitespace   = True,
-                fix_sentence_endings = False,
-                break_long_words     = True,
-                drop_whitespace      = True,
-                break_on_hyphens     = True,
-                tabsize              = 8,
-                max_lines            = None
-            )
-            tComm = textwrap.TextWrapper(
-                width                = self.wordWrap-2,
-                initial_indent       = "",
-                subsequent_indent    = "",
-                expand_tabs          = True,
-                replace_whitespace   = True,
-                fix_sentence_endings = False,
-                break_long_words     = True,
-                drop_whitespace      = True,
-                break_on_hyphens     = True,
-                tabsize              = 8,
-                max_lines            = None
-            )
-
         self.theResult = ""
         thisPar = []
         for tType, tText, tFormat, tAlign in self.theTokens:
@@ -101,14 +60,6 @@ class ToLaTeX(Tokenizer):
 
             tLen = len(tText)
 
-            # The text can now be word wrapped, if we have requested this and it's needed.
-            if self.wordWrap > 0 and tLen > self.wordWrap:
-                if tType == self.T_COMMENT:
-                    aText = tComm.wrap(tText)
-                    tText = "\n% ".join(aText)
-                else:
-                    tText = tWrap.fill(tText)
-
             # Then the text can receive final formatting before we append it to the results.
             # We also store text lines in a buffer and merge them only when we find an empty line
             # indicating a new paragraph.
@@ -122,25 +73,25 @@ class ToLaTeX(Tokenizer):
 
             elif tType == self.T_HEAD1:
                 self.theResult += begText
-                self.theResult += "{\\Huge %s}\n" % tText
+                self.theResult += "{\\Huge %s}\n" % self._escapeUnicode(tText)
                 self.theResult += endText
 
             elif tType == self.T_HEAD2:
-                self.theResult += "\\chapter*{%s}\n\n" % tText
+                self.theResult += "\\chapter*{%s}\n\n" % self._escapeUnicode(tText)
 
             elif tType == self.T_HEAD3:
-                self.theResult += "\\section*{%s}\n\n" % tText
+                self.theResult += "\\section*{%s}\n\n" % self._escapeUnicode(tText)
 
             elif tType == self.T_HEAD4:
-                self.theResult += "\\subsection*{%s}\n\n" % tText
+                self.theResult += "\\subsection*{%s}\n\n" % self._escapeUnicode(tText)
 
             elif tType == self.T_SEP:
                 self.theResult += begText
-                self.theResult += "%s\n" % tText
+                self.theResult += "%s\n" % self._escapeUnicode(tText)
                 self.theResult += endText
 
             elif tType == self.T_TEXT:
-                thisPar.append(tText)
+                thisPar.append(self._escapeUnicode(tText))
 
             elif tType == self.T_PBREAK:
                 self.theResult += "\\newpage\n\n"
@@ -153,4 +104,12 @@ class ToLaTeX(Tokenizer):
 
         return
 
+    def _escapeUnicode(self, theText):
+        try:
+            import latexcodec
+            return codecs.encode(theText, "ulatex")
+        except:
+            self.texCodecFail = True
+            return theText
+
 # END Class ToLaTeX

nw/gui/export.py

@@ -32,7 +32,7 @@ from nw.convert.file.markdown import MarkdownFile
 from nw.convert.file.latex    import LaTeXFile
 from nw.convert.file.concat   import ConcatFile
 from nw.constants             import nwFiles
-from nw.enum                  import nwItemType
+from nw.enum                  import nwItemType, nwAlert
 
 logger = logging.getLogger(__name__)
 
@@ -167,11 +167,19 @@ class GuiExport(QDialog):
             nDone += 1
 
         outFile.closeFile()
-
         self.exportProgress.setValue(nDone)
         self.exportStatus.setText("Export to %s complete" % outFile.fileName)
         logger.verbose("Export to %s complete" % outFile.fileName)
 
+        if eFormat == GuiExportMain.FMT_TEX:
+            # Check that encoding was successful
+            if outFile.texCodecFail:
+                self.theParent.makeAlert((
+                    "Failed to escape unicode characters while writing LaTeX file. "
+                    "The genrated .tex file may not build properly. "
+                    "Make sure the python package 'latexcodec' is installed and working."
+                ), nwAlert.WARN)
+
         return
 
     def _doClose(self):
@@ -384,7 +392,7 @@ class GuiExportMain(QWidget):
         self.fixedWidth.setMaximum(999)
         self.fixedWidth.setSingleStep(1)
         self.fixedWidth.setValue(self.optState.getSetting("fixWidth"))
-        self.fixedWidth.setToolTip("Applies to .txt, .md and .tex files. 0 disables the feature.")
+        self.fixedWidth.setToolTip("Applies to .txt and .md files. A value of '0' disables the feature.")
 
         self.addSettingsForm.addWidget(QLabel("Fixed width"), 0, 0)
         self.addSettingsForm.addWidget(self.fixedWidth,       0, 1)

requirements.txt

@@ -3,3 +3,4 @@ appdirs
 lxml
 pyenchant
 pycountry
+latexcodec