Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improvements to saving meta data, and added counting of words, etc

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2019-04-19 23:42:30 +02:00
parent 99257b82cd
commit f68a64503e
6 changed files with 177 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1 +1,2 @@
__pycache__
*.bak
nw/project/document.py
+36 -9
View File
@@ -13,8 +13,9 @@
import logging
import nw
from os import path, mkdir
from lxml import etree, html
from os import path, mkdir, rename, unlink
from nw.tools.analyse import TextAnalysis
logger = logging.getLogger(__name__)
@@ -27,35 +28,61 @@ class NWDoc():
self.mainConf = nw.CONFIG
self.theProject = theProject
self.docHandle = None
self.theItem = None
return
def openDocument(self, tHandle):
self.docHandle = tHandle
self.theItem = self.theProject.getItem(tHandle)
docDir, docFile = self._assemblePath(self.FILE_MN)
logger.debug("Opening document %s" % path.join(docDir,docFile))
dataDir = path.join(self.theProject.projPath, docDir)
docPath = path.join(dataDir, docFile)
if path.isfile(docPath):
with open(docPath,mode="r") as inFile:
return inFile.read()
else:
logger.debug("The requested document does not exist.")
return ""
return None
def saveDocument(self, docHtml):
def saveDocument(self, docText):
if self.docHandle is None:
return False
docDir, docFile = self._assemblePath(self.FILE_MN)
logger.debug("Saving document %s" % path.join(docDir,docFile))
dataDir = path.join(self.theProject.projPath, docDir)
docPath = path.join(dataDir, docFile)
if not path.isdir(dataDir):
mkdir(dataDir)
logger.debug("Created folder %s" % dataDir)
dataPath = path.join(self.theProject.projPath, docDir)
docPath = path.join(dataPath, docFile)
if not path.isdir(dataPath):
mkdir(dataPath)
logger.debug("Created folder %s" % dataPath)
docTemp = path.join(dataPath,docFile[:-3]+"tmp")
docBack = path.join(dataPath,docFile[:-3]+"bak")
if path.isfile(docTemp): unlink(docTemp)
if path.isfile(docBack): rename(docBack,docTemp)
if path.isfile(docPath): rename(docPath,docBack)
with open(docPath,mode="w") as outFile:
outFile.write(docHtml)
outFile.write(docText)
if path.isfile(docTemp): unlink(docTemp)
docAna = TextAnalysis(docText,"en_GB")
wC, sC, pC = docAna.getStats()
self.theItem.setWordCount(wC)
self.theItem.setSentCount(sC)
self.theItem.setParaCount(pC)
return True
##
nw/project/item.py
+91 -10
View File
@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
class NWItem():
MAXDEPTH = 8
MAX_DEPTH = 8
def __init__(self):
@@ -34,20 +34,68 @@ class NWItem():
self.itemType = nwItemType.NONE
self.itemClass = nwItemClass.NONE
self.itemDepth = None
self.isExpanded = False
self.hasChildren = False
self.isExpanded = False
self.wordCount = None
self.sentCount = None
self.paraCount = None
return
##
# XML Pack
##
def packXML(self, xParent):
xPack = etree.SubElement(xParent,"item",attrib={
"handle" : str(self.itemHandle),
"parent" : str(self.parHandle),
"order" : str(self.itemOrder),
})
xSub = self._subPack(xPack,"name", text=str(self.itemName))
xSub = self._subPack(xPack,"type", text=str(self.itemType.name))
xSub = self._subPack(xPack,"class", text=str(self.itemClass.name))
xSub = self._subPack(xPack,"depth", text=str(self.itemDepth))
xSub = self._subPack(xPack,"children", text=str(self.hasChildren))
xSub = self._subPack(xPack,"expanded", text=str(self.isExpanded))
xSub = self._subPack(xPack,"wordCount", text=str(self.wordCount), none=False)
xSub = self._subPack(xPack,"sentCount", text=str(self.sentCount), none=False)
xSub = self._subPack(xPack,"paraCount", text=str(self.paraCount), none=False)
return xPack
def _subPack(self, xParent, name, attrib=None, text=None, none=True):
if not none and (text == None or text == "None"):
return None
xSub = etree.SubElement(xParent,name,attrib=attrib)
if text is not None:
xSub.text = text
return xSub
##
# Settings Wrapper
##
def setFromTag(self, tagName, tagValue):
logger.verbose("Setting tag '%s' to value '%s'" % (tagName, str(tagValue)))
if tagName == "name": self.setName(tagValue)
elif tagName == "order": self.setOrder(tagValue)
elif tagName == "type": self.setType(tagValue)
elif tagName == "class": self.setClass(tagValue)
elif tagName == "expanded": self.setExpanded(tagValue)
if tagName == "name": self.setName(tagValue)
elif tagName == "order": self.setOrder(tagValue)
elif tagName == "type": self.setType(tagValue)
elif tagName == "class": self.setClass(tagValue)
elif tagName == "depth": self.setDepth(tagValue)
elif tagName == "children": self.setChildren(tagValue)
elif tagName == "expanded": self.setExpanded(tagValue)
elif tagName == "wordCount": self.setWordCount(tagValue)
elif tagName == "sentCount": self.setSentCount(tagValue)
elif tagName == "paraCount": self.setParaCount(tagValue)
else:
logger.error("Unknown tag '%s'" % tagName)
return
##
# Set Item Values
##
def setName(self, theName):
self.itemName = theName.strip()
return
@@ -89,12 +137,17 @@ class NWItem():
return
def setDepth(self, theDepth):
if theDepth >= 0 and theDepth <= self.MAXDEPTH:
theDepth = self._checkInt(theDepth,-1)
if theDepth >= 0 and theDepth <= self.MAX_DEPTH:
self.itemDepth = theDepth
else:
logger.error("Invalid item depth %d" % theDepth)
return
def setChildren(self, hasChildren):
self.hasChildren = hasChildren
return
def setExpanded(self, expState):
if isinstance(expState, str):
self.isExpanded = expState == str(True)
@@ -102,8 +155,36 @@ class NWItem():
self.isExpanded = expState
return
def setHasChildren(self, hasChildren):
self.hasChildren = hasChildren
##
# Set Stats
##
def setWordCount(self, theCount):
theCount = self._checkInt(theCount,None,True)
self.wordCount = theCount
return
def setSentCount(self, theCount):
theCount = self._checkInt(theCount,None,True)
self.sentCount = theCount
return
def setParaCount(self, theCount):
theCount = self._checkInt(theCount,None,True)
self.paraCount = theCount
return
##
# Internal Functions
##
def _checkInt(self,checkValue,defaultValue,allowNone=False):
if allowNone:
if checkValue == None: return None
if checkValue == "None": return None
try:
return int(checkValue)
except:
return defaultValue
# END Class NWItem
nw/project/project.py
+10 -28
View File
@@ -78,11 +78,11 @@ class NWProject():
self.projName = ""
self.bookTitle = ""
self.bookAuthors = []
hNovel = self.newRoot("Novel", nwItemClass.NOVEL)
hChars = self.newRoot("Characters",nwItemClass.CHARACTER)
hWorld = self.newRoot("World", nwItemClass.WORLD)
hChapt = self.newFolder("New Chapter", nwItemClass.CHAPTER ,hNovel)
hScene = self.newFile("New Scene", nwItemClass.NONE, hChapt)
hNovel = self.newRoot("Novel", nwItemClass.NOVEL)
hChars = self.newRoot("Characters", nwItemClass.CHARACTER)
hWorld = self.newRoot("World", nwItemClass.WORLD)
hChapt = self.newFolder("New Chapter", nwItemClass.NOVEL, hNovel)
hScene = self.newFile("New Scene", nwItemClass.NOVEL, hChapt)
return
def openProject(self, fileName):
@@ -178,26 +178,8 @@ class NWProject():
# Save Tree Content
logger.debug("Writing project content")
xContent = etree.SubElement(nwXML,"content",attrib={"count":str(len(self.projTree))})
itemIdx = 0
for tHandle in self.treeOrder:
nwItem = self.projTree[tHandle]
xItem = etree.SubElement(xContent,"item",attrib={
"handle" : str(tHandle),
"parent" : str(nwItem.parHandle),
"order" : str(nwItem.itemOrder),
})
xItemValue = etree.SubElement(xItem,"name")
xItemValue.text = str(nwItem.itemName)
xItemValue = etree.SubElement(xItem,"type")
xItemValue.text = str(nwItem.itemType.name)
xItemValue = etree.SubElement(xItem,"class")
xItemValue.text = str(nwItem.itemClass.name)
xItemValue = etree.SubElement(xItem,"depth")
xItemValue.text = str(nwItem.itemDepth)
xItemValue = etree.SubElement(xItem,"expanded")
xItemValue.text = str(nwItem.isExpanded)
xItemValue = etree.SubElement(xItem,"children")
xItemValue.text = str(nwItem.hasChildren)
self.projTree[tHandle].packXML(xContent)
# Write the xml tree to file
with open(path.join(self.projPath,self.projFile),"wb") as outFile:
@@ -299,14 +281,14 @@ class NWProject():
if tType == nwItemType.FILE:
validActions[nwItemAction.SPLIT] = {}
else:
if tDepth < NWItem.MAXDEPTH-1 and (
if tDepth < NWItem.MAX_DEPTH-1 and (
tType == nwItemType.ROOT or tType == nwItemType.FOLDER
):
validActions[nwItemAction.ADD_FOLDER] = {
"Type" : nwItemType.FOLDER,
"Class" : tClass
}
if tDepth < NWItem.MAXDEPTH:
if tDepth < NWItem.MAX_DEPTH:
validActions[nwItemAction.ADD_FILE] = {
"Type" : nwItemType.FILE,
"Class" : tClass
@@ -342,7 +324,7 @@ class NWProject():
while nwItem.parHandle is not None:
theDepth += 1
nwItem = self.getItem(nwItem.parHandle)
if theDepth > NWItem.MAXDEPTH:
if theDepth > NWItem.MAX_DEPTH:
return None
return theDepth
@@ -357,7 +339,7 @@ class NWProject():
self.projTree[tHandle] = nwItem
self.treeOrder.append(tHandle)
if pHandle is not None:
self.projTree[pHandle].setHasChildren(True)
self.projTree[pHandle].setChildren(True)
if nwItem.itemType == nwItemType.ROOT:
logger.verbose("Entry %s is a root item" % str(tHandle))
nw/tools/analyse.py
+27 -26
View File
@@ -18,34 +18,35 @@ logger = logging.getLogger(__name__)
class TextAnalysis():
def __init__(self, langCode):
def __init__(self, theText, langCode):
self.theText = theText
self.langCode = langCode
return
def getStats(self, theText):
tStart = time()
wordCount = self._countWords(theText)
tEnd = time()-tStart
print("Words: %7d in %7.3f µs" % (wordCount,tEnd*1e6))
tStart = time()
sentCount = self._countSentences(theText)
tEnd = time()-tStart
print("Sentences: %7d in %7.3f µs" % (sentCount,tEnd*1e6))
tStart = time()
paraCount = self._countParagraphs(theText)
tEnd = time()-tStart
print("Paragraphs: %7d in %7.3f µs" % (paraCount,tEnd*1e6))
def getStats(self):
# tStart = time()
wordCount = self._countWords()
# tEnd = time()-tStart
# print("Words: %7d in %7.3f µs" % (wordCount,tEnd*1e6))
# tStart = time()
sentCount = self._countSentences()
# tEnd = time()-tStart
# print("Sentences: %7d in %7.3f µs" % (sentCount,tEnd*1e6))
# tStart = time()
paraCount = self._countParagraphs()
# tEnd = time()-tStart
# print("Paragraphs: %7d in %7.3f µs" % (paraCount,tEnd*1e6))
return wordCount, sentCount, paraCount
def getReadabilityScore(self, theText):
def getReadabilityScore(self):
"""
Calculate Flesch--Kincaid Readability Score.
"""
tStart = time()
wordCount = self._countWords(theText)
sentCount = self._countSentences(theText)
wordCount = self._countWords(self.theText)
sentCount = self._countSentences(self.theText)
if self.langCode[:3] == "en_":
ratSyllWord = self._countSyllablesEN(theText)
ratSyllWord = self._countSyllablesEN(self.theText)
else:
ratSyllWord = -1.0
rScore = 206.835 - 1.015*(wordCount/sentCount) - 84.6*(ratSyllWord)
@@ -77,20 +78,20 @@ class TextAnalysis():
# Internal Functions
#
def _countWords(self, theText):
def _countWords(self):
"""
Counts the number of words in a text by simply splitting on all white spaces.
"""
return len(theText.strip().split())
return len(self.theText.strip().split())
def _countSentences(self, theText):
def _countSentences(self):
"""
Counts the number of non-repeated sentence endings seen in the text.
Note: This will count filenames and urls as multiple sentences.
"""
nSent = 0
sawEnd = False
for ch in theText.strip():
for ch in self.theText.strip():
if ch in ".!?":
if not sawEnd:
sawEnd = True
@@ -99,13 +100,13 @@ class TextAnalysis():
sawEnd = False
return nSent
def _countParagraphs(self, theText, pThreshold=2):
def _countParagraphs(self, pThreshold=2):
"""
Counts the number of paragraphs by counting repeated line breaks.
"""
nPara = 1
sawEnd = 0
for ch in theText.strip():
for ch in self.theText.strip():
if ch == "\r": # Ignore Windows line end chars
continue
if ch == "\n": # Count endlines
@@ -116,7 +117,7 @@ class TextAnalysis():
sawEnd = 0
return nPara
def _countSyllablesEN(self, theText):
def _countSyllablesEN(self):
"""
Attempt to count the syllables in a piece of English language text.
This function tends to slightly over-estimate the number of syllables as it doesn't handle
@@ -124,7 +125,7 @@ class TextAnalysis():
"""
cleanText = ""
for ch in theText:
for ch in self.theText:
if ch in "abcdefghijklmnopqrstuvwxyz'’":
cleanText += ch
else:
sample/sampleNovel/nwProject.nwx
+12 -9
View File
@@ -1,5 +1,5 @@
<?xml version='1.0' encoding='utf-8'?>
<novelWriterXML appVersion="0.0.1" fileVersion="1.0" timeStamp="2019-04-15 21:51:16">
<novelWriterXML appVersion="0.0.1" fileVersion="1.0" timeStamp="2019-04-19 23:41:53">
<project>
<name>Sample Project</name>
<title>Sample Project</title>
@@ -12,64 +12,67 @@
<type>ROOT</type>
<class>NOVEL</class>
<depth>0</depth>
<expanded>True</expanded>
<children>True</children>
<expanded>True</expanded>
</item>
<item handle="e7ded148d6e4a" order="0" parent="7031beac91f75">
<name>New Chapter</name>
<type>FOLDER</type>
<class>NOVEL</class>
<depth>1</depth>
<expanded>True</expanded>
<children>True</children>
<expanded>True</expanded>
</item>
<item handle="96b68994dfa3d" order="0" parent="e7ded148d6e4a">
<name>New Scene</name>
<type>FILE</type>
<class>NOVEL</class>
<depth>2</depth>
<expanded>False</expanded>
<children>False</children>
<expanded>False</expanded>
<wordCount>381</wordCount>
<sentCount>51</sentCount>
<paraCount>9</paraCount>
</item>
<item handle="636b6aa9b697b" order="1" parent="e7ded148d6e4a">
<name>New File</name>
<type>FILE</type>
<class>NOVEL</class>
<depth>2</depth>
<expanded>False</expanded>
<children>False</children>
<expanded>False</expanded>
</item>
<item handle="f6622b4617424" order="1" parent="None">
<name>Characters</name>
<type>ROOT</type>
<class>CHARACTER</class>
<depth>0</depth>
<expanded>True</expanded>
<children>True</children>
<expanded>True</expanded>
</item>
<item handle="14298de4d9524" order="0" parent="f6622b4617424">
<name>Jon Smith</name>
<type>FILE</type>
<class>CHARACTER</class>
<depth>1</depth>
<expanded>False</expanded>
<children>False</children>
<expanded>False</expanded>
</item>
<item handle="bb2c23b3c42cc" order="1" parent="f6622b4617424">
<name>Jane Smith</name>
<type>FILE</type>
<class>CHARACTER</class>
<depth>1</depth>
<expanded>False</expanded>
<children>False</children>
<expanded>False</expanded>
</item>
<item handle="73eee34351f85" order="2" parent="None">
<name>World</name>
<type>ROOT</type>
<class>NONE</class>
<depth>0</depth>
<expanded>False</expanded>
<children>False</children>
<expanded>False</expanded>
</item>
</content>
</novelWriterXML>