Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated text analysis class

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2019-04-20 14:17:00 +02:00
parent f68a64503e
commit 31cb799171
1 changed files with 22 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/tools/analyse.py
+22 -27
View File
@@ -24,37 +24,36 @@ class TextAnalysis():
return return
def getStats(self): def getStats(self):
# tStart = time() tStart = time()
wordCount = self._countWords() wordCount = self._countWords()
# tEnd = time()-tStart tEnd = time()-tStart
# print("Words: %7d in %7.3f µs" % (wordCount,tEnd*1e6)) logger.verbose("Words: %7d in %8.3f ms" % (wordCount,tEnd*1e3))
# tStart = time() tStart = time()
sentCount = self._countSentences() sentCount = self._countSentences()
# tEnd = time()-tStart tEnd = time()-tStart
# print("Sentences: %7d in %7.3f µs" % (sentCount,tEnd*1e6)) logger.verbose("Sentences: %7d in %8.3f ms" % (sentCount,tEnd*1e3))
# tStart = time() tStart = time()
paraCount = self._countParagraphs() paraCount = self._countParagraphs()
# tEnd = time()-tStart tEnd = time()-tStart
# print("Paragraphs: %7d in %7.3f µs" % (paraCount,tEnd*1e6)) logger.verbose("Paragraphs: %7d in %8.3f ms" % (paraCount,tEnd*1e3))
return wordCount, sentCount, paraCount return wordCount, sentCount, paraCount
def getReadabilityScore(self): def getReadabilityScore(self):
""" """Calculate Flesch--Kincaid Readability Score.
Calculate Flesch--Kincaid Readability Score.
""" """
tStart = time() tStart = time()
wordCount = self._countWords(self.theText) wordCount = self._countWords()
sentCount = self._countSentences(self.theText) sentCount = self._countSentences()
if self.langCode[:3] == "en_": if self.langCode[:3] == "en_":
ratSyllWord = self._countSyllablesEN(self.theText) ratSyllWord = self._countSyllablesEN()
else: else:
ratSyllWord = -1.0 ratSyllWord = -1.0
rScore = 206.835 - 1.015*(wordCount/sentCount) - 84.6*(ratSyllWord) rScore = 206.835 - 1.015*(wordCount/sentCount) - 84.6*(ratSyllWord)
gLevel = -15.59 + 0.390*(wordCount/sentCount) + 11.8*(ratSyllWord) gLevel = -15.59 + 0.390*(wordCount/sentCount) + 11.8*(ratSyllWord)
tEnd = time()-tStart tEnd = time()-tStart
print("Readability: %7.3f in %7.3f ms" % (rScore,tEnd*1e3)) logger.verbose("Readability: %7.3f in %8.3f ms" % (rScore,tEnd*1e3))
print("Grade Level: %7.3f in %7.3f ms" % (gLevel,tEnd*1e3)) logger.verbose("Grade Level: %7.3f in %8.3f ms" % (gLevel,tEnd*1e3))
print("Assessment: %s" % self.getReadabilityText(rScore)) logger.verbose("Assessment: %s" % self.getReadabilityText(rScore))
return rScore, gLevel return rScore, gLevel
@@ -79,14 +78,12 @@ class TextAnalysis():
# #
def _countWords(self): def _countWords(self):
""" """Counts the number of words in a text by simply splitting on all white spaces.
Counts the number of words in a text by simply splitting on all white spaces.
""" """
return len(self.theText.strip().split()) return len(self.theText.strip().split())
def _countSentences(self): def _countSentences(self):
""" """Counts the number of non-repeated sentence endings seen in the text.
Counts the number of non-repeated sentence endings seen in the text.
Note: This will count filenames and urls as multiple sentences. Note: This will count filenames and urls as multiple sentences.
""" """
nSent = 0 nSent = 0
@@ -101,8 +98,7 @@ class TextAnalysis():
return nSent return nSent
def _countParagraphs(self, pThreshold=2): def _countParagraphs(self, pThreshold=2):
""" """Counts the number of paragraphs by counting repeated line breaks.
Counts the number of paragraphs by counting repeated line breaks.
""" """
nPara = 1 nPara = 1
sawEnd = 0 sawEnd = 0
@@ -118,8 +114,7 @@ class TextAnalysis():
return nPara return nPara
def _countSyllablesEN(self): def _countSyllablesEN(self):
""" """Attempt to count the syllables in a piece of English language text.
Attempt to count the syllables in a piece of English language text.
This function tends to slightly over-estimate the number of syllables as it doesn't handle This function tends to slightly over-estimate the number of syllables as it doesn't handle
the complexity of silent vowels in endings very well. It will count them all. the complexity of silent vowels in endings very well. It will count them all.
""" """
@@ -132,7 +127,7 @@ class TextAnalysis():
cleanText += " " cleanText += " "
asVow = "aeiouy'’" asVow = "aeiouy'’"
dExep = ("ei","ie","ua","ia","eo") dExept = ("ei","ie","ua","ia","eo")
theWords = cleanText.lower().split() theWords = cleanText.lower().split()
allSylls = 0 allSylls = 0
for inWord in theWords: for inWord in theWords:
@@ -155,7 +150,7 @@ class TextAnalysis():
nSyll -= 1 nSyll -= 1
if isVow and wasY: if isVow and wasY:
nSyll -= 1 nSyll -= 1
if inWord[c:c+2] in dExep: if inWord[c:c+2] in dExept:
nSyll += 1 nSyll += 1
wasVow = isVow wasVow = isVow
wasY = inWord[c] == "y" wasY = inWord[c] == "y"