Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made some fixes to the index (including a highlighting bug), some cleanup, and added more tests

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2020-05-19 00:03:52 +02:00
parent 0ae853618c
commit 59e6b725ba
3 changed files with 287 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/core/index.py
+38 -45
View File
@@ -41,7 +41,7 @@ logger = logging.getLogger(__name__)
class NWIndex(): class NWIndex():
VALID_KEYS = [ VALID_KEYS = set([
nwKeyWords.TAG_KEY, nwKeyWords.TAG_KEY,
nwKeyWords.PLOT_KEY, nwKeyWords.PLOT_KEY,
nwKeyWords.POV_KEY, nwKeyWords.POV_KEY,
@@ -51,7 +51,7 @@ class NWIndex():
nwKeyWords.OBJECT_KEY, nwKeyWords.OBJECT_KEY,
nwKeyWords.ENTITY_KEY, nwKeyWords.ENTITY_KEY,
nwKeyWords.CUSTOM_KEY nwKeyWords.CUSTOM_KEY
] ])
TAG_CLASS = { TAG_CLASS = {
nwKeyWords.CHAR_KEY : nwItemClass.CHARACTER, nwKeyWords.CHAR_KEY : nwItemClass.CHARACTER,
nwKeyWords.POV_KEY : nwItemClass.CHARACTER, nwKeyWords.POV_KEY : nwItemClass.CHARACTER,
@@ -107,7 +107,6 @@ class NWIndex():
def deleteHandle(self, tHandle): def deleteHandle(self, tHandle):
"""Delete all entries of a given document handle. """Delete all entries of a given document handle.
""" """
delTags = [] delTags = []
for tTag in self.tagIndex: for tTag in self.tagIndex:
if self.tagIndex[tTag][1] == tHandle: if self.tagIndex[tTag][1] == tHandle:
@@ -130,7 +129,6 @@ class NWIndex():
def loadIndex(self): def loadIndex(self):
"""Load index from last session from the project meta folder. """Load index from last session from the project meta folder.
""" """
theData = {} theData = {}
indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE) indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE)
@@ -169,7 +167,6 @@ class NWIndex():
"""Save the current index as a json file in the project meta """Save the current index as a json file in the project meta
data folder. data folder.
""" """
indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE) indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE)
logger.debug("Saving index file") logger.debug("Saving index file")
@@ -179,7 +176,7 @@ class NWIndex():
nIndent = None nIndent = None
try: try:
with open(indexFile,mode="w+",encoding="utf8") as outFile: with open(indexFile, mode="w+", encoding="utf8") as outFile:
outFile.write(json.dumps({ outFile.write(json.dumps({
"tagIndex" : self.tagIndex, "tagIndex" : self.tagIndex,
"refIndex" : self.refIndex, "refIndex" : self.refIndex,
@@ -198,7 +195,6 @@ class NWIndex():
"""Check that the entries in the index are valid and contain the """Check that the entries in the index are valid and contain the
elements it should. elements it should.
""" """
self.indexBroken = False self.indexBroken = False
try: try:
@@ -265,7 +261,7 @@ class NWIndex():
logger.debug("Indexing item with handle %s" % tHandle) logger.debug("Indexing item with handle %s" % tHandle)
# Check file type, and reset its old index # Check file type, and reset its old index
# Also add an entry for T0 in case the file has no title # Also add a dummy entry for T0 in case the file has no title
self.refIndex[tHandle] = {} self.refIndex[tHandle] = {}
self.refIndex[tHandle]["T0"] = { self.refIndex[tHandle]["T0"] = {
"tags" : [], "tags" : [],
@@ -297,16 +293,16 @@ class NWIndex():
continue continue
if aLine.startswith(r"#"): if aLine.startswith(r"#"):
isTitle = self.indexTitle(tHandle, isNovel, aLine, nLine, itemLayout) isTitle = self._indexTitle(tHandle, isNovel, aLine, nLine, itemLayout)
if isTitle and nLine > 0: if isTitle and nLine > 0:
if nTitle > 0: if nTitle > 0:
lastText = "\n".join(theLines[nTitle-1:nLine-1]) lastText = "\n".join(theLines[nTitle-1:nLine-1])
self.indexWordCounts(tHandle, isNovel, lastText, nTitle) self._indexWordCounts(tHandle, isNovel, lastText, nTitle)
nTitle = nLine nTitle = nLine
elif aLine.startswith(r"@"): elif aLine.startswith(r"@"):
self.indexNoteRef(tHandle, aLine, nLine, nTitle) self._indexNoteRef(tHandle, aLine, nLine, nTitle)
self.indexTag(tHandle, aLine, nLine, itemClass) self._indexTag(tHandle, aLine, nLine, itemClass)
elif aLine.startswith(r"%"): elif aLine.startswith(r"%"):
if nTitle > 0: if nTitle > 0:
@@ -315,12 +311,12 @@ class NWIndex():
cLen = len(toCheck) cLen = len(toCheck)
cOff = tLen - cLen cOff = tLen - cLen
if toCheck.startswith("synopsis:"): if toCheck.startswith("synopsis:"):
self.indexSynopsis(tHandle, isNovel, aLine[cOff+9:].strip(), nTitle) self._indexSynopsis(tHandle, isNovel, aLine[cOff+9:].strip(), nTitle)
# Count words for remaining text after last heading # Count words for remaining text after last heading
if nTitle > 0: if nTitle > 0:
lastText = "\n".join(theLines[nTitle-1:nLine-1]) lastText = "\n".join(theLines[nTitle-1:nLine-1])
self.indexWordCounts(tHandle, isNovel, lastText, nTitle) self._indexWordCounts(tHandle, isNovel, lastText, nTitle)
# Run word counter for whole text # Run word counter for whole text
cC, wC, pC = countWords(theText) cC, wC, pC = countWords(theText)
@@ -336,11 +332,14 @@ class NWIndex():
return True return True
def indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout): ##
"""Save information about the title and its location in the # Internal Indexers
file. ##
"""
def _indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout):
"""Save information about the title and its location in the
file to the index.
"""
if aLine.startswith("# "): if aLine.startswith("# "):
hDepth = "H1" hDepth = "H1"
hText = aLine[2:].strip() hText = aLine[2:].strip()
@@ -382,7 +381,9 @@ class NWIndex():
return True return True
def indexWordCounts(self, tHandle, isNovel, theText, nTitle): def _indexWordCounts(self, tHandle, isNovel, theText, nTitle):
"""Count text stats and save the counts to the index.
"""
cC, wC, pC = countWords(theText) cC, wC, pC = countWords(theText)
sTitle = "T%d" % nTitle sTitle = "T%d" % nTitle
if isNovel: if isNovel:
@@ -401,7 +402,9 @@ class NWIndex():
self.noteIndex[tHandle][sTitle]["updated"] = time() self.noteIndex[tHandle][sTitle]["updated"] = time()
return return
def indexSynopsis(self, tHandle, isNovel, theText, nTitle): def _indexSynopsis(self, tHandle, isNovel, theText, nTitle):
"""Save the synopsis to the index.
"""
sTitle = "T%d" % nTitle sTitle = "T%d" % nTitle
if isNovel: if isNovel:
if tHandle in self.novelIndex: if tHandle in self.novelIndex:
@@ -415,11 +418,10 @@ class NWIndex():
self.noteIndex[tHandle][sTitle]["updated"] = time() self.noteIndex[tHandle][sTitle]["updated"] = time()
return return
def indexNoteRef(self, tHandle, aLine, nLine, nTitle): def _indexNoteRef(self, tHandle, aLine, nLine, nTitle):
"""Validate and save the information about a reference to a tag """Validate and save the information about a reference to a tag
in another file. in another file.
""" """
isValid, theBits, thePos = self.scanThis(aLine) isValid, theBits, thePos = self.scanThis(aLine)
if not isValid or len(theBits) == 0: if not isValid or len(theBits) == 0:
return False return False
@@ -435,10 +437,9 @@ class NWIndex():
return True return True
def indexTag(self, tHandle, aLine, nLine, itemClass): def _indexTag(self, tHandle, aLine, nLine, itemClass):
"""Validate and save the information from a tag. """Validate and save the information from a tag.
""" """
isValid, theBits, thePos = self.scanThis(aLine) isValid, theBits, thePos = self.scanThis(aLine)
if not isValid or len(theBits) != 2: if not isValid or len(theBits) != 2:
return False return False
@@ -453,42 +454,39 @@ class NWIndex():
## ##
def scanThis(self, aLine): def scanThis(self, aLine):
"""Scan a line starting with @ to check that it's valid and to """Scan a line starting with @ to check that it's valid. Then
split up its elements into an array and an array of positions. split it up into its elements and positions as two arrays.
The latter is needed for the syntax highlighter.
""" """
theBits = [] # The elements of the string
thePos = [] # The absolute position of each element
theBits = [] aLine = aLine.rstrip() # Remove all trailing white spaces
thePos = []
aLine = aLine.strip()
nChar = len(aLine) nChar = len(aLine)
if nChar < 2: if nChar < 2:
return False, theBits, thePos return False, theBits, thePos
if aLine[0] != "@": if aLine[0] != "@":
return False, theBits, thePos return False, theBits, thePos
cPos = 0 cKey, _, cVals = aLine.partition(":")
cKey, cSep, cVals = aLine.partition(":")
sKey = cKey.strip() sKey = cKey.strip()
if sKey == "@": if sKey == "@":
return False, theBits, thePos return False, theBits, thePos
cPos = 0
theBits.append(sKey) theBits.append(sKey)
thePos.append(cPos) thePos.append(cPos)
cPos += len(sKey) + 1 cPos += len(cKey) + 1
if cVals == "": if not cVals:
# No values, so we're done # No values, so we're done
return True, theBits, thePos return True, theBits, thePos
aVals = cVals.split(",") for cVal in cVals.split(","):
for cVal in aVals:
sVal = cVal.strip() sVal = cVal.strip()
rLen = len(cVal.lstrip()) rLen = len(cVal.lstrip())
tLen = len(cVal) tLen = len(cVal)
theBits.append(sVal) theBits.append(sVal)
thePos.append(cPos+tLen-rLen) thePos.append(cPos + tLen - rLen)
cPos += tLen + 1 cPos += tLen + 1
return True, theBits, thePos return True, theBits, thePos
@@ -497,8 +495,7 @@ class NWIndex():
"""Check the tags against the index to see if they are valid """Check the tags against the index to see if they are valid
tags. This is needed for syntax highlighting. tags. This is needed for syntax highlighting.
""" """
nBits = len(theBits)
nBits = len(theBits)
isGood = [False]*nBits isGood = [False]*nBits
if nBits == 0: if nBits == 0:
return [] return []
@@ -512,7 +509,7 @@ class NWIndex():
# is ignored # is ignored
if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1: if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1:
isGood[0] = True isGood[0] = True
if theBits[1] in self.tagIndex.keys(): if theBits[1] in self.tagIndex:
if self.tagIndex[theBits[1]][1] == tItem.itemHandle: if self.tagIndex[theBits[1]][1] == tItem.itemHandle:
isGood[1] = True isGood[1] = True
else: else:
@@ -537,7 +534,6 @@ class NWIndex():
order as they appear in the tree view and in the respective order as they appear in the tree view and in the respective
document files, but skipping all note files. document files, but skipping all note files.
""" """
theStructure = [] theStructure = []
for tHandle in self.theProject.projTree.handles(): for tHandle in self.theProject.projTree.handles():
if tHandle not in self.novelIndex: if tHandle not in self.novelIndex:
@@ -551,7 +547,6 @@ class NWIndex():
"""Returns the counts for a file, or a section of a file """Returns the counts for a file, or a section of a file
starting at title nTitle. starting at title nTitle.
""" """
cC = 0 cC = 0
wC = 0 wC = 0
pC = 0 pC = 0
@@ -579,7 +574,6 @@ class NWIndex():
"""Extract all references made in a file, and optionally title """Extract all references made in a file, and optionally title
section. sTitle must be a string. section. sTitle must be a string.
""" """
theRefs = {} theRefs = {}
for tKey in self.TAG_CLASS: for tKey in self.TAG_CLASS:
theRefs[tKey] = [] theRefs[tKey] = []
@@ -602,7 +596,6 @@ class NWIndex():
"""Build a list of files referring back to our file, specified """Build a list of files referring back to our file, specified
by tHandle. by tHandle.
""" """
theRefs = {} theRefs = {}
tItem = self.theProject.projTree[tHandle] tItem = self.theProject.projTree[tHandle]
tests/reference/proj/3_nwProject.nwx
+142
View File
@@ -0,0 +1,142 @@
<?xml version='1.0' encoding='utf-8'?>
<novelWriterXML appVersion="0.5.1" hexVersion="0x000501f0" fileVersion="1.0" saveCount="5" autoCount="0" timeStamp="2020-05-18 23:33:37">
<project>
<name></name>
<title></title>
<backup>True</backup>
</project>
<settings>
<spellCheck>False</spellCheck>
<autoOutline>True</autoOutline>
<lastEdited>None</lastEdited>
<lastViewed>None</lastViewed>
<lastWordCount>0</lastWordCount>
<autoReplace/>
<titleFormat>
<title>%title%</title>
<chapter>Chapter %num%\\%title%</chapter>
<unnumbered>%title%</unnumbered>
<scene>* * *</scene>
<section></section>
<withSynopsis>False</withSynopsis>
<withComments>False</withComments>
<withKeywords>False</withKeywords>
</titleFormat>
<status>
<entry blue="100" green="100" red="100">New</entry>
<entry blue="0" green="50" red="200">Note</entry>
<entry blue="0" green="150" red="200">Draft</entry>
<entry blue="0" green="200" red="50">Finished</entry>
</status>
<importance>
<entry blue="100" green="100" red="100">New</entry>
<entry blue="0" green="50" red="200">Minor</entry>
<entry blue="0" green="150" red="200">Major</entry>
<entry blue="0" green="200" red="50">Main</entry>
</importance>
</settings>
<content count="12">
<item handle="73475cb40a568" order="None" parent="None">
<name>Novel</name>
<type>ROOT</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="44cb730c42048" order="None" parent="None">
<name>Characters</name>
<type>ROOT</type>
<class>CHARACTER</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="71ee45a3c0db9" order="None" parent="None">
<name>Plot</name>
<type>ROOT</type>
<class>PLOT</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="811786ad1ae74" order="None" parent="None">
<name>World</name>
<type>ROOT</type>
<class>WORLD</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="25fc0e7096fc6" order="None" parent="73475cb40a568">
<name>New Chapter</name>
<type>FOLDER</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="31489056e0916" order="None" parent="25fc0e7096fc6">
<name>New Scene</name>
<type>FILE</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
<exported>True</exported>
<layout>SCENE</layout>
<charCount>0</charCount>
<wordCount>0</wordCount>
<paraCount>0</paraCount>
<cursorPos>0</cursorPos>
</item>
<item handle="98010bd9270f9" order="None" parent="None">
<name>Timeline</name>
<type>ROOT</type>
<class>TIMELINE</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="0e17daca5f3e1" order="None" parent="None">
<name>Object</name>
<type>ROOT</type>
<class>OBJECT</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="1a6562590ef19" order="None" parent="None">
<name>Custom1</name>
<type>ROOT</type>
<class>CUSTOM</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="031b4af5197ec" order="None" parent="None">
<name>Custom2</name>
<type>ROOT</type>
<class>CUSTOM</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="41cfc0d1f2d12" order="None" parent="73475cb40a568">
<name>Hello</name>
<type>FILE</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
<exported>True</exported>
<layout>SCENE</layout>
<charCount>0</charCount>
<wordCount>0</wordCount>
<paraCount>0</paraCount>
<cursorPos>0</cursorPos>
</item>
<item handle="2858dcd1057d3" order="None" parent="44cb730c42048">
<name>Jane</name>
<type>FILE</type>
<class>CHARACTER</class>
<status>New</status>
<expanded>False</expanded>
<exported>True</exported>
<layout>NOTE</layout>
<charCount>0</charCount>
<wordCount>0</wordCount>
<paraCount>0</paraCount>
<cursorPos>0</cursorPos>
</item>
</content>
</novelWriterXML>
tests/test_project.py
+107 -1
View File
@@ -76,20 +76,39 @@ def testProjectNewRoot(nwTempProj,nwRef):
assert cmpFiles(projFile, refFile, [2]) assert cmpFiles(projFile, refFile, [2])
assert not theProject.projChanged assert not theProject.projChanged
@pytest.mark.project
def testProjectNewFile(nwTempProj,nwRef):
projFile = path.join(nwTempProj,"nwProject.nwx")
refFile = path.join(nwRef,"proj","3_nwProject.nwx")
assert theProject.openProject(projFile)
assert isinstance(theProject.newFile("Hello", nwItemClass.NOVEL, "73475cb40a568"), str)
assert isinstance(theProject.newFile("Jane", nwItemClass.CHARACTER, "44cb730c42048"), str)
assert theProject.projChanged
assert theProject.saveProject()
assert theProject.closeProject()
assert cmpFiles(projFile, refFile, [2])
assert not theProject.projChanged
@pytest.mark.project @pytest.mark.project
def testIndexScanThis(nwTempProj): def testIndexScanThis(nwTempProj):
projFile = path.join(nwTempProj,"nwProject.nwx") projFile = path.join(nwTempProj,"nwProject.nwx")
assert theProject.openProject(projFile) assert theProject.openProject(projFile)
theIndex = NWIndex(theProject,theMain) theIndex = NWIndex(theProject, theMain)
tHandle = "31489056e0916" tHandle = "31489056e0916"
isValid, theBits, thePos = theIndex.scanThis("tag: this, and this") isValid, theBits, thePos = theIndex.scanThis("tag: this, and this")
assert not isValid assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@:") isValid, theBits, thePos = theIndex.scanThis("@:")
assert not isValid assert not isValid
isValid, theBits, thePos = theIndex.scanThis(" @a: b")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@a:") isValid, theBits, thePos = theIndex.scanThis("@a:")
assert isValid assert isValid
assert str(theBits) == "['@a']" assert str(theBits) == "['@a']"
@@ -105,9 +124,96 @@ def testIndexScanThis(nwTempProj):
assert str(theBits) == "['@a', 'b', 'c', 'd']" assert str(theBits) == "['@a', 'b', 'c', 'd']"
assert str(thePos) == "[0, 3, 5, 7]" assert str(thePos) == "[0, 3, 5, 7]"
isValid, theBits, thePos = theIndex.scanThis("@a : b , c , d")
assert isValid
assert str(theBits) == "['@a', 'b', 'c', 'd']"
assert str(thePos) == "[0, 5, 9, 13]"
isValid, theBits, thePos = theIndex.scanThis("@tag: this, and this") isValid, theBits, thePos = theIndex.scanThis("@tag: this, and this")
assert isValid assert isValid
assert str(theBits) == "['@tag', 'this', 'and this']" assert str(theBits) == "['@tag', 'this', 'and this']"
assert str(thePos) == "[0, 6, 12]" assert str(thePos) == "[0, 6, 12]"
assert theProject.closeProject() assert theProject.closeProject()
@pytest.mark.project
def testIndexCheckThese(nwTempProj):
projFile = path.join(nwTempProj,"nwProject.nwx")
assert theProject.openProject(projFile)
theIndex = NWIndex(theProject, theMain)
nHandle = "41cfc0d1f2d12"
nItem = theProject.projTree[nHandle]
cHandle = "2858dcd1057d3"
cItem = theProject.projTree[cHandle]
assert theIndex.scanText(cHandle, (
"# Jane Smith\n"
"@tag: Jane"
))
assert theIndex.scanText(nHandle, (
"# Hello World!\n"
"@pov: Jane"
))
assert str(theIndex.tagIndex) == "{'Jane': [2, '2858dcd1057d3', 'CHARACTER']}"
assert theIndex.novelIndex[nHandle]["T1"]["title"] == "Hello World!"
assert str(theIndex.checkThese(["@tag", "Jane"], cItem)) == "[True, True]"
assert str(theIndex.checkThese(["@tag", "John"], cItem)) == "[True, True]"
assert str(theIndex.checkThese(["@tag", "Jane"], nItem)) == "[True, False]"
assert str(theIndex.checkThese(["@tag", "John"], nItem)) == "[True, True]"
assert str(theIndex.checkThese(["@pov", "John"], nItem)) == "[True, False]"
assert str(theIndex.checkThese(["@pov", "Jane"], nItem)) == "[True, True]"
assert str(theIndex.checkThese(["@ pov", "Jane"], nItem)) == "[False, False]"
assert str(theIndex.checkThese(["@what", "Jane"], nItem)) == "[False, False]"
assert theProject.closeProject()
@pytest.mark.project
def testIndexMeta(nwTempProj):
projFile = path.join(nwTempProj,"nwProject.nwx")
assert theProject.openProject(projFile)
theIndex = NWIndex(theProject, theMain)
nHandle = "41cfc0d1f2d12"
nItem = theProject.projTree[nHandle]
cHandle = "2858dcd1057d3"
cItem = theProject.projTree[cHandle]
assert theIndex.scanText(cHandle, (
"# Jane Smith\n"
"@tag: Jane\n"
))
assert theIndex.scanText(nHandle, (
"# Hello World!\n"
"@pov: Jane\n"
"@char: Jane\n"
"\n"
"% this is a comment\n"
"\n"
"This is a story about Jane Smith.\n"
"\n"
"Well, not really.\n"
))
assert str(theIndex.tagIndex) == "{'Jane': [2, '2858dcd1057d3', 'CHARACTER']}"
assert theIndex.novelIndex[nHandle]["T1"]["title"] == "Hello World!"
# The novel structure should contain the pointer to the novel file header
assert str(theIndex.getNovelStructure()) == "['41cfc0d1f2d12:T1']"
# The novel file should have the correct counts
cC, wC, pC = theIndex.getCounts(nHandle)
assert cC == 62 # Characters in text and title only
assert wC == 12 # Words in text and title only
assert pC == 2 # Paragraphs in text only
# The novel file should now refer to Jane as @pov and @char
theRefs = theIndex.getReferences(nHandle)
assert str(theRefs["@pov"]) == "['Jane']"
assert str(theRefs["@char"]) == "['Jane']"
# The character file should have a record of the reference from the novel file
theRefs = theIndex.getBackReferenceList(cHandle)
assert str(theRefs) == "{'41cfc0d1f2d12': 3}"
assert theProject.closeProject()