Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made some fixes to the index (including a highlighting bug), some cleanup, and added more tests

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2020-05-19 00:03:52 +02:00
parent 0ae853618c
commit 59e6b725ba
3 changed files with 287 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/core/index.py
+38 -45
View File
@@ -41,7 +41,7 @@ logger = logging.getLogger(__name__)
class NWIndex():
VALID_KEYS = [
VALID_KEYS = set([
nwKeyWords.TAG_KEY,
nwKeyWords.PLOT_KEY,
nwKeyWords.POV_KEY,
@@ -51,7 +51,7 @@ class NWIndex():
nwKeyWords.OBJECT_KEY,
nwKeyWords.ENTITY_KEY,
nwKeyWords.CUSTOM_KEY
]
])
TAG_CLASS = {
nwKeyWords.CHAR_KEY : nwItemClass.CHARACTER,
nwKeyWords.POV_KEY : nwItemClass.CHARACTER,
@@ -107,7 +107,6 @@ class NWIndex():
def deleteHandle(self, tHandle):
"""Delete all entries of a given document handle.
"""
delTags = []
for tTag in self.tagIndex:
if self.tagIndex[tTag][1] == tHandle:
@@ -130,7 +129,6 @@ class NWIndex():
def loadIndex(self):
"""Load index from last session from the project meta folder.
"""
theData = {}
indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE)
@@ -169,7 +167,6 @@ class NWIndex():
"""Save the current index as a json file in the project meta
data folder.
"""
indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE)
logger.debug("Saving index file")
@@ -179,7 +176,7 @@ class NWIndex():
nIndent = None
try:
with open(indexFile,mode="w+",encoding="utf8") as outFile:
with open(indexFile, mode="w+", encoding="utf8") as outFile:
outFile.write(json.dumps({
"tagIndex" : self.tagIndex,
"refIndex" : self.refIndex,
@@ -198,7 +195,6 @@ class NWIndex():
"""Check that the entries in the index are valid and contain the
elements it should.
"""
self.indexBroken = False
try:
@@ -265,7 +261,7 @@ class NWIndex():
logger.debug("Indexing item with handle %s" % tHandle)
# Check file type, and reset its old index
# Also add an entry for T0 in case the file has no title
# Also add a dummy entry for T0 in case the file has no title
self.refIndex[tHandle] = {}
self.refIndex[tHandle]["T0"] = {
"tags" : [],
@@ -297,16 +293,16 @@ class NWIndex():
continue
if aLine.startswith(r"#"):
isTitle = self.indexTitle(tHandle, isNovel, aLine, nLine, itemLayout)
isTitle = self._indexTitle(tHandle, isNovel, aLine, nLine, itemLayout)
if isTitle and nLine > 0:
if nTitle > 0:
lastText = "\n".join(theLines[nTitle-1:nLine-1])
self.indexWordCounts(tHandle, isNovel, lastText, nTitle)
self._indexWordCounts(tHandle, isNovel, lastText, nTitle)
nTitle = nLine
elif aLine.startswith(r"@"):
self.indexNoteRef(tHandle, aLine, nLine, nTitle)
self.indexTag(tHandle, aLine, nLine, itemClass)
self._indexNoteRef(tHandle, aLine, nLine, nTitle)
self._indexTag(tHandle, aLine, nLine, itemClass)
elif aLine.startswith(r"%"):
if nTitle > 0:
@@ -315,12 +311,12 @@ class NWIndex():
cLen = len(toCheck)
cOff = tLen - cLen
if toCheck.startswith("synopsis:"):
self.indexSynopsis(tHandle, isNovel, aLine[cOff+9:].strip(), nTitle)
self._indexSynopsis(tHandle, isNovel, aLine[cOff+9:].strip(), nTitle)
# Count words for remaining text after last heading
if nTitle > 0:
lastText = "\n".join(theLines[nTitle-1:nLine-1])
self.indexWordCounts(tHandle, isNovel, lastText, nTitle)
self._indexWordCounts(tHandle, isNovel, lastText, nTitle)
# Run word counter for whole text
cC, wC, pC = countWords(theText)
@@ -336,11 +332,14 @@ class NWIndex():
return True
def indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout):
"""Save information about the title and its location in the
file.
"""
##
# Internal Indexers
##
def _indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout):
"""Save information about the title and its location in the
file to the index.
"""
if aLine.startswith("# "):
hDepth = "H1"
hText = aLine[2:].strip()
@@ -382,7 +381,9 @@ class NWIndex():
return True
def indexWordCounts(self, tHandle, isNovel, theText, nTitle):
def _indexWordCounts(self, tHandle, isNovel, theText, nTitle):
"""Count text stats and save the counts to the index.
"""
cC, wC, pC = countWords(theText)
sTitle = "T%d" % nTitle
if isNovel:
@@ -401,7 +402,9 @@ class NWIndex():
self.noteIndex[tHandle][sTitle]["updated"] = time()
return
def indexSynopsis(self, tHandle, isNovel, theText, nTitle):
def _indexSynopsis(self, tHandle, isNovel, theText, nTitle):
"""Save the synopsis to the index.
"""
sTitle = "T%d" % nTitle
if isNovel:
if tHandle in self.novelIndex:
@@ -415,11 +418,10 @@ class NWIndex():
self.noteIndex[tHandle][sTitle]["updated"] = time()
return
def indexNoteRef(self, tHandle, aLine, nLine, nTitle):
def _indexNoteRef(self, tHandle, aLine, nLine, nTitle):
"""Validate and save the information about a reference to a tag
in another file.
"""
isValid, theBits, thePos = self.scanThis(aLine)
if not isValid or len(theBits) == 0:
return False
@@ -435,10 +437,9 @@ class NWIndex():
return True
def indexTag(self, tHandle, aLine, nLine, itemClass):
def _indexTag(self, tHandle, aLine, nLine, itemClass):
"""Validate and save the information from a tag.
"""
isValid, theBits, thePos = self.scanThis(aLine)
if not isValid or len(theBits) != 2:
return False
@@ -453,42 +454,39 @@ class NWIndex():
##
def scanThis(self, aLine):
"""Scan a line starting with @ to check that it's valid and to
split up its elements into an array and an array of positions.
The latter is needed for the syntax highlighter.
"""Scan a line starting with @ to check that it's valid. Then
split it up into its elements and positions as two arrays.
"""
theBits = [] # The elements of the string
thePos = [] # The absolute position of each element
theBits = []
thePos = []
aLine = aLine.strip()
aLine = aLine.rstrip() # Remove all trailing white spaces
nChar = len(aLine)
if nChar < 2:
return False, theBits, thePos
if aLine[0] != "@":
return False, theBits, thePos
cPos = 0
cKey, cSep, cVals = aLine.partition(":")
cKey, _, cVals = aLine.partition(":")
sKey = cKey.strip()
if sKey == "@":
return False, theBits, thePos
cPos = 0
theBits.append(sKey)
thePos.append(cPos)
cPos += len(sKey) + 1
cPos += len(cKey) + 1
if cVals == "":
if not cVals:
# No values, so we're done
return True, theBits, thePos
aVals = cVals.split(",")
for cVal in aVals:
for cVal in cVals.split(","):
sVal = cVal.strip()
rLen = len(cVal.lstrip())
tLen = len(cVal)
theBits.append(sVal)
thePos.append(cPos+tLen-rLen)
thePos.append(cPos + tLen - rLen)
cPos += tLen + 1
return True, theBits, thePos
@@ -497,8 +495,7 @@ class NWIndex():
"""Check the tags against the index to see if they are valid
tags. This is needed for syntax highlighting.
"""
nBits = len(theBits)
nBits = len(theBits)
isGood = [False]*nBits
if nBits == 0:
return []
@@ -512,7 +509,7 @@ class NWIndex():
# is ignored
if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1:
isGood[0] = True
if theBits[1] in self.tagIndex.keys():
if theBits[1] in self.tagIndex:
if self.tagIndex[theBits[1]][1] == tItem.itemHandle:
isGood[1] = True
else:
@@ -537,7 +534,6 @@ class NWIndex():
order as they appear in the tree view and in the respective
document files, but skipping all note files.
"""
theStructure = []
for tHandle in self.theProject.projTree.handles():
if tHandle not in self.novelIndex:
@@ -551,7 +547,6 @@ class NWIndex():
"""Returns the counts for a file, or a section of a file
starting at title nTitle.
"""
cC = 0
wC = 0
pC = 0
@@ -579,7 +574,6 @@ class NWIndex():
"""Extract all references made in a file, and optionally title
section. sTitle must be a string.
"""
theRefs = {}
for tKey in self.TAG_CLASS:
theRefs[tKey] = []
@@ -602,7 +596,6 @@ class NWIndex():
"""Build a list of files referring back to our file, specified
by tHandle.
"""
theRefs = {}
tItem = self.theProject.projTree[tHandle]
tests/reference/proj/3_nwProject.nwx
+142
View File
@@ -0,0 +1,142 @@
<?xml version='1.0' encoding='utf-8'?>
<novelWriterXML appVersion="0.5.1" hexVersion="0x000501f0" fileVersion="1.0" saveCount="5" autoCount="0" timeStamp="2020-05-18 23:33:37">
<project>
<name></name>
<title></title>
<backup>True</backup>
</project>
<settings>
<spellCheck>False</spellCheck>
<autoOutline>True</autoOutline>
<lastEdited>None</lastEdited>
<lastViewed>None</lastViewed>
<lastWordCount>0</lastWordCount>
<autoReplace/>
<titleFormat>
<title>%title%</title>
<chapter>Chapter %num%\\%title%</chapter>
<unnumbered>%title%</unnumbered>
<scene>* * *</scene>
<section></section>
<withSynopsis>False</withSynopsis>
<withComments>False</withComments>
<withKeywords>False</withKeywords>
</titleFormat>
<status>
<entry blue="100" green="100" red="100">New</entry>
<entry blue="0" green="50" red="200">Note</entry>
<entry blue="0" green="150" red="200">Draft</entry>
<entry blue="0" green="200" red="50">Finished</entry>
</status>
<importance>
<entry blue="100" green="100" red="100">New</entry>
<entry blue="0" green="50" red="200">Minor</entry>
<entry blue="0" green="150" red="200">Major</entry>
<entry blue="0" green="200" red="50">Main</entry>
</importance>
</settings>
<content count="12">
<item handle="73475cb40a568" order="None" parent="None">
<name>Novel</name>
<type>ROOT</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="44cb730c42048" order="None" parent="None">
<name>Characters</name>
<type>ROOT</type>
<class>CHARACTER</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="71ee45a3c0db9" order="None" parent="None">
<name>Plot</name>
<type>ROOT</type>
<class>PLOT</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="811786ad1ae74" order="None" parent="None">
<name>World</name>
<type>ROOT</type>
<class>WORLD</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="25fc0e7096fc6" order="None" parent="73475cb40a568">
<name>New Chapter</name>
<type>FOLDER</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="31489056e0916" order="None" parent="25fc0e7096fc6">
<name>New Scene</name>
<type>FILE</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
<exported>True</exported>
<layout>SCENE</layout>
<charCount>0</charCount>
<wordCount>0</wordCount>
<paraCount>0</paraCount>
<cursorPos>0</cursorPos>
</item>
<item handle="98010bd9270f9" order="None" parent="None">
<name>Timeline</name>
<type>ROOT</type>
<class>TIMELINE</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="0e17daca5f3e1" order="None" parent="None">
<name>Object</name>
<type>ROOT</type>
<class>OBJECT</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="1a6562590ef19" order="None" parent="None">
<name>Custom1</name>
<type>ROOT</type>
<class>CUSTOM</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="031b4af5197ec" order="None" parent="None">
<name>Custom2</name>
<type>ROOT</type>
<class>CUSTOM</class>
<status>New</status>
<expanded>False</expanded>
</item>
<item handle="41cfc0d1f2d12" order="None" parent="73475cb40a568">
<name>Hello</name>
<type>FILE</type>
<class>NOVEL</class>
<status>New</status>
<expanded>False</expanded>
<exported>True</exported>
<layout>SCENE</layout>
<charCount>0</charCount>
<wordCount>0</wordCount>
<paraCount>0</paraCount>
<cursorPos>0</cursorPos>
</item>
<item handle="2858dcd1057d3" order="None" parent="44cb730c42048">
<name>Jane</name>
<type>FILE</type>
<class>CHARACTER</class>
<status>New</status>
<expanded>False</expanded>
<exported>True</exported>
<layout>NOTE</layout>
<charCount>0</charCount>
<wordCount>0</wordCount>
<paraCount>0</paraCount>
<cursorPos>0</cursorPos>
</item>
</content>
</novelWriterXML>
tests/test_project.py
+107 -1
View File
@@ -76,20 +76,39 @@ def testProjectNewRoot(nwTempProj,nwRef):
assert cmpFiles(projFile, refFile, [2])
assert not theProject.projChanged
@pytest.mark.project
def testProjectNewFile(nwTempProj,nwRef):
projFile = path.join(nwTempProj,"nwProject.nwx")
refFile = path.join(nwRef,"proj","3_nwProject.nwx")
assert theProject.openProject(projFile)
assert isinstance(theProject.newFile("Hello", nwItemClass.NOVEL, "73475cb40a568"), str)
assert isinstance(theProject.newFile("Jane", nwItemClass.CHARACTER, "44cb730c42048"), str)
assert theProject.projChanged
assert theProject.saveProject()
assert theProject.closeProject()
assert cmpFiles(projFile, refFile, [2])
assert not theProject.projChanged
@pytest.mark.project
def testIndexScanThis(nwTempProj):
projFile = path.join(nwTempProj,"nwProject.nwx")
assert theProject.openProject(projFile)
theIndex = NWIndex(theProject,theMain)
theIndex = NWIndex(theProject, theMain)
tHandle = "31489056e0916"
isValid, theBits, thePos = theIndex.scanThis("tag: this, and this")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@:")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis(" @a: b")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@a:")
assert isValid
assert str(theBits) == "['@a']"
@@ -105,9 +124,96 @@ def testIndexScanThis(nwTempProj):
assert str(theBits) == "['@a', 'b', 'c', 'd']"
assert str(thePos) == "[0, 3, 5, 7]"
isValid, theBits, thePos = theIndex.scanThis("@a : b , c , d")
assert isValid
assert str(theBits) == "['@a', 'b', 'c', 'd']"
assert str(thePos) == "[0, 5, 9, 13]"
isValid, theBits, thePos = theIndex.scanThis("@tag: this, and this")
assert isValid
assert str(theBits) == "['@tag', 'this', 'and this']"
assert str(thePos) == "[0, 6, 12]"
assert theProject.closeProject()
@pytest.mark.project
def testIndexCheckThese(nwTempProj):
projFile = path.join(nwTempProj,"nwProject.nwx")
assert theProject.openProject(projFile)
theIndex = NWIndex(theProject, theMain)
nHandle = "41cfc0d1f2d12"
nItem = theProject.projTree[nHandle]
cHandle = "2858dcd1057d3"
cItem = theProject.projTree[cHandle]
assert theIndex.scanText(cHandle, (
"# Jane Smith\n"
"@tag: Jane"
))
assert theIndex.scanText(nHandle, (
"# Hello World!\n"
"@pov: Jane"
))
assert str(theIndex.tagIndex) == "{'Jane': [2, '2858dcd1057d3', 'CHARACTER']}"
assert theIndex.novelIndex[nHandle]["T1"]["title"] == "Hello World!"
assert str(theIndex.checkThese(["@tag", "Jane"], cItem)) == "[True, True]"
assert str(theIndex.checkThese(["@tag", "John"], cItem)) == "[True, True]"
assert str(theIndex.checkThese(["@tag", "Jane"], nItem)) == "[True, False]"
assert str(theIndex.checkThese(["@tag", "John"], nItem)) == "[True, True]"
assert str(theIndex.checkThese(["@pov", "John"], nItem)) == "[True, False]"
assert str(theIndex.checkThese(["@pov", "Jane"], nItem)) == "[True, True]"
assert str(theIndex.checkThese(["@ pov", "Jane"], nItem)) == "[False, False]"
assert str(theIndex.checkThese(["@what", "Jane"], nItem)) == "[False, False]"
assert theProject.closeProject()
@pytest.mark.project
def testIndexMeta(nwTempProj):
projFile = path.join(nwTempProj,"nwProject.nwx")
assert theProject.openProject(projFile)
theIndex = NWIndex(theProject, theMain)
nHandle = "41cfc0d1f2d12"
nItem = theProject.projTree[nHandle]
cHandle = "2858dcd1057d3"
cItem = theProject.projTree[cHandle]
assert theIndex.scanText(cHandle, (
"# Jane Smith\n"
"@tag: Jane\n"
))
assert theIndex.scanText(nHandle, (
"# Hello World!\n"
"@pov: Jane\n"
"@char: Jane\n"
"\n"
"% this is a comment\n"
"\n"
"This is a story about Jane Smith.\n"
"\n"
"Well, not really.\n"
))
assert str(theIndex.tagIndex) == "{'Jane': [2, '2858dcd1057d3', 'CHARACTER']}"
assert theIndex.novelIndex[nHandle]["T1"]["title"] == "Hello World!"
# The novel structure should contain the pointer to the novel file header
assert str(theIndex.getNovelStructure()) == "['41cfc0d1f2d12:T1']"
# The novel file should have the correct counts
cC, wC, pC = theIndex.getCounts(nHandle)
assert cC == 62 # Characters in text and title only
assert wC == 12 # Words in text and title only
assert pC == 2 # Paragraphs in text only
# The novel file should now refer to Jane as @pov and @char
theRefs = theIndex.getReferences(nHandle)
assert str(theRefs["@pov"]) == "['Jane']"
assert str(theRefs["@char"]) == "['Jane']"
# The character file should have a record of the reference from the novel file
theRefs = theIndex.getBackReferenceList(cHandle)
assert str(theRefs) == "{'41cfc0d1f2d12': 3}"
assert theProject.closeProject()