From 59e6b725bab01aac6548dbdf892256845bfa2387 Mon Sep 17 00:00:00 2001 From: "Veronica K. B. Olsen" <1619840+vkbo@users.noreply.github.com> Date: Tue, 19 May 2020 00:03:52 +0200 Subject: [PATCH] Made some fixes to the index (including a highlighting bug), some cleanup, and added more tests --- nw/core/index.py | 83 +++++++--------- tests/reference/proj/3_nwProject.nwx | 142 +++++++++++++++++++++++++++ tests/test_project.py | 108 +++++++++++++++++++- 3 files changed, 287 insertions(+), 46 deletions(-) create mode 100644 tests/reference/proj/3_nwProject.nwx diff --git a/nw/core/index.py b/nw/core/index.py index 617ddb65..60129b35 100644 --- a/nw/core/index.py +++ b/nw/core/index.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) class NWIndex(): - VALID_KEYS = [ + VALID_KEYS = set([ nwKeyWords.TAG_KEY, nwKeyWords.PLOT_KEY, nwKeyWords.POV_KEY, @@ -51,7 +51,7 @@ class NWIndex(): nwKeyWords.OBJECT_KEY, nwKeyWords.ENTITY_KEY, nwKeyWords.CUSTOM_KEY - ] + ]) TAG_CLASS = { nwKeyWords.CHAR_KEY : nwItemClass.CHARACTER, nwKeyWords.POV_KEY : nwItemClass.CHARACTER, @@ -107,7 +107,6 @@ class NWIndex(): def deleteHandle(self, tHandle): """Delete all entries of a given document handle. """ - delTags = [] for tTag in self.tagIndex: if self.tagIndex[tTag][1] == tHandle: @@ -130,7 +129,6 @@ class NWIndex(): def loadIndex(self): """Load index from last session from the project meta folder. """ - theData = {} indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE) @@ -169,7 +167,6 @@ class NWIndex(): """Save the current index as a json file in the project meta data folder. """ - indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE) logger.debug("Saving index file") @@ -179,7 +176,7 @@ class NWIndex(): nIndent = None try: - with open(indexFile,mode="w+",encoding="utf8") as outFile: + with open(indexFile, mode="w+", encoding="utf8") as outFile: outFile.write(json.dumps({ "tagIndex" : self.tagIndex, "refIndex" : self.refIndex, @@ -198,7 +195,6 @@ class NWIndex(): """Check that the entries in the index are valid and contain the elements it should. """ - self.indexBroken = False try: @@ -265,7 +261,7 @@ class NWIndex(): logger.debug("Indexing item with handle %s" % tHandle) # Check file type, and reset its old index - # Also add an entry for T0 in case the file has no title + # Also add a dummy entry for T0 in case the file has no title self.refIndex[tHandle] = {} self.refIndex[tHandle]["T0"] = { "tags" : [], @@ -297,16 +293,16 @@ class NWIndex(): continue if aLine.startswith(r"#"): - isTitle = self.indexTitle(tHandle, isNovel, aLine, nLine, itemLayout) + isTitle = self._indexTitle(tHandle, isNovel, aLine, nLine, itemLayout) if isTitle and nLine > 0: if nTitle > 0: lastText = "\n".join(theLines[nTitle-1:nLine-1]) - self.indexWordCounts(tHandle, isNovel, lastText, nTitle) + self._indexWordCounts(tHandle, isNovel, lastText, nTitle) nTitle = nLine elif aLine.startswith(r"@"): - self.indexNoteRef(tHandle, aLine, nLine, nTitle) - self.indexTag(tHandle, aLine, nLine, itemClass) + self._indexNoteRef(tHandle, aLine, nLine, nTitle) + self._indexTag(tHandle, aLine, nLine, itemClass) elif aLine.startswith(r"%"): if nTitle > 0: @@ -315,12 +311,12 @@ class NWIndex(): cLen = len(toCheck) cOff = tLen - cLen if toCheck.startswith("synopsis:"): - self.indexSynopsis(tHandle, isNovel, aLine[cOff+9:].strip(), nTitle) + self._indexSynopsis(tHandle, isNovel, aLine[cOff+9:].strip(), nTitle) # Count words for remaining text after last heading if nTitle > 0: lastText = "\n".join(theLines[nTitle-1:nLine-1]) - self.indexWordCounts(tHandle, isNovel, lastText, nTitle) + self._indexWordCounts(tHandle, isNovel, lastText, nTitle) # Run word counter for whole text cC, wC, pC = countWords(theText) @@ -336,11 +332,14 @@ class NWIndex(): return True - def indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout): - """Save information about the title and its location in the - file. - """ + ## + # Internal Indexers + ## + def _indexTitle(self, tHandle, isNovel, aLine, nLine, itemLayout): + """Save information about the title and its location in the + file to the index. + """ if aLine.startswith("# "): hDepth = "H1" hText = aLine[2:].strip() @@ -382,7 +381,9 @@ class NWIndex(): return True - def indexWordCounts(self, tHandle, isNovel, theText, nTitle): + def _indexWordCounts(self, tHandle, isNovel, theText, nTitle): + """Count text stats and save the counts to the index. + """ cC, wC, pC = countWords(theText) sTitle = "T%d" % nTitle if isNovel: @@ -401,7 +402,9 @@ class NWIndex(): self.noteIndex[tHandle][sTitle]["updated"] = time() return - def indexSynopsis(self, tHandle, isNovel, theText, nTitle): + def _indexSynopsis(self, tHandle, isNovel, theText, nTitle): + """Save the synopsis to the index. + """ sTitle = "T%d" % nTitle if isNovel: if tHandle in self.novelIndex: @@ -415,11 +418,10 @@ class NWIndex(): self.noteIndex[tHandle][sTitle]["updated"] = time() return - def indexNoteRef(self, tHandle, aLine, nLine, nTitle): + def _indexNoteRef(self, tHandle, aLine, nLine, nTitle): """Validate and save the information about a reference to a tag in another file. """ - isValid, theBits, thePos = self.scanThis(aLine) if not isValid or len(theBits) == 0: return False @@ -435,10 +437,9 @@ class NWIndex(): return True - def indexTag(self, tHandle, aLine, nLine, itemClass): + def _indexTag(self, tHandle, aLine, nLine, itemClass): """Validate and save the information from a tag. """ - isValid, theBits, thePos = self.scanThis(aLine) if not isValid or len(theBits) != 2: return False @@ -453,42 +454,39 @@ class NWIndex(): ## def scanThis(self, aLine): - """Scan a line starting with @ to check that it's valid and to - split up its elements into an array and an array of positions. - The latter is needed for the syntax highlighter. + """Scan a line starting with @ to check that it's valid. Then + split it up into its elements and positions as two arrays. """ + theBits = [] # The elements of the string + thePos = [] # The absolute position of each element - theBits = [] - thePos = [] - - aLine = aLine.strip() + aLine = aLine.rstrip() # Remove all trailing white spaces nChar = len(aLine) if nChar < 2: return False, theBits, thePos if aLine[0] != "@": return False, theBits, thePos - cPos = 0 - cKey, cSep, cVals = aLine.partition(":") + cKey, _, cVals = aLine.partition(":") sKey = cKey.strip() if sKey == "@": return False, theBits, thePos + cPos = 0 theBits.append(sKey) thePos.append(cPos) - cPos += len(sKey) + 1 + cPos += len(cKey) + 1 - if cVals == "": + if not cVals: # No values, so we're done return True, theBits, thePos - aVals = cVals.split(",") - for cVal in aVals: + for cVal in cVals.split(","): sVal = cVal.strip() rLen = len(cVal.lstrip()) tLen = len(cVal) theBits.append(sVal) - thePos.append(cPos+tLen-rLen) + thePos.append(cPos + tLen - rLen) cPos += tLen + 1 return True, theBits, thePos @@ -497,8 +495,7 @@ class NWIndex(): """Check the tags against the index to see if they are valid tags. This is needed for syntax highlighting. """ - - nBits = len(theBits) + nBits = len(theBits) isGood = [False]*nBits if nBits == 0: return [] @@ -512,7 +509,7 @@ class NWIndex(): # is ignored if theBits[0] == nwKeyWords.TAG_KEY and nBits > 1: isGood[0] = True - if theBits[1] in self.tagIndex.keys(): + if theBits[1] in self.tagIndex: if self.tagIndex[theBits[1]][1] == tItem.itemHandle: isGood[1] = True else: @@ -537,7 +534,6 @@ class NWIndex(): order as they appear in the tree view and in the respective document files, but skipping all note files. """ - theStructure = [] for tHandle in self.theProject.projTree.handles(): if tHandle not in self.novelIndex: @@ -551,7 +547,6 @@ class NWIndex(): """Returns the counts for a file, or a section of a file starting at title nTitle. """ - cC = 0 wC = 0 pC = 0 @@ -579,7 +574,6 @@ class NWIndex(): """Extract all references made in a file, and optionally title section. sTitle must be a string. """ - theRefs = {} for tKey in self.TAG_CLASS: theRefs[tKey] = [] @@ -602,7 +596,6 @@ class NWIndex(): """Build a list of files referring back to our file, specified by tHandle. """ - theRefs = {} tItem = self.theProject.projTree[tHandle] diff --git a/tests/reference/proj/3_nwProject.nwx b/tests/reference/proj/3_nwProject.nwx new file mode 100644 index 00000000..5238e40d --- /dev/null +++ b/tests/reference/proj/3_nwProject.nwx @@ -0,0 +1,142 @@ + + + + + + True + + + False + True + None + None + 0 + + + %title% + Chapter %num%\\%title% + %title% + * * * +
+ False + False + False + + + New + Note + Draft + Finished + + + New + Minor + Major + Main + + + + + Novel + ROOT + NOVEL + New + False + + + Characters + ROOT + CHARACTER + New + False + + + Plot + ROOT + PLOT + New + False + + + World + ROOT + WORLD + New + False + + + New Chapter + FOLDER + NOVEL + New + False + + + New Scene + FILE + NOVEL + New + False + True + SCENE + 0 + 0 + 0 + 0 + + + Timeline + ROOT + TIMELINE + New + False + + + Object + ROOT + OBJECT + New + False + + + Custom1 + ROOT + CUSTOM + New + False + + + Custom2 + ROOT + CUSTOM + New + False + + + Hello + FILE + NOVEL + New + False + True + SCENE + 0 + 0 + 0 + 0 + + + Jane + FILE + CHARACTER + New + False + True + NOTE + 0 + 0 + 0 + 0 + + + diff --git a/tests/test_project.py b/tests/test_project.py index 7d920a37..a49cba7d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -76,20 +76,39 @@ def testProjectNewRoot(nwTempProj,nwRef): assert cmpFiles(projFile, refFile, [2]) assert not theProject.projChanged +@pytest.mark.project +def testProjectNewFile(nwTempProj,nwRef): + projFile = path.join(nwTempProj,"nwProject.nwx") + refFile = path.join(nwRef,"proj","3_nwProject.nwx") + assert theProject.openProject(projFile) + assert isinstance(theProject.newFile("Hello", nwItemClass.NOVEL, "73475cb40a568"), str) + assert isinstance(theProject.newFile("Jane", nwItemClass.CHARACTER, "44cb730c42048"), str) + assert theProject.projChanged + assert theProject.saveProject() + assert theProject.closeProject() + assert cmpFiles(projFile, refFile, [2]) + assert not theProject.projChanged + @pytest.mark.project def testIndexScanThis(nwTempProj): projFile = path.join(nwTempProj,"nwProject.nwx") assert theProject.openProject(projFile) - theIndex = NWIndex(theProject,theMain) + theIndex = NWIndex(theProject, theMain) tHandle = "31489056e0916" isValid, theBits, thePos = theIndex.scanThis("tag: this, and this") assert not isValid + isValid, theBits, thePos = theIndex.scanThis("@") + assert not isValid + isValid, theBits, thePos = theIndex.scanThis("@:") assert not isValid + isValid, theBits, thePos = theIndex.scanThis(" @a: b") + assert not isValid + isValid, theBits, thePos = theIndex.scanThis("@a:") assert isValid assert str(theBits) == "['@a']" @@ -105,9 +124,96 @@ def testIndexScanThis(nwTempProj): assert str(theBits) == "['@a', 'b', 'c', 'd']" assert str(thePos) == "[0, 3, 5, 7]" + isValid, theBits, thePos = theIndex.scanThis("@a : b , c , d") + assert isValid + assert str(theBits) == "['@a', 'b', 'c', 'd']" + assert str(thePos) == "[0, 5, 9, 13]" + isValid, theBits, thePos = theIndex.scanThis("@tag: this, and this") assert isValid assert str(theBits) == "['@tag', 'this', 'and this']" assert str(thePos) == "[0, 6, 12]" assert theProject.closeProject() + +@pytest.mark.project +def testIndexCheckThese(nwTempProj): + projFile = path.join(nwTempProj,"nwProject.nwx") + assert theProject.openProject(projFile) + + theIndex = NWIndex(theProject, theMain) + nHandle = "41cfc0d1f2d12" + nItem = theProject.projTree[nHandle] + cHandle = "2858dcd1057d3" + cItem = theProject.projTree[cHandle] + + assert theIndex.scanText(cHandle, ( + "# Jane Smith\n" + "@tag: Jane" + )) + assert theIndex.scanText(nHandle, ( + "# Hello World!\n" + "@pov: Jane" + )) + assert str(theIndex.tagIndex) == "{'Jane': [2, '2858dcd1057d3', 'CHARACTER']}" + assert theIndex.novelIndex[nHandle]["T1"]["title"] == "Hello World!" + + assert str(theIndex.checkThese(["@tag", "Jane"], cItem)) == "[True, True]" + assert str(theIndex.checkThese(["@tag", "John"], cItem)) == "[True, True]" + assert str(theIndex.checkThese(["@tag", "Jane"], nItem)) == "[True, False]" + assert str(theIndex.checkThese(["@tag", "John"], nItem)) == "[True, True]" + assert str(theIndex.checkThese(["@pov", "John"], nItem)) == "[True, False]" + assert str(theIndex.checkThese(["@pov", "Jane"], nItem)) == "[True, True]" + assert str(theIndex.checkThese(["@ pov", "Jane"], nItem)) == "[False, False]" + assert str(theIndex.checkThese(["@what", "Jane"], nItem)) == "[False, False]" + + assert theProject.closeProject() + +@pytest.mark.project +def testIndexMeta(nwTempProj): + projFile = path.join(nwTempProj,"nwProject.nwx") + assert theProject.openProject(projFile) + + theIndex = NWIndex(theProject, theMain) + nHandle = "41cfc0d1f2d12" + nItem = theProject.projTree[nHandle] + cHandle = "2858dcd1057d3" + cItem = theProject.projTree[cHandle] + + assert theIndex.scanText(cHandle, ( + "# Jane Smith\n" + "@tag: Jane\n" + )) + assert theIndex.scanText(nHandle, ( + "# Hello World!\n" + "@pov: Jane\n" + "@char: Jane\n" + "\n" + "% this is a comment\n" + "\n" + "This is a story about Jane Smith.\n" + "\n" + "Well, not really.\n" + )) + assert str(theIndex.tagIndex) == "{'Jane': [2, '2858dcd1057d3', 'CHARACTER']}" + assert theIndex.novelIndex[nHandle]["T1"]["title"] == "Hello World!" + + # The novel structure should contain the pointer to the novel file header + assert str(theIndex.getNovelStructure()) == "['41cfc0d1f2d12:T1']" + + # The novel file should have the correct counts + cC, wC, pC = theIndex.getCounts(nHandle) + assert cC == 62 # Characters in text and title only + assert wC == 12 # Words in text and title only + assert pC == 2 # Paragraphs in text only + + # The novel file should now refer to Jane as @pov and @char + theRefs = theIndex.getReferences(nHandle) + assert str(theRefs["@pov"]) == "['Jane']" + assert str(theRefs["@char"]) == "['Jane']" + + # The character file should have a record of the reference from the novel file + theRefs = theIndex.getBackReferenceList(cHandle) + assert str(theRefs) == "{'41cfc0d1f2d12': 3}" + + assert theProject.closeProject()