diff --git a/nw/gui/winmain.py b/nw/gui/winmain.py index 2521a3d0..d605f8ec 100644 --- a/nw/gui/winmain.py +++ b/nw/gui/winmain.py @@ -385,6 +385,7 @@ class GuiMain(QMainWindow): logger.debug("Rebuilding indices ...") self.treeView.saveTreeOrder() + self.tagIndex.clearIndex() nItems = len(self.theProject.treeOrder) dlgProg = QProgressDialog("Scanning files ...", "Cancel", 0, nItems, self) diff --git a/nw/project/index.py b/nw/project/index.py index 2e1616f8..c15409aa 100644 --- a/nw/project/index.py +++ b/nw/project/index.py @@ -29,18 +29,32 @@ import nw from os import path from nw.project.document import NWDoc -from nw.enum import nwItemType +from nw.enum import nwItemType, nwItemClass from nw.constants import nwFiles logger = logging.getLogger(__name__) class NWIndex(): - TAG_KEY = "tag" - NOTE_KEYS = ["pov","char","plot","time","location","object","custom"] - NOVEL_KEYS = ["scene","chapter","part"] + TAG_KEY = "@tag" + POV_KEY = "@pov" + CHAR_KEY = "@char" + PLOT_KEY = "@plot" + TIME_KEY = "@time" + WORLD_KEY = "@location" + OBJECT_KEY = "@object" + CUSTOM_KEY = "@custom" - VALID_KEYS = [TAG_KEY] + NOTE_KEYS + NOVEL_KEYS + NOTE_KEYS = [PLOT_KEY, POV_KEY, CHAR_KEY, WORLD_KEY, TIME_KEY, OBJECT_KEY, CUSTOM_KEY] + VALID_CLASS = { + nwItemClass.NOVEL : [], + nwItemClass.PLOT : [PLOT_KEY], + nwItemClass.CHARACTER : [POV_KEY, CHAR_KEY], + nwItemClass.WORLD : [WORLD_KEY], + nwItemClass.TIMELINE : [TIME_KEY], + nwItemClass.OBJECT : [OBJECT_KEY], + nwItemClass.CUSTOM : [CUSTOM_KEY], + } def __init__(self, theProject, theParent): @@ -50,28 +64,44 @@ class NWIndex(): self.mainConf = self.theParent.mainConf # Indices - self.itemIndex = {} + self.tagIndex = {} + self.noteIndex = {} + self.novelIndex = {} return def clearIndex(self): - self.itemIndex = {} + self.tagIndex = {} + self.noteIndex = {} + self.novelIndex = {} return + ## + # Load and Save Index to/from File + ## + def loadIndex(self): + theData = {} indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE) if path.isfile(indexFile): logger.debug("Loading index file") try: with open(indexFile,mode="r") as inFile: theJson = inFile.read() - self.itemIndex = json.loads(theJson) + theData = json.loads(theJson) except Exception as e: logger.error("Failed to load index file") logger.error(str(e)) return False + if "tagIndex" in theData.keys(): + self.tagIndex = theData["tagIndex"] + if "noteIndex" in theData.keys(): + self.noteIndex = theData["noteIndex"] + if "novelIndex" in theData.keys(): + self.novelIndex = theData["novelIndex"] + return True return False @@ -86,7 +116,11 @@ class NWIndex(): nIndent = None try: with open(indexFile,mode="w+") as outFile: - outFile.write(json.dumps(self.itemIndex, indent=nIndent)) + outFile.write(json.dumps({ + "tagIndex" : self.tagIndex, + "noteIndex" : self.noteIndex, + "novelIndex" : self.novelIndex, + }, indent=nIndent)) except Exception as e: logger.error("Failed to save index file") logger.error(str(e)) @@ -94,64 +128,130 @@ class NWIndex(): return True + ## + # Index Building + ## + def scanText(self, tHandle, theText): theItem = self.theProject.getItem(tHandle) - if theItem is None: - return False - if theItem.itemType != nwItemType.FILE: - return False + if theItem is None: return False + if theItem.itemType != nwItemType.FILE: return False + itemClass = theItem.itemClass logger.debug("Indexing item with handle %s" % tHandle) - self.itemIndex[tHandle] = {} + # Check file type, and reset its old index + if itemClass == nwItemClass.NOVEL: + self.novelIndex[tHandle] = [] + self.noteIndex[tHandle] = [] + isNovel = True + else: + isNovel = False + + # Also clear references to file in tag index + for aTag in self.tagIndex: + if self.tagIndex[aTag][1] == tHandle: + self.tagIndex.pop(aTag) nLine = 0 for aLine in theText.splitlines(): aLine = aLine.strip() nLine += 1 nChar = len(aLine) - if nChar > 0 and aLine[0] == "@": - self.indexThis(tHandle, aLine, nLine, theItem) + if nChar == 0: continue + if aLine[0] == "#": + if isNovel: + self.indexTitle(tHandle, aLine, nLine) + elif aLine[0] == "@": + if isNovel: + self.indexNoteRef(tHandle, aLine, nLine) + else: + self.indexTag(tHandle, aLine, nLine, itemClass) return True - def indexThis(self, tHandle, aLine, nLine, theItem): + def indexTitle(self, tHandle, aLine, nLine): - nChar = len(aLine) - nPos = aLine.find(":") - if nPos < 2 or nChar < nPos+2: - return False - - aKey = aLine[1:nPos].strip().lower() - tVal = aLine[nPos+1:].strip().lower() - if aKey not in self.VALID_KEYS: - logger.verbose("Not a valid key '%s'" % aKey) - return False - - logger.verbose("Found valid key '%s'" % aKey) - if aKey == self.TAG_KEY: - if tVal.find(",") >- 0: - return False - self._addItem(tHandle, aKey, nLine, tVal) + if aLine.startswith("# "): + hDepth = 1 + hText = aLine[2:].strip() + elif aLine.startswith("## "): + hDepth = 2 + hText = aLine[3:].strip() + elif aLine.startswith("### "): + hDepth = 3 + hText = aLine[4:].strip() + elif aLine.startswith("#### "): + hDepth = 4 + hText = aLine[5:].strip() else: - kVal = tVal.split(",") - cVal = [aVal.strip() for aVal in kVal] - if len(cVal) > 0: - self._addItem(tHandle, aKey, nLine, cVal) - else: - return False + return False + + if hText != "": + self.novelIndex[tHandle].append([nLine, hDepth, hText]) return True - ## - # Internal Functions - ## + def indexNoteRef(self, tHandle, aLine, nLine): - def _addItem(self, tHandle, tKey, tLine, tVal): - if tKey not in self.itemIndex[tHandle].keys(): - self.itemIndex[tHandle][tKey] = [] - self.itemIndex[tHandle][tKey].append([tLine, tVal]) - return + isValid, theBits, thePos = self.scanThis(aLine) + if not isValid or len(theBits) == 0: + return False + + theKey = theBits[0] + if theKey in self.NOTE_KEYS: + for aVal in theBits[1:]: + self.noteIndex[tHandle].append([nLine, theKey, aVal]) + + return True + + def indexTag(self, tHandle, aLine, nLine, itemClass): + + isValid, theBits, thePos = self.scanThis(aLine) + if not isValid or len(theBits) != 2: + return False + + if theBits[0] == self.TAG_KEY: + self.tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name] + + return True + + def scanThis(self, aLine): + + theBits = [] + thePos = [] + + aLine = aLine.strip() + nChar = len(aLine) + if nChar < 2: + return False, theBits, thePos + if aLine[0] != "@": + return False, theBits, thePos + + cPos = 0 + cKey, cSep, cVals = aLine.partition(":") + sKey = cKey.strip() + if sKey == "@": + return False, theBits, thePos + + theBits.append(sKey.lower()) + thePos.append(cPos) + cPos += len(sKey) + 1 + + if cVals == "": + # No values, so we're done + return True, theBits, thePos + + aVals = cVals.split(",") + for cVal in aVals: + sVal = cVal.strip() + rLen = len(cVal.lstrip()) + tLen = len(cVal) + theBits.append(sVal.lower()) + thePos.append(cPos+tLen-rLen) + cPos += tLen + 1 + + return True, theBits, thePos # END Class NWIndex diff --git a/sample/sampleNovel/data_1/4298de4d9524_main.nwd b/sample/sampleNovel/data_1/4298de4d9524_main.nwd index 11127dab..9bd4961b 100644 --- a/sample/sampleNovel/data_1/4298de4d9524_main.nwd +++ b/sample/sampleNovel/data_1/4298de4d9524_main.nwd @@ -1,3 +1,5 @@ # John Smith +@tag: John + He’s pretty cool. Not Brad Pitt though. \ No newline at end of file diff --git a/sample/sampleNovel/data_6/36b6aa9b697b_main.nwd b/sample/sampleNovel/data_6/36b6aa9b697b_main.nwd index eb71a80b..de9fae73 100644 --- a/sample/sampleNovel/data_6/36b6aa9b697b_main.nwd +++ b/sample/sampleNovel/data_6/36b6aa9b697b_main.nwd @@ -3,8 +3,8 @@ ## This is the Subtitle % Begin Meta -@POV: Sam -@Chars: Sam, Adam, Scott +@POV: Jane +@Char: Jane, John % End Meta Some text here would look good as well, and maybe some "dialogue"? @@ -17,8 +17,3 @@ This paragraph is also meaningless. At least a bit. It’s also very short. But This one is a bit longer. “It also has some dialogue in it” she said, before she moved on to check if the spellchecker worked. It did. “Cool,” she concluded. - -@ToDo: Stuff that will be done at some point - - - diff --git a/sample/sampleNovel/data_b/b2c23b3c42cc_main.nwd b/sample/sampleNovel/data_b/b2c23b3c42cc_main.nwd index 409a428a..1a91f10a 100644 --- a/sample/sampleNovel/data_b/b2c23b3c42cc_main.nwd +++ b/sample/sampleNovel/data_b/b2c23b3c42cc_main.nwd @@ -1,3 +1,5 @@ # Jane Smith +@tag: Jane + She’s pretty cool. Not Angelina Jolie though. \ No newline at end of file diff --git a/sample/sampleNovel/meta/sessionInfo.log b/sample/sampleNovel/meta/sessionInfo.log index 79d3266e..908773f6 100644 --- a/sample/sampleNovel/meta/sessionInfo.log +++ b/sample/sampleNovel/meta/sessionInfo.log @@ -13,3 +13,91 @@ Start: 2019-05-26 15:37:47 End: 2019-05-26 15:38:07 Words: 538 Start: 2019-05-26 15:38:11 End: 2019-05-26 15:43:07 Words: 2 Start: 2019-05-26 15:49:18 End: 2019-05-26 15:49:47 Words: 0 Start: 2019-05-26 15:52:39 End: 2019-05-26 15:52:56 Words: 0 +Start: 2019-05-27 21:23:07 End: 2019-05-27 21:26:41 Words: 0 +Start: 2019-05-27 21:26:56 End: 2019-05-27 21:27:48 Words: 0 +Start: 2019-05-27 21:29:11 End: 2019-05-27 21:30:30 Words: 0 +Start: 2019-05-27 21:33:30 End: 2019-05-27 21:35:14 Words: 0 +Start: 2019-05-27 21:35:20 End: 2019-05-27 21:36:03 Words: 0 +Start: 2019-05-27 21:36:07 End: 2019-05-27 21:36:23 Words: 0 +Start: 2019-05-27 21:36:27 End: 2019-05-27 21:37:46 Words: 0 +Start: 2019-05-27 21:37:50 End: 2019-05-27 21:38:05 Words: 0 +Start: 2019-05-27 21:38:14 End: 2019-05-27 21:38:24 Words: 0 +Start: 2019-05-27 21:39:08 End: 2019-05-27 21:39:23 Words: 0 +Start: 2019-05-27 21:42:21 End: 2019-05-27 21:42:39 Words: 0 +Start: 2019-05-27 21:45:07 End: 2019-05-27 21:45:18 Words: 0 +Start: 2019-05-27 21:58:30 End: 2019-05-27 21:58:43 Words: 0 +Start: 2019-05-27 22:02:01 End: 2019-05-27 22:02:09 Words: 0 +Start: 2019-05-27 22:05:47 End: 2019-05-27 22:06:02 Words: 0 +Start: 2019-05-27 22:08:25 End: 2019-05-27 22:09:35 Words: 0 +Start: 2019-05-27 22:11:40 End: 2019-05-27 22:14:18 Words: 3 +Start: 2019-05-27 22:20:55 End: 2019-05-27 22:21:40 Words: -2 +Start: 2019-05-27 22:22:38 End: 2019-05-27 22:23:12 Words: 16 +Start: 2019-05-27 22:55:34 End: 2019-05-27 22:56:09 Words: 0 +Start: 2019-05-27 22:58:47 End: 2019-05-27 23:00:47 Words: 17 +Start: 2019-05-28 18:20:15 End: 2019-05-28 18:20:30 Words: -1 +Start: 2019-05-28 18:21:04 End: 2019-05-28 18:21:24 Words: 0 +Start: 2019-05-28 18:29:41 End: 2019-05-28 18:30:16 Words: 0 +Start: 2019-05-28 18:30:59 End: 2019-05-28 18:32:06 Words: 0 +Start: 2019-05-28 18:34:33 End: 2019-05-28 18:34:48 Words: 1 +Start: 2019-05-28 18:43:08 End: 2019-05-28 18:43:21 Words: 0 +Start: 2019-05-28 18:49:29 End: 2019-05-28 18:49:40 Words: 0 +Start: 2019-05-28 19:13:20 End: 2019-05-28 19:13:24 Words: -1 +Start: 2019-05-28 19:17:05 End: 2019-05-28 19:17:13 Words: 1 +Start: 2019-05-28 19:49:44 End: 2019-05-28 19:52:23 Words: -1 +Start: 2019-05-28 19:52:29 End: 2019-05-28 19:54:42 Words: 0 +Start: 2019-05-28 19:55:33 End: 2019-05-28 19:55:56 Words: 0 +Start: 2019-05-28 19:57:29 End: 2019-05-28 20:02:33 Words: 0 +Start: 2019-05-28 20:02:38 End: 2019-05-28 20:05:19 Words: 0 +Start: 2019-05-28 20:05:23 End: 2019-05-28 20:06:48 Words: 0 +Start: 2019-05-28 20:07:00 End: 2019-05-28 20:12:26 Words: 0 +Start: 2019-05-28 20:12:31 End: 2019-05-28 20:17:58 Words: -16 +Start: 2019-05-28 20:28:31 End: 2019-05-28 20:28:53 Words: 0 +Start: 2019-05-28 20:28:57 End: 2019-05-28 20:29:05 Words: 16 +Start: 2019-05-28 21:00:49 End: 2019-05-28 21:01:45 Words: 1 +Start: 2019-05-28 21:02:01 End: 2019-05-28 21:02:06 Words: 0 +Start: 2019-05-28 21:17:57 End: 2019-05-28 21:25:48 Words: 0 +Start: 2019-05-28 21:35:39 End: 2019-05-28 21:35:47 Words: 0 +Start: 2019-05-28 22:22:54 End: 2019-05-28 22:23:12 Words: 0 +Start: 2019-05-30 11:58:19 End: 2019-05-30 11:59:15 Words: 0 +Start: 2019-05-30 11:59:19 End: 2019-05-30 11:59:26 Words: 0 +Start: 2019-05-30 11:59:55 End: 2019-05-30 12:00:09 Words: 0 +Start: 2019-05-30 12:01:47 End: 2019-05-30 12:02:08 Words: 0 +Start: 2019-05-30 12:03:56 End: 2019-05-30 12:04:23 Words: 0 +Start: 2019-05-30 12:09:24 End: 2019-05-30 12:09:47 Words: 0 +Start: 2019-05-30 12:10:13 End: 2019-05-30 12:10:40 Words: 0 +Start: 2019-05-30 12:12:14 End: 2019-05-30 12:12:33 Words: 0 +Start: 2019-05-30 12:13:56 End: 2019-05-30 12:14:08 Words: 0 +Start: 2019-05-30 12:14:24 End: 2019-05-30 12:14:44 Words: 0 +Start: 2019-05-30 12:14:53 End: 2019-05-30 12:15:31 Words: 0 +Start: 2019-05-30 12:17:36 End: 2019-05-30 12:17:49 Words: 0 +Start: 2019-05-30 12:20:12 End: 2019-05-30 12:20:24 Words: 0 +Start: 2019-05-30 12:36:42 End: 2019-05-30 12:37:21 Words: 0 +Start: 2019-05-30 12:37:42 End: 2019-05-30 12:38:01 Words: 0 +Start: 2019-05-30 12:47:13 End: 2019-05-30 12:47:27 Words: 0 +Start: 2019-05-30 12:47:47 End: 2019-05-30 12:48:03 Words: 0 +Start: 2019-05-30 12:50:01 End: 2019-05-30 12:50:15 Words: 0 +Start: 2019-05-30 12:54:26 End: 2019-05-30 12:54:34 Words: 0 +Start: 2019-05-30 15:25:50 End: 2019-05-30 15:25:59 Words: 0 +Start: 2019-05-30 15:43:36 End: 2019-05-30 15:43:44 Words: 0 +Start: 2019-05-30 15:52:46 End: 2019-05-30 15:53:12 Words: 0 +Start: 2019-05-30 15:53:44 End: 2019-05-30 15:53:56 Words: 0 +Start: 2019-05-30 15:56:29 End: 2019-05-30 15:57:35 Words: 0 +Start: 2019-05-30 15:57:41 End: 2019-05-30 15:59:58 Words: 0 +Start: 2019-05-30 16:00:01 End: 2019-05-30 16:00:28 Words: 0 +Start: 2019-05-30 16:00:56 End: 2019-05-30 16:01:06 Words: 0 +Start: 2019-05-30 16:01:32 End: 2019-05-30 16:01:48 Words: 0 +Start: 2019-05-30 16:02:35 End: 2019-05-30 16:03:03 Words: 0 +Start: 2019-05-30 17:00:23 End: 2019-05-30 17:21:07 Words: 0 +Start: 2019-05-30 17:21:11 End: 2019-05-30 17:21:24 Words: 0 +Start: 2019-05-30 17:24:41 End: 2019-05-30 17:24:46 Words: 0 +Start: 2019-05-30 17:29:08 End: 2019-05-30 17:29:19 Words: 0 +Start: 2019-05-30 18:12:59 End: 2019-05-30 18:13:07 Words: 0 +Start: 2019-05-30 18:13:32 End: 2019-05-30 18:13:42 Words: 0 +Start: 2019-05-30 18:16:13 End: 2019-05-30 18:16:18 Words: 0 +Start: 2019-05-30 18:22:24 End: 2019-05-30 18:22:30 Words: 0 +Start: 2019-05-30 18:22:43 End: 2019-05-30 18:24:11 Words: 0 +Start: 2019-05-30 18:25:40 End: 2019-05-30 18:25:45 Words: 0 +Start: 2019-05-30 18:26:01 End: 2019-05-30 18:27:56 Words: 0 +Start: 2019-05-30 18:28:00 End: 2019-05-30 18:30:22 Words: 0 +Start: 2019-05-30 18:42:16 End: 2019-05-30 18:43:10 Words: 0 +Start: 2019-05-30 18:43:24 End: 2019-05-30 18:43:31 Words: 0 diff --git a/sample/sampleNovel/nwProject.nwx b/sample/sampleNovel/nwProject.nwx index 448da65a..c7108caf 100644 --- a/sample/sampleNovel/nwProject.nwx +++ b/sample/sampleNovel/nwProject.nwx @@ -1,5 +1,5 @@ - + Sample Project Sample Project @@ -10,7 +10,7 @@ True 636b6aa9b697b None - 540 + 558 New Notes @@ -76,7 +76,7 @@ 656 121 5 - 573 + 729 New File @@ -85,8 +85,8 @@ Notes False SCENE - 69 - 17 + 82 + 19 1 0 @@ -111,10 +111,10 @@ Minor False NOTE - 42 - 8 + 49 + 9 1 - 0 + 24 Jane Smith @@ -123,10 +123,10 @@ Major False NOTE - 51 + 55 9 1 - 0 + 71 Locations @@ -142,9 +142,9 @@ None False NOTE - 0 - 0 - 0 + 76 + 15 + 1 0 diff --git a/tests/test_project.py b/tests/test_project.py index db925c02..9ef638c8 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -11,6 +11,7 @@ from nwdummy import DummyMain from nw.config import Config from nw.project.project import NWProject from nw.project.item import NWItem +from nw.project.index import NWIndex from nw.enum import nwItemClass theConf = Config() @@ -60,3 +61,39 @@ def testProjectNewRoot(nwTempProj,nwRef): assert theProject.saveProject() assert cmpFiles(projFile, refFile, [2]) assert not theProject.projChanged + +@pytest.mark.project +def testIndexScanThis(nwTempProj): + projFile = path.join(nwTempProj,"nwProject.nwx") + assert theProject.openProject(projFile) + + theIndex = NWIndex(theProject,theMain) + tHandle = "31489056e0916" + + isValid, theBits, thePos = theIndex.scanThis("tag: this, and this") + assert not isValid + + isValid, theBits, thePos = theIndex.scanThis("@:") + assert not isValid + + isValid, theBits, thePos = theIndex.scanThis("@a:") + assert isValid + assert str(theBits) == "['@a']" + assert str(thePos) == "[0]" + + isValid, theBits, thePos = theIndex.scanThis("@a:b") + assert isValid + assert str(theBits) == "['@a', 'b']" + assert str(thePos) == "[0, 3]" + + isValid, theBits, thePos = theIndex.scanThis("@a:b,c,d") + assert isValid + assert str(theBits) == "['@a', 'b', 'c', 'd']" + assert str(thePos) == "[0, 3, 5, 7]" + + isValid, theBits, thePos = theIndex.scanThis("@tag: this, and this") + assert isValid + assert str(theBits) == "['@tag', 'this', 'and this']" + assert str(thePos) == "[0, 6, 12]" + + # assert False