Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Got a working version of index builder that seems to do what is needed.

Browse Source
This commit is contained in:
Veronica K. B. Olsen
2019-05-30 18:45:17 +02:00
parent 2962ab4070
commit b526b0f5d9
8 changed files with 292 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nw/gui/winmain.py
+1
View File
@@ -385,6 +385,7 @@ class GuiMain(QMainWindow):
logger.debug("Rebuilding indices ...")
self.treeView.saveTreeOrder()
self.tagIndex.clearIndex()
nItems = len(self.theProject.treeOrder)
dlgProg = QProgressDialog("Scanning files ...", "Cancel", 0, nItems, self)
nw/project/index.py
+147 -47
View File
@@ -29,18 +29,32 @@ import nw
from os import path
from nw.project.document import NWDoc
from nw.enum import nwItemType
from nw.enum import nwItemType, nwItemClass
from nw.constants import nwFiles
logger = logging.getLogger(__name__)
class NWIndex():
TAG_KEY = "tag"
NOTE_KEYS = ["pov","char","plot","time","location","object","custom"]
NOVEL_KEYS = ["scene","chapter","part"]
TAG_KEY = "@tag"
POV_KEY = "@pov"
CHAR_KEY = "@char"
PLOT_KEY = "@plot"
TIME_KEY = "@time"
WORLD_KEY = "@location"
OBJECT_KEY = "@object"
CUSTOM_KEY = "@custom"
VALID_KEYS = [TAG_KEY] + NOTE_KEYS + NOVEL_KEYS
NOTE_KEYS = [PLOT_KEY, POV_KEY, CHAR_KEY, WORLD_KEY, TIME_KEY, OBJECT_KEY, CUSTOM_KEY]
VALID_CLASS = {
nwItemClass.NOVEL : [],
nwItemClass.PLOT : [PLOT_KEY],
nwItemClass.CHARACTER : [POV_KEY, CHAR_KEY],
nwItemClass.WORLD : [WORLD_KEY],
nwItemClass.TIMELINE : [TIME_KEY],
nwItemClass.OBJECT : [OBJECT_KEY],
nwItemClass.CUSTOM : [CUSTOM_KEY],
}
def __init__(self, theProject, theParent):
@@ -50,28 +64,44 @@ class NWIndex():
self.mainConf = self.theParent.mainConf
# Indices
self.itemIndex = {}
self.tagIndex = {}
self.noteIndex = {}
self.novelIndex = {}
return
def clearIndex(self):
self.itemIndex = {}
self.tagIndex = {}
self.noteIndex = {}
self.novelIndex = {}
return
##
# Load and Save Index to/from File
##
def loadIndex(self):
theData = {}
indexFile = path.join(self.theProject.projMeta, nwFiles.INDEX_FILE)
if path.isfile(indexFile):
logger.debug("Loading index file")
try:
with open(indexFile,mode="r") as inFile:
theJson = inFile.read()
self.itemIndex = json.loads(theJson)
theData = json.loads(theJson)
except Exception as e:
logger.error("Failed to load index file")
logger.error(str(e))
return False
if "tagIndex" in theData.keys():
self.tagIndex = theData["tagIndex"]
if "noteIndex" in theData.keys():
self.noteIndex = theData["noteIndex"]
if "novelIndex" in theData.keys():
self.novelIndex = theData["novelIndex"]
return True
return False
@@ -86,7 +116,11 @@ class NWIndex():
nIndent = None
try:
with open(indexFile,mode="w+") as outFile:
outFile.write(json.dumps(self.itemIndex, indent=nIndent))
outFile.write(json.dumps({
"tagIndex" : self.tagIndex,
"noteIndex" : self.noteIndex,
"novelIndex" : self.novelIndex,
}, indent=nIndent))
except Exception as e:
logger.error("Failed to save index file")
logger.error(str(e))
@@ -94,64 +128,130 @@ class NWIndex():
return True
##
# Index Building
##
def scanText(self, tHandle, theText):
theItem = self.theProject.getItem(tHandle)
if theItem is None:
return False
if theItem.itemType != nwItemType.FILE:
return False
if theItem is None: return False
if theItem.itemType != nwItemType.FILE: return False
itemClass = theItem.itemClass
logger.debug("Indexing item with handle %s" % tHandle)
self.itemIndex[tHandle] = {}
# Check file type, and reset its old index
if itemClass == nwItemClass.NOVEL:
self.novelIndex[tHandle] = []
self.noteIndex[tHandle] = []
isNovel = True
else:
isNovel = False
# Also clear references to file in tag index
for aTag in self.tagIndex:
if self.tagIndex[aTag][1] == tHandle:
self.tagIndex.pop(aTag)
nLine = 0
for aLine in theText.splitlines():
aLine = aLine.strip()
nLine += 1
nChar = len(aLine)
if nChar > 0 and aLine[0] == "@":
self.indexThis(tHandle, aLine, nLine, theItem)
if nChar == 0: continue
if aLine[0] == "#":
if isNovel:
self.indexTitle(tHandle, aLine, nLine)
elif aLine[0] == "@":
if isNovel:
self.indexNoteRef(tHandle, aLine, nLine)
else:
self.indexTag(tHandle, aLine, nLine, itemClass)
return True
def indexThis(self, tHandle, aLine, nLine, theItem):
def indexTitle(self, tHandle, aLine, nLine):
nChar = len(aLine)
nPos = aLine.find(":")
if nPos < 2 or nChar < nPos+2:
return False
aKey = aLine[1:nPos].strip().lower()
tVal = aLine[nPos+1:].strip().lower()
if aKey not in self.VALID_KEYS:
logger.verbose("Not a valid key '%s'" % aKey)
return False
logger.verbose("Found valid key '%s'" % aKey)
if aKey == self.TAG_KEY:
if tVal.find(",") >- 0:
return False
self._addItem(tHandle, aKey, nLine, tVal)
if aLine.startswith("# "):
hDepth = 1
hText = aLine[2:].strip()
elif aLine.startswith("## "):
hDepth = 2
hText = aLine[3:].strip()
elif aLine.startswith("### "):
hDepth = 3
hText = aLine[4:].strip()
elif aLine.startswith("#### "):
hDepth = 4
hText = aLine[5:].strip()
else:
kVal = tVal.split(",")
cVal = [aVal.strip() for aVal in kVal]
if len(cVal) > 0:
self._addItem(tHandle, aKey, nLine, cVal)
else:
return False
return False
if hText != "":
self.novelIndex[tHandle].append([nLine, hDepth, hText])
return True
##
# Internal Functions
##
def indexNoteRef(self, tHandle, aLine, nLine):
def _addItem(self, tHandle, tKey, tLine, tVal):
if tKey not in self.itemIndex[tHandle].keys():
self.itemIndex[tHandle][tKey] = []
self.itemIndex[tHandle][tKey].append([tLine, tVal])
return
isValid, theBits, thePos = self.scanThis(aLine)
if not isValid or len(theBits) == 0:
return False
theKey = theBits[0]
if theKey in self.NOTE_KEYS:
for aVal in theBits[1:]:
self.noteIndex[tHandle].append([nLine, theKey, aVal])
return True
def indexTag(self, tHandle, aLine, nLine, itemClass):
isValid, theBits, thePos = self.scanThis(aLine)
if not isValid or len(theBits) != 2:
return False
if theBits[0] == self.TAG_KEY:
self.tagIndex[theBits[1]] = [nLine, tHandle, itemClass.name]
return True
def scanThis(self, aLine):
theBits = []
thePos = []
aLine = aLine.strip()
nChar = len(aLine)
if nChar < 2:
return False, theBits, thePos
if aLine[0] != "@":
return False, theBits, thePos
cPos = 0
cKey, cSep, cVals = aLine.partition(":")
sKey = cKey.strip()
if sKey == "@":
return False, theBits, thePos
theBits.append(sKey.lower())
thePos.append(cPos)
cPos += len(sKey) + 1
if cVals == "":
# No values, so we're done
return True, theBits, thePos
aVals = cVals.split(",")
for cVal in aVals:
sVal = cVal.strip()
rLen = len(cVal.lstrip())
tLen = len(cVal)
theBits.append(sVal.lower())
thePos.append(cPos+tLen-rLen)
cPos += tLen + 1
return True, theBits, thePos
# END Class NWIndex
sample/sampleNovel/data_1/4298de4d9524_main.nwd
+2
View File
@@ -1,3 +1,5 @@
# John Smith
@tag: John
He’s pretty cool. Not Brad Pitt though.
sample/sampleNovel/data_6/36b6aa9b697b_main.nwd
+2 -7
View File
@@ -3,8 +3,8 @@
## This is the Subtitle
% Begin Meta
@POV: Sam
@Chars: Sam, Adam, Scott
@POV: Jane
@Char: Jane, John
% End Meta
Some text here would look good as well, and maybe some "dialogue"?
@@ -17,8 +17,3 @@ This paragraph is also meaningless. At least a bit. It’s also very short. But
This one is a bit longer. “It also has some dialogue in it” she said, before she moved on to check if the spellchecker worked. It did. “Cool,” she concluded.
@ToDo: Stuff that will be done at some point
sample/sampleNovel/data_b/b2c23b3c42cc_main.nwd
+2
View File
@@ -1,3 +1,5 @@
# Jane Smith
@tag: Jane
She’s pretty cool. Not Angelina Jolie though.
sample/sampleNovel/meta/sessionInfo.log
+88
View File
@@ -13,3 +13,91 @@ Start: 2019-05-26 15:37:47 End: 2019-05-26 15:38:07 Words: 538
Start: 2019-05-26 15:38:11 End: 2019-05-26 15:43:07 Words: 2
Start: 2019-05-26 15:49:18 End: 2019-05-26 15:49:47 Words: 0
Start: 2019-05-26 15:52:39 End: 2019-05-26 15:52:56 Words: 0
Start: 2019-05-27 21:23:07 End: 2019-05-27 21:26:41 Words: 0
Start: 2019-05-27 21:26:56 End: 2019-05-27 21:27:48 Words: 0
Start: 2019-05-27 21:29:11 End: 2019-05-27 21:30:30 Words: 0
Start: 2019-05-27 21:33:30 End: 2019-05-27 21:35:14 Words: 0
Start: 2019-05-27 21:35:20 End: 2019-05-27 21:36:03 Words: 0
Start: 2019-05-27 21:36:07 End: 2019-05-27 21:36:23 Words: 0
Start: 2019-05-27 21:36:27 End: 2019-05-27 21:37:46 Words: 0
Start: 2019-05-27 21:37:50 End: 2019-05-27 21:38:05 Words: 0
Start: 2019-05-27 21:38:14 End: 2019-05-27 21:38:24 Words: 0
Start: 2019-05-27 21:39:08 End: 2019-05-27 21:39:23 Words: 0
Start: 2019-05-27 21:42:21 End: 2019-05-27 21:42:39 Words: 0
Start: 2019-05-27 21:45:07 End: 2019-05-27 21:45:18 Words: 0
Start: 2019-05-27 21:58:30 End: 2019-05-27 21:58:43 Words: 0
Start: 2019-05-27 22:02:01 End: 2019-05-27 22:02:09 Words: 0
Start: 2019-05-27 22:05:47 End: 2019-05-27 22:06:02 Words: 0
Start: 2019-05-27 22:08:25 End: 2019-05-27 22:09:35 Words: 0
Start: 2019-05-27 22:11:40 End: 2019-05-27 22:14:18 Words: 3
Start: 2019-05-27 22:20:55 End: 2019-05-27 22:21:40 Words: -2
Start: 2019-05-27 22:22:38 End: 2019-05-27 22:23:12 Words: 16
Start: 2019-05-27 22:55:34 End: 2019-05-27 22:56:09 Words: 0
Start: 2019-05-27 22:58:47 End: 2019-05-27 23:00:47 Words: 17
Start: 2019-05-28 18:20:15 End: 2019-05-28 18:20:30 Words: -1
Start: 2019-05-28 18:21:04 End: 2019-05-28 18:21:24 Words: 0
Start: 2019-05-28 18:29:41 End: 2019-05-28 18:30:16 Words: 0
Start: 2019-05-28 18:30:59 End: 2019-05-28 18:32:06 Words: 0
Start: 2019-05-28 18:34:33 End: 2019-05-28 18:34:48 Words: 1
Start: 2019-05-28 18:43:08 End: 2019-05-28 18:43:21 Words: 0
Start: 2019-05-28 18:49:29 End: 2019-05-28 18:49:40 Words: 0
Start: 2019-05-28 19:13:20 End: 2019-05-28 19:13:24 Words: -1
Start: 2019-05-28 19:17:05 End: 2019-05-28 19:17:13 Words: 1
Start: 2019-05-28 19:49:44 End: 2019-05-28 19:52:23 Words: -1
Start: 2019-05-28 19:52:29 End: 2019-05-28 19:54:42 Words: 0
Start: 2019-05-28 19:55:33 End: 2019-05-28 19:55:56 Words: 0
Start: 2019-05-28 19:57:29 End: 2019-05-28 20:02:33 Words: 0
Start: 2019-05-28 20:02:38 End: 2019-05-28 20:05:19 Words: 0
Start: 2019-05-28 20:05:23 End: 2019-05-28 20:06:48 Words: 0
Start: 2019-05-28 20:07:00 End: 2019-05-28 20:12:26 Words: 0
Start: 2019-05-28 20:12:31 End: 2019-05-28 20:17:58 Words: -16
Start: 2019-05-28 20:28:31 End: 2019-05-28 20:28:53 Words: 0
Start: 2019-05-28 20:28:57 End: 2019-05-28 20:29:05 Words: 16
Start: 2019-05-28 21:00:49 End: 2019-05-28 21:01:45 Words: 1
Start: 2019-05-28 21:02:01 End: 2019-05-28 21:02:06 Words: 0
Start: 2019-05-28 21:17:57 End: 2019-05-28 21:25:48 Words: 0
Start: 2019-05-28 21:35:39 End: 2019-05-28 21:35:47 Words: 0
Start: 2019-05-28 22:22:54 End: 2019-05-28 22:23:12 Words: 0
Start: 2019-05-30 11:58:19 End: 2019-05-30 11:59:15 Words: 0
Start: 2019-05-30 11:59:19 End: 2019-05-30 11:59:26 Words: 0
Start: 2019-05-30 11:59:55 End: 2019-05-30 12:00:09 Words: 0
Start: 2019-05-30 12:01:47 End: 2019-05-30 12:02:08 Words: 0
Start: 2019-05-30 12:03:56 End: 2019-05-30 12:04:23 Words: 0
Start: 2019-05-30 12:09:24 End: 2019-05-30 12:09:47 Words: 0
Start: 2019-05-30 12:10:13 End: 2019-05-30 12:10:40 Words: 0
Start: 2019-05-30 12:12:14 End: 2019-05-30 12:12:33 Words: 0
Start: 2019-05-30 12:13:56 End: 2019-05-30 12:14:08 Words: 0
Start: 2019-05-30 12:14:24 End: 2019-05-30 12:14:44 Words: 0
Start: 2019-05-30 12:14:53 End: 2019-05-30 12:15:31 Words: 0
Start: 2019-05-30 12:17:36 End: 2019-05-30 12:17:49 Words: 0
Start: 2019-05-30 12:20:12 End: 2019-05-30 12:20:24 Words: 0
Start: 2019-05-30 12:36:42 End: 2019-05-30 12:37:21 Words: 0
Start: 2019-05-30 12:37:42 End: 2019-05-30 12:38:01 Words: 0
Start: 2019-05-30 12:47:13 End: 2019-05-30 12:47:27 Words: 0
Start: 2019-05-30 12:47:47 End: 2019-05-30 12:48:03 Words: 0
Start: 2019-05-30 12:50:01 End: 2019-05-30 12:50:15 Words: 0
Start: 2019-05-30 12:54:26 End: 2019-05-30 12:54:34 Words: 0
Start: 2019-05-30 15:25:50 End: 2019-05-30 15:25:59 Words: 0
Start: 2019-05-30 15:43:36 End: 2019-05-30 15:43:44 Words: 0
Start: 2019-05-30 15:52:46 End: 2019-05-30 15:53:12 Words: 0
Start: 2019-05-30 15:53:44 End: 2019-05-30 15:53:56 Words: 0
Start: 2019-05-30 15:56:29 End: 2019-05-30 15:57:35 Words: 0
Start: 2019-05-30 15:57:41 End: 2019-05-30 15:59:58 Words: 0
Start: 2019-05-30 16:00:01 End: 2019-05-30 16:00:28 Words: 0
Start: 2019-05-30 16:00:56 End: 2019-05-30 16:01:06 Words: 0
Start: 2019-05-30 16:01:32 End: 2019-05-30 16:01:48 Words: 0
Start: 2019-05-30 16:02:35 End: 2019-05-30 16:03:03 Words: 0
Start: 2019-05-30 17:00:23 End: 2019-05-30 17:21:07 Words: 0
Start: 2019-05-30 17:21:11 End: 2019-05-30 17:21:24 Words: 0
Start: 2019-05-30 17:24:41 End: 2019-05-30 17:24:46 Words: 0
Start: 2019-05-30 17:29:08 End: 2019-05-30 17:29:19 Words: 0
Start: 2019-05-30 18:12:59 End: 2019-05-30 18:13:07 Words: 0
Start: 2019-05-30 18:13:32 End: 2019-05-30 18:13:42 Words: 0
Start: 2019-05-30 18:16:13 End: 2019-05-30 18:16:18 Words: 0
Start: 2019-05-30 18:22:24 End: 2019-05-30 18:22:30 Words: 0
Start: 2019-05-30 18:22:43 End: 2019-05-30 18:24:11 Words: 0
Start: 2019-05-30 18:25:40 End: 2019-05-30 18:25:45 Words: 0
Start: 2019-05-30 18:26:01 End: 2019-05-30 18:27:56 Words: 0
Start: 2019-05-30 18:28:00 End: 2019-05-30 18:30:22 Words: 0
Start: 2019-05-30 18:42:16 End: 2019-05-30 18:43:10 Words: 0
Start: 2019-05-30 18:43:24 End: 2019-05-30 18:43:31 Words: 0
sample/sampleNovel/nwProject.nwx
+13 -13
View File
@@ -1,5 +1,5 @@
<?xml version='1.0' encoding='utf-8'?>
<novelWriterXML appVersion="0.1.4" fileVersion="1.0" timeStamp="2019-05-26 15:52:53">
<novelWriterXML appVersion="0.1.4" fileVersion="1.0" timeStamp="2019-05-30 18:43:31">
<project>
<name>Sample Project</name>
<title>Sample Project</title>
@@ -10,7 +10,7 @@
<spellCheck>True</spellCheck>
<lastEdited>636b6aa9b697b</lastEdited>
<lastViewed>None</lastViewed>
<lastWordCount>540</lastWordCount>
<lastWordCount>558</lastWordCount>
<status>
<entry blue="100" green="100" red="100">New</entry>
<entry blue="0" green="50" red="200">Notes</entry>
@@ -76,7 +76,7 @@
<charCount>656</charCount>
<wordCount>121</wordCount>
<paraCount>5</paraCount>
<cursorPos>573</cursorPos>
<cursorPos>729</cursorPos>
</item>
<item handle="bc0cbd2a407f3" order="2" parent="e7ded148d6e4a">
<name>New File</name>
@@ -85,8 +85,8 @@
<status>Notes</status>
<expanded>False</expanded>
<layout>SCENE</layout>
<charCount>69</charCount>
<wordCount>17</wordCount>
<charCount>82</charCount>
<wordCount>19</wordCount>
<paraCount>1</paraCount>
<cursorPos>0</cursorPos>
</item>
@@ -111,10 +111,10 @@
<status>Minor</status>
<expanded>False</expanded>
<layout>NOTE</layout>
<charCount>42</charCount>
<wordCount>8</wordCount>
<charCount>49</charCount>
<wordCount>9</wordCount>
<paraCount>1</paraCount>
<cursorPos>0</cursorPos>
<cursorPos>24</cursorPos>
</item>
<item handle="bb2c23b3c42cc" order="1" parent="f7e2d9f330615">
<name>Jane Smith</name>
@@ -123,10 +123,10 @@
<status>Major</status>
<expanded>False</expanded>
<layout>NOTE</layout>
<charCount>51</charCount>
<charCount>55</charCount>
<wordCount>9</wordCount>
<paraCount>1</paraCount>
<cursorPos>0</cursorPos>
<cursorPos>71</cursorPos>
</item>
<item handle="15c4492bd5107" order="2" parent="None">
<name>Locations</name>
@@ -142,9 +142,9 @@
<status>None</status>
<expanded>False</expanded>
<layout>NOTE</layout>
<charCount>0</charCount>
<wordCount>0</wordCount>
<paraCount>0</paraCount>
<charCount>76</charCount>
<wordCount>15</wordCount>
<paraCount>1</paraCount>
<cursorPos>0</cursorPos>
</item>
<item handle="98acd8c76c93a" order="3" parent="None">
tests/test_project.py
+37
View File
@@ -11,6 +11,7 @@ from nwdummy import DummyMain
from nw.config import Config
from nw.project.project import NWProject
from nw.project.item import NWItem
from nw.project.index import NWIndex
from nw.enum import nwItemClass
theConf = Config()
@@ -60,3 +61,39 @@ def testProjectNewRoot(nwTempProj,nwRef):
assert theProject.saveProject()
assert cmpFiles(projFile, refFile, [2])
assert not theProject.projChanged
@pytest.mark.project
def testIndexScanThis(nwTempProj):
projFile = path.join(nwTempProj,"nwProject.nwx")
assert theProject.openProject(projFile)
theIndex = NWIndex(theProject,theMain)
tHandle = "31489056e0916"
isValid, theBits, thePos = theIndex.scanThis("tag: this, and this")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@:")
assert not isValid
isValid, theBits, thePos = theIndex.scanThis("@a:")
assert isValid
assert str(theBits) == "['@a']"
assert str(thePos) == "[0]"
isValid, theBits, thePos = theIndex.scanThis("@a:b")
assert isValid
assert str(theBits) == "['@a', 'b']"
assert str(thePos) == "[0, 3]"
isValid, theBits, thePos = theIndex.scanThis("@a:b,c,d")
assert isValid
assert str(theBits) == "['@a', 'b', 'c', 'd']"
assert str(thePos) == "[0, 3, 5, 7]"
isValid, theBits, thePos = theIndex.scanThis("@tag: this, and this")
assert isValid
assert str(theBits) == "['@tag', 'this', 'and this']"
assert str(thePos) == "[0, 6, 12]"
# assert False