Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up variables and annotations in index classes

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2023-09-17 20:29:33 +02:00
parent 29518a33f2
commit 2e0204d577
2 changed files with 98 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/core/index.py
+95 -98
View File
@@ -169,15 +169,13 @@ class NWIndex:
if not isinstance(indexFile, Path): if not isinstance(indexFile, Path):
return False return False
theData = {}
tStart = time() tStart = time()
self._indexBroken = False self._indexBroken = False
if indexFile.exists(): if indexFile.exists():
logger.debug("Loading index file") logger.debug("Loading index file")
try: try:
with open(indexFile, mode="r", encoding="utf-8") as inFile: with open(indexFile, mode="r", encoding="utf-8") as inFile:
theData = json.load(inFile) data = json.load(inFile)
except Exception: except Exception:
logger.error("Failed to load index file") logger.error("Failed to load index file")
logException() logException()
@@ -185,8 +183,8 @@ class NWIndex:
return False return False
try: try:
self._tagsIndex.unpackData(theData["novelWriter.tagsIndex"]) self._tagsIndex.unpackData(data["novelWriter.tagsIndex"])
self._itemIndex.unpackData(theData["novelWriter.itemIndex"]) self._itemIndex.unpackData(data["novelWriter.itemIndex"])
except Exception: except Exception:
logger.error("The index content is invalid") logger.error("The index content is invalid")
logException() logException()
@@ -298,14 +296,14 @@ class NWIndex:
pTitle = TT_NONE # Tag of the previous title pTitle = TT_NONE # Tag of the previous title
canSetHeader = True # First header has not yet been set canSetHeader = True # First header has not yet been set
theLines = text.splitlines() lines = text.splitlines()
for nLine, aLine in enumerate(theLines, start=1): for n, line in enumerate(lines, start=1):
if aLine.strip() == "": if line.strip() == "":
continue continue
if aLine.startswith("#"): if line.startswith("#"):
hDepth, hText = self._splitHeading(aLine) hDepth, hText = self._splitHeading(line)
if hDepth == "H0": if hDepth == "H0":
continue continue
@@ -313,33 +311,33 @@ class NWIndex:
nwItem.setMainHeading(hDepth) nwItem.setMainHeading(hDepth)
canSetHeader = False canSetHeader = False
cTitle = self._itemIndex.addItemHeading(tHandle, nLine, hDepth, hText) cTitle = self._itemIndex.addItemHeading(tHandle, n, hDepth, hText)
if cTitle != TT_NONE: if cTitle != TT_NONE:
if nTitle > 0: if nTitle > 0:
# We have a new title, so we need to count the words of the previous one # We have a new title, so we need to count the words of the previous one
lastText = "\n".join(theLines[nTitle-1:nLine-1]) lastText = "\n".join(lines[nTitle-1:n-1])
self._indexWordCounts(tHandle, lastText, pTitle) self._indexWordCounts(tHandle, lastText, pTitle)
nTitle = nLine nTitle = n
pTitle = cTitle pTitle = cTitle
elif aLine.startswith("@"): elif line.startswith("@"):
if cTitle != TT_NONE: if cTitle != TT_NONE:
self._indexKeyword(tHandle, aLine, cTitle, nwItem.itemClass, tags) self._indexKeyword(tHandle, line, cTitle, nwItem.itemClass, tags)
elif aLine.startswith("%"): elif line.startswith("%"):
if cTitle != TT_NONE: if cTitle != TT_NONE:
toCheck = aLine[1:].lstrip() toCheck = line[1:].lstrip()
synTag = toCheck[:9].lower() synTag = toCheck[:9].lower()
tLen = len(aLine) tLen = len(line)
cLen = len(toCheck) cLen = len(toCheck)
cOff = tLen - cLen cOff = tLen - cLen
if synTag == "synopsis:": if synTag == "synopsis:":
sText = aLine[cOff+9:].strip() sText = line[cOff+9:].strip()
self._itemIndex.setHeadingSynopsis(tHandle, cTitle, sText) self._itemIndex.setHeadingSynopsis(tHandle, cTitle, sText)
# Count words for remaining text after last heading # Count words for remaining text after last heading
if pTitle != TT_NONE: if pTitle != TT_NONE:
lastText = "\n".join(theLines[nTitle-1:]) lastText = "\n".join(lines[nTitle-1:])
self._indexWordCounts(tHandle, lastText, pTitle) self._indexWordCounts(tHandle, lastText, pTitle)
# Also count words on a page with no titles # Also count words on a page with no titles
@@ -356,9 +354,9 @@ class NWIndex:
def _scanInactive(self, nwItem: NWItem, text: str) -> None: def _scanInactive(self, nwItem: NWItem, text: str) -> None:
"""Scan an inactive document for meta data.""" """Scan an inactive document for meta data."""
for aLine in text.splitlines(): for line in text.splitlines():
if aLine.startswith("#"): if line.startswith("#"):
hDepth, _ = self._splitHeading(aLine) hDepth, _ = self._splitHeading(line)
if hDepth != "H0": if hDepth != "H0":
nwItem.setMainHeading(hDepth) nwItem.setMainHeading(hDepth)
break break
@@ -380,7 +378,7 @@ class NWIndex:
return "H2", line[4:].strip() return "H2", line[4:].strip()
return "H0", "" return "H0", ""
def _indexWordCounts(self, tHandle: str, text: str, sTitle: str): def _indexWordCounts(self, tHandle: str, text: str, sTitle: str) -> None:
"""Count text stats and save the counts to the index.""" """Count text stats and save the counts to the index."""
cC, wC, pC = countWords(text) cC, wC, pC = countWords(text)
self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC) self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC)
@@ -393,22 +391,22 @@ class NWIndex:
of active tags is updated so that no longer used tags can be of active tags is updated so that no longer used tags can be
pruned later. pruned later.
""" """
isValid, theBits, _ = self.scanThis(line) isValid, tBits, _ = self.scanThis(line)
if not isValid or len(theBits) < 2: if not isValid or len(tBits) < 2:
logger.warning("Skipping keyword with %d value(s) in '%s'", len(theBits), tHandle) logger.warning("Skipping keyword with %d value(s) in '%s'", len(tBits), tHandle)
return return
if theBits[0] not in nwKeyWords.VALID_KEYS: if tBits[0] not in nwKeyWords.VALID_KEYS:
logger.warning("Skipping invalid keyword '%s' in '%s'", theBits[0], tHandle) logger.warning("Skipping invalid keyword '%s' in '%s'", tBits[0], tHandle)
return return
if theBits[0] == nwKeyWords.TAG_KEY: if tBits[0] == nwKeyWords.TAG_KEY:
tagName = theBits[1] tagName = tBits[1]
self._tagsIndex.add(tagName, tHandle, sTitle, itemClass) self._tagsIndex.add(tagName, tHandle, sTitle, itemClass)
self._itemIndex.setHeadingTag(tHandle, sTitle, tagName) self._itemIndex.setHeadingTag(tHandle, sTitle, tagName)
tags[tagName] = True tags[tagName] = True
else: else:
self._itemIndex.addHeadingReferences(tHandle, sTitle, theBits[1:], theBits[0]) self._itemIndex.addHeadingRef(tHandle, sTitle, tBits[1:], tBits[0])
return return
@@ -506,8 +504,8 @@ class NWIndex:
they appear in the tree view and in the respective document they appear in the tree view and in the respective document
files, but skipping all note files. files, but skipping all note files.
""" """
novStruct = self._itemIndex.iterNovelStructure(rHandle=rootHandle, skipExcl=skipExcl) structure = self._itemIndex.iterNovelStructure(rHandle=rootHandle, skipExcl=skipExcl)
for tHandle, sTitle, hItem in novStruct: for tHandle, sTitle, hItem in structure:
yield f"{tHandle}:{sTitle}", tHandle, sTitle, hItem yield f"{tHandle}:{sTitle}", tHandle, sTitle, hItem
return return
@@ -557,14 +555,14 @@ class NWIndex:
"words": hItem.wordCount, "words": hItem.wordCount,
} }
theToC = [( result = [(
tKey, tKey,
tData[tKey]["level"], tData[tKey]["level"],
tData[tKey]["title"], tData[tKey]["title"],
tData[tKey]["words"] tData[tKey]["words"]
) for tKey in tOrder] ) for tKey in tOrder]
return theToC return result
def getCounts(self, tHandle: str, sTitle: str | None = None) -> tuple[int, int, int]: def getCounts(self, tHandle: str, sTitle: str | None = None) -> tuple[int, int, int]:
"""Return the counts for a file, or a section of a file, """Return the counts for a file, or a section of a file,
@@ -745,7 +743,7 @@ class ItemIndex:
__slots__ = ("_project", "_items") __slots__ = ("_project", "_items")
def __init__(self, project: NWProject): def __init__(self, project: NWProject) -> None:
self._project = project self._project = project
self._items: dict[str, IndexItem] = {} self._items: dict[str, IndexItem] = {}
return return
@@ -753,7 +751,7 @@ class ItemIndex:
def __contains__(self, tHandle: str) -> bool: def __contains__(self, tHandle: str) -> bool:
return tHandle in self._items return tHandle in self._items
def __delitem__(self, tHandle: str): def __delitem__(self, tHandle: str) -> None:
self._items.pop(tHandle, None) self._items.pop(tHandle, None)
return return
@@ -764,12 +762,12 @@ class ItemIndex:
# Methods # Methods
## ##
def clear(self): def clear(self) -> None:
"""Clear the index.""" """Clear the index."""
self._items = {} self._items = {}
return return
def add(self, tHandle: str, nwItem: NWItem): def add(self, tHandle: str, nwItem: NWItem) -> None:
"""Add a new item to the index. This will overwrite the item if """Add a new item to the index. This will overwrite the item if
it already exists. it already exists.
""" """
@@ -837,7 +835,7 @@ class ItemIndex:
return sTitle return sTitle
return TT_NONE return TT_NONE
def setHeadingCounts(self, tHandle: str, sTitle: str, cC: int, wC: int, pC: int): def setHeadingCounts(self, tHandle: str, sTitle: str, cC: int, wC: int, pC: int) -> None:
"""Set the character, word and paragraph counts of a heading """Set the character, word and paragraph counts of a heading
on a given item. on a given item.
""" """
@@ -845,22 +843,22 @@ class ItemIndex:
self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC) self._items[tHandle].setHeadingCounts(sTitle, cC, wC, pC)
return return
def setHeadingSynopsis(self, tHandle: str, sTitle: str, text: str): def setHeadingSynopsis(self, tHandle: str, sTitle: str, text: str) -> None:
"""Set the synopsis text for a heading on a given item.""" """Set the synopsis text for a heading on a given item."""
if tHandle in self._items: if tHandle in self._items:
self._items[tHandle].setHeadingSynopsis(sTitle, text) self._items[tHandle].setHeadingSynopsis(sTitle, text)
return return
def setHeadingTag(self, tHandle: str, sTitle: str, tagKey: str): def setHeadingTag(self, tHandle: str, sTitle: str, tagKey: str) -> None:
"""Set the main tag for a heading on a given item.""" """Set the main tag for a heading on a given item."""
if tHandle in self._items: if tHandle in self._items:
self._items[tHandle].setHeadingTag(sTitle, tagKey) self._items[tHandle].setHeadingTag(sTitle, tagKey)
return return
def addHeadingReferences(self, tHandle: str, sTitle: str, tagKeys: list[str], refType: str): def addHeadingRef(self, tHandle: str, sTitle: str, tagKeys: list[str], refType: str) -> None:
"""Set the reference tags for a heading on a given item.""" """Set the reference tags for a heading on a given item."""
if tHandle in self._items: if tHandle in self._items:
self._items[tHandle].addHeadingReferences(sTitle, tagKeys, refType) self._items[tHandle].addHeadingRef(sTitle, tagKeys, refType)
return return
## ##
@@ -871,7 +869,7 @@ class ItemIndex:
"""Pack all the data of the index into a single dictionary.""" """Pack all the data of the index into a single dictionary."""
return {handle: item.packData() for handle, item in self._items.items()} return {handle: item.packData() for handle, item in self._items.items()}
def unpackData(self, data: dict): def unpackData(self, data: dict) -> None:
"""Iterate through the itemIndex loaded from cache and check """Iterate through the itemIndex loaded from cache and check
that it's valid. This will raise errors if there is a problem. that it's valid. This will raise errors if there is a problem.
""" """
@@ -904,17 +902,13 @@ class IndexItem:
must be reset each time the item is re-indexed. must be reset each time the item is re-indexed.
""" """
__slots__ = ("_handle", "_item", "_headings", "_headings", "_count") __slots__ = ("_handle", "_item", "_headings", "_count")
def __init__(self, tHandle: str, nwItem: NWItem): def __init__(self, tHandle: str, nwItem: NWItem) -> None:
self._handle = tHandle self._handle = tHandle
self._item = nwItem self._item = nwItem
self._headings: dict[str, IndexHeading] = {} self._headings: dict[str, IndexHeading] = {TT_NONE: IndexHeading(TT_NONE)}
self._count = 0 self._count = 0
# Add a placeholder heading
self._headings[TT_NONE] = IndexHeading(TT_NONE)
return return
def __repr__(self) -> str: def __repr__(self) -> str:
@@ -935,13 +929,14 @@ class IndexItem:
@property @property
def item(self) -> NWItem: def item(self) -> NWItem:
"""Return the project item of the index item."""
return self._item return self._item
## ##
# Setters # Setters
## ##
def addHeading(self, tHeading: IndexHeading): def addHeading(self, tHeading: IndexHeading) -> None:
"""Add a heading to the item. Also remove the placeholder entry """Add a heading to the item. Also remove the placeholder entry
if it exists. if it exists.
""" """
@@ -950,25 +945,25 @@ class IndexItem:
self._headings[tHeading.key] = tHeading self._headings[tHeading.key] = tHeading
return return
def setHeadingCounts(self, sTitle: str, cCount: int, wCount: int, pCount: int): def setHeadingCounts(self, sTitle: str, cCount: int, wCount: int, pCount: int) -> None:
"""Set the character, word and paragraph count of a heading.""" """Set the character, word and paragraph count of a heading."""
if sTitle in self._headings: if sTitle in self._headings:
self._headings[sTitle].setCounts(cCount, wCount, pCount) self._headings[sTitle].setCounts(cCount, wCount, pCount)
return return
def setHeadingSynopsis(self, sTitle: str, text: str): def setHeadingSynopsis(self, sTitle: str, text: str) -> None:
"""Set the synopsis text of a heading.""" """Set the synopsis text of a heading."""
if sTitle in self._headings: if sTitle in self._headings:
self._headings[sTitle].setSynopsis(text) self._headings[sTitle].setSynopsis(text)
return return
def setHeadingTag(self, sTitle: str, tagKey: str): def setHeadingTag(self, sTitle: str, tagKey: str) -> None:
"""Set the tag of a heading.""" """Set the tag of a heading."""
if sTitle in self._headings: if sTitle in self._headings:
self._headings[sTitle].setTag(tagKey) self._headings[sTitle].setTag(tagKey)
return return
def addHeadingReferences(self, sTitle: str, tagKeys: list[str], refType: str): def addHeadingRef(self, sTitle: str, tagKeys: list[str], refType: str) -> None:
"""Add a reference key and all its types to a heading.""" """Add a reference key and all its types to a heading."""
if sTitle in self._headings: if sTitle in self._headings:
for tagKey in tagKeys: for tagKey in tagKeys:
@@ -980,9 +975,11 @@ class IndexItem:
## ##
def items(self) -> ItemsView[str, IndexHeading]: def items(self) -> ItemsView[str, IndexHeading]:
"""Return IndexHeading items."""
return self._headings.items() return self._headings.items()
def headings(self) -> list[str]: def headings(self) -> list[str]:
"""Return heading keys in sorted order."""
return sorted(self._headings.keys()) return sorted(self._headings.keys())
def allTags(self) -> list[str]: def allTags(self) -> list[str]:
@@ -1015,7 +1012,7 @@ class IndexItem:
return data return data
def unpackData(self, data: dict): def unpackData(self, data: dict) -> None:
"""Unpack an item entry from the data.""" """Unpack an item entry from the data."""
references = data.get("references", {}) references = data.get("references", {})
for sTitle, hData in data.get("headings", {}).items(): for sTitle, hData in data.get("headings", {}).items():
@@ -1043,7 +1040,7 @@ class IndexHeading:
"_paraCount", "_synopsis", "_tag", "_refs", "_paraCount", "_synopsis", "_tag", "_refs",
) )
def __init__(self, key: str, line: int = 0, level: str = "H0", title: str = ""): def __init__(self, key: str, line: int = 0, level: str = "H0", title: str = "") -> None:
self._key = key self._key = key
self._line = line self._line = line
self._level = level self._level = level
@@ -1110,18 +1107,18 @@ class IndexHeading:
# Setters # Setters
## ##
def setLevel(self, level: str): def setLevel(self, level: str) -> None:
"""Set the level of the header if it's a valid value.""" """Set the level of the header if it's a valid value."""
if level in nwHeaders.H_VALID: if level in nwHeaders.H_VALID:
self._level = level self._level = level
return return
def setLine(self, line: int): def setLine(self, line: int) -> None:
"""Set the line number of a heading.""" """Set the line number of a heading."""
self._line = max(0, checkInt(line, 0)) self._line = max(0, checkInt(line, 0))
return return
def setCounts(self, charCount: int, wordCount: int, paraCount: int): def setCounts(self, charCount: int, wordCount: int, paraCount: int) -> None:
"""Set the character, word and paragraph count. Make sure the """Set the character, word and paragraph count. Make sure the
value is an integer and is not smaller than 0. value is an integer and is not smaller than 0.
""" """
@@ -1130,17 +1127,17 @@ class IndexHeading:
self._paraCount = max(0, checkInt(paraCount, 0)) self._paraCount = max(0, checkInt(paraCount, 0))
return return
def setSynopsis(self, text: str): def setSynopsis(self, text: str) -> None:
"""Set the synopsis text and make sure it is a string.""" """Set the synopsis text and make sure it is a string."""
self._synopsis = str(text) self._synopsis = str(text)
return return
def setTag(self, tagKey: str): def setTag(self, tagKey: str) -> None:
"""Set the tag for references, and make sure it is a string.""" """Set the tag for references, and make sure it is a string."""
self._tag = str(tagKey).lower() self._tag = str(tagKey).lower()
return return
def addReference(self, tagKey: str, refType: str): def addReference(self, tagKey: str, refType: str) -> None:
"""Add a record of a reference tag, and what keyword types it is """Add a record of a reference tag, and what keyword types it is
associated with. associated with.
""" """
@@ -1176,7 +1173,7 @@ class IndexHeading:
""" """
return {key: ",".join(sorted(list(value))) for key, value in self._refs.items()} return {key: ",".join(sorted(list(value))) for key, value in self._refs.items()}
def unpackData(self, data: dict): def unpackData(self, data: dict) -> None:
"""Unpack a heading entry from a dictionary.""" """Unpack a heading entry from a dictionary."""
self.setLevel(data.get("level", "H0")) self.setLevel(data.get("level", "H0"))
self._title = str(data.get("title", "")) self._title = str(data.get("title", ""))
@@ -1190,7 +1187,7 @@ class IndexHeading:
self._synopsis = str(data.get("synopsis", "")) self._synopsis = str(data.get("synopsis", ""))
return return
def unpackReferences(self, data: dict): def unpackReferences(self, data: dict) -> None:
"""Unpack a set of references from a dictionary.""" """Unpack a set of references from a dictionary."""
for tagKey, refTypes in data.items(): for tagKey, refTypes in data.items():
if not isinstance(tagKey, str): if not isinstance(tagKey, str):
@@ -1232,55 +1229,55 @@ def countWords(text: str) -> tuple[int, int, int]:
if nwUnicode.U_EMDASH in text: if nwUnicode.U_EMDASH in text:
text = text.replace(nwUnicode.U_EMDASH, " ") text = text.replace(nwUnicode.U_EMDASH, " ")
for aLine in text.splitlines(): for line in text.splitlines():
countPara = True countPara = True
if not aLine: if not line:
prevEmpty = True prevEmpty = True
continue continue
if aLine[0] == "@" or aLine[0] == "%": if line[0] == "@" or line[0] == "%":
continue continue
if aLine[0] == "[": if line[0] == "[":
if aLine.startswith(("[NEWPAGE]", "[NEW PAGE]", "[VSPACE]")): if line.startswith(("[NEWPAGE]", "[NEW PAGE]", "[VSPACE]")):
continue continue
elif aLine.startswith("[VSPACE:") and aLine.endswith("]"): elif line.startswith("[VSPACE:") and line.endswith("]"):
continue continue
elif aLine[0] == "#": elif line[0] == "#":
if aLine[:5] == "#### ": if line[:5] == "#### ":
aLine = aLine[5:] line = line[5:]
countPara = False countPara = False
elif aLine[:4] == "### ": elif line[:4] == "### ":
aLine = aLine[4:] line = line[4:]
countPara = False countPara = False
elif aLine[:3] == "## ": elif line[:3] == "## ":
aLine = aLine[3:] line = line[3:]
countPara = False countPara = False
elif aLine[:2] == "# ": elif line[:2] == "# ":
aLine = aLine[2:] line = line[2:]
countPara = False countPara = False
elif aLine[:3] == "#! ": elif line[:3] == "#! ":
aLine = aLine[3:] line = line[3:]
countPara = False countPara = False
elif aLine[:4] == "##! ": elif line[:4] == "##! ":
aLine = aLine[4:] line = line[4:]
countPara = False countPara = False
elif aLine[0] == ">" or aLine[-1] == "<": elif line[0] == ">" or line[-1] == "<":
if aLine[:2] == ">>": if line[:2] == ">>":
aLine = aLine[2:].lstrip(" ") line = line[2:].lstrip(" ")
elif aLine[:1] == ">": elif line[:1] == ">":
aLine = aLine[1:].lstrip(" ") line = line[1:].lstrip(" ")
if aLine[-2:] == "<<": if line[-2:] == "<<":
aLine = aLine[:-2].rstrip(" ") line = line[:-2].rstrip(" ")
elif aLine[-1:] == "<": elif line[-1:] == "<":
aLine = aLine[:-1].rstrip(" ") line = line[:-1].rstrip(" ")
wordCount += len(aLine.split()) wordCount += len(line.split())
charCount += len(aLine) charCount += len(line)
if countPara and prevEmpty: if countPara and prevEmpty:
paraCount += 1 paraCount += 1
tests/test_core/test_core_index.py
+3 -3
View File
@@ -1027,9 +1027,9 @@ def testCoreIndex_ItemIndex(mockGUI, fncPath, mockRnd):
itemIndex.setHeadingCounts(cHandle, "T0001", 60, 10, 2) itemIndex.setHeadingCounts(cHandle, "T0001", 60, 10, 2)
itemIndex.setHeadingSynopsis(cHandle, "T0001", "In the beginning ...") itemIndex.setHeadingSynopsis(cHandle, "T0001", "In the beginning ...")
itemIndex.setHeadingTag(cHandle, "T0001", "One") itemIndex.setHeadingTag(cHandle, "T0001", "One")
itemIndex.addHeadingReferences(cHandle, "T0001", ["Jane"], "@pov") itemIndex.addHeadingRef(cHandle, "T0001", ["Jane"], "@pov")
itemIndex.addHeadingReferences(cHandle, "T0001", ["Jane"], "@focus") itemIndex.addHeadingRef(cHandle, "T0001", ["Jane"], "@focus")
itemIndex.addHeadingReferences(cHandle, "T0001", ["Jane", "John"], "@char") itemIndex.addHeadingRef(cHandle, "T0001", ["Jane", "John"], "@char")
idxData = itemIndex.packData() idxData = itemIndex.packData()
assert idxData[cHandle]["headings"]["T0001"] == { assert idxData[cHandle]["headings"]["T0001"] == {