Reproog/novelWriter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move session log handling to a separate class, and use jsonl format

Browse Source
This commit is contained in:
Veronica Berglyd Olsen
2023-06-12 01:50:27 +02:00
parent aea41032ef
commit eba6783816
7 changed files with 199 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
novelwriter/constants.py
+1
View File
@@ -91,6 +91,7 @@ class nwFiles:
OPTS_FILE = "options.json"
PROJ_DICT = "wordlist.txt"
SESS_STATS = "sessionStats.log"
SESS_FILE = "sessions.jsonl"
# END Class nwFiles
novelwriter/core/coretools.py
+1 -3
View File
@@ -1,7 +1,6 @@
"""
novelWriter – Project Document Tools
====================================
A collection of tools to create and manipulate documents
File History:
Created: 2022-10-02 [2.0rc1] DocMerger
@@ -28,7 +27,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
import shutil
import logging
from time import time
from functools import partial
from PyQt5.QtCore import QCoreApplication
@@ -319,7 +317,7 @@ class ProjectBuilder:
project.data.setTitle(projTitle)
project.data.setAuthor(projAuthor)
project.setDefaultStatusImport()
project._projOpened = int(time())
project.session.startSession()
# Add Root Folders
hNovelRoot = project.newRoot(nwItemClass.NOVEL)
novelwriter/core/project.py
+17 -58
View File
@@ -22,6 +22,7 @@ General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import json
import logging
@@ -33,9 +34,10 @@ from functools import partial
from PyQt5.QtCore import QCoreApplication, QObject, pyqtSignal
from novelwriter import CONFIG, __version__, __hexversion__
from novelwriter.core.sessions import NWSessionLog
from novelwriter.enum import nwItemType, nwItemClass, nwItemLayout, nwAlert
from novelwriter.error import logException
from novelwriter.constants import trConst, nwFiles, nwLabels
from novelwriter.constants import trConst, nwLabels
from novelwriter.core.tree import NWTree
from novelwriter.core.item import NWItem
from novelwriter.core.index import NWIndex
@@ -66,12 +68,12 @@ class NWProject(QObject):
self._data = NWProjectData(self) # The project settings
self._tree = NWTree(self) # The project tree
self._index = NWIndex(self) # The projecty index
self._session = NWSessionLog(self) # The session record
# Data Cache
self._langData = {} # Localisation data
# Project Status
self._projOpened = 0 # The time stamp of when the project file was opened
self._projChanged = False # The project has unsaved changes
self._lockedBy = None # Data on which computer has the project open
self._projFiles = [] # A list of all files in the content folder on load
@@ -108,9 +110,13 @@ class NWProject(QObject):
def index(self):
return self._index
@property
def session(self) -> NWSessionLog:
return self._session
@property
def projOpened(self):
return self._projOpened
return self._session.start
@property
def projChanged(self):
@@ -228,7 +234,6 @@ class NWProject(QObject):
default values.
"""
# Project Status
self._projOpened = 0
self._projChanged = False
# Project Tree
@@ -236,6 +241,7 @@ class NWProject(QObject):
self._tree.clear()
self._index.clearIndex()
self._data = NWProjectData(self)
self._session = NWSessionLog(self)
# Project Settings
self._projFiles = []
@@ -370,8 +376,7 @@ class NWProject(QObject):
self._index.rebuildIndex()
self.updateWordCounts()
self._projOpened = time()
self._session.startSession()
self._storage.writeLockFile()
self.setProjectChanged(False)
self.mainGui.setStatus(self.tr("Opened Project: {0}").format(self._data.name))
@@ -407,7 +412,7 @@ class NWProject(QObject):
return False
saveTime = time()
editTime = int(self._data.editTime + saveTime - self._projOpened)
editTime = self._data.editTime + max(round(saveTime - self._session.start), 0)
content = self._tree.pack()
if not xmlWriter.write(self._data, content, saveTime, editTime):
self.mainGui.makeAlert(self.tr(
@@ -437,7 +442,7 @@ class NWProject(QObject):
logger.info("Closing project")
self._options.saveSettings()
self._tree.writeToCFile()
self._appendSessionStats(idleTime)
self._session.appendSession(idleTime)
self._storage.clearLockFile()
self._storage.closeSession()
self.clearProject()
@@ -560,18 +565,17 @@ class NWProject(QObject):
# Getters
##
def getLockStatus(self):
"""Return the project lock information for the project.
"""
def getLockStatus(self) -> list | None:
"""Return the project lock information for the project."""
if isinstance(self._lockedBy, list) and len(self._lockedBy) == 4:
return self._lockedBy
return None
def getCurrentEditTime(self):
def getCurrentEditTime(self) -> int:
"""Get the total project edit time, including the time spent in
the current session.
"""
return round(self._data.editTime + time() - self._projOpened)
return self._data.editTime + round(time() - self._session.start)
def getProjectItems(self):
"""This function ensures that the item tree loaded is sent to
@@ -798,49 +802,4 @@ class NWProject(QObject):
return True
def _appendSessionStats(self, idleTime):
"""Append session statistics to the sessions log file.
"""
sessionFile = self._storage.getMetaFile(nwFiles.SESS_STATS)
if not isinstance(sessionFile, Path):
return False
nowTime = time()
iNovel, iNotes = self._data.initCounts
cNovel, cNotes = self._data.currCounts
iTotal = iNovel + iNotes
sessDiff = cNovel + cNotes - iTotal
sessTime = nowTime - self._projOpened
logger.info("The session lasted %d sec and added %d words", int(sessTime), sessDiff)
if sessTime < 300 and sessDiff == 0:
logger.info("Session too short, skipping log entry")
return False
try:
isFile = sessionFile.exists() # We must save the state before we open
with open(sessionFile, mode="a+", encoding="utf-8") as outFile:
if not isFile:
# It's a new file, so add a header
if iTotal > 0:
outFile.write("# Offset %d\n" % iTotal)
outFile.write("# %-17s %-19s %8s %8s %8s\n" % (
"Start Time", "End Time", "Novel", "Notes", "Idle"
))
outFile.write("%-19s %-19s %8d %8d %8d\n" % (
formatTimeStamp(self._projOpened),
formatTimeStamp(nowTime),
cNovel,
cNotes,
int(idleTime),
))
except Exception:
logger.error("Failed to write session stats file")
logException()
return False
return True
# END Class NWProject
novelwriter/core/sessions.py
+125
View File
@@ -0,0 +1,125 @@
"""
novelWriter – Project Session Log Class
=======================================
File History:
Created: 2023-06-11 [2.1b1]
This file is a part of novelWriter
Copyright 2018–2023, Veronica Berglyd Olsen
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from __future__ import annotations
import json
import logging
from time import time
from typing import TYPE_CHECKING
from pathlib import Path
from novelwriter.error import logException
from novelwriter.common import formatTimeStamp
from novelwriter.constants import nwFiles
if TYPE_CHECKING:
from novelwriter.core.project import NWProject
logger = logging.getLogger(__name__)
class NWSessionLog:
"""Core: Session JSON Lines Log File
The class that wraps the session log file, which is in JSON Lines
format. That is, one JSON object per line.
"""
def __init__(self, project: NWProject):
self._project = project
self._start = 0.0
return
##
# Properties
##
@property
def start(self) -> float:
"""The session start time."""
return self._start
##
# Methods
##
def startSession(self):
"""Start the writng session."""
self._start = time()
return
def appendSession(self, idleTime: float) -> bool:
"""Append session statistics to the sessions log file."""
sessFile = self._project.storage.getMetaFile(nwFiles.SESS_FILE)
if not isinstance(sessFile, Path):
return False
now = time()
iNovel, iNotes = self._project.data.initCounts
cNovel, cNotes = self._project.data.currCounts
iTotal = iNovel + iNotes
wDiff = cNovel + cNotes - iTotal
sTime = now - self._start
logger.info("The session lasted %d sec and added %d words", int(sTime), wDiff)
if sTime < 300 and wDiff == 0:
logger.info("Session too short, skipping log entry")
return False
try:
if not sessFile.exists():
with open(sessFile, mode="w", encoding="utf-8") as fObj:
fObj.write(self.createInitial(iTotal))
with open(sessFile, mode="a+", encoding="utf-8") as fObj:
fObj.write(self.createRecord(
start=formatTimeStamp(self._start),
end=formatTimeStamp(now),
novel=cNovel,
notes=cNotes,
idle=round(idleTime)
))
except Exception:
logger.error("Failed to write to session stats file")
logException()
return False
return True
def createInitial(self, total: int) -> str:
"""Low level function to create the initial log file record."""
data = json.dumps({"type": "initial", "offset": total})
return f"{data}\n"
def createRecord(self, start: str, end: str, novel: int, notes: int, idle: int) -> str:
"""Low level function to create a log record."""
data = json.dumps({
"type": "record", "start": start, "end": end,
"novel": novel, "notes": notes, "idle": idle,
})
return f"{data}\n"
# END Class NWSessionLog
novelwriter/core/storage.py
+46 -1
View File
@@ -254,7 +254,7 @@ class NWStorage:
(baseMeta / nwFiles.INDEX_FILE, f"meta/{nwFiles.INDEX_FILE}"),
(baseMeta / nwFiles.OPTS_FILE, f"meta/{nwFiles.OPTS_FILE}"),
(baseMeta / nwFiles.PROJ_DICT, f"meta/{nwFiles.PROJ_DICT}"),
(baseMeta / nwFiles.SESS_STATS, f"meta/{nwFiles.SESS_STATS}"),
(baseMeta / nwFiles.SESS_FILE, f"meta/{nwFiles.SESS_FILE}"),
]
for contItem in baseCont.iterdir():
name = contItem.name
@@ -374,6 +374,10 @@ class NWStorage:
def _deprecatedFiles(self, path: Path):
"""Handle files that are no longer used by novelWriter."""
sessLog = path / "meta" / "sessionStats.log"
if sessLog.is_file():
self._convertOldLogFile(sessLog, path / "meta" / nwFiles.SESS_FILE)
remove = [
path / "meta" / "tagsIndex.json", # Renamed in 2.1 Beta 1
path / "meta" / "mainOptions.json", # Replaced in 0.5
@@ -409,4 +413,45 @@ class NWStorage:
return
def _convertOldLogFile(self, sessLog: Path, sessJson: Path) -> bool:
"""Convert the old text log file format to the new JSON Lines
format.
"""
if sessJson.exists() or not sessLog.exists():
# If the new file already exists, we won't overwrite it
return True
try:
data = []
offset = 0
session = self._project.session
with open(sessLog, mode="r", encoding="utf-8") as fObj:
for record in fObj:
bits = record.split()
nBits = len(bits)
if record.startswith("# Offset") and nBits == 3:
offset = int(bits[2])
elif not record.startswith("#") and nBits > 5:
data.append(session.createRecord(
start=f"{bits[0]} {bits[1]}",
end=f"{bits[2]} {bits[3]}",
novel=int(bits[4]),
notes=int(bits[5]),
idle=int(bits[6]) if nBits > 6 else -1,
))
with open(sessJson, mode="a+", encoding="utf-8") as fObj:
fObj.write(session.createInitial(offset))
fObj.write("".join(data))
# If we're here, we remove the old file
sessLog.unlink()
except Exception:
logger.error("Failed to convert old stats file")
logException()
return False
return True
# END Class NWStorage
tests/test_core/test_core_project.py
+8 -8
View File
@@ -487,7 +487,7 @@ def testCoreProject_Methods(monkeypatch, mockGUI, fncPath, mockRnd):
# Edit Time
theProject.data.setEditTime(1234)
theProject._projOpened = 1600000000
theProject._session._start = 1600000000
with monkeypatch.context() as mp:
mp.setattr("novelwriter.core.project.time", lambda: 1600005600)
assert theProject.getCurrentEditTime() == 6834
@@ -585,32 +585,32 @@ def testCoreProject_Methods(monkeypatch, mockGUI, fncPath, mockRnd):
# No path for writing
with monkeypatch.context() as mp:
mp.setattr("novelwriter.core.storage.NWStorage.getMetaFile", lambda *a: None)
assert theProject._appendSessionStats(idleTime=0) is False
assert theProject.session.appendSession(idleTime=0) is False
# Block open
with monkeypatch.context() as mp:
mp.setattr("builtins.open", causeOSError)
assert theProject._appendSessionStats(idleTime=0) is False
assert theProject.session.appendSession(idleTime=0) is False
# Session too short
theProject._projOpened = time()
theProject._session._start = time()
theProject.data.setInitCounts(50, 50)
theProject.data.setCurrCounts(50, 50)
assert theProject._appendSessionStats(idleTime=0) is False
assert theProject.session.appendSession(idleTime=0) is False
# Write entry
statsFile = theProject.storage.getMetaFile(nwFiles.SESS_STATS)
statsFile = theProject.storage.getMetaFile(nwFiles.SESS_FILE)
assert isinstance(statsFile, Path)
if statsFile.exists():
statsFile.unlink()
theProject._projOpened = 1600002000
theProject._session._start = 1600002000
theProject.data._initCounts = [50, 50]
theProject.data._currCounts = [200, 100]
with monkeypatch.context() as mp:
mp.setattr("novelwriter.core.project.time", lambda: 1600005600)
assert theProject._appendSessionStats(idleTime=99)
assert theProject.session.appendSession(idleTime=99)
assert statsFile.read_text(encoding="utf-8") == (
"# Offset 100\n"
tests/tools.py
+1 -2
View File
@@ -19,7 +19,6 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
import time
import shutil
from pathlib import Path
@@ -201,7 +200,7 @@ def buildTestProject(theObject, projPath):
aDoc.writeDocument("### %s\n\n" % theProject.tr("New Scene"))
theProject.index.reIndexHandle(xHandle[8])
theProject._projOpened = time.time()
theProject.session.startSession()
theProject.setProjectChanged(True)
theProject.saveProject(autoSave=True)