Move session log handling to a separate class, and use jsonl format

novelwriter/constants.py

@@ -91,6 +91,7 @@ class nwFiles:
     OPTS_FILE   = "options.json"
     PROJ_DICT   = "wordlist.txt"
     SESS_STATS  = "sessionStats.log"
+    SESS_FILE   = "sessions.jsonl"
 
 # END Class nwFiles
 

novelwriter/core/coretools.py

@@ -1,7 +1,6 @@
 """
 novelWriter – Project Document Tools
 ====================================
-A collection of tools to create and manipulate documents
 
 File History:
 Created: 2022-10-02 [2.0rc1] DocMerger
@@ -28,7 +27,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 import shutil
 import logging
 
-from time import time
 from functools import partial
 
 from PyQt5.QtCore import QCoreApplication
@@ -319,7 +317,7 @@ class ProjectBuilder:
         project.data.setTitle(projTitle)
         project.data.setAuthor(projAuthor)
         project.setDefaultStatusImport()
-        project._projOpened = int(time())
+        project.session.startSession()
 
         # Add Root Folders
         hNovelRoot = project.newRoot(nwItemClass.NOVEL)

novelwriter/core/project.py

@@ -22,6 +22,7 @@ General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
+from __future__ import annotations
 
 import json
 import logging
@@ -33,9 +34,10 @@ from functools import partial
 from PyQt5.QtCore import QCoreApplication, QObject, pyqtSignal
 
 from novelwriter import CONFIG, __version__, __hexversion__
+from novelwriter.core.sessions import NWSessionLog
 from novelwriter.enum import nwItemType, nwItemClass, nwItemLayout, nwAlert
 from novelwriter.error import logException
-from novelwriter.constants import trConst, nwFiles, nwLabels
+from novelwriter.constants import trConst, nwLabels
 from novelwriter.core.tree import NWTree
 from novelwriter.core.item import NWItem
 from novelwriter.core.index import NWIndex
@@ -66,12 +68,12 @@ class NWProject(QObject):
         self._data    = NWProjectData(self)  # The project settings
         self._tree    = NWTree(self)         # The project tree
         self._index   = NWIndex(self)        # The projecty index
+        self._session = NWSessionLog(self)   # The session record
 
         # Data Cache
         self._langData = {}  # Localisation data
 
         # Project Status
-        self._projOpened  = 0      # The time stamp of when the project file was opened
         self._projChanged = False  # The project has unsaved changes
         self._lockedBy    = None   # Data on which computer has the project open
         self._projFiles   = []     # A list of all files in the content folder on load
@@ -108,9 +110,13 @@ class NWProject(QObject):
     def index(self):
         return self._index
 
+    @property
+    def session(self) -> NWSessionLog:
+        return self._session
+
     @property
     def projOpened(self):
-        return self._projOpened
+        return self._session.start
 
     @property
     def projChanged(self):
@@ -228,7 +234,6 @@ class NWProject(QObject):
         default values.
         """
         # Project Status
-        self._projOpened  = 0
         self._projChanged = False
 
         # Project Tree
@@ -236,6 +241,7 @@ class NWProject(QObject):
         self._tree.clear()
         self._index.clearIndex()
         self._data = NWProjectData(self)
+        self._session = NWSessionLog(self)
 
         # Project Settings
         self._projFiles = []
@@ -370,8 +376,7 @@ class NWProject(QObject):
             self._index.rebuildIndex()
 
         self.updateWordCounts()
-        self._projOpened = time()
+        self._session.startSession()
-
         self._storage.writeLockFile()
         self.setProjectChanged(False)
         self.mainGui.setStatus(self.tr("Opened Project: {0}").format(self._data.name))
@@ -407,7 +412,7 @@ class NWProject(QObject):
             return False
 
         saveTime = time()
-        editTime = int(self._data.editTime + saveTime - self._projOpened)
+        editTime = self._data.editTime + max(round(saveTime - self._session.start), 0)
         content = self._tree.pack()
         if not xmlWriter.write(self._data, content, saveTime, editTime):
             self.mainGui.makeAlert(self.tr(
@@ -437,7 +442,7 @@ class NWProject(QObject):
         logger.info("Closing project")
         self._options.saveSettings()
         self._tree.writeToCFile()
-        self._appendSessionStats(idleTime)
+        self._session.appendSession(idleTime)
         self._storage.clearLockFile()
         self._storage.closeSession()
         self.clearProject()
@@ -560,18 +565,17 @@ class NWProject(QObject):
     #  Getters
     ##
 
-    def getLockStatus(self):
+    def getLockStatus(self) -> list | None:
-        """Return the project lock information for the project.
+        """Return the project lock information for the project."""
-        """
         if isinstance(self._lockedBy, list) and len(self._lockedBy) == 4:
             return self._lockedBy
         return None
 
-    def getCurrentEditTime(self):
+    def getCurrentEditTime(self) -> int:
         """Get the total project edit time, including the time spent in
         the current session.
         """
-        return round(self._data.editTime + time() - self._projOpened)
+        return self._data.editTime + round(time() - self._session.start)
 
     def getProjectItems(self):
         """This function ensures that the item tree loaded is sent to
@@ -798,49 +802,4 @@ class NWProject(QObject):
 
         return True
 
-    def _appendSessionStats(self, idleTime):
-        """Append session statistics to the sessions log file.
-        """
-        sessionFile = self._storage.getMetaFile(nwFiles.SESS_STATS)
-        if not isinstance(sessionFile, Path):
-            return False
-
-        nowTime = time()
-        iNovel, iNotes = self._data.initCounts
-        cNovel, cNotes = self._data.currCounts
-        iTotal = iNovel + iNotes
-        sessDiff = cNovel + cNotes - iTotal
-        sessTime = nowTime - self._projOpened
-
-        logger.info("The session lasted %d sec and added %d words", int(sessTime), sessDiff)
-        if sessTime < 300 and sessDiff == 0:
-            logger.info("Session too short, skipping log entry")
-            return False
-
-        try:
-            isFile = sessionFile.exists()  # We must save the state before we open
-            with open(sessionFile, mode="a+", encoding="utf-8") as outFile:
-                if not isFile:
-                    # It's a new file, so add a header
-                    if iTotal > 0:
-                        outFile.write("# Offset %d\n" % iTotal)
-                    outFile.write("# %-17s  %-19s  %8s  %8s  %8s\n" % (
-                        "Start Time", "End Time", "Novel", "Notes", "Idle"
-                    ))
-
-                outFile.write("%-19s  %-19s  %8d  %8d  %8d\n" % (
-                    formatTimeStamp(self._projOpened),
-                    formatTimeStamp(nowTime),
-                    cNovel,
-                    cNotes,
-                    int(idleTime),
-                ))
-
-        except Exception:
-            logger.error("Failed to write session stats file")
-            logException()
-            return False
-
-        return True
-
 # END Class NWProject

novelwriter/core/sessions.py

@@ -0,0 +1,125 @@
+"""
+novelWriter – Project Session Log Class
+=======================================
+
+File History:
+Created:   2023-06-11 [2.1b1]
+
+This file is a part of novelWriter
+Copyright 2018–2023, Veronica Berglyd Olsen
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+from __future__ import annotations
+
+import json
+import logging
+
+from time import time
+from typing import TYPE_CHECKING
+from pathlib import Path
+
+from novelwriter.error import logException
+from novelwriter.common import formatTimeStamp
+from novelwriter.constants import nwFiles
+
+if TYPE_CHECKING:
+    from novelwriter.core.project import NWProject
+
+logger = logging.getLogger(__name__)
+
+
+class NWSessionLog:
+    """Core: Session JSON Lines Log File
+
+    The class that wraps the session log file, which is in JSON Lines
+    format. That is, one JSON object per line.
+    """
+
+    def __init__(self, project: NWProject):
+        self._project = project
+        self._start = 0.0
+        return
+
+    ##
+    #  Properties
+    ##
+
+    @property
+    def start(self) -> float:
+        """The session start time."""
+        return self._start
+
+    ##
+    #  Methods
+    ##
+
+    def startSession(self):
+        """Start the writng session."""
+        self._start = time()
+        return
+
+    def appendSession(self, idleTime: float) -> bool:
+        """Append session statistics to the sessions log file."""
+        sessFile = self._project.storage.getMetaFile(nwFiles.SESS_FILE)
+        if not isinstance(sessFile, Path):
+            return False
+
+        now = time()
+        iNovel, iNotes = self._project.data.initCounts
+        cNovel, cNotes = self._project.data.currCounts
+        iTotal = iNovel + iNotes
+        wDiff = cNovel + cNotes - iTotal
+        sTime = now - self._start
+
+        logger.info("The session lasted %d sec and added %d words", int(sTime), wDiff)
+        if sTime < 300 and wDiff == 0:
+            logger.info("Session too short, skipping log entry")
+            return False
+
+        try:
+            if not sessFile.exists():
+                with open(sessFile, mode="w", encoding="utf-8") as fObj:
+                    fObj.write(self.createInitial(iTotal))
+
+            with open(sessFile, mode="a+", encoding="utf-8") as fObj:
+                fObj.write(self.createRecord(
+                    start=formatTimeStamp(self._start),
+                    end=formatTimeStamp(now),
+                    novel=cNovel,
+                    notes=cNotes,
+                    idle=round(idleTime)
+                ))
+
+        except Exception:
+            logger.error("Failed to write to session stats file")
+            logException()
+            return False
+
+        return True
+
+    def createInitial(self, total: int) -> str:
+        """Low level function to create the initial log file record."""
+        data = json.dumps({"type": "initial", "offset": total})
+        return f"{data}\n"
+
+    def createRecord(self, start: str, end: str, novel: int, notes: int, idle: int) -> str:
+        """Low level function to create a log record."""
+        data = json.dumps({
+            "type": "record", "start": start, "end": end,
+            "novel": novel, "notes": notes, "idle": idle,
+        })
+        return f"{data}\n"
+
+# END Class NWSessionLog

novelwriter/core/storage.py

@@ -254,7 +254,7 @@ class NWStorage:
             (baseMeta / nwFiles.INDEX_FILE,  f"meta/{nwFiles.INDEX_FILE}"),
             (baseMeta / nwFiles.OPTS_FILE,   f"meta/{nwFiles.OPTS_FILE}"),
             (baseMeta / nwFiles.PROJ_DICT,   f"meta/{nwFiles.PROJ_DICT}"),
-            (baseMeta / nwFiles.SESS_STATS,  f"meta/{nwFiles.SESS_STATS}"),
+            (baseMeta / nwFiles.SESS_FILE,   f"meta/{nwFiles.SESS_FILE}"),
         ]
         for contItem in baseCont.iterdir():
             name = contItem.name
@@ -374,6 +374,10 @@ class NWStorage:
 
     def _deprecatedFiles(self, path: Path):
         """Handle files that are no longer used by novelWriter."""
+        sessLog = path / "meta" / "sessionStats.log"
+        if sessLog.is_file():
+            self._convertOldLogFile(sessLog, path / "meta" / nwFiles.SESS_FILE)
+
         remove = [
             path / "meta" / "tagsIndex.json",          # Renamed in 2.1 Beta 1
             path / "meta" / "mainOptions.json",        # Replaced in 0.5
@@ -409,4 +413,45 @@ class NWStorage:
 
         return
 
+    def _convertOldLogFile(self, sessLog: Path, sessJson: Path) -> bool:
+        """Convert the old text log file format to the new JSON Lines
+        format.
+        """
+        if sessJson.exists() or not sessLog.exists():
+            # If the new file already exists, we won't overwrite it
+            return True
+
+        try:
+            data = []
+            offset = 0
+            session = self._project.session
+            with open(sessLog, mode="r", encoding="utf-8") as fObj:
+                for record in fObj:
+                    bits = record.split()
+                    nBits = len(bits)
+                    if record.startswith("# Offset") and nBits == 3:
+                        offset = int(bits[2])
+                    elif not record.startswith("#") and nBits > 5:
+                        data.append(session.createRecord(
+                            start=f"{bits[0]} {bits[1]}",
+                            end=f"{bits[2]} {bits[3]}",
+                            novel=int(bits[4]),
+                            notes=int(bits[5]),
+                            idle=int(bits[6]) if nBits > 6 else -1,
+                        ))
+
+            with open(sessJson, mode="a+", encoding="utf-8") as fObj:
+                fObj.write(session.createInitial(offset))
+                fObj.write("".join(data))
+
+            # If we're here, we remove the old file
+            sessLog.unlink()
+
+        except Exception:
+            logger.error("Failed to convert old stats file")
+            logException()
+            return False
+
+        return True
+
 # END Class NWStorage

tests/test_core/test_core_project.py

@@ -487,7 +487,7 @@ def testCoreProject_Methods(monkeypatch, mockGUI, fncPath, mockRnd):
 
     # Edit Time
     theProject.data.setEditTime(1234)
-    theProject._projOpened = 1600000000
+    theProject._session._start = 1600000000
     with monkeypatch.context() as mp:
         mp.setattr("novelwriter.core.project.time", lambda: 1600005600)
         assert theProject.getCurrentEditTime() == 6834
@@ -585,32 +585,32 @@ def testCoreProject_Methods(monkeypatch, mockGUI, fncPath, mockRnd):
     # No path for writing
     with monkeypatch.context() as mp:
         mp.setattr("novelwriter.core.storage.NWStorage.getMetaFile", lambda *a: None)
-        assert theProject._appendSessionStats(idleTime=0) is False
+        assert theProject.session.appendSession(idleTime=0) is False
 
     # Block open
     with monkeypatch.context() as mp:
         mp.setattr("builtins.open", causeOSError)
-        assert theProject._appendSessionStats(idleTime=0) is False
+        assert theProject.session.appendSession(idleTime=0) is False
 
     # Session too short
-    theProject._projOpened = time()
+    theProject._session._start = time()
     theProject.data.setInitCounts(50, 50)
     theProject.data.setCurrCounts(50, 50)
-    assert theProject._appendSessionStats(idleTime=0) is False
+    assert theProject.session.appendSession(idleTime=0) is False
 
     # Write entry
-    statsFile = theProject.storage.getMetaFile(nwFiles.SESS_STATS)
+    statsFile = theProject.storage.getMetaFile(nwFiles.SESS_FILE)
     assert isinstance(statsFile, Path)
     if statsFile.exists():
         statsFile.unlink()
 
-    theProject._projOpened = 1600002000
+    theProject._session._start = 1600002000
     theProject.data._initCounts = [50, 50]
     theProject.data._currCounts = [200, 100]
 
     with monkeypatch.context() as mp:
         mp.setattr("novelwriter.core.project.time", lambda: 1600005600)
-        assert theProject._appendSessionStats(idleTime=99)
+        assert theProject.session.appendSession(idleTime=99)
 
     assert statsFile.read_text(encoding="utf-8") == (
         "# Offset 100\n"

tests/tools.py

@@ -19,7 +19,6 @@ You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
 
-import time
 import shutil
 
 from pathlib import Path
@@ -201,7 +200,7 @@ def buildTestProject(theObject, projPath):
     aDoc.writeDocument("### %s\n\n" % theProject.tr("New Scene"))
     theProject.index.reIndexHandle(xHandle[8])
 
-    theProject._projOpened = time.time()
+    theProject.session.startSession()
     theProject.setProjectChanged(True)
     theProject.saveProject(autoSave=True)