Merge the Report._files/_chunks

Instead of maintaining two separate fields, this merges the previous `_files` which was a dict from filename to `totals`, and the `_chunks` which was either a fully parsed `ReportFile`, or just a shallow `list` of lines.

Maintaining only a single structure for the list of files makes a ton of internals of the `Report` simpler.

It also solves the problem of `file_totals` getting out of sync with the `ReportFile.totals`.

Author: Swatinem

shared/reports/carryforward.py

@@ -2,7 +2,6 @@
 import re
 from typing import Mapping, Sequence
 
-from shared.reports.editable import EditableReport
 from shared.reports.resources import Report
 from shared.utils.match import Matcher
 from shared.utils.sessions import SessionType
@@ -36,7 +35,7 @@ def generate_carryforward_report(
     flags: Sequence[str],
     paths: Sequence[str],
     session_extras: Mapping[str, str] | None = None,
-) -> EditableReport:
+) -> Report:
     """
         Generates a carriedforward report starting from report `report`, flags `flags`
             and paths `paths`
@@ -66,19 +65,16 @@ def generate_carryforward_report(
     Returns:
         EditableReport: A new report with only the info related to `flags` on it, as described above
     """
-    new_report = EditableReport(
-        chunks=report._chunks,
-        files=report._files,
-        sessions=report.sessions,
-        totals=None,
-    )
     if paths:
         matcher = Matcher(paths)
-        for filename in new_report.files:
-            if not matcher.match(filename):
-                del new_report[filename]
+        files_to_delete = {
+            filename for filename in report._files.keys() if not matcher.match(filename)
+        }
+        for filename in files_to_delete:
+            del report[filename]
+
     sessions_to_delete = []
-    for sid, session in new_report.sessions.items():
+    for sid, session in report.sessions.items():
         if not contain_any_of_the_flags(flags, session.flags):
             sessions_to_delete.append(int(sid))
         else:
@@ -89,8 +85,8 @@ def generate_carryforward_report(
         "Removing sessions that are not supposed to carryforward",
         extra=dict(deleted_sessions=sessions_to_delete),
     )
-    new_report.delete_multiple_sessions(sessions_to_delete)
-    return new_report
+    report.delete_multiple_sessions(sessions_to_delete)
+    return report
 
 
 def contain_any_of_the_flags(expected_flags, actual_flags):

shared/reports/readonly.py

@@ -65,8 +65,7 @@ def from_chunks(cls, files=None, sessions=None, totals=None, chunks=None):
         )
         totals = inner_report._totals
         filename_mapping = {
-            filename: file_summary.file_index
-            for (filename, file_summary) in inner_report._files.items()
+            filename: idx for idx, filename in enumerate(inner_report._files.keys())
         }
         session_mapping = {
             sid: (session.flags or []) for sid, session in inner_report.sessions.items()

shared/reports/reportfile.py

@@ -13,12 +13,20 @@
 log = logging.getLogger(__name__)
 
 
-class ReportFile(object):
+class ReportFile:
+    name: str
+    _totals: ReportTotals | None
+    diff_totals: ReportTotals | None
+    _lines: list[None | str | ReportLine]
+    _details: dict[str, Any]
+    __present_sessions: set[int] | None
+
     def __init__(
         self,
         name: str,
         totals: ReportTotals | list | None = None,
         lines: list[None | str | ReportLine] | str | None = None,
+        diff_totals: ReportTotals | list | None = None,
         ignore=None,
     ):
         """
@@ -32,18 +40,20 @@ def __init__(
             {eof:N, lines:[1,10]}
         """
         self.name = name
-        self._details: dict[str, Any] = {}
+        self._totals = None
+        self.diff_totals = None
+        self._lines = []
+        self._details = {}
+        self.__present_sessions = None
 
-        # lines = [<details dict()>, <Line #1>, ....]
-        self._lines: list[None | str | ReportLine] = []
         if lines:
             if isinstance(lines, list):
                 self._lines = lines
 
             else:
                 lines = lines.splitlines()
                 if detailsline := lines.pop(0):
-                    self._details = orjson.loads(detailsline) or {}
+                    self._details = orjson.loads(detailsline or b"null") or {}
                 self._lines = lines
 
         self._ignore = _ignore_to_func(ignore) if ignore else None
@@ -54,18 +64,22 @@ def __init__(
         # All mutating methods (like `append`, `merge`, etc) will either re-calculate these values
         # directly, or clear them so the `@property` accessors re-calculate them when needed.
 
-        self._totals: ReportTotals | None = None
         if isinstance(totals, ReportTotals):
             self._totals = totals
         elif totals:
             self._totals = ReportTotals(*totals)
 
-        self.__present_sessions: set[int] | None = None
+        if isinstance(diff_totals, ReportTotals):
+            self.diff_totals = diff_totals
+        elif diff_totals:
+            self.diff_totals = ReportTotals(*diff_totals)
+
         if present_sessions := self._details.get("present_sessions"):
             self.__present_sessions = set(present_sessions)
 
     def _invalidate_caches(self):
         self._totals = None
+        self.diff_totals = None
         self.__present_sessions = None
 
     @property
@@ -84,12 +98,9 @@ def details(self):
     @property
     def totals(self):
         if not self._totals:
-            self._totals = self._process_totals()
+            self._totals = get_line_totals(line for _ln, line in self.lines)
         return self._totals
 
-    def _process_totals(self) -> ReportTotals:
-        return get_line_totals(line for _ln, line in self.lines)
-
     def __repr__(self):
         try:
             return "<%s name=%s lines=%s>" % (