Remove more unused Report code, change serde code to use str (#575)

Author: Swatinem

shared/reports/readonly.py

@@ -12,7 +12,6 @@
     END_OF_HEADER,
     Report,
     ReportTotals,
-    chunks_from_storage_contains_header,
 )
 from shared.utils.match import Matcher
 
@@ -23,8 +22,9 @@ class LazyRustReport(object):
     def __init__(self, filename_mapping, chunks, session_mapping):
         # Because Rust can't parse the header. It doesn't need it either,
         # So it's simpler to just never sent it.
-        if chunks_from_storage_contains_header(chunks):
-            _, chunks = chunks.split(END_OF_HEADER, 1)
+        splits = chunks.split(END_OF_HEADER, maxsplit=1)
+        if len(splits) > 1:
+            chunks = splits[1]
         self._chunks = chunks
         self._filename_mapping = filename_mapping
         self._session_mapping = session_mapping

shared/reports/resources.py

@@ -13,15 +13,14 @@
 from shared.reports.exceptions import LabelIndexNotFoundError, LabelNotFoundError
 from shared.reports.filtered import FilteredReport
 from shared.reports.reportfile import ReportFile
-from shared.reports.serde import orjson_option, report_default
-from shared.reports.types import TOTALS_MAP, ReportHeader, ReportTotals
+from shared.reports.types import ReportHeader, ReportTotals
 from shared.utils.flare import report_to_flare
 from shared.utils.make_network_file import make_network_file
 from shared.utils.migrate import migrate_totals
 from shared.utils.sessions import Session, SessionType
 from shared.utils.totals import agg_totals
 
-from .serde import _encode_chunk, serialize_report
+from .serde import END_OF_CHUNK, END_OF_HEADER, serialize_report
 
 log = logging.getLogger(__name__)
 
@@ -37,21 +36,6 @@ def unique_everseen(iterable):
         yield element
 
 
-END_OF_CHUNK = "\n<<<<< end_of_chunk >>>>>\n"
-END_OF_HEADER = "\n<<<<< end_of_header >>>>>\n"
-
-
-def chunks_from_storage_contains_header(chunks: str) -> bool:
-    try:
-        first_line_end = chunks.index("\n")
-        second_line_end = chunks.index("\n", first_line_end + 1)
-    except ValueError:
-        return False
-    # If the header is present then the END_OF_HEADER marker
-    # is in the 2nd line of the report
-    return chunks[first_line_end : second_line_end + 1] == END_OF_HEADER
-
-
 class Report:
     sessions: dict[int, Session]
     _header: ReportHeader
@@ -80,14 +64,14 @@ def __init__(
                 for sid, session in sessions.items()
             }
 
-        _chunks: list[bytes] = []
+        _chunks: list[str] = []
         if chunks:
-            if isinstance(chunks, str):
-                chunks = chunks.encode()
             if isinstance(chunks, bytes):
-                splits = chunks.split(b"\n<<<<< end_of_header >>>>>\n", maxsplit=1)
+                chunks = chunks.decode()
+            if isinstance(chunks, str):
+                splits = chunks.split(END_OF_HEADER, maxsplit=1)
                 if len(splits) > 1:
-                    _header = orjson.loads(splits[0] or b"{}")
+                    _header = orjson.loads(splits[0] or "{}")
                     self._header = ReportHeader(
                         labels_index={
                             int(k): v
@@ -96,7 +80,7 @@ def __init__(
                     )
                     chunks = splits[1]
 
-                _chunks = chunks.split(b"\n<<<<< end_of_chunk >>>>>\n")
+                _chunks = chunks.split(END_OF_CHUNK)
             else:
                 _chunks = chunks
 
@@ -112,8 +96,7 @@ def __init__(
                     file_diff_totals = None
 
                 try:
-                    _lines = _chunks[chunks_index]
-                    lines = _lines.decode() if isinstance(_lines, bytes) else _lines
+                    lines = _chunks[chunks_index]
                 except IndexError:
                     lines = ""
 
@@ -345,42 +328,12 @@ def is_empty(self):
     def __bool__(self):
         return self.is_empty() is False
 
-    @sentry_sdk.trace
-    def to_archive(self, with_header=True):
-        # TODO: confirm removing encoding here is fine
-        chunks = END_OF_CHUNK.join(
-            _encode_chunk(chunk).decode() for chunk in self._files.values()
-        )
-        if with_header:
-            # When saving to database we want this
-            return END_OF_HEADER.join(
-                [orjson.dumps(self._header, option=orjson_option).decode(), chunks]
-            )
-        # This is helpful to build ReadOnlyReport
-        # Because Rust can't parse the header. It doesn't need it either,
-        # So it's simpler to just never sent it.
-        return chunks
-
-    @sentry_sdk.trace
-    def to_database(self):
-        """returns (totals, report) to be stored in database"""
-        totals = dict(zip(TOTALS_MAP, self.totals))
-        totals["diff"] = self.diff_totals
-
-        files = {
-            file.name: [i, file.totals, None, file.diff_totals]
-            for i, file in enumerate(self._files.values())
-        }
-        return (
-            totals,
-            orjson.dumps(
-                {"files": files, "sessions": self.sessions},
-                default=report_default,
-                option=orjson_option,
-            ).decode(),
-        )
-
     def serialize(self, with_totals=True) -> tuple[bytes, bytes, ReportTotals | None]:
+        """
+        Serializes a report as `(report_json, chunks, totals)`.
+
+        The `totals` is either a `ReportTotals`, or `None`, depending on the `with_totals` flag.
+        """
         return serialize_report(self, with_totals)
 
     @sentry_sdk.trace
@@ -584,10 +537,6 @@ def get_uploaded_flags(self):
                 flags.update(sess.flags)
         return flags
 
-    @sentry_sdk.trace
-    def repack(self):
-        pass
-
     def delete_labels(
         self, sessionids: list[int] | set[int], labels_to_delete: list[int] | set[int]
     ):

shared/reports/serde.py

@@ -16,8 +16,8 @@
     from .resources import Report
 
 
-END_OF_CHUNK = b"\n<<<<< end_of_chunk >>>>>\n"
-END_OF_HEADER = b"\n<<<<< end_of_header >>>>>\n"
+END_OF_CHUNK = "\n<<<<< end_of_chunk >>>>>\n"
+END_OF_HEADER = "\n<<<<< end_of_header >>>>>\n"
 
 
 @sentry_sdk.trace
@@ -33,7 +33,7 @@ def serialize_report(
     indexed_files = list(enumerate(report._files.values()))
 
     chunks = (
-        orjson.dumps(report._header, option=orjson_option)
+        orjson.dumps(report._header, option=orjson_option).decode()
         + END_OF_HEADER
         + END_OF_CHUNK.join(_encode_chunk(file) for i, file in indexed_files)
     )
@@ -57,7 +57,7 @@ def serialize_report(
         option=orjson_option,
     )
 
-    return (report_json, chunks, totals)
+    return (report_json, chunks.encode(), totals)
 
 
 def report_default(obj):
@@ -81,18 +81,18 @@ def report_default(obj):
 orjson_option = orjson.OPT_PASSTHROUGH_DATACLASS | orjson.OPT_NON_STR_KEYS
 
 
-def _dumps_not_none(value) -> bytes:
+def _dumps_not_none(value) -> str:
     if isinstance(value, list):
         return orjson.dumps(
             _rstrip_none(list(value)), default=report_default, option=orjson_option
-        )
+        ).decode()
     if isinstance(value, ReportLine):
         return orjson.dumps(
             _rstrip_none(list(value.astuple())),
             default=report_default,
             option=orjson_option,
-        )
-    return value.encode() if value and str(value) != "null" else b""
+        ).decode()
+    return value if value and value != "null" else ""
 
 
 def _rstrip_none(lst):
@@ -107,21 +107,19 @@ def chunk_default(obj):
     return obj
 
 
-def _encode_chunk(chunk) -> bytes:
+def _encode_chunk(chunk) -> str:
     if chunk is None:
-        return b"null"
+        return "null"
     elif isinstance(chunk, ReportFile):
         if isinstance(chunk._raw_lines, str):
-            return chunk._raw_lines.encode()
+            return chunk._raw_lines
         else:
             return (
-                orjson.dumps(chunk.details, option=orjson_option)
-                + b"\n"
-                + b"\n".join(_dumps_not_none(line) for line in chunk._lines)
+                orjson.dumps(chunk.details, option=orjson_option).decode()
+                + "\n"
+                + "\n".join(_dumps_not_none(line) for line in chunk._lines)
             )
     elif isinstance(chunk, (list, dict)):
-        return orjson.dumps(chunk, default=chunk_default, option=orjson_option)
-    elif isinstance(chunk, str):
-        return chunk.encode()
+        return orjson.dumps(chunk, default=chunk_default, option=orjson_option).decode()
     else:
         return chunk

tests/integration/test_report.py

@@ -402,42 +402,6 @@ def test_non_zero(r, boolean):
     assert bool(r) is boolean
 
 
-@pytest.mark.integration
-def test_to_archive():
-    chunks = Report(
-        files={"file.py": [0, ReportTotals()]},
-        chunks="null\n[1]\n[1]\n[1]\n<<<<< end_of_chunk >>>>>\nnull\n[1]\n[1]\n[1]",
-    ).to_archive()
-    assert chunks == """{}\n<<<<< end_of_header >>>>>\nnull\n[1]\n[1]\n[1]"""
-
-
-@pytest.mark.integration
-def test_to_database():
-    totals, report_json = Report(
-        files={"file.py": [0, ReportTotals()]},
-        chunks="null\n[1]\n[1]\n[1]\n<<<<< end_of_chunk >>>>>\nnull\n[1]\n[1]\n[1]",
-    ).to_database()
-    assert totals == {
-        "M": 0,
-        "c": None,
-        "b": 0,
-        "d": 0,
-        "f": 1,
-        "h": 0,
-        "m": 0,
-        "C": 0,
-        "n": 0,
-        "p": 0,
-        "s": 0,
-        "diff": None,
-        "N": 0,
-    }
-    assert (
-        report_json
-        == '{"files":{"file.py":[0,[0,0,0,0,0,0,0,0,0,0,0,0,0],null,null]},"sessions":{}}'
-    )
-
-
 def test_serialize(mocker):
     report = Report(
         files={"file.py": [0, ReportTotals()]},

tests/unit/reports/test_carryforward.py

@@ -1,6 +1,3 @@
-import dataclasses
-from json import loads
-
 import pytest
 
 from shared.reports.carryforward import (
@@ -9,6 +6,7 @@
 )
 from shared.reports.resources import Report, ReportFile, Session
 from shared.reports.types import LineSession, ReportLine
+from tests.unit.reports.utils import convert_report_to_better_readable
 
 
 @pytest.fixture
@@ -65,50 +63,10 @@ def test_carriedforward_session_name(self):
             carriedforward_session_name("CF CF CF CF CF CF CF Dude") == "CF[8] - Dude"
         )
 
-    def convert_report_to_better_readable(self, report):
-        totals_dict, report_dict = report.to_database()
-        report_dict = loads(report_dict)
-        archive_dict = {}
-        for filename in report.files:
-            file_report = report.get(filename)
-            lines = []
-            for line_number, line in file_report.lines:
-                (
-                    coverage,
-                    line_type,
-                    sessions,
-                    messages,
-                    complexity,
-                    datapoints,
-                ) = dataclasses.astuple(line)
-                sessions = [list(s) for s in sessions]
-                lines.append(
-                    (
-                        line_number,
-                        coverage,
-                        line_type,
-                        sessions,
-                        messages,
-                        complexity,
-                        datapoints,
-                    )
-                    if datapoints is not None
-                    else (
-                        line_number,
-                        coverage,
-                        line_type,
-                        sessions,
-                        messages,
-                        complexity,
-                    )
-                )
-            archive_dict[filename] = lines
-        return {"totals": totals_dict, "report": report_dict, "archive": archive_dict}
-
     def test_generate_carryforward_report(self, sample_report):
         res = generate_carryforward_report(sample_report, flags=["simple"], paths=None)
         assert res.files == ["file_1.go", "file_2.py"]
-        readable_report = self.convert_report_to_better_readable(res)
+        readable_report = convert_report_to_better_readable(res)
         expected_result = {
             "archive": {
                 "file_1.go": [
@@ -199,7 +157,7 @@ def test_generate_carryforward_report_with_paths(self, sample_report):
             sample_report, flags=["simple"], paths=["file_1.*"]
         )
         assert res.files == ["file_1.go"]
-        readable_report = self.convert_report_to_better_readable(res)
+        readable_report = convert_report_to_better_readable(res)
         expected_result = {
             "archive": {
                 "file_1.go": [
@@ -267,7 +225,7 @@ def test_generate_carryforward_report_with_path_none_matches(self, sample_report
             sample_report, flags=["simple"], paths=["file_\\W.*"]
         )
         assert res.files == []
-        readable_report = self.convert_report_to_better_readable(res)
+        readable_report = convert_report_to_better_readable(res)
         expected_result = {
             "archive": {},
             "report": {
@@ -320,7 +278,7 @@ def test_generate_carryforward_report_with_path_two_patterns(self, sample_report
             sample_report, flags=["simple"], paths=[".*\\.cpp", ".*_2\\..*"]
         )
         assert res.files == ["file_2.py"]
-        readable_report = self.convert_report_to_better_readable(res)
+        readable_report = convert_report_to_better_readable(res)
         assert readable_report == {
             "archive": {
                 "file_2.py": [
@@ -375,7 +333,7 @@ def test_generate_carryforward_report_with_path_two_patterns(self, sample_report
     def test_generate_carryforward_report_one_file_not_covered(self, sample_report):
         res = generate_carryforward_report(sample_report, flags=["complex"], paths=None)
         assert res.files == ["file_1.go"]
-        readable_report = self.convert_report_to_better_readable(res)
+        readable_report = convert_report_to_better_readable(res)
 
         expected_result = {
             "archive": {
@@ -447,7 +405,7 @@ def test_generate_carryforward_report_session_extras(self, sample_report):
             session_extras={"cfed_parent": "0f9ab1fe6c879bc49a9e559b23f49fd033daadb0"},
         )
         assert res.files == ["file_1.go"]
-        readable_report = self.convert_report_to_better_readable(res)
+        readable_report = convert_report_to_better_readable(res)
 
         expected_result = {
             "archive": {