diff --git a/services/notification/notifiers/tests/conftest.py b/services/notification/notifiers/tests/conftest.py index 6577919bb..73d2a4544 100644 --- a/services/notification/notifiers/tests/conftest.py +++ b/services/notification/notifiers/tests/conftest.py @@ -7,11 +7,7 @@ CommitFactory, OwnerFactory, PullFactory, - ReportDetailsFactory, - ReportFactory, RepositoryFactory, - RepositoryFlagFactory, - UploadFactory, ) from services.archive import ArchiveService from services.comparison import ComparisonProxy @@ -97,144 +93,164 @@ def sample_report(): @pytest.fixture def sample_commit_with_report_already_carriedforward(dbsession, mock_storage): + sessions_dict = { + "0": { + "N": None, + "a": None, + "c": None, + "d": None, + "e": None, + "f": [], + "j": None, + "n": None, + "p": None, + "st": "uploaded", + "t": None, + "u": None, + }, + "1": { + "N": None, + "a": None, + "c": None, + "d": None, + "e": None, + "f": ["unit"], + "j": None, + "n": None, + "p": None, + "st": "uploaded", + "t": None, + "u": None, + }, + "2": { + "N": None, + "a": None, + "c": None, + "d": None, + "e": None, + "f": ["enterprise"], + "j": None, + "n": None, + "p": None, + "st": "carriedforward", + "t": None, + "u": None, + "se": {"carriedforward_from": "123456789sha"}, + }, + "3": { + "N": None, + "a": None, + "c": None, + "d": None, + "e": None, + "f": ["integration"], + "j": None, + "n": None, + "p": None, + "st": "carriedforward", + "t": None, + "u": None, + }, + } + file_headers = { + "file_00.py": [ + 0, + [0, 14, 12, 0, 2, "85.71429", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 14, 12, 0, 2, "85.71429", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_01.py": [ + 1, + [0, 11, 8, 0, 3, "72.72727", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 11, 8, 0, 3, "72.72727", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_10.py": [ + 10, + [0, 10, 6, 1, 3, "60.00000", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 10, 6, 1, 3, "60.00000", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_11.py": [ + 11, + [0, 23, 15, 1, 7, "65.21739", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 23, 15, 1, 7, "65.21739", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_12.py": [ + 12, + [0, 14, 8, 0, 6, "57.14286", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 14, 8, 0, 6, "57.14286", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_13.py": [ + 13, + [0, 15, 9, 0, 6, "60.00000", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 15, 9, 0, 6, "60.00000", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_14.py": [ + 14, + [0, 23, 13, 0, 10, "56.52174", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 23, 13, 0, 10, "56.52174", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_02.py": [ + 2, + [0, 13, 9, 0, 4, "69.23077", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 13, 9, 0, 4, "69.23077", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_03.py": [ + 3, + [0, 16, 8, 0, 8, "50.00000", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 16, 8, 0, 8, "50.00000", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_04.py": [ + 4, + [0, 10, 6, 0, 4, "60.00000", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 10, 6, 0, 4, "60.00000", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_05.py": [ + 5, + [0, 14, 10, 0, 4, "71.42857", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 14, 10, 0, 4, "71.42857", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_06.py": [ + 6, + [0, 9, 7, 1, 1, "77.77778", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 9, 7, 1, 1, "77.77778", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_07.py": [ + 7, + [0, 11, 9, 0, 2, "81.81818", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 11, 9, 0, 2, "81.81818", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_08.py": [ + 8, + [0, 11, 6, 0, 5, "54.54545", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 11, 6, 0, 5, "54.54545", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + "file_09.py": [ + 9, + [0, 14, 10, 1, 3, "71.42857", 0, 0, 0, 0, 0, 0, 0], + [None, None, None, [0, 14, 10, 1, 3, "71.42857", 0, 0, 0, 0, 0, 0, 0]], + None, + ], + } commit = CommitFactory.create( + _report_json={"sessions": sessions_dict, "files": file_headers}, repository__owner__service="github", repository__owner__integration_id="10000", repository__using_integration=True, ) dbsession.add(commit) - - unit_flag = RepositoryFlagFactory(repository=commit.repository, flag_name="unit") - dbsession.add(unit_flag) - enterprise_flag = RepositoryFlagFactory( - repository=commit.repository, flag_name="enterprise" - ) - dbsession.add(enterprise_flag) - integration_flag = RepositoryFlagFactory( - repository=commit.repository, flag_name="integration" - ) - dbsession.add(integration_flag) - - report = ReportFactory(commit=commit) - dbsession.add(report) - - upload0 = UploadFactory(report=report, order_number=0, upload_type="uploaded") - dbsession.add(upload0) - upload1 = UploadFactory( - report=report, order_number=1, upload_type="uploaded", flags=[unit_flag] - ) - dbsession.add(upload1) - upload2 = UploadFactory( - report=report, - order_number=2, - upload_type="carriedforward", - upload_extras={"carriedforward_from": "123456789sha"}, - flags=[enterprise_flag], - ) - dbsession.add(upload2) - upload3 = UploadFactory( - report=report, - order_number=3, - upload_type="carriedforward", - flags=[integration_flag], - ) - dbsession.add(upload3) - - files_array = [ - { - "filename": "file_00.py", - "file_index": 0, - "file_totals": [0, 14, 12, 0, 2, "85.71429", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_01.py", - "file_index": 1, - "file_totals": [0, 11, 8, 0, 3, "72.72727", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_10.py", - "file_index": 10, - "file_totals": [0, 10, 6, 1, 3, "60.00000", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_11.py", - "file_index": 11, - "file_totals": [0, 23, 15, 1, 7, "65.21739", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_12.py", - "file_index": 12, - "file_totals": [0, 14, 8, 0, 6, "57.14286", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_13.py", - "file_index": 13, - "file_totals": [0, 15, 9, 0, 6, "60.00000", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_14.py", - "file_index": 14, - "file_totals": [0, 23, 13, 0, 10, "56.52174", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_02.py", - "file_index": 2, - "file_totals": [0, 13, 9, 0, 4, "69.23077", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_03.py", - "file_index": 3, - "file_totals": [0, 16, 8, 0, 8, "50.00000", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_04.py", - "file_index": 4, - "file_totals": [0, 10, 6, 0, 4, "60.00000", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_05.py", - "file_index": 5, - "file_totals": [0, 14, 10, 0, 4, "71.42857", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_06.py", - "file_index": 6, - "file_totals": [0, 9, 7, 1, 1, "77.77778", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_07.py", - "file_index": 7, - "file_totals": [0, 11, 9, 0, 2, "81.81818", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_08.py", - "file_index": 8, - "file_totals": [0, 11, 6, 0, 5, "54.54545", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "file_09.py", - "file_index": 9, - "file_totals": [0, 14, 10, 1, 3, "71.42857", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - ] - details = ReportDetailsFactory(report=report, _files_array=files_array) - dbsession.add(details) - dbsession.flush() with open("tasks/tests/samples/sample_chunks_4_sessions.txt") as f: @@ -368,7 +384,7 @@ def sample_comparison_coverage_carriedforward( dbsession.flush() yaml_dict = {"flags": {"enterprise": {"carryforward": True}}} - report = ReportService(yaml_dict).build_report_from_commit(head_commit) + report = ReportService(yaml_dict).get_existing_report_for_commit(head_commit) report._totals = ( None # need to reset the report to get it to recalculate totals correctly ) diff --git a/services/report/__init__.py b/services/report/__init__.py index 5f732ca44..1cc6a4629 100644 --- a/services/report/__init__.py +++ b/services/report/__init__.py @@ -6,19 +6,17 @@ from dataclasses import dataclass from json import loads from time import time -from typing import Any, Mapping, Optional, Sequence +from typing import Any, Mapping, Sequence import sentry_sdk from asgiref.sync import async_to_sync from celery.exceptions import SoftTimeLimitExceeded -from shared.config import get_config from shared.django_apps.reports.models import ReportType from shared.metrics import metrics from shared.reports.carryforward import generate_carryforward_report from shared.reports.editable import EditableReport from shared.reports.enums import UploadState, UploadType from shared.reports.resources import Report -from shared.reports.types import ReportFileSummary, ReportTotals from shared.storage.exceptions import FileNotInStorageError from shared.torngit.base import TorngitBaseAdapter from shared.torngit.exceptions import TorngitError @@ -29,21 +27,18 @@ from database.models import Commit, Repository, Upload, UploadError from database.models.reports import ( - AbstractTotals, CommitReport, ReportDetails, ReportLevelTotals, RepositoryFlag, UploadLevelTotals, ) -from helpers.environment import Environment, get_current_env from helpers.exceptions import ( OwnerWithoutValidBotError, ReportEmptyError, ReportExpiredException, RepositoryWithoutValidBotError, ) -from helpers.labels import get_labels_per_session from helpers.telemetry import MetricContext from rollouts import ( CARRYFORWARD_BASE_SEARCH_RANGE_BY_OWNER, @@ -103,7 +98,7 @@ class BaseReportService: It's always the user yaml, but might have different uses on different places """ - def __init__(self, current_yaml: UserYaml): + def __init__(self, current_yaml: UserYaml | dict): if isinstance(current_yaml, dict): current_yaml = UserYaml(current_yaml) self.current_yaml = current_yaml @@ -177,20 +172,15 @@ class ReportService(BaseReportService): metrics_prefix = "services.report" def __init__( - self, current_yaml: UserYaml, gh_app_installation_name: str | None = None + self, current_yaml: UserYaml | dict, gh_app_installation_name: str | None = None ): super().__init__(current_yaml) - self.flag_dict = None + self.flag_dict: dict[str, RepositoryFlag] | None = None self.gh_app_installation_name = gh_app_installation_name def has_initialized_report(self, commit: Commit) -> bool: - """Says whether a commit has already initialized its report or not - - Args: - commit (Commit): The commit we want to know about - - Returns: - bool: Whether the commit already has initialized a report + """ + Says whether a commit has already initialized its report or not """ return ( commit._report_json is not None @@ -254,7 +244,7 @@ def initialize_and_save_report( db_session.add(report_details) db_session.flush() - actual_report = self.get_existing_report_for_commit_from_legacy_data( + actual_report = self.get_existing_report_for_commit( commit, report_code=report_code ) if actual_report is not None: @@ -294,7 +284,8 @@ def create_report_upload( self, normalized_arguments: Mapping[str, str], commit_report: CommitReport ) -> Upload: upload = super().create_report_upload(normalized_arguments, commit_report) - flags = normalized_arguments.get("flags", "").split(",") + flags_str = normalized_arguments.get("flags") + flags = flags_str.split(",") if flags_str else [] self._attach_flags_to_upload(upload, flags) # Insert entry in user measurements table only @@ -329,128 +320,30 @@ def _attach_flags_to_upload(self, upload: Upload, flag_names: Sequence[str]): all_flags = [] db_session = upload.get_db_session() repoid = upload.report.commit.repoid - - if self.flag_dict is None: - self.fetch_repo_flags(db_session, repoid) + flag_dict = self.fetch_repo_flags(db_session, repoid) for individual_flag in flag_names: - flag_obj = self.flag_dict.get(individual_flag, None) + flag_obj = flag_dict.get(individual_flag, None) if flag_obj is None: flag_obj = RepositoryFlag( repository_id=repoid, flag_name=individual_flag ) db_session.add(flag_obj) db_session.flush() - self.flag_dict[individual_flag] = flag_obj + flag_dict[individual_flag] = flag_obj all_flags.append(flag_obj) + upload.flags = all_flags db_session.flush() return all_flags - def fetch_repo_flags(self, db_session, repoid): - existing_flags_on_repo = ( - db_session.query(RepositoryFlag).filter_by(repository_id=repoid).all() - ) - self.flag_dict = {flag.flag_name: flag for flag in existing_flags_on_repo} - - def build_files( - self, report_details: ReportDetails - ) -> dict[str, ReportFileSummary]: - return { - file["filename"]: ReportFileSummary( - file_index=file["file_index"], - file_totals=ReportTotals(*file["file_totals"]), - diff_totals=file["diff_totals"], + def fetch_repo_flags(self, db_session, repoid: int) -> dict[str, RepositoryFlag]: + if self.flag_dict is None: + existing_flags_on_repo = ( + db_session.query(RepositoryFlag).filter_by(repository_id=repoid).all() ) - for file in report_details.files_array - } - - def build_totals(self, totals: AbstractTotals) -> ReportTotals: - """ - Build a `shared.reports.types.ReportTotals` instance from one of the - various database totals records. - """ - return ReportTotals( - files=totals.files, - lines=totals.lines, - hits=totals.hits, - misses=totals.misses, - partials=totals.partials, - coverage=totals.coverage, - branches=totals.branches, - methods=totals.methods, - ) - - def build_session(self, upload: Upload): - """ - Build a `shared.utils.sessions.Session` from a database `reports_upload` record. - """ - totals = self.build_totals(upload.totals) if upload.totals is not None else None - - return Session( - id=upload.order_number, - totals=totals, - time=int(upload.created_at.timestamp()), - archive=upload.storage_path, - flags=upload.flag_names, - provider=upload.provider, - build=upload.build_code, - job=upload.job_code, - url=upload.build_url, - state=upload.state, - env=upload.env, - name=upload.name, - session_type=SessionType.get_from_string(upload.upload_type), - session_extras=upload.upload_extras, - ) - - def build_sessions(self, commit: Commit) -> dict[int, Session]: - """ - Build mapping of report number -> session that can be passed to the report class. - Does not include CF sessions if there is also an upload session with the same - flag name. - """ - sessions: dict[int, Session] = {} - - carryforward_sessions = {} - uploaded_flags = set() - - commit_report = commit.report - if not commit_report: - log.warning("Missing commit report", extra=dict(commit=commit.commitid)) - return sessions - - db_session = commit.get_db_session() - report_uploads = db_session.query(Upload).filter( - (Upload.report_id == commit_report.id_) - & ((Upload.state == "processed") | (Upload.state == "complete")) - ) - - for upload in report_uploads: - session = self.build_session(upload) - if session.session_type == SessionType.carriedforward: - carryforward_sessions[session.id] = session - else: - sessions[session.id] = session - uploaded_flags |= set(session.flags) - - for sid, session in carryforward_sessions.items(): - overlapping_flags = uploaded_flags & set(session.flags) - if len(overlapping_flags) == 0 or self._is_labels_flags(overlapping_flags): - # we can include this CF session since there are no direct uploads - # with the same flag name OR we're carrying forward labels - sessions[sid] = session - - log.info( - "Building report sessions from upload records", - extra=dict( - commit=commit.commitid, - upload_count=report_uploads.count(), - session_ids=list(sessions.keys()), - ), - ) - - return sessions + self.flag_dict = {flag.flag_name: flag for flag in existing_flags_on_repo} + return self.flag_dict @sentry_sdk.trace def build_report( @@ -466,6 +359,7 @@ def build_report( # sess is an encoded dict if sess.get("st") == "carriedforward": report_class = EditableReport + return report_class.from_chunks( chunks=chunks, files=files, sessions=sessions, totals=totals ) @@ -473,18 +367,14 @@ def build_report( def get_archive_service(self, repository: Repository) -> ArchiveService: return ArchiveService(repository) - def build_report_from_commit(self, commit) -> Report: - report = self.get_existing_report_for_commit(commit) - if report is not None: - return report - return self.create_new_report_for_commit(commit) - - def get_existing_report_for_commit_from_legacy_data( - self, commit: Commit, report_class=None, *, report_code=None - ) -> Optional[Report]: + @sentry_sdk.trace + def get_existing_report_for_commit( + self, commit: Commit, report_class=None, report_code=None + ) -> Report | None: commitid = commit.commitid - if commit._report_json is None and commit._report_json_storage_path is None: + if not self.has_initialized_report(commit): return None + try: archive_service = self.get_archive_service(commit.repository) chunks = archive_service.read_chunks(commitid, report_code) @@ -496,8 +386,10 @@ def get_existing_report_for_commit_from_legacy_data( ), ) return None + if chunks is None: return None + files = commit.report_json["files"] sessions = commit.report_json["sessions"] totals = commit.totals @@ -506,104 +398,9 @@ def get_existing_report_for_commit_from_legacy_data( ) return res - def _is_labels_flags(self, flags: Sequence[str]) -> bool: - return len(flags) > 0 and all( - [ - (self.current_yaml.get_flag_configuration(flag) or {}).get( - "carryforward_mode" - ) - == "labels" - for flag in flags - ] - ) - - @sentry_sdk.trace - def get_existing_report_for_commit( - self, commit: Commit, report_class=None, *, report_code=None - ) -> Optional[Report]: - commit_report = commit.report - if commit_report is None: - log.warning( - "Building report from legacy data", - extra=dict(commitid=commit.commitid), - ) - return self.get_existing_report_for_commit_from_legacy_data( - commit, report_class=report_class, report_code=report_code - ) - - # TODO: this can be removed once confirmed working well on prod - report_builder_repo_ids = get_config( - "setup", "report_builder", "repo_ids", default=[] - ) - new_report_builder_enabled = ( - get_current_env() == Environment.local - or commit.repoid in report_builder_repo_ids - ) - if not new_report_builder_enabled: - return self.get_existing_report_for_commit_from_legacy_data( - commit, report_class=report_class, report_code=report_code - ) - - commitid = commit.commitid - totals = None - files = {} - sessions = self.build_sessions(commit) - if commit_report.details: - files = self.build_files(commit_report.details) - if commit_report.totals: - totals = self.build_totals(commit_report.totals) - try: - archive_service = self.get_archive_service(commit.repository) - chunks = archive_service.read_chunks(commitid, report_code) - except FileNotInStorageError: - log.warning( - "File for chunks not found in storage", - extra=dict( - commit=commitid, repo=commit.repoid, report_code=report_code - ), - ) - return None - if chunks is None: - return None - - report = self.build_report( - chunks, files, sessions, totals, report_class=report_class - ) - - sessions_to_delete = [] - for sid, session in report.sessions.items(): - # this mimics behavior in the `adjust_sessions` function from - # `services/report/raw_upload_processor.py` - we need to delete - # label sessions for which there are no labels - # TODO: ultimately use `reports_upload.state` once the - # `PARTIALLY_OVERWRITTEN` and `FULLY_OVERWRITTEN` states are being saved - labels_session = self._is_labels_flags(session.flags) - if labels_session: - labels = get_labels_per_session(report, sid) - if not labels: - sessions_to_delete.append(sid) - - if len(sessions_to_delete) > 0: - log.info( - "Deleting empty labels sessions", - extra=dict(commit=commit.commitid, session_ids=sessions_to_delete), - ) - for sid in sessions_to_delete: - del sessions[sid] - - # rebuild the report since we deleted some sessions - report = self.build_report( - chunks, files, sessions, totals, report_class=report_class - ) - - return report - - @metrics.timer( - "services.report.ReportService.get_appropriate_commit_to_carryforward_from" - ) def get_appropriate_commit_to_carryforward_from( self, commit: Commit, max_parenthood_deepness: int = 10 - ) -> Optional[Commit]: + ) -> Commit | None: parent_commit = commit.get_parent_commit() parent_commit_tracking = [] count = 1 # `parent_commit` is already the first parent diff --git a/services/tests/test_report.py b/services/tests/test_report.py index 4349b57c1..4925a15da 100644 --- a/services/tests/test_report.py +++ b/services/tests/test_report.py @@ -1,4 +1,3 @@ -from asyncio import Future from decimal import Decimal import mock @@ -11,15 +10,7 @@ from shared.yaml import UserYaml from database.models import CommitReport, ReportDetails, RepositoryFlag, Upload -from database.tests.factories import ( - CommitFactory, - ReportDetailsFactory, - ReportFactory, - ReportLevelTotalsFactory, - RepositoryFlagFactory, - UploadFactory, - UploadLevelTotalsFactory, -) +from database.tests.factories import CommitFactory, UploadFactory from helpers.exceptions import RepositoryWithoutValidBotError from services.archive import ArchiveService from services.report import ( @@ -423,386 +414,6 @@ def sample_commit_with_report_big_already_carriedforward(dbsession, mock_storage class TestReportService(BaseTestCase): - def test_build_report_from_commit_no_report_saved(self, dbsession, mocker): - commit = CommitFactory(_report_json=None) - dbsession.add(commit) - dbsession.commit() - res = ReportService({}).build_report_from_commit(commit) - assert res is not None - assert res.files == [] - assert tuple(res.totals) == (0, 0, 0, 0, 0, None, 0, 0, 0, 0, 0, 0, 0) - - def test_build_report_from_commit(self, dbsession, mock_storage): - commit = CommitFactory(_report_json=None) - dbsession.add(commit) - report = ReportFactory(commit=commit) - dbsession.add(report) - - details = ReportDetailsFactory( - report=report, - _files_array=[ - { - "filename": "awesome/__init__.py", - "file_index": 2, - "file_totals": [0, 10, 8, 2, 0, "80.00000", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": [0, 2, 1, 1, 0, "50.00000", 0, 0, 0, 0, 0, 0, 0], - }, - { - "filename": "tests/__init__.py", - "file_index": 0, - "file_totals": [0, 3, 2, 1, 0, "66.66667", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "tests/test_sample.py", - "file_index": 1, - "file_totals": [0, 7, 7, 0, 0, "100", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - ], - ) - dbsession.add(details) - totals = ReportLevelTotalsFactory( - report=report, - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=85.0, - branches=0, - methods=0, - ) - dbsession.add(totals) - - upload = UploadFactory(report=report, order_number=0, upload_type="upload") - dbsession.add(upload) - upload_totals = UploadLevelTotalsFactory( - upload=upload, - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=85.0, - branches=0, - methods=0, - ) - dbsession.add(upload_totals) - dbsession.commit() - dbsession.flush() - - with open("tasks/tests/samples/sample_chunks_1.txt") as f: - content = f.read().encode() - archive_hash = ArchiveService.get_archive_hash(commit.repository) - chunks_url = f"v4/repos/{archive_hash}/commits/{commit.commitid}/chunks.txt" - mock_storage.write_file("archive", chunks_url, content) - - report = ReportService({}).build_report_from_commit(commit) - assert report is not None - assert report.files == [ - "awesome/__init__.py", - "tests/__init__.py", - "tests/test_sample.py", - ] - assert report.totals == ReportTotals( - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=Decimal("85.00"), - branches=0, - methods=0, - messages=0, - sessions=0, - complexity=0, - complexity_total=0, - diff=0, - ) - - assert len(report.sessions) == 1 - assert report.sessions[0].flags == [] - assert report.sessions[0].session_type == SessionType.uploaded - assert report.sessions[0].totals == ReportTotals( - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=Decimal("85.00"), - branches=0, - methods=0, - messages=0, - sessions=0, - complexity=0, - complexity_total=0, - diff=0, - ) - - # make sure report is still serializable - ReportService({}).save_report(commit, report) - - def test_build_report_from_commit_with_flags(self, dbsession, mock_storage): - commit = CommitFactory(_report_json=None) - dbsession.add(commit) - report = ReportFactory(commit=commit) - dbsession.add(report) - - details = ReportDetailsFactory( - report=report, - _files_array=[ - { - "filename": "awesome/__init__.py", - "file_index": 2, - "file_totals": [0, 10, 8, 2, 0, "80.00000", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": [0, 2, 1, 1, 0, "50.00000", 0, 0, 0, 0, 0, 0, 0], - }, - { - "filename": "tests/__init__.py", - "file_index": 0, - "file_totals": [0, 3, 2, 1, 0, "66.66667", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - { - "filename": "tests/test_sample.py", - "file_index": 1, - "file_totals": [0, 7, 7, 0, 0, "100", 0, 0, 0, 0, 0, 0, 0], - "diff_totals": None, - }, - ], - ) - dbsession.add(details) - totals = ReportLevelTotalsFactory( - report=report, - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=85.0, - branches=0, - methods=0, - ) - dbsession.add(totals) - - flag1 = RepositoryFlagFactory( - repository=commit.repository, - flag_name="unit", - ) - dbsession.add(flag1) - flag2 = RepositoryFlagFactory( - repository=commit.repository, - flag_name="integration", - ) - dbsession.add(flag2) - flag3 = RepositoryFlagFactory( - repository=commit.repository, - flag_name="labels-flag", - ) - dbsession.add(flag3) - - upload1 = UploadFactory( - report=report, flags=[flag1], order_number=0, upload_type="upload" - ) - dbsession.add(upload1) - upload_totals1 = UploadLevelTotalsFactory( - upload=upload1, - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=85.0, - branches=0, - methods=0, - ) - dbsession.add(upload_totals1) - dbsession.commit() - - upload2 = UploadFactory( - report=report, flags=[flag1], order_number=1, upload_type="carriedforward" - ) - dbsession.add(upload2) - upload_totals2 = UploadLevelTotalsFactory( - upload=upload2, - files=3, - lines=20, - hits=20, - misses=0, - partials=0, - coverage=100.0, - branches=0, - methods=0, - ) - dbsession.add(upload_totals2) - dbsession.commit() - - upload3 = UploadFactory( - report=report, flags=[flag2], order_number=2, upload_type="carriedforward" - ) - dbsession.add(upload3) - upload_totals3 = UploadLevelTotalsFactory( - upload=upload3, - files=3, - lines=20, - hits=20, - misses=0, - partials=0, - coverage=100.0, - branches=0, - methods=0, - ) - dbsession.add(upload_totals3) - dbsession.commit() - dbsession.flush() - - upload4 = UploadFactory( - report=report, flags=[flag3], order_number=3, upload_type="upload" - ) - dbsession.add(upload4) - upload_totals4 = UploadLevelTotalsFactory( - upload=upload4, - files=3, - lines=20, - hits=20, - misses=0, - partials=0, - coverage=100.0, - branches=0, - methods=0, - ) - dbsession.add(upload_totals4) - dbsession.commit() - dbsession.flush() - - with open("tasks/tests/samples/sample_chunks_1.txt") as f: - content = f.read().encode() - archive_hash = ArchiveService.get_archive_hash(commit.repository) - chunks_url = f"v4/repos/{archive_hash}/commits/{commit.commitid}/chunks.txt" - mock_storage.write_file("archive", chunks_url, content) - - yaml = { - "flag_management": { - "individual_flags": [ - { - "name": "labels-flag", - "carryforward": True, - "carryforward_mode": "labels", - } - ] - } - } - report = ReportService(yaml).build_report_from_commit(commit) - assert report is not None - assert report.files == [ - "awesome/__init__.py", - "tests/__init__.py", - "tests/test_sample.py", - ] - assert report.totals == ReportTotals( - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=Decimal("85.00"), - branches=0, - methods=0, - messages=0, - sessions=0, - complexity=0, - complexity_total=0, - diff=0, - ) - - assert len(report.sessions) == 2 - assert report.sessions[0].flags == ["unit"] - assert report.sessions[0].session_type == SessionType.uploaded - assert report.sessions[0].totals == ReportTotals( - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage=Decimal("85.00"), - branches=0, - methods=0, - messages=0, - sessions=0, - complexity=0, - complexity_total=0, - diff=0, - ) - assert 1 not in report.sessions # CF w/ equivalent direct upload - assert report.sessions[2].flags == ["integration"] - assert report.sessions[2].session_type == SessionType.carriedforward - assert report.sessions[2].totals == ReportTotals( - files=3, - lines=20, - hits=20, - misses=0, - partials=0, - coverage=Decimal("100.00"), - branches=0, - methods=0, - messages=0, - sessions=0, - complexity=0, - complexity_total=0, - diff=0, - ) - assert 3 not in report.sessions # labels flag w/ empty label set - - # make sure report is still serializable - ReportService({}).save_report(commit, report) - - def test_build_report_from_commit_fallback(self, dbsession, mocker, mock_storage): - commit = CommitFactory() - dbsession.add(commit) - dbsession.commit() - with open("tasks/tests/samples/sample_chunks_1.txt") as f: - content = f.read().encode() - archive_hash = ArchiveService.get_archive_hash(commit.repository) - chunks_url = f"v4/repos/{archive_hash}/commits/{commit.commitid}/chunks.txt" - mock_storage.write_file("archive", chunks_url, content) - res = ReportService({}).build_report_from_commit(commit) - assert res is not None - assert res.files == [ - "awesome/__init__.py", - "tests/__init__.py", - "tests/test_sample.py", - ] - assert res.totals == ReportTotals( - files=3, - lines=20, - hits=17, - misses=3, - partials=0, - coverage="85.00000", - branches=0, - methods=0, - messages=0, - sessions=1, - complexity=0, - complexity_total=0, - diff=[1, 2, 1, 1, 0, "50.00000", 0, 0, 0, 0, 0, 0, 0], - ) - res._totals = None - assert res.totals.files == 3 - assert res.totals.lines == 20 - assert res.totals.hits == 17 - assert res.totals.misses == 3 - assert res.totals.partials == 0 - assert res.totals.coverage == "85.00000" - assert res.totals.branches == 0 - assert res.totals.methods == 0 - assert res.totals.messages == 0 - assert res.totals.sessions == 1 - assert res.totals.complexity == 0 - assert res.totals.complexity_total == 0 - # notice we dont compare the diff since that one comes from git information we lost on the reset - @pytest.mark.django_db(databases={"default", "timeseries"}) def test_create_new_report_for_commit( self, @@ -2261,21 +1872,6 @@ def test_create_new_report_for_commit_with_labels( assert expected_results["report"] == readable_report["report"] assert expected_results == readable_report - def test_create_new_report_for_commit_is_called_as_generate( - self, dbsession, mocker - ): - commit = CommitFactory.create(_report_json=None) - dbsession.add(commit) - dbsession.flush() - mocked_create_new_report_for_commit = mocker.patch.object( - ReportService, "create_new_report_for_commit", return_value=Future() - ) - mocked_create_new_report_for_commit.return_value.set_result("report") - yaml_dict = {"flags": {"enterprise": {"carryforward": True}}} - report_service = ReportService(UserYaml(yaml_dict)) - report = report_service.build_report_from_commit(commit) - assert report == mocked_create_new_report_for_commit.return_value - @pytest.mark.django_db(databases={"default", "timeseries"}) def test_build_report_from_commit_carriedforward_add_sessions( self, dbsession, sample_commit_with_report_big, mocker @@ -2363,14 +1959,16 @@ def fake_possibly_shift(report, base, head): assert readable_report["report"] == expected_results["report"] assert readable_report == expected_results - def test_build_report_from_commit_already_carriedforward_add_sessions( + def test_get_existing_report_for_commit_already_carriedforward_add_sessions( self, dbsession, sample_commit_with_report_big_already_carriedforward ): commit = sample_commit_with_report_big_already_carriedforward dbsession.add(commit) dbsession.flush() yaml_dict = {"flags": {"enterprise": {"carryforward": True}}} - report = ReportService(UserYaml(yaml_dict)).build_report_from_commit(commit) + report = ReportService(UserYaml(yaml_dict)).get_existing_report_for_commit( + commit + ) assert report is not None assert len(report.files) == 15 assert sorted(report.sessions.keys()) == [0, 1, 2, 3] diff --git a/tasks/backfill_commit_data_to_storage.py b/tasks/backfill_commit_data_to_storage.py index ae9b770dd..7d439dbde 100644 --- a/tasks/backfill_commit_data_to_storage.py +++ b/tasks/backfill_commit_data_to_storage.py @@ -122,10 +122,8 @@ def handle_single_report_row( repo_yaml = get_repo_yaml(commit.repository) report_service = ReportService(current_yaml=repo_yaml) - actual_report = ( - report_service.get_existing_report_for_commit_from_legacy_data( - commit, report_code=None - ) + actual_report = report_service.get_existing_report_for_commit( + commit, report_code=None ) if actual_report is not None: report_service.save_report(commit, actual_report) diff --git a/tasks/tests/unit/test_backfill_commit_data_to_storage_task.py b/tasks/tests/unit/test_backfill_commit_data_to_storage_task.py index 85f4ce9c9..aadb2b852 100644 --- a/tasks/tests/unit/test_backfill_commit_data_to_storage_task.py +++ b/tasks/tests/unit/test_backfill_commit_data_to_storage_task.py @@ -192,7 +192,7 @@ def test_handle_single_report_row_ReportDetails_missing_data( @patch("tasks.backfill_commit_data_to_storage.ReportService.save_report") @patch( - "tasks.backfill_commit_data_to_storage.ReportService.get_existing_report_for_commit_from_legacy_data", + "tasks.backfill_commit_data_to_storage.ReportService.get_existing_report_for_commit", return_value="the existing report", ) def test_handle_single_report_row_create_ReportDetails(