diff --git a/backend/analytics_server/mhq/api/incidents.py b/backend/analytics_server/mhq/api/incidents.py index 2821dc48..d25356f9 100644 --- a/backend/analytics_server/mhq/api/incidents.py +++ b/backend/analytics_server/mhq/api/incidents.py @@ -36,19 +36,26 @@ { Required("from_time"): All(str, Coerce(datetime.fromisoformat)), Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), } ), ) -def get_resolved_incidents(team_id: str, from_time: datetime, to_time: datetime): +def get_resolved_incidents( + team_id: str, from_time: datetime, to_time: datetime, pr_filter: dict = None +): query_validator = get_query_validator() interval = query_validator.interval_validator(from_time, to_time) query_validator.team_validator(team_id) + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + incident_service = get_incident_service() resolved_incidents: List[Incident] = incident_service.get_resolved_team_incidents( - team_id, interval + team_id, interval, pr_filter ) # ToDo: Generate a user map @@ -90,7 +97,9 @@ def get_deployments_with_related_incidents( incident_service = get_incident_service() - incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) + incidents: List[Incident] = incident_service.get_team_incidents( + team_id, interval, pr_filter + ) deployment_incidents_map: Dict[Deployment, List[Incident]] = ( incident_service.get_deployment_incidents_map(deployments, incidents) @@ -112,18 +121,25 @@ def get_deployments_with_related_incidents( { Required("from_time"): All(str, Coerce(datetime.fromisoformat)), Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), } ), ) -def get_team_mttr(team_id: str, from_time: datetime, to_time: datetime): +def get_team_mttr( + team_id: str, from_time: datetime, to_time: datetime, pr_filter: dict = None +): query_validator = get_query_validator() interval = query_validator.interval_validator(from_time, to_time) query_validator.team_validator(team_id) + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + incident_service = get_incident_service() team_mean_time_to_recovery_metrics = ( - incident_service.get_team_mean_time_to_recovery(team_id, interval) + incident_service.get_team_mean_time_to_recovery(team_id, interval, pr_filter) ) return adapt_mean_time_to_recovery_metrics(team_mean_time_to_recovery_metrics) @@ -135,18 +151,27 @@ def get_team_mttr(team_id: str, from_time: datetime, to_time: datetime): { Required("from_time"): All(str, Coerce(datetime.fromisoformat)), Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), } ), ) -def get_team_mttr_trends(team_id: str, from_time: datetime, to_time: datetime): +def get_team_mttr_trends( + team_id: str, from_time: datetime, to_time: datetime, pr_filter: dict = None +): query_validator = get_query_validator() interval = query_validator.interval_validator(from_time, to_time) query_validator.team_validator(team_id) + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + incident_service = get_incident_service() weekly_mean_time_to_recovery_metrics = ( - incident_service.get_team_mean_time_to_recovery_trends(team_id, interval) + incident_service.get_team_mean_time_to_recovery_trends( + team_id, interval, pr_filter + ) ) return { @@ -192,7 +217,9 @@ def get_team_cfr( incident_service = get_incident_service() - incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) + incidents: List[Incident] = incident_service.get_team_incidents( + team_id, interval, pr_filter + ) team_change_failure_rate: ChangeFailureRateMetrics = ( incident_service.get_change_failure_rate_metrics(deployments, incidents) @@ -236,7 +263,9 @@ def get_team_cfr_trends( incident_service = get_incident_service() - incidents: List[Incident] = incident_service.get_team_incidents(team_id, interval) + incidents: List[Incident] = incident_service.get_team_incidents( + team_id, interval, pr_filter + ) team_weekly_change_failure_rate: Dict[datetime, ChangeFailureRateMetrics] = ( incident_service.get_weekly_change_failure_rate( diff --git a/backend/analytics_server/mhq/api/resources/settings_resource.py b/backend/analytics_server/mhq/api/resources/settings_resource.py index 192ea2ed..f64b4ce2 100644 --- a/backend/analytics_server/mhq/api/resources/settings_resource.py +++ b/backend/analytics_server/mhq/api/resources/settings_resource.py @@ -1,10 +1,11 @@ from mhq.service.settings.models import ( ConfigurationSettings, - DefaultSyncDaysSetting, IncidentSettings, ExcludedPRsSetting, IncidentTypesSetting, IncidentSourcesSetting, + DefaultSyncDaysSetting, + IncidentPrsSetting, ) from mhq.store.models import EntityType @@ -55,6 +56,15 @@ def _add_setting_data(config_settings: ConfigurationSettings, response): "default_sync_days": config_settings.specific_settings.default_sync_days } + if isinstance(config_settings.specific_settings, IncidentPrsSetting): + response["setting"] = { + "include_revert_prs": config_settings.specific_settings.include_revert_prs, + "title_filters": config_settings.specific_settings.title_filters, + "head_branch_filters": config_settings.specific_settings.head_branch_filters, + "pr_mapping_field": config_settings.specific_settings.pr_mapping_field, + "pr_mapping_pattern": config_settings.specific_settings.pr_mapping_pattern, + } + # ADD NEW API ADAPTER HERE return response diff --git a/backend/analytics_server/mhq/service/incidents/incident_filter.py b/backend/analytics_server/mhq/service/incidents/incident_filter.py index f2bc8c7a..8ed91619 100644 --- a/backend/analytics_server/mhq/service/incidents/incident_filter.py +++ b/backend/analytics_server/mhq/service/incidents/incident_filter.py @@ -1,9 +1,11 @@ from typing import Dict, List, Any, Optional +from mhq.store.models.incidents.enums import IncidentType from mhq.store.models.settings.configuration_settings import SettingType from mhq.service.settings.configuration_settings import ( get_settings_service, IncidentSettings, IncidentTypesSetting, + IncidentPrsSetting, ) from mhq.store.models.incidents import IncidentFilter @@ -106,4 +108,19 @@ def __incident_type_setting(self) -> List[str]: incident_types = [] if setting and isinstance(setting, IncidentTypesSetting): incident_types = setting.incident_types + + if SettingType.INCIDENT_PRS_SETTING in self.setting_types: + incident_prs_setting: Optional[IncidentPrsSetting] = ( + self.setting_type_to_settings_map.get(SettingType.INCIDENT_PRS_SETTING) + ) + if ( + isinstance(incident_prs_setting, IncidentPrsSetting) + and not incident_prs_setting.include_revert_prs + ): + incident_types = [ + incident_type + for incident_type in incident_types + if incident_type != IncidentType.REVERT_PR + ] + return incident_types diff --git a/backend/analytics_server/mhq/service/incidents/incidents.py b/backend/analytics_server/mhq/service/incidents/incidents.py index 20b89e39..4c1a0914 100644 --- a/backend/analytics_server/mhq/service/incidents/incidents.py +++ b/backend/analytics_server/mhq/service/incidents/incidents.py @@ -1,6 +1,12 @@ from collections import defaultdict from datetime import datetime -from typing import List, Dict, Tuple +from typing import List, Dict, Tuple, Optional +import re +from mhq.service.settings.models import IncidentPrsSetting +from mhq.store.models.code.filter import PRFilter +from mhq.store.models.code.pull_requests import PullRequest +from mhq.store.models.incidents.enums import IncidentStatus, IncidentType +from mhq.store.repos.code import CodeRepoService from mhq.service.incidents.models.mean_time_to_recovery import ( ChangeFailureRateMetrics, MeanTimeToRecoveryMetrics, @@ -15,13 +21,15 @@ generate_expanded_buckets, get_given_weeks_monday, ) - +from mhq.utils.regex import check_regex from mhq.store.models.incidents import Incident from mhq.service.settings.configuration_settings import ( SettingsService, get_settings_service, ) from mhq.store.repos.incidents import IncidentsRepoService +from mhq.service.incidents.sync.etl_git_incidents_handler import GitIncidentsETLHandler +from mhq.store.models.incidents.enums import PR_FILTER_PATTERNS class IncidentService: @@ -29,12 +37,14 @@ def __init__( self, incidents_repo_service: IncidentsRepoService, settings_service: SettingsService, + code_repo_service: CodeRepoService, ): self._incidents_repo_service = incidents_repo_service self._settings_service = settings_service + self._code_repo_service = code_repo_service def get_resolved_team_incidents( - self, team_id: str, interval: Interval + self, team_id: str, interval: Interval, pr_filter: PRFilter ) -> List[Incident]: incident_filter: IncidentFilter = apply_incident_filter( entity_type=EntityType.TEAM, @@ -42,25 +52,212 @@ def get_resolved_team_incidents( setting_types=[ SettingType.INCIDENT_SETTING, SettingType.INCIDENT_TYPES_SETTING, + SettingType.INCIDENT_PRS_SETTING, ], ) - return self._incidents_repo_service.get_resolved_team_incidents( + resolved_incidents = self._incidents_repo_service.get_resolved_team_incidents( team_id, interval, incident_filter ) + resolved_pr_incidents = self.get_team_pr_incidents(team_id, interval, pr_filter) + + return resolved_incidents + resolved_pr_incidents + + def get_team_incidents( + self, team_id: str, interval: Interval, pr_filter: PRFilter + ) -> List[Incident]: - def get_team_incidents(self, team_id: str, interval: Interval) -> List[Incident]: incident_filter: IncidentFilter = apply_incident_filter( entity_type=EntityType.TEAM, entity_id=team_id, setting_types=[ SettingType.INCIDENT_SETTING, SettingType.INCIDENT_TYPES_SETTING, + SettingType.INCIDENT_PRS_SETTING, ], ) - return self._incidents_repo_service.get_team_incidents( + incidents = self._incidents_repo_service.get_team_incidents( team_id, interval, incident_filter ) + pr_incidents: List[Incident] = self.get_team_pr_incidents( + team_id, interval, pr_filter + ) + + return incidents + pr_incidents + + def get_team_pr_incidents( + self, team_id: str, interval: Interval, pr_filter: PRFilter + ) -> List[Incident]: + incident_prs_setting: IncidentPrsSetting = ( + self._settings_service.get_or_set_default_settings( + setting_type=SettingType.INCIDENT_PRS_SETTING, + entity_type=EntityType.TEAM, + entity_id=team_id, + ).specific_settings + ) + + if not ( + incident_prs_setting.head_branch_filters + or incident_prs_setting.title_filters + or incident_prs_setting.pr_mapping_field + or incident_prs_setting.pr_mapping_pattern + ): + return [] + + team_repo_ids = ( + tr.org_repo_id + for tr in self._code_repo_service.get_active_team_repos_by_team_id(team_id) + ) + prs = self._code_repo_service.get_prs_merged_in_interval( + team_repo_ids, interval, pr_filter + ) + repo_id_to_pr_number_to_pr_map: Dict[str, Dict[str, PullRequest]] = {} + + for pr in prs: + if str(pr.repo_id) not in repo_id_to_pr_number_to_pr_map: + repo_id_to_pr_number_to_pr_map[str(pr.repo_id)] = {} + repo_id_to_pr_number_to_pr_map[str(pr.repo_id)][pr.number] = pr + + mapped_incident_prs: List[Incident] = self._get_mapped_incident_prs( + prs, incident_prs_setting, team_repo_ids, repo_id_to_pr_number_to_pr_map + ) + + filtered_incident_prs: List[Incident] = self._filter_incident_prs( + incident_prs_setting, repo_id_to_pr_number_to_pr_map + ) + + return mapped_incident_prs + filtered_incident_prs + + def _get_mapped_incident_prs( + self, + prs: List[PullRequest], + incident_prs_setting: IncidentPrsSetting, + team_repo_ids: List[str], + repo_id_to_pr_number_to_pr_map: Dict[str, Dict[str, PullRequest]], + ) -> List[Incident]: + + if ( + not incident_prs_setting.pr_mapping_field + or not incident_prs_setting.pr_mapping_pattern + ): + return [] + + pr_incidents: List[Incident] = [] + + mapping_field = incident_prs_setting.pr_mapping_field + pattern = incident_prs_setting.pr_mapping_pattern + pr_numbers_match_strings: List[str] = [] + + for pr in prs: + if pr_number := self._extract_pr_number_from_pattern( + getattr(pr, mapping_field), pattern + ): + if pr_number in repo_id_to_pr_number_to_pr_map[str(pr.repo_id)].keys(): + incident_pr = repo_id_to_pr_number_to_pr_map[str(pr.repo_id)][ + pr_number + ] + adapted_incident_pr = self._adapt_incident_pr(incident_pr, pr) + pr_incidents.append(adapted_incident_pr) + repo_id_to_pr_number_to_pr_map[str(pr.repo_id)].pop(pr.number) + repo_id_to_pr_number_to_pr_map[str(pr.repo_id)].pop( + incident_pr.number + ) + else: + match_string = pattern.replace("1234", str(pr.number)) + pr_numbers_match_strings.append(match_string) + + matched_string_prs = ( + self._code_repo_service.get_prs_by_head_branch_match_strings( + list(team_repo_ids), pr_numbers_match_strings + ) + if mapping_field == "head_branch" + else self._code_repo_service.get_prs_by_title_match_strings( + list(team_repo_ids), pr_numbers_match_strings + ) + ) + + for pr in matched_string_prs: + if pr_number := self._extract_pr_number_from_pattern( + getattr(pr, mapping_field), pattern + ): + if pr_number in repo_id_to_pr_number_to_pr_map[str(pr.repo_id)].keys(): + original_pr = repo_id_to_pr_number_to_pr_map[str(pr.repo_id)][ + pr_number + ] + adapted_incident_pr = self._adapt_incident_pr(original_pr, pr) + pr_incidents.append(adapted_incident_pr) + repo_id_to_pr_number_to_pr_map[str(pr.repo_id)].pop( + original_pr.number + ) + + return pr_incidents + + def _filter_incident_prs( + self, + incident_prs_setting: IncidentPrsSetting, + repo_id_to_pr_number_to_pr_map: Dict[str, Dict[str, PullRequest]], + ) -> List[Incident]: + + if not ( + incident_prs_setting.head_branch_filters + or incident_prs_setting.title_filters + ): + return [] + + pr_incidents: List[Incident] = [] + + for repo_id in repo_id_to_pr_number_to_pr_map: + original_prs: List[PullRequest] = list( + repo_id_to_pr_number_to_pr_map[repo_id].values() + ) + original_prs = sorted(original_prs, key=lambda x: x.state_changed_at) + for prev_pr, current_pr in zip(original_prs, original_prs[1:]): + matches = any( + branch in current_pr.head_branch + for branch in incident_prs_setting.head_branch_filters + ) or any( + title in current_pr.title + for title in incident_prs_setting.title_filters + ) + if matches: + adapted_pr_incident = self._adapt_incident_pr(prev_pr, current_pr) + pr_incidents.append(adapted_pr_incident) + + return pr_incidents + + def _extract_pr_number_from_pattern(self, text: str, pattern: str) -> Optional[str]: + regex_pattern = PR_FILTER_PATTERNS.get(pattern) + if regex_pattern and check_regex(regex_pattern): + match = re.search(regex_pattern, text) + if match: + pr_number = int(match.group(1)) + return str(pr_number) + return None + + def _adapt_incident_pr( + self, incident_pr: PullRequest, resolution_pr: PullRequest + ) -> Incident: + return Incident( + id=incident_pr.id, + provider=incident_pr.provider, + title=incident_pr.title, + incident_number=int(incident_pr.number), + status=IncidentStatus.RESOLVED.value, + creation_date=incident_pr.state_changed_at, + acknowledged_date=resolution_pr.created_at, + resolved_date=resolution_pr.state_changed_at, + assigned_to=resolution_pr.author, + assignees=[resolution_pr.author], + url=incident_pr.url, + meta={ + "revert_pr": GitIncidentsETLHandler._adapt_pr_to_json(resolution_pr), + "original_pr": GitIncidentsETLHandler._adapt_pr_to_json(incident_pr), + "created_at": resolution_pr.created_at.isoformat(), + "updated_at": resolution_pr.updated_at.isoformat(), + }, + incident_type=IncidentType.REVERT_PR, + ) + def get_deployment_incidents_map( self, deployments: List[Deployment], incidents: List[Incident] ): @@ -100,18 +297,22 @@ def get_deployment_incidents_map( return deployment_incidents_map def get_team_mean_time_to_recovery( - self, team_id: str, interval: Interval + self, team_id: str, interval: Interval, pr_filter: PRFilter ) -> MeanTimeToRecoveryMetrics: - resolved_team_incidents = self.get_resolved_team_incidents(team_id, interval) + resolved_team_incidents = self.get_resolved_team_incidents( + team_id, interval, pr_filter + ) return self._get_incidents_mean_time_to_recovery(resolved_team_incidents) def get_team_mean_time_to_recovery_trends( - self, team_id: str, interval: Interval + self, team_id: str, interval: Interval, pr_filter: PRFilter ) -> MeanTimeToRecoveryMetrics: - resolved_team_incidents = self.get_resolved_team_incidents(team_id, interval) + resolved_team_incidents = self.get_resolved_team_incidents( + team_id, interval, pr_filter + ) return self._get_incidents_mean_time_to_recovery_trends( resolved_team_incidents, interval @@ -218,4 +419,6 @@ def _get_incidents_mean_time_to_recovery_trends( def get_incident_service(): - return IncidentService(IncidentsRepoService(), get_settings_service()) + return IncidentService( + IncidentsRepoService(), get_settings_service(), CodeRepoService() + ) diff --git a/backend/analytics_server/mhq/service/settings/configuration_settings.py b/backend/analytics_server/mhq/service/settings/configuration_settings.py index 41975312..b36a6c2e 100644 --- a/backend/analytics_server/mhq/service/settings/configuration_settings.py +++ b/backend/analytics_server/mhq/service/settings/configuration_settings.py @@ -4,11 +4,12 @@ from mhq.service.settings.default_settings_data import get_default_setting_data from mhq.service.settings.models import ( ConfigurationSettings, - DefaultSyncDaysSetting, ExcludedPRsSetting, IncidentSettings, IncidentSourcesSetting, IncidentTypesSetting, + DefaultSyncDaysSetting, + IncidentPrsSetting, ) from mhq.store.models.core.users import Users from mhq.store.models.incidents import IncidentSource, IncidentType @@ -66,6 +67,17 @@ def _adapt_default_sync_days_setting_from_setting_data(self, data: Dict[str, any default_sync_days=data.get("default_sync_days", None) ) + def _adapt_incident_prs_setting_setting_from_setting_data( + self, data: Dict[str, any] + ): + return IncidentPrsSetting( + include_revert_prs=data.get("include_revert_prs", True), + title_filters=data.get("title_filters", []), + head_branch_filters=data.get("head_branch_filters", []), + pr_mapping_field=data.get("pr_mapping_field", ""), + pr_mapping_pattern=data.get("pr_mapping_pattern", ""), + ) + # ADD NEW DICT TO DATACLASS ADAPTERS HERE def _handle_config_setting_from_db_setting( @@ -88,6 +100,11 @@ def _handle_config_setting_from_db_setting( if setting_type == SettingType.DEFAULT_SYNC_DAYS_SETTING: return self._adapt_default_sync_days_setting_from_setting_data(setting_data) + if setting_type == SettingType.INCIDENT_PRS_SETTING: + return self._adapt_incident_prs_setting_setting_from_setting_data( + setting_data + ) + # ADD NEW HANDLE FROM DB SETTINGS HERE raise Exception(f"Invalid Setting Type: {setting_type}") @@ -131,7 +148,14 @@ def get_or_set_default_settings( setting = self.get_settings(setting_type, entity_type, entity_id) if not setting: - setting = self.save_settings(setting_type, entity_type, entity_id) + try: + setting = self.save_settings(setting_type, entity_type, entity_id) + except Exception as e: + if "UniqueViolation" in str(e) and "Settings_pkey" in str(e): + # If another concurrent request already created the settings, fetch that settings + setting = self.get_settings(setting_type, entity_type, entity_id) + else: + raise e return setting @@ -182,6 +206,15 @@ def _adapt_default_sync_days_setting_from_json(self, data: Dict[str, any]): default_sync_days=data.get("default_sync_days", None) ) + def _adapt_incident_prs_setting_setting_from_json(self, data: Dict[str, any]): + return IncidentPrsSetting( + include_revert_prs=data.get("include_revert_prs", True), + title_filters=data.get("title_filters", []), + head_branch_filters=data.get("head_branch_filters", []), + pr_mapping_field=data.get("pr_mapping_field", ""), + pr_mapping_pattern=data.get("pr_mapping_pattern", ""), + ) + # ADD NEW DICT TO API ADAPTERS HERE def _handle_config_setting_from_json_data( @@ -204,6 +237,9 @@ def _handle_config_setting_from_json_data( if setting_type == SettingType.DEFAULT_SYNC_DAYS_SETTING: return self._adapt_default_sync_days_setting_from_json(setting_data) + if setting_type == SettingType.INCIDENT_PRS_SETTING: + return self._adapt_incident_prs_setting_setting_from_json(setting_data) + # ADD NEW HANDLE FROM JSON DATA HERE raise Exception(f"Invalid Setting Type: {setting_type}") @@ -242,6 +278,17 @@ def _adapt_default_sync_days_setting_json_data( ): return {"default_sync_days": specific_setting.default_sync_days} + def _adapt_incident_prs_setting_setting_json_data( + self, specific_setting: IncidentPrsSetting + ): + return { + "include_revert_prs": specific_setting.include_revert_prs, + "title_filters": specific_setting.title_filters, + "head_branch_filters": specific_setting.head_branch_filters, + "pr_mapping_field": specific_setting.pr_mapping_field, + "pr_mapping_pattern": specific_setting.pr_mapping_pattern, + } + # ADD NEW DATACLASS TO JSON DATA ADAPTERS HERE def _handle_config_setting_to_db_setting( @@ -273,6 +320,11 @@ def _handle_config_setting_to_db_setting( ): return self._adapt_default_sync_days_setting_json_data(specific_setting) + if setting_type == SettingType.INCIDENT_PRS_SETTING and isinstance( + specific_setting, IncidentPrsSetting + ): + return self._adapt_incident_prs_setting_setting_json_data(specific_setting) + # ADD NEW HANDLE TO DB SETTINGS HERE raise Exception(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/mhq/service/settings/default_settings_data.py b/backend/analytics_server/mhq/service/settings/default_settings_data.py index 8d133b1c..8c9350fc 100644 --- a/backend/analytics_server/mhq/service/settings/default_settings_data.py +++ b/backend/analytics_server/mhq/service/settings/default_settings_data.py @@ -29,6 +29,15 @@ def get_default_setting_data(setting_type: SettingType): if setting_type == SettingType.DEFAULT_SYNC_DAYS_SETTING: return {"default_sync_days": 31} + if setting_type == SettingType.INCIDENT_PRS_SETTING: + return { + "include_revert_prs": True, + "title_filters": [], + "head_branch_filters": [], + "pr_mapping_field": "", + "pr_mapping_pattern": "", + } + # ADD NEW DEFAULT SETTING HERE raise Exception(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/mhq/service/settings/models.py b/backend/analytics_server/mhq/service/settings/models.py index 8522d79b..a154f3f4 100644 --- a/backend/analytics_server/mhq/service/settings/models.py +++ b/backend/analytics_server/mhq/service/settings/models.py @@ -46,6 +46,15 @@ class DefaultSyncDaysSetting(BaseSetting): default_sync_days: int +@dataclass +class IncidentPrsSetting(BaseSetting): + include_revert_prs: bool + title_filters: List[str] + head_branch_filters: List[str] + pr_mapping_field: str + pr_mapping_pattern: str + + # ADD NEW SETTING CLASS HERE # Sample Future Settings diff --git a/backend/analytics_server/mhq/service/settings/setting_type_validator.py b/backend/analytics_server/mhq/service/settings/setting_type_validator.py index 008e4a9f..7d71ba7e 100644 --- a/backend/analytics_server/mhq/service/settings/setting_type_validator.py +++ b/backend/analytics_server/mhq/service/settings/setting_type_validator.py @@ -19,6 +19,9 @@ def settings_type_validator(setting_type: str): if setting_type == SettingType.DEFAULT_SYNC_DAYS_SETTING.value: return SettingType.DEFAULT_SYNC_DAYS_SETTING + if setting_type == SettingType.INCIDENT_PRS_SETTING.value: + return SettingType.INCIDENT_PRS_SETTING + # ADD NEW VALIDATOR HERE raise BadRequest(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/mhq/store/models/incidents/enums.py b/backend/analytics_server/mhq/store/models/incidents/enums.py index 6b46cf80..fdfa749e 100644 --- a/backend/analytics_server/mhq/store/models/incidents/enums.py +++ b/backend/analytics_server/mhq/store/models/incidents/enums.py @@ -34,3 +34,10 @@ class IncidentType(Enum): class IncidentBookmarkType(Enum): SERVICE = "SERVICE" + + +PR_FILTER_PATTERNS = { + "fix #1234": r"(?i)fix #(\d+)", + "fix(1234)": r"(?i)fix\((\d+)\)", + "fix-1234": r"(?i)fix-(\d+)", +} diff --git a/backend/analytics_server/mhq/store/models/settings/configuration_settings.py b/backend/analytics_server/mhq/store/models/settings/configuration_settings.py index 4f81eae7..27050b60 100644 --- a/backend/analytics_server/mhq/store/models/settings/configuration_settings.py +++ b/backend/analytics_server/mhq/store/models/settings/configuration_settings.py @@ -16,6 +16,7 @@ class SettingType(Enum): INCIDENT_TYPES_SETTING = "INCIDENT_TYPES_SETTING" INCIDENT_SOURCES_SETTING = "INCIDENT_SOURCES_SETTING" EXCLUDED_PRS_SETTING = "EXCLUDED_PRS_SETTING" + INCIDENT_PRS_SETTING = "INCIDENT_PRS_SETTING" DEFAULT_SYNC_DAYS_SETTING = "DEFAULT_SYNC_DAYS_SETTING" # ADD NEW SETTING TYPE ENUM HERE diff --git a/backend/analytics_server/mhq/store/repos/code.py b/backend/analytics_server/mhq/store/repos/code.py index 115ad675..ad1a3fb0 100644 --- a/backend/analytics_server/mhq/store/repos/code.py +++ b/backend/analytics_server/mhq/store/repos/code.py @@ -241,6 +241,29 @@ def get_prs_by_head_branch_match_strings( return query.all() + @rollback_on_exc + def get_prs_by_title_match_strings( + self, repo_ids: List[str], match_strings: List[str] + ) -> List[PullRequest]: + query = ( + self._db.session.query(PullRequest) + .options(defer(PullRequest.data)) + .filter( + and_( + PullRequest.repo_id.in_(repo_ids), + or_( + *[ + PullRequest.title.ilike(f"{match_string}%") + for match_string in match_strings + ] + ), + ) + ) + .order_by(PullRequest.updated_in_db_at.desc()) + ) + + return query.all() + @rollback_on_exc def get_reverted_prs_by_numbers( self, repo_ids: List[str], numbers: List[str] diff --git a/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py b/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py index e09a711a..22dc0671 100644 --- a/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py +++ b/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py @@ -16,7 +16,7 @@ def test_get_incidents_mean_time_to_recovery_for_no_incidents(): - incident_service = IncidentService(None, None) + incident_service = IncidentService(None, None, None) mean_time_to_recovery = incident_service._get_incidents_mean_time_to_recovery([]) assert get_mean_time_to_recovery_metrics(None, 0) == mean_time_to_recovery @@ -24,7 +24,7 @@ def test_get_incidents_mean_time_to_recovery_for_no_incidents(): def test_get_incidents_mean_time_to_recovery_for_incidents(): - incident_service = IncidentService(None, None) + incident_service = IncidentService(None, None, None) incident_1_resolution_time = timedelta(seconds=100) incident_1 = get_incident( diff --git a/web-server/pages/api/internal/team/[team_id]/incident_prs_filter.ts b/web-server/pages/api/internal/team/[team_id]/incident_prs_filter.ts new file mode 100644 index 00000000..4c92acaf --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/incident_prs_filter.ts @@ -0,0 +1,61 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { + TeamIncidentPRsSettingApiResponse, + TeamIncidentPRsSettingsResponse +} from '@/types/resources'; + +const getSchema = yup.object(); +const pathSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const putSchema = yup.object().shape({ + setting: yup.object().shape({ + include_revert_prs: yup.boolean(), + title_filters: yup.array().of(yup.string()).required(), + head_branch_filters: yup.array().of(yup.string()).required(), + pr_mapping_field: yup.string(), + pr_mapping_pattern: yup.string().when('pr_mapping_field', { + is: (pr_mapping_field: string) => pr_mapping_field !== '', + then: yup.string().required() + }) + }) +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + const { team_id } = req.payload; + const { setting } = await handleRequest( + `/teams/${team_id}/settings`, + { + method: 'GET', + params: { + setting_type: 'INCIDENT_PRS_SETTING' + } + } + ); + return res.send({ setting } as TeamIncidentPRsSettingsResponse); +}); + +endpoint.handle.PUT(putSchema, async (req, res) => { + const { team_id, setting } = req.payload; + const response = await handleRequest( + `/teams/${team_id}/settings`, + { + method: 'PUT', + data: { + setting_type: 'INCIDENT_PRS_SETTING', + setting_data: setting + } + } + ); + return res.send({ + setting: response.setting + } as TeamIncidentPRsSettingsResponse); +}); + +export default endpoint.serve(); diff --git a/web-server/src/components/DoraMetricsConfigurationSettings.tsx b/web-server/src/components/DoraMetricsConfigurationSettings.tsx index 558abc80..068b66a8 100644 --- a/web-server/src/components/DoraMetricsConfigurationSettings.tsx +++ b/web-server/src/components/DoraMetricsConfigurationSettings.tsx @@ -3,13 +3,17 @@ import { Button, Menu, MenuItem } from '@mui/material'; import { useCallback, useRef, useEffect } from 'react'; import { FlexBox } from '@/components/FlexBox'; +import { TeamIncidentPRsFilter } from '@/components/TeamIncidentPRsFilter'; import { TeamProductionBranchSelector } from '@/components/TeamProductionBranchSelector'; import { isRoleLessThanEM } from '@/constants/useRoute'; import { useModal } from '@/contexts/ModalContext'; import { useAuth } from '@/hooks/useAuth'; import { useBoolState } from '@/hooks/useEasyState'; import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; -import { fetchTeamReposProductionBranches } from '@/slices/team'; +import { + fetchTeamIncidentPRsFilter, + fetchTeamReposProductionBranches +} from '@/slices/team'; import { useDispatch } from '@/store'; export const DoraMetricsConfigurationSettings = () => { @@ -20,6 +24,7 @@ export const DoraMetricsConfigurationSettings = () => { const isEng = isRoleLessThanEM(role); useEffect(() => { dispatch(fetchTeamReposProductionBranches({ team_id: singleTeamId })); + dispatch(fetchTeamIncidentPRsFilter({ team_id: singleTeamId })); }, [dispatch, singleTeamId]); const openProductionBranchSelectorModal = useCallback(async () => { @@ -32,6 +37,14 @@ export const DoraMetricsConfigurationSettings = () => { }); }, [addModal, closeModal]); + const openIncidentPRFilterModal = useCallback(async () => { + const modal = addModal({ + title: `Configure Filters for Incident PRs`, + body: closeModal(modal.key)} />, + showCloseIcon: true + }); + }, [addModal, closeModal]); + const anchorEl = useRef(null); const open = useBoolState(false); @@ -74,6 +87,14 @@ export const DoraMetricsConfigurationSettings = () => { > Configure Production Branches + { + open.false(); + openIncidentPRFilterModal(); + }} + > + Configure Filters for Incident PRs + ); diff --git a/web-server/src/components/TeamIncidentPRsFilter.tsx b/web-server/src/components/TeamIncidentPRsFilter.tsx new file mode 100644 index 00000000..c73c923e --- /dev/null +++ b/web-server/src/components/TeamIncidentPRsFilter.tsx @@ -0,0 +1,267 @@ +import { yupResolver } from '@hookform/resolvers/yup'; +import { HelpOutlineRounded } from '@mui/icons-material'; +import { LoadingButton } from '@mui/lab'; +import { Divider, Select, MenuItem } from '@mui/material'; +import { useSnackbar } from 'notistack'; +import { FC } from 'react'; +import { FormProvider, useForm } from 'react-hook-form'; +import * as yup from 'yup'; + +import { useAuth } from '@/hooks/useAuth'; +import { useEasyState } from '@/hooks/useEasyState'; +import { + useSingleTeamConfig, + useStateBranchConfig +} from '@/hooks/useStateTeamConfig'; +import { fetchTeamDoraMetrics } from '@/slices/dora_metrics'; +import { updateTeamIncidentPRsFilter } from '@/slices/team'; +import { useDispatch, useSelector } from '@/store'; +import { ActiveBranchMode, IncidentPRMappingOptions } from '@/types/resources'; + +import { ChipInput } from './ChipInput'; +import { FlexBox } from './FlexBox'; +import { IOSSwitch } from './Shared'; +import { Line } from './Text'; + +const incidentPRFilterFormSchema = yup + .object({ + setting: yup + .object({ + include_revert_prs: yup.boolean(), + title_filters: yup.array().of(yup.string()).required(), + head_branch_filters: yup.array().of(yup.string()).required(), + pr_mapping_field: yup.string(), + pr_mapping_pattern: yup.string().when('pr_mapping_field', { + is: (pr_mapping_field: string) => pr_mapping_field !== '', + then: yup.string().required() + }) + }) + .required() + }) + .required(); + +type incidentPRFilterFormSchema = yup.InferType< + typeof incidentPRFilterFormSchema +>; + +export const TeamIncidentPRsFilter: FC<{ + onClose: () => void; +}> = ({ onClose }) => { + const dispatch = useDispatch(); + const { orgId } = useAuth(); + const { singleTeamId, dates } = useSingleTeamConfig(); + const branches = useStateBranchConfig(); + const isSaving = useEasyState(false); + const { enqueueSnackbar } = useSnackbar(); + const activeBranchMode = useSelector((s) => s.app.branchMode); + const teamIncidentPRsFilters = useSelector( + (s) => s.team.teamIncidentPRsFilters + )?.setting; + + const addUserMethods = useForm({ + resolver: yupResolver(incidentPRFilterFormSchema), + mode: 'onChange', + defaultValues: { + setting: teamIncidentPRsFilters + } + }); + + const { + watch, + formState: { isDirty, isValid }, + setValue + } = addUserMethods; + + const settings = watch('setting'); + + const handleSave = async (e: any) => { + const updateConfArgs = { + team_id: singleTeamId, + setting: settings + }; + + e.preventDefault(); + isSaving.set(true); + + await dispatch(updateTeamIncidentPRsFilter(updateConfArgs)).then( + async (response) => { + if (response.meta.requestStatus === 'fulfilled') { + const fetchDoraArgs = { + orgId: orgId, + teamId: singleTeamId, + fromDate: dates.start, + toDate: dates.end, + branches: + activeBranchMode === ActiveBranchMode.PROD ? null : branches + }; + await dispatch(fetchTeamDoraMetrics(fetchDoraArgs)); + enqueueSnackbar('Updated Successfully', { + variant: 'success', + autoHideDuration: 3000 + }); + } else { + enqueueSnackbar('Something went wrong', { + variant: 'error', + autoHideDuration: 3000 + }); + } + } + ); + + isSaving.set(false); + onClose(); + }; + + return ( + + + Define filters to include only PRs that are relevant to Incidents for + better tracking and insights. 🚀 + + + + + + Field + + + Filters (case sensitive) + + + + + + + Include Reverted PRs + + { + const isEnabled = e.target.value === 'true' ? true : false; + setValue(`setting.include_revert_prs`, !isEnabled, { + shouldValidate: true, + shouldDirty: true + }); + }} + > + + + + Head Branch Includes + + + setValue(`setting.head_branch_filters`, updatedValues, { + shouldValidate: true, + shouldDirty: true + }) + } + /> + + + + Title Includes + + + setValue(`setting.title_filters`, updatedValues, { + shouldValidate: true, + shouldDirty: true + }) + } + /> + + + + + PR Mapping + + + Helps to link Resolution PRs to their corresponding Incident + PRs + + } + darkTip + > + + + + + {settings.pr_mapping_field && ( + + )} + + + + + Save + + + + + ); +}; diff --git a/web-server/src/slices/team.ts b/web-server/src/slices/team.ts index 36126449..0bf55c8b 100644 --- a/web-server/src/slices/team.ts +++ b/web-server/src/slices/team.ts @@ -14,7 +14,9 @@ import { PR, BaseRepo, RepoUniqueDetails, - DB_OrgRepo + DB_OrgRepo, + TeamIncidentPRsSettingsResponse, + IncidentPRsSettings } from '@/types/resources'; import { addFetchCasesToReducer } from '@/utils/redux'; import { getUrlParam } from '@/utils/url'; @@ -32,6 +34,7 @@ type State = StateFetchConfig<{ teamRepos: DB_OrgRepo[]; teamReposProductionBranches: TeamRepoBranchDetails[]; teamIncidentFilters: null | TeamIncidentSettingsResponse; + teamIncidentPRsFilters: null | TeamIncidentPRsSettingsResponse; excludedPrs: PR[]; teamReposMaps: null | Record; }>; @@ -47,6 +50,7 @@ const initialState: State = { teamRepos: [], teamReposProductionBranches: [], teamIncidentFilters: null, + teamIncidentPRsFilters: null, excludedPrs: [], teamReposMaps: {} }; @@ -114,6 +118,22 @@ export const teamSlice = createSlice({ state.teamIncidentFilters = action.payload; } ); + addFetchCasesToReducer( + builder, + fetchTeamIncidentPRsFilter, + 'teamIncidentPRsFilters', + (state, action) => { + state.teamIncidentPRsFilters = action.payload; + } + ); + addFetchCasesToReducer( + builder, + updateTeamIncidentPRsFilter, + 'teamIncidentPRsFilters', + (state, action) => { + state.teamIncidentPRsFilters = action.payload; + } + ); addFetchCasesToReducer( builder, fetchExcludedPrs, @@ -248,6 +268,32 @@ export const updateTeamIncidentsFilter = createAsyncThunk( ); } ); + +export const fetchTeamIncidentPRsFilter = createAsyncThunk( + 'teams/fetchTeamIncidentPRsFilter', + async (params: { team_id: ID }) => { + return await handleApi( + `/internal/team/${params.team_id}/incident_prs_filter`, + { + method: 'GET' + } + ); + } +); + +export const updateTeamIncidentPRsFilter = createAsyncThunk( + 'teams/updateTeamIncidentPRsFilter', + async (params: { team_id: ID; setting: IncidentPRsSettings }) => { + return await handleApi( + `/internal/team/${params.team_id}/incident_prs_filter`, + { + method: 'PUT', + data: { setting: params.setting } + } + ); + } +); + export const fetchExcludedPrs = createAsyncThunk( 'teams/fetchExcludedPrs', async (params: { teamId: ID }) => { diff --git a/web-server/src/types/resources.ts b/web-server/src/types/resources.ts index 6bdcd574..a567016d 100644 --- a/web-server/src/types/resources.ts +++ b/web-server/src/types/resources.ts @@ -620,6 +620,27 @@ export type TeamIncidentSettingsResponse = { setting: IncidentSettings; }; +export type IncidentPRsSettings = { + include_revert_prs: boolean; + title_filters: string[]; + head_branch_filters: string[]; + pr_mapping_field: string; + pr_mapping_pattern: string; +}; + +export type TeamIncidentPRsSettingApiResponse = { + created_at: Date; + updated_at: Date; + team_id: ID; + setting: IncidentPRsSettings; +}; + +export type TeamIncidentPRsSettingsResponse = { + setting: IncidentPRsSettings; +}; + +export const IncidentPRMappingOptions = ['fix #1234', 'fix(1234)', 'fix-1234']; + export enum TeamSettings { TEAM_MEMBER_METRICS_FILTER_SETTING = 'TEAM_MEMBER_METRICS_FILTER_SETTING', EXCLUDED_TICKET_TYPES_SETTING = 'EXCLUDED_TICKET_TYPES_SETTING' diff --git a/web-server/src/utils/cockpitMetricUtils.ts b/web-server/src/utils/cockpitMetricUtils.ts index d925b25e..e1b2f47d 100644 --- a/web-server/src/utils/cockpitMetricUtils.ts +++ b/web-server/src/utils/cockpitMetricUtils.ts @@ -97,6 +97,7 @@ export const fetchMeanTimeToRestoreStats = async (params: { teamId, currTrendsTimeObject, prevTrendsTimeObject, + prFilter, currStatsTimeObject, prevStatsTimeObject } = params; @@ -110,28 +111,30 @@ export const fetchMeanTimeToRestoreStats = async (params: { handleRequest( `/teams/${teamId}/mean_time_to_recovery`, { - params: { ...currStatsTimeObject } + params: { ...currStatsTimeObject, ...prFilter } } ), handleRequest( `/teams/${teamId}/mean_time_to_recovery`, { params: { - ...prevStatsTimeObject + ...prevStatsTimeObject, + ...prFilter } } ), handleRequest( `/teams/${teamId}/mean_time_to_recovery/trends`, { - params: { ...currTrendsTimeObject } + params: { ...currTrendsTimeObject, ...prFilter } } ), handleRequest( `/teams/${teamId}/mean_time_to_recovery/trends`, { params: { - ...prevTrendsTimeObject + ...prevTrendsTimeObject, + ...prFilter } } )