From c1bec8959c879551b9c4eccdbc1adb704dac448d Mon Sep 17 00:00:00 2001 From: BIGWangYuDong Date: Tue, 28 Feb 2023 21:58:58 +0800 Subject: [PATCH 1/5] [Refactor] Refactor CityScapesMetric by using MMEval --- .../_base_/datasets/cityscapes_instance.py | 7 +- mmdet/evaluation/metrics/cityscapes_metric.py | 187 ++++++++++-------- 2 files changed, 105 insertions(+), 89 deletions(-) diff --git a/configs/_base_/datasets/cityscapes_instance.py b/configs/_base_/datasets/cityscapes_instance.py index 0254af3f97a..3ac97e295b6 100644 --- a/configs/_base_/datasets/cityscapes_instance.py +++ b/configs/_base_/datasets/cityscapes_instance.py @@ -66,9 +66,10 @@ metric=['bbox', 'segm']), dict( type='CityScapesMetric', - ann_file=data_root + - 'annotations/instancesonly_filtered_gtFine_val.json', - seg_prefix=data_root + '/gtFine/val', + seg_prefix=data_root + 'gtFine/val', + keep_results=False, + keep_gt_json=False, + classwise=True, outfile_prefix='./work_dirs/cityscapes_metric/instance') ] diff --git a/mmdet/evaluation/metrics/cityscapes_metric.py b/mmdet/evaluation/metrics/cityscapes_metric.py index 2b28100aff4..3ab21650128 100644 --- a/mmdet/evaluation/metrics/cityscapes_metric.py +++ b/mmdet/evaluation/metrics/cityscapes_metric.py @@ -1,37 +1,30 @@ # Copyright (c) OpenMMLab. All rights reserved. -import os +import itertools import os.path as osp -import shutil -from collections import OrderedDict -from typing import Dict, Optional, Sequence +import warnings +from typing import Optional, Sequence import mmcv import numpy as np -from mmengine.dist import is_main_process, master_only -from mmengine.evaluator import BaseMetric -from mmengine.logging import MMLogger +from mmengine.logging import MMLogger, print_log +from mmeval import CityScapesDetection +from terminaltables import AsciiTable from mmdet.registry import METRICS try: - import cityscapesscripts - from cityscapesscripts.evaluation import \ - evalInstanceLevelSemanticLabeling as CSEval - from cityscapesscripts.helpers import labels as CSLabels + import cityscapesscripts.helpers.labels as CSLabels except ImportError: - cityscapesscripts = None CSLabels = None - CSEval = None @METRICS.register_module() -class CityScapesMetric(BaseMetric): - """CityScapes metric for instance segmentation. +class CityScapesMetric(CityScapesDetection): + """A wrapper of :class:`mmeval.CityScapesDetection`. Args: - outfile_prefix (str): The prefix of txt and png files. The txt and - png file will be save in a directory whose path is - "outfile_prefix.results/". + outfile_prefix (str): The prefix of txt and png files. It is the + saving path of txt and png file, e.g. "a/b/prefix". seg_prefix (str, optional): Path to the directory which contains the cityscapes instance segmentation masks. It's necessary when training and validation. It could be None when infer on test @@ -42,6 +35,8 @@ class CityScapesMetric(BaseMetric): Defaults to False. keep_results (bool): Whether to keep the results. When ``format_only`` is True, ``keep_results`` must be True. Defaults to False. + classwise (bool): Whether to return the computed results of each + class. Defaults to True. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. @@ -53,36 +48,41 @@ class CityScapesMetric(BaseMetric): default_prefix: Optional[str] = 'cityscapes' def __init__(self, - outfile_prefix: str, + outfile_prefix: Optional[str] = None, seg_prefix: Optional[str] = None, format_only: bool = False, keep_results: bool = False, - collect_device: str = 'cpu', - prefix: Optional[str] = None) -> None: - if cityscapesscripts is None: + classwise: bool = True, + keep_gt_json: bool = False, + prefix: Optional[str] = None, + dist_backend: str = 'torch_cuda', + **kwargs) -> None: + + if CSLabels is None: raise RuntimeError('Please run "pip install cityscapesscripts" to ' 'install cityscapesscripts first.') - assert outfile_prefix, 'outfile_prefix must be not None.' - - if format_only: - assert keep_results, 'keep_results must be True when ' - 'format_only is True' + collect_device = kwargs.pop('collect_device', None) + if collect_device is not None: + warnings.warn( + 'DeprecationWarning: The `collect_device` parameter of ' + '`CocoMetric` is deprecated, use `dist_backend` instead.') - super().__init__(collect_device=collect_device, prefix=prefix) - self.format_only = format_only - self.keep_results = keep_results - self.seg_out_dir = osp.abspath(f'{outfile_prefix}.results') - self.seg_prefix = seg_prefix + logger = MMLogger.get_current_instance() - if is_main_process(): - os.makedirs(self.seg_out_dir, exist_ok=True) + super().__init__( + outfile_prefix=outfile_prefix, + seg_prefix=seg_prefix, + format_only=format_only, + keep_results=keep_results, + classwise=classwise, + keep_gt_json=keep_gt_json, + logger=logger, + dist_backend=dist_backend, + **kwargs) - @master_only - def __del__(self) -> None: - """Clean up.""" - if not self.keep_results: - shutil.rmtree(self.seg_out_dir) + self.prefix = prefix or self.default_prefix + self.classes = None # TODO: data_batch is no longer needed, consider adjusting the # parameter position @@ -96,77 +96,92 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ + if self.classes is None: + self._get_classes() + predictions, groundtruths = [], [] + for data_sample in data_samples: # parse pred - result = dict() - pred = data_sample['pred_instances'] + pred = dict() + pred_instances = data_sample['pred_instances'] filename = data_sample['img_path'] basename = osp.splitext(osp.basename(filename))[0] - pred_txt = osp.join(self.seg_out_dir, basename + '_pred.txt') - result['pred_txt'] = pred_txt - labels = pred['labels'].cpu().numpy() - masks = pred['masks'].cpu().numpy().astype(np.uint8) - if 'mask_scores' in pred: + pred_txt = osp.join(self.outfile_prefix, basename + '_pred.txt') + pred['pred_txt'] = pred_txt + + labels = pred_instances['labels'].cpu().numpy() + masks = pred_instances['masks'].cpu().numpy().astype(np.uint8) + if 'mask_scores' in pred_instances: # some detectors use different scores for bbox and mask - mask_scores = pred['mask_scores'].cpu().numpy() + mask_scores = pred_instances['mask_scores'].cpu().numpy() else: - mask_scores = pred['scores'].cpu().numpy() + mask_scores = pred_instances['scores'].cpu().numpy() with open(pred_txt, 'w') as f: for i, (label, mask, mask_score) in enumerate( zip(labels, masks, mask_scores)): - class_name = self.dataset_meta['classes'][label] + class_name = self.classes[label] class_id = CSLabels.name2label[class_name].id png_filename = osp.join( - self.seg_out_dir, basename + f'_{i}_{class_name}.png') + self.outfile_prefix, + basename + f'_{i}_{class_name}.png') mmcv.imwrite(mask, png_filename) f.write(f'{osp.basename(png_filename)} ' f'{class_id} {mask_score}\n') + predictions.append(pred) # parse gt gt = dict() img_path = filename.replace('leftImg8bit.png', 'gtFine_instanceIds.png') img_path = img_path.replace('leftImg8bit', 'gtFine') - gt['file_name'] = osp.join(self.seg_prefix, img_path) - - self.results.append((gt, result)) + gt['file_name'] = img_path + groundtruths.append(gt) - def compute_metrics(self, results: list) -> Dict[str, float]: - """Compute the metrics from processed results. + self.add(predictions, groundtruths) - Args: - results (list): The processed results of each batch. + def evaluate(self, *args, **kwargs) -> dict: + """Returns metric results and print pretty table of metrics per class. - Returns: - Dict[str, float]: The computed metrics. The keys are the names of - the metrics, and the values are corresponding results. + This method would be invoked by ``mmengine.Evaluator``. """ - logger: MMLogger = MMLogger.get_current_instance() + metric_results = self.compute(*args, **kwargs) + self.reset() if self.format_only: - logger.info( - f'results are saved to {osp.dirname(self.seg_out_dir)}') - return OrderedDict() - logger.info('starts to compute metric') - - gts, preds = zip(*results) - # set global states in cityscapes evaluation API - CSEval.args.cityscapesPath = osp.join(self.seg_prefix, '../..') - CSEval.args.predictionPath = self.seg_out_dir - CSEval.args.predictionWalk = None - CSEval.args.JSONOutput = False - CSEval.args.colorized = False - CSEval.args.gtInstancesFile = osp.join(self.seg_out_dir, - 'gtInstances.json') - - groundTruthImgList = [gt['file_name'] for gt in gts] - predictionImgList = [pred['pred_txt'] for pred in preds] - CSEval_results = CSEval.evaluateImgLists(predictionImgList, - groundTruthImgList, - CSEval.args)['averages'] - eval_results = OrderedDict() - eval_results['mAP'] = CSEval_results['allAp'] - eval_results['AP@50'] = CSEval_results['allAp50%'] - - return eval_results + return metric_results + + result = metric_results.pop('results_list', None) + if result is None: + return metric_results + + header = ['class', 'AP(%)', 'AP50(%)'] + table_title = ' Cityscapes Results' + + results_flatten = list(itertools.chain(*result)) + + results_2d = itertools.zip_longest( + *[results_flatten[i::3] for i in range(3)]) + table_data = [header] + table_data += [result for result in results_2d] + table = AsciiTable(table_data, title=table_title) + table.inner_footing_row_border = True + print_log('\n' + table.table, logger='current') + evaluate_results = { + f'{self.prefix}/{k}(%)': round(float(v) * 100, 4) + for k, v in metric_results.items() + } + return evaluate_results + + def _get_classes(self): + + if self.dataset_meta and 'classes' in self.dataset_meta: + self.classes = self.dataset_meta['classes'] + elif self.dataset_meta and 'CLASSES' in self.dataset_meta: + self.classes = self.dataset_meta['CLASSES'] + warnings.warn( + 'DeprecationWarning: The `CLASSES` in `dataset_meta` is ' + 'deprecated, use `classes` instead!') + else: + raise RuntimeError('Could not find `classes` in dataset_meta: ' + f'{self.dataset_meta}') From b85922aa3680b70b4277a743df1207f87a408bc7 Mon Sep 17 00:00:00 2001 From: BIGWangYuDong Date: Wed, 1 Mar 2023 19:44:39 +0800 Subject: [PATCH 2/5] update logic and UT --- mmdet/evaluation/metrics/cityscapes_metric.py | 43 +---- .../test_metrics/test_cityscapes_metric.py | 159 ++++++++++++------ 2 files changed, 113 insertions(+), 89 deletions(-) diff --git a/mmdet/evaluation/metrics/cityscapes_metric.py b/mmdet/evaluation/metrics/cityscapes_metric.py index 3ab21650128..15b391bd651 100644 --- a/mmdet/evaluation/metrics/cityscapes_metric.py +++ b/mmdet/evaluation/metrics/cityscapes_metric.py @@ -4,7 +4,6 @@ import warnings from typing import Optional, Sequence -import mmcv import numpy as np from mmengine.logging import MMLogger, print_log from mmeval import CityScapesDetection @@ -13,9 +12,9 @@ from mmdet.registry import METRICS try: - import cityscapesscripts.helpers.labels as CSLabels + import cityscapesscripts except ImportError: - CSLabels = None + cityscapesscripts = None @METRICS.register_module() @@ -58,8 +57,8 @@ def __init__(self, dist_backend: str = 'torch_cuda', **kwargs) -> None: - if CSLabels is None: - raise RuntimeError('Please run "pip install cityscapesscripts" to ' + if cityscapesscripts is None: + raise RuntimeError('Please run `pip install cityscapesscripts` to ' 'install cityscapesscripts first.') collect_device = kwargs.pop('collect_device', None) @@ -82,7 +81,6 @@ def __init__(self, **kwargs) self.prefix = prefix or self.default_prefix - self.classes = None # TODO: data_batch is no longer needed, consider adjusting the # parameter position @@ -96,8 +94,6 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ - if self.classes is None: - self._get_classes() predictions, groundtruths = [], [] for data_sample in data_samples: @@ -106,9 +102,6 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: pred_instances = data_sample['pred_instances'] filename = data_sample['img_path'] basename = osp.splitext(osp.basename(filename))[0] - pred_txt = osp.join(self.outfile_prefix, basename + '_pred.txt') - pred['pred_txt'] = pred_txt - labels = pred_instances['labels'].cpu().numpy() masks = pred_instances['masks'].cpu().numpy().astype(np.uint8) if 'mask_scores' in pred_instances: @@ -117,17 +110,10 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: else: mask_scores = pred_instances['scores'].cpu().numpy() - with open(pred_txt, 'w') as f: - for i, (label, mask, mask_score) in enumerate( - zip(labels, masks, mask_scores)): - class_name = self.classes[label] - class_id = CSLabels.name2label[class_name].id - png_filename = osp.join( - self.outfile_prefix, - basename + f'_{i}_{class_name}.png') - mmcv.imwrite(mask, png_filename) - f.write(f'{osp.basename(png_filename)} ' - f'{class_id} {mask_score}\n') + pred['labels'] = labels + pred['masks'] = masks + pred['mask_scores'] = mask_scores + pred['basename'] = basename predictions.append(pred) # parse gt @@ -172,16 +158,3 @@ def evaluate(self, *args, **kwargs) -> dict: for k, v in metric_results.items() } return evaluate_results - - def _get_classes(self): - - if self.dataset_meta and 'classes' in self.dataset_meta: - self.classes = self.dataset_meta['classes'] - elif self.dataset_meta and 'CLASSES' in self.dataset_meta: - self.classes = self.dataset_meta['CLASSES'] - warnings.warn( - 'DeprecationWarning: The `CLASSES` in `dataset_meta` is ' - 'deprecated, use `classes` instead!') - else: - raise RuntimeError('Could not find `classes` in dataset_meta: ' - f'{self.dataset_meta}') diff --git a/tests/test_evaluation/test_metrics/test_cityscapes_metric.py b/tests/test_evaluation/test_metrics/test_cityscapes_metric.py index 91a4f745dd6..d2d6975381c 100644 --- a/tests/test_evaluation/test_metrics/test_cityscapes_metric.py +++ b/tests/test_evaluation/test_metrics/test_cityscapes_metric.py @@ -1,3 +1,4 @@ +import math import os import os.path as osp import tempfile @@ -30,82 +31,132 @@ def test_init(self): with self.assertRaises(AssertionError): CityScapesMetric(outfile_prefix=None) + # test with seg_prefix = None + with self.assertRaises(AssertionError): + CityScapesMetric( + outfile_prefix='tmp/cityscapes/results', seg_prefix=None) + # test with format_only=True, keep_results=False with self.assertRaises(AssertionError): CityScapesMetric( - outfile_prefix=self.tmp_dir.name + 'test', + outfile_prefix='tmp/cityscapes/results', format_only=True, keep_results=False) @unittest.skipIf(cityscapesscripts is None, 'cityscapesscripts is not installed.') def test_evaluate(self): - dummy_mask1 = np.zeros((1, 20, 20), dtype=np.uint8) - dummy_mask1[:, :10, :10] = 1 - dummy_mask2 = np.zeros((1, 20, 20), dtype=np.uint8) - dummy_mask2[:, :10, :10] = 1 + tmp_dir = tempfile.TemporaryDirectory() - self.outfile_prefix = osp.join(self.tmp_dir.name, 'test') - self.seg_prefix = osp.join(self.tmp_dir.name, 'cityscapes/gtFine/val') - city = 'lindau' - sequenceNb = '000000' - frameNb = '000019' - img_name1 = f'{city}_{sequenceNb}_{frameNb}_gtFine_instanceIds.png' - img_path1 = osp.join(self.seg_prefix, city, img_name1) + dataset_metas = { + 'classes': ('person', 'rider', 'car', 'truck', 'bus', 'train', + 'motorcycle', 'bicycle') + } - frameNb = '000020' - img_name2 = f'{city}_{sequenceNb}_{frameNb}_gtFine_instanceIds.png' - img_path2 = osp.join(self.seg_prefix, city, img_name2) - os.makedirs(osp.join(self.seg_prefix, city)) + # create dummy data + self.seg_prefix = osp.join(tmp_dir.name, 'cityscapes', 'gtFine', 'val') + os.makedirs(self.seg_prefix, exist_ok=True) + data_samples = self._gen_fake_datasamples() - masks1 = np.zeros((20, 20), dtype=np.int32) - masks1[:10, :10] = 24 * 1000 - Image.fromarray(masks1).save(img_path1) + # test single evaluation + metric = CityScapesMetric( + dataset_meta=dataset_metas, + outfile_prefix=osp.join(tmp_dir.name, 'test'), + seg_prefix=self.seg_prefix, + keep_results=False, + keep_gt_json=False, + classwise=False) - masks2 = np.zeros((20, 20), dtype=np.int32) - masks2[:10, :10] = 24 * 1000 + 1 - Image.fromarray(masks2).save(img_path2) + metric.process({}, data_samples) + results = metric.evaluate() + targets = {'cityscapes/mAP(%)': 50.0, 'cityscapes/AP50(%)': 50.0} + self.assertDictEqual(results, targets) - data_samples = [{ - 'img_path': img_path1, - 'pred_instances': { - 'scores': torch.from_numpy(np.array([1.0])), - 'labels': torch.from_numpy(np.array([0])), - 'masks': torch.from_numpy(dummy_mask1) - } - }, { - 'img_path': img_path2, - 'pred_instances': { - 'scores': torch.from_numpy(np.array([0.98])), - 'labels': torch.from_numpy(np.array([1])), - 'masks': torch.from_numpy(dummy_mask2) - } - }] - - target = {'cityscapes/mAP': 0.5, 'cityscapes/AP@50': 0.5} + # test classwise result evaluation metric = CityScapesMetric( + dataset_meta=dataset_metas, + outfile_prefix=osp.join(tmp_dir.name, 'test'), seg_prefix=self.seg_prefix, - format_only=False, keep_results=False, - outfile_prefix=self.outfile_prefix) - metric.dataset_meta = dict( - classes=('person', 'rider', 'car', 'truck', 'bus', 'train', - 'motorcycle', 'bicycle')) + keep_gt_json=False, + classwise=True) + metric.process({}, data_samples) - results = metric.evaluate(size=2) - self.assertDictEqual(results, target) - del metric - self.assertTrue(not osp.exists('{self.outfile_prefix}.results')) + results = metric.evaluate() + mAP = results.pop('cityscapes/mAP(%)') + AP50 = results.pop('cityscapes/AP50(%)') + self.assertEqual(mAP, 50.0) + self.assertEqual(AP50, 50.0) + + # except person, others classes ap or ap50 should be nan + person_ap = results.pop('cityscapes/person_ap(%)') + person_ap50 = results.pop('cityscapes/person_ap50(%)') + self.assertEqual(person_ap, 50.0) + self.assertEqual(person_ap50, 50.0) + for v in results.values(): + self.assertTrue(math.isnan(v)) # test format_only metric = CityScapesMetric( - seg_prefix=self.seg_prefix, + dataset_meta=dataset_metas, format_only=True, + outfile_prefix=osp.join(tmp_dir.name, 'test'), + seg_prefix=self.seg_prefix, keep_results=True, - outfile_prefix=self.outfile_prefix) - metric.dataset_meta = dict( - classes=('person', 'rider', 'car', 'truck', 'bus', 'train', - 'motorcycle', 'bicycle')) + keep_gt_json=True, + classwise=True) + metric.process({}, data_samples) - results = metric.evaluate(size=2) + results = metric.evaluate() + self.assertTrue(osp.exists(f'{osp.join(tmp_dir.name, "test")}')) self.assertDictEqual(results, dict()) + + def _gen_fake_datasamples(self): + city = 'lindau' + os.makedirs(osp.join(self.seg_prefix, city), exist_ok=True) + + sequenceNb = '000000' + frameNb1 = '000019' + img_name1 = f'{city}_{sequenceNb}_{frameNb1}_gtFine_instanceIds.png' + img_path1 = osp.join(self.seg_prefix, city, img_name1) + + masks1 = np.zeros((20, 20), dtype=np.int32) + masks1[:10, :10] = 24 * 1000 + Image.fromarray(masks1).save(img_path1) + + dummy_mask1 = np.zeros((1, 20, 20), dtype=np.uint8) + dummy_mask1[:, :10, :10] = 1 + prediction1 = { + 'mask_scores': torch.from_numpy(np.array([1.0])), + 'labels': torch.from_numpy(np.array([0])), + 'masks': torch.from_numpy(dummy_mask1) + } + + frameNb2 = '000020' + img_name2 = f'{city}_{sequenceNb}_{frameNb2}_gtFine_instanceIds.png' + img_path2 = osp.join(self.seg_prefix, city, img_name2) + + masks2 = np.zeros((20, 20), dtype=np.int32) + masks2[:10, :10] = 24 * 1000 + 1 + Image.fromarray(masks2).save(img_path2) + + dummy_mask2 = np.zeros((1, 20, 20), dtype=np.uint8) + dummy_mask2[:, :10, :10] = 1 + prediction2 = { + 'mask_scores': torch.from_numpy(np.array([0.98])), + 'labels': torch.from_numpy(np.array([1])), + 'masks': torch.from_numpy(dummy_mask2) + } + + data_samples = [ + { + 'pred_instances': prediction1, + 'img_path': img_path1 + }, + { + 'pred_instances': prediction2, + 'img_path': img_path2 + }, + ] + + return data_samples From 3f7a06d451f4a9bfd2d74038fafc1d1fee20d6bb Mon Sep 17 00:00:00 2001 From: BIGWangYuDong Date: Wed, 1 Mar 2023 20:11:33 +0800 Subject: [PATCH 3/5] fix typo --- mmdet/evaluation/metrics/cityscapes_metric.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmdet/evaluation/metrics/cityscapes_metric.py b/mmdet/evaluation/metrics/cityscapes_metric.py index 15b391bd651..ba389c285d3 100644 --- a/mmdet/evaluation/metrics/cityscapes_metric.py +++ b/mmdet/evaluation/metrics/cityscapes_metric.py @@ -65,7 +65,8 @@ def __init__(self, if collect_device is not None: warnings.warn( 'DeprecationWarning: The `collect_device` parameter of ' - '`CocoMetric` is deprecated, use `dist_backend` instead.') + '`CityScapesMetric` is deprecated, use `dist_backend` ' + 'instead.') logger = MMLogger.get_current_instance() From a401419005f419fb12f2a8fb6830e7b372a0d72a Mon Sep 17 00:00:00 2001 From: BIGWangYuDong Date: Tue, 7 Mar 2023 22:32:03 +0800 Subject: [PATCH 4/5] sync with mmeval --- .../_base_/datasets/cityscapes_instance.py | 5 +--- mmdet/evaluation/metrics/cityscapes_metric.py | 26 +------------------ 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/configs/_base_/datasets/cityscapes_instance.py b/configs/_base_/datasets/cityscapes_instance.py index 3ac97e295b6..8e71957bbdd 100644 --- a/configs/_base_/datasets/cityscapes_instance.py +++ b/configs/_base_/datasets/cityscapes_instance.py @@ -67,10 +67,7 @@ dict( type='CityScapesMetric', seg_prefix=data_root + 'gtFine/val', - keep_results=False, - keep_gt_json=False, - classwise=True, - outfile_prefix='./work_dirs/cityscapes_metric/instance') + classwise=True) ] test_evaluator = val_evaluator diff --git a/mmdet/evaluation/metrics/cityscapes_metric.py b/mmdet/evaluation/metrics/cityscapes_metric.py index ba389c285d3..82f9057aa78 100644 --- a/mmdet/evaluation/metrics/cityscapes_metric.py +++ b/mmdet/evaluation/metrics/cityscapes_metric.py @@ -1,13 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. -import itertools import os.path as osp import warnings from typing import Optional, Sequence import numpy as np -from mmengine.logging import MMLogger, print_log +from mmengine.logging import MMLogger from mmeval import CityScapesDetection -from terminaltables import AsciiTable from mmdet.registry import METRICS @@ -32,8 +30,6 @@ class CityScapesMetric(CityScapesDetection): evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. - keep_results (bool): Whether to keep the results. When ``format_only`` - is True, ``keep_results`` must be True. Defaults to False. classwise (bool): Whether to return the computed results of each class. Defaults to True. collect_device (str): Device name used for collecting results from @@ -50,9 +46,7 @@ def __init__(self, outfile_prefix: Optional[str] = None, seg_prefix: Optional[str] = None, format_only: bool = False, - keep_results: bool = False, classwise: bool = True, - keep_gt_json: bool = False, prefix: Optional[str] = None, dist_backend: str = 'torch_cuda', **kwargs) -> None: @@ -74,9 +68,7 @@ def __init__(self, outfile_prefix=outfile_prefix, seg_prefix=seg_prefix, format_only=format_only, - keep_results=keep_results, classwise=classwise, - keep_gt_json=keep_gt_json, logger=logger, dist_backend=dist_backend, **kwargs) @@ -138,22 +130,6 @@ def evaluate(self, *args, **kwargs) -> dict: if self.format_only: return metric_results - result = metric_results.pop('results_list', None) - if result is None: - return metric_results - - header = ['class', 'AP(%)', 'AP50(%)'] - table_title = ' Cityscapes Results' - - results_flatten = list(itertools.chain(*result)) - - results_2d = itertools.zip_longest( - *[results_flatten[i::3] for i in range(3)]) - table_data = [header] - table_data += [result for result in results_2d] - table = AsciiTable(table_data, title=table_title) - table.inner_footing_row_border = True - print_log('\n' + table.table, logger='current') evaluate_results = { f'{self.prefix}/{k}(%)': round(float(v) * 100, 4) for k, v in metric_results.items() From c9798182a3247d884b7b9b229acfc057f69f389d Mon Sep 17 00:00:00 2001 From: BigDong Date: Wed, 15 Mar 2023 13:08:33 +0800 Subject: [PATCH 5/5] Update mmdet/evaluation/metrics/cityscapes_metric.py Co-authored-by: RangiLyu --- mmdet/evaluation/metrics/cityscapes_metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmdet/evaluation/metrics/cityscapes_metric.py b/mmdet/evaluation/metrics/cityscapes_metric.py index 82f9057aa78..34e4d2d60f9 100644 --- a/mmdet/evaluation/metrics/cityscapes_metric.py +++ b/mmdet/evaluation/metrics/cityscapes_metric.py @@ -30,7 +30,7 @@ class CityScapesMetric(CityScapesDetection): evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. - classwise (bool): Whether to return the computed results of each + classwise (bool): Whether to return the computed results of each class. Defaults to True. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or