diff --git a/configs/_base_/datasets/cityscapes_instance.py b/configs/_base_/datasets/cityscapes_instance.py index 0254af3f97a..8e71957bbdd 100644 --- a/configs/_base_/datasets/cityscapes_instance.py +++ b/configs/_base_/datasets/cityscapes_instance.py @@ -66,10 +66,8 @@ metric=['bbox', 'segm']), dict( type='CityScapesMetric', - ann_file=data_root + - 'annotations/instancesonly_filtered_gtFine_val.json', - seg_prefix=data_root + '/gtFine/val', - outfile_prefix='./work_dirs/cityscapes_metric/instance') + seg_prefix=data_root + 'gtFine/val', + classwise=True) ] test_evaluator = val_evaluator diff --git a/mmdet/evaluation/metrics/cityscapes_metric.py b/mmdet/evaluation/metrics/cityscapes_metric.py index 2b28100aff4..34e4d2d60f9 100644 --- a/mmdet/evaluation/metrics/cityscapes_metric.py +++ b/mmdet/evaluation/metrics/cityscapes_metric.py @@ -1,37 +1,27 @@ # Copyright (c) OpenMMLab. All rights reserved. -import os import os.path as osp -import shutil -from collections import OrderedDict -from typing import Dict, Optional, Sequence +import warnings +from typing import Optional, Sequence -import mmcv import numpy as np -from mmengine.dist import is_main_process, master_only -from mmengine.evaluator import BaseMetric from mmengine.logging import MMLogger +from mmeval import CityScapesDetection from mmdet.registry import METRICS try: import cityscapesscripts - from cityscapesscripts.evaluation import \ - evalInstanceLevelSemanticLabeling as CSEval - from cityscapesscripts.helpers import labels as CSLabels except ImportError: cityscapesscripts = None - CSLabels = None - CSEval = None @METRICS.register_module() -class CityScapesMetric(BaseMetric): - """CityScapes metric for instance segmentation. +class CityScapesMetric(CityScapesDetection): + """A wrapper of :class:`mmeval.CityScapesDetection`. Args: - outfile_prefix (str): The prefix of txt and png files. The txt and - png file will be save in a directory whose path is - "outfile_prefix.results/". + outfile_prefix (str): The prefix of txt and png files. It is the + saving path of txt and png file, e.g. "a/b/prefix". seg_prefix (str, optional): Path to the directory which contains the cityscapes instance segmentation masks. It's necessary when training and validation. It could be None when infer on test @@ -40,8 +30,8 @@ class CityScapesMetric(BaseMetric): evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. - keep_results (bool): Whether to keep the results. When ``format_only`` - is True, ``keep_results`` must be True. Defaults to False. + classwise (bool): Whether to return the computed results of each + class. Defaults to True. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. @@ -53,36 +43,37 @@ class CityScapesMetric(BaseMetric): default_prefix: Optional[str] = 'cityscapes' def __init__(self, - outfile_prefix: str, + outfile_prefix: Optional[str] = None, seg_prefix: Optional[str] = None, format_only: bool = False, - keep_results: bool = False, - collect_device: str = 'cpu', - prefix: Optional[str] = None) -> None: + classwise: bool = True, + prefix: Optional[str] = None, + dist_backend: str = 'torch_cuda', + **kwargs) -> None: + if cityscapesscripts is None: - raise RuntimeError('Please run "pip install cityscapesscripts" to ' + raise RuntimeError('Please run `pip install cityscapesscripts` to ' 'install cityscapesscripts first.') - assert outfile_prefix, 'outfile_prefix must be not None.' - - if format_only: - assert keep_results, 'keep_results must be True when ' - 'format_only is True' + collect_device = kwargs.pop('collect_device', None) + if collect_device is not None: + warnings.warn( + 'DeprecationWarning: The `collect_device` parameter of ' + '`CityScapesMetric` is deprecated, use `dist_backend` ' + 'instead.') - super().__init__(collect_device=collect_device, prefix=prefix) - self.format_only = format_only - self.keep_results = keep_results - self.seg_out_dir = osp.abspath(f'{outfile_prefix}.results') - self.seg_prefix = seg_prefix + logger = MMLogger.get_current_instance() - if is_main_process(): - os.makedirs(self.seg_out_dir, exist_ok=True) + super().__init__( + outfile_prefix=outfile_prefix, + seg_prefix=seg_prefix, + format_only=format_only, + classwise=classwise, + logger=logger, + dist_backend=dist_backend, + **kwargs) - @master_only - def __del__(self) -> None: - """Clean up.""" - if not self.keep_results: - shutil.rmtree(self.seg_out_dir) + self.prefix = prefix or self.default_prefix # TODO: data_batch is no longer needed, consider adjusting the # parameter position @@ -96,77 +87,51 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ + predictions, groundtruths = [], [] + for data_sample in data_samples: # parse pred - result = dict() - pred = data_sample['pred_instances'] + pred = dict() + pred_instances = data_sample['pred_instances'] filename = data_sample['img_path'] basename = osp.splitext(osp.basename(filename))[0] - pred_txt = osp.join(self.seg_out_dir, basename + '_pred.txt') - result['pred_txt'] = pred_txt - labels = pred['labels'].cpu().numpy() - masks = pred['masks'].cpu().numpy().astype(np.uint8) - if 'mask_scores' in pred: + labels = pred_instances['labels'].cpu().numpy() + masks = pred_instances['masks'].cpu().numpy().astype(np.uint8) + if 'mask_scores' in pred_instances: # some detectors use different scores for bbox and mask - mask_scores = pred['mask_scores'].cpu().numpy() + mask_scores = pred_instances['mask_scores'].cpu().numpy() else: - mask_scores = pred['scores'].cpu().numpy() - - with open(pred_txt, 'w') as f: - for i, (label, mask, mask_score) in enumerate( - zip(labels, masks, mask_scores)): - class_name = self.dataset_meta['classes'][label] - class_id = CSLabels.name2label[class_name].id - png_filename = osp.join( - self.seg_out_dir, basename + f'_{i}_{class_name}.png') - mmcv.imwrite(mask, png_filename) - f.write(f'{osp.basename(png_filename)} ' - f'{class_id} {mask_score}\n') + mask_scores = pred_instances['scores'].cpu().numpy() + + pred['labels'] = labels + pred['masks'] = masks + pred['mask_scores'] = mask_scores + pred['basename'] = basename + predictions.append(pred) # parse gt gt = dict() img_path = filename.replace('leftImg8bit.png', 'gtFine_instanceIds.png') img_path = img_path.replace('leftImg8bit', 'gtFine') - gt['file_name'] = osp.join(self.seg_prefix, img_path) + gt['file_name'] = img_path + groundtruths.append(gt) - self.results.append((gt, result)) + self.add(predictions, groundtruths) - def compute_metrics(self, results: list) -> Dict[str, float]: - """Compute the metrics from processed results. + def evaluate(self, *args, **kwargs) -> dict: + """Returns metric results and print pretty table of metrics per class. - Args: - results (list): The processed results of each batch. - - Returns: - Dict[str, float]: The computed metrics. The keys are the names of - the metrics, and the values are corresponding results. + This method would be invoked by ``mmengine.Evaluator``. """ - logger: MMLogger = MMLogger.get_current_instance() + metric_results = self.compute(*args, **kwargs) + self.reset() if self.format_only: - logger.info( - f'results are saved to {osp.dirname(self.seg_out_dir)}') - return OrderedDict() - logger.info('starts to compute metric') - - gts, preds = zip(*results) - # set global states in cityscapes evaluation API - CSEval.args.cityscapesPath = osp.join(self.seg_prefix, '../..') - CSEval.args.predictionPath = self.seg_out_dir - CSEval.args.predictionWalk = None - CSEval.args.JSONOutput = False - CSEval.args.colorized = False - CSEval.args.gtInstancesFile = osp.join(self.seg_out_dir, - 'gtInstances.json') - - groundTruthImgList = [gt['file_name'] for gt in gts] - predictionImgList = [pred['pred_txt'] for pred in preds] - CSEval_results = CSEval.evaluateImgLists(predictionImgList, - groundTruthImgList, - CSEval.args)['averages'] - eval_results = OrderedDict() - eval_results['mAP'] = CSEval_results['allAp'] - eval_results['AP@50'] = CSEval_results['allAp50%'] - - return eval_results + return metric_results + + evaluate_results = { + f'{self.prefix}/{k}(%)': round(float(v) * 100, 4) + for k, v in metric_results.items() + } + return evaluate_results diff --git a/tests/test_evaluation/test_metrics/test_cityscapes_metric.py b/tests/test_evaluation/test_metrics/test_cityscapes_metric.py index 91a4f745dd6..d2d6975381c 100644 --- a/tests/test_evaluation/test_metrics/test_cityscapes_metric.py +++ b/tests/test_evaluation/test_metrics/test_cityscapes_metric.py @@ -1,3 +1,4 @@ +import math import os import os.path as osp import tempfile @@ -30,82 +31,132 @@ def test_init(self): with self.assertRaises(AssertionError): CityScapesMetric(outfile_prefix=None) + # test with seg_prefix = None + with self.assertRaises(AssertionError): + CityScapesMetric( + outfile_prefix='tmp/cityscapes/results', seg_prefix=None) + # test with format_only=True, keep_results=False with self.assertRaises(AssertionError): CityScapesMetric( - outfile_prefix=self.tmp_dir.name + 'test', + outfile_prefix='tmp/cityscapes/results', format_only=True, keep_results=False) @unittest.skipIf(cityscapesscripts is None, 'cityscapesscripts is not installed.') def test_evaluate(self): - dummy_mask1 = np.zeros((1, 20, 20), dtype=np.uint8) - dummy_mask1[:, :10, :10] = 1 - dummy_mask2 = np.zeros((1, 20, 20), dtype=np.uint8) - dummy_mask2[:, :10, :10] = 1 + tmp_dir = tempfile.TemporaryDirectory() - self.outfile_prefix = osp.join(self.tmp_dir.name, 'test') - self.seg_prefix = osp.join(self.tmp_dir.name, 'cityscapes/gtFine/val') - city = 'lindau' - sequenceNb = '000000' - frameNb = '000019' - img_name1 = f'{city}_{sequenceNb}_{frameNb}_gtFine_instanceIds.png' - img_path1 = osp.join(self.seg_prefix, city, img_name1) + dataset_metas = { + 'classes': ('person', 'rider', 'car', 'truck', 'bus', 'train', + 'motorcycle', 'bicycle') + } - frameNb = '000020' - img_name2 = f'{city}_{sequenceNb}_{frameNb}_gtFine_instanceIds.png' - img_path2 = osp.join(self.seg_prefix, city, img_name2) - os.makedirs(osp.join(self.seg_prefix, city)) + # create dummy data + self.seg_prefix = osp.join(tmp_dir.name, 'cityscapes', 'gtFine', 'val') + os.makedirs(self.seg_prefix, exist_ok=True) + data_samples = self._gen_fake_datasamples() - masks1 = np.zeros((20, 20), dtype=np.int32) - masks1[:10, :10] = 24 * 1000 - Image.fromarray(masks1).save(img_path1) + # test single evaluation + metric = CityScapesMetric( + dataset_meta=dataset_metas, + outfile_prefix=osp.join(tmp_dir.name, 'test'), + seg_prefix=self.seg_prefix, + keep_results=False, + keep_gt_json=False, + classwise=False) - masks2 = np.zeros((20, 20), dtype=np.int32) - masks2[:10, :10] = 24 * 1000 + 1 - Image.fromarray(masks2).save(img_path2) + metric.process({}, data_samples) + results = metric.evaluate() + targets = {'cityscapes/mAP(%)': 50.0, 'cityscapes/AP50(%)': 50.0} + self.assertDictEqual(results, targets) - data_samples = [{ - 'img_path': img_path1, - 'pred_instances': { - 'scores': torch.from_numpy(np.array([1.0])), - 'labels': torch.from_numpy(np.array([0])), - 'masks': torch.from_numpy(dummy_mask1) - } - }, { - 'img_path': img_path2, - 'pred_instances': { - 'scores': torch.from_numpy(np.array([0.98])), - 'labels': torch.from_numpy(np.array([1])), - 'masks': torch.from_numpy(dummy_mask2) - } - }] - - target = {'cityscapes/mAP': 0.5, 'cityscapes/AP@50': 0.5} + # test classwise result evaluation metric = CityScapesMetric( + dataset_meta=dataset_metas, + outfile_prefix=osp.join(tmp_dir.name, 'test'), seg_prefix=self.seg_prefix, - format_only=False, keep_results=False, - outfile_prefix=self.outfile_prefix) - metric.dataset_meta = dict( - classes=('person', 'rider', 'car', 'truck', 'bus', 'train', - 'motorcycle', 'bicycle')) + keep_gt_json=False, + classwise=True) + metric.process({}, data_samples) - results = metric.evaluate(size=2) - self.assertDictEqual(results, target) - del metric - self.assertTrue(not osp.exists('{self.outfile_prefix}.results')) + results = metric.evaluate() + mAP = results.pop('cityscapes/mAP(%)') + AP50 = results.pop('cityscapes/AP50(%)') + self.assertEqual(mAP, 50.0) + self.assertEqual(AP50, 50.0) + + # except person, others classes ap or ap50 should be nan + person_ap = results.pop('cityscapes/person_ap(%)') + person_ap50 = results.pop('cityscapes/person_ap50(%)') + self.assertEqual(person_ap, 50.0) + self.assertEqual(person_ap50, 50.0) + for v in results.values(): + self.assertTrue(math.isnan(v)) # test format_only metric = CityScapesMetric( - seg_prefix=self.seg_prefix, + dataset_meta=dataset_metas, format_only=True, + outfile_prefix=osp.join(tmp_dir.name, 'test'), + seg_prefix=self.seg_prefix, keep_results=True, - outfile_prefix=self.outfile_prefix) - metric.dataset_meta = dict( - classes=('person', 'rider', 'car', 'truck', 'bus', 'train', - 'motorcycle', 'bicycle')) + keep_gt_json=True, + classwise=True) + metric.process({}, data_samples) - results = metric.evaluate(size=2) + results = metric.evaluate() + self.assertTrue(osp.exists(f'{osp.join(tmp_dir.name, "test")}')) self.assertDictEqual(results, dict()) + + def _gen_fake_datasamples(self): + city = 'lindau' + os.makedirs(osp.join(self.seg_prefix, city), exist_ok=True) + + sequenceNb = '000000' + frameNb1 = '000019' + img_name1 = f'{city}_{sequenceNb}_{frameNb1}_gtFine_instanceIds.png' + img_path1 = osp.join(self.seg_prefix, city, img_name1) + + masks1 = np.zeros((20, 20), dtype=np.int32) + masks1[:10, :10] = 24 * 1000 + Image.fromarray(masks1).save(img_path1) + + dummy_mask1 = np.zeros((1, 20, 20), dtype=np.uint8) + dummy_mask1[:, :10, :10] = 1 + prediction1 = { + 'mask_scores': torch.from_numpy(np.array([1.0])), + 'labels': torch.from_numpy(np.array([0])), + 'masks': torch.from_numpy(dummy_mask1) + } + + frameNb2 = '000020' + img_name2 = f'{city}_{sequenceNb}_{frameNb2}_gtFine_instanceIds.png' + img_path2 = osp.join(self.seg_prefix, city, img_name2) + + masks2 = np.zeros((20, 20), dtype=np.int32) + masks2[:10, :10] = 24 * 1000 + 1 + Image.fromarray(masks2).save(img_path2) + + dummy_mask2 = np.zeros((1, 20, 20), dtype=np.uint8) + dummy_mask2[:, :10, :10] = 1 + prediction2 = { + 'mask_scores': torch.from_numpy(np.array([0.98])), + 'labels': torch.from_numpy(np.array([1])), + 'masks': torch.from_numpy(dummy_mask2) + } + + data_samples = [ + { + 'pred_instances': prediction1, + 'img_path': img_path1 + }, + { + 'pred_instances': prediction2, + 'img_path': img_path2 + }, + ] + + return data_samples