Allow matching empty frames in quality checks (#8652)

### Motivation and context  Depends on #8634 Added a quality check option to consider frames matching, if both GT and job annotations have no annotations on a frame. This affects quality metrics and total counts in reports, but confusion matrices stay unchanged. This allows to use both positive and negative validation frames. ### How has this been tested?  Unit tests ### Checklist  - [ ] I submit my changes into the `develop` branch - [ ] I have created a changelog fragment  - [ ] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [ ] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning)) ### License - [ ] I submit _my code changes_ under the same [MIT License]( https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern.  ## Summary by CodeRabbit ## Release Notes - **New Features** - Introduced a quality setting for comparing point groups without bounding boxes. - Added an option to consider empty frames as matching in quality checks. - Enhanced quality settings form with new options for `matchEmptyFrames` and `useBboxSizeForPoints`. - **Bug Fixes** - Corrected property name from `peoject_id` to `project_id` in the API quality reports filter. - **Documentation** - Updated API schemas to include new quality settings properties. These changes improve the flexibility and accuracy of quality assessments within the application.
cvat-ai · Nov 11, 2024 · d315485 · d315485
1 parent ed33852
commit d315485
Show file tree

Hide file tree

Showing 13 changed files with 226 additions and 40 deletions.
diff --git a/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md b/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md
@@ -0,0 +1,4 @@
+### Added
+
+- A quality check option to consider empty frames matching
+  (<https://github.com/cvat-ai/cvat/pull/8652>)
diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts
@@ -38,6 +38,7 @@ export default class QualitySettings {
     #objectVisibilityThreshold: number;
     #panopticComparison: boolean;
     #compareAttributes: boolean;
+    #matchEmptyFrames: boolean;
     #descriptions: Record<string, string>;
 
     constructor(initialData: SerializedQualitySettingsData) {
@@ -59,6 +60,7 @@ export default class QualitySettings {
         this.#objectVisibilityThreshold = initialData.object_visibility_threshold;
         this.#panopticComparison = initialData.panoptic_comparison;
         this.#compareAttributes = initialData.compare_attributes;
+        this.#matchEmptyFrames = initialData.match_empty_frames;
         this.#descriptions = initialData.descriptions;
     }
 
@@ -198,6 +200,14 @@ export default class QualitySettings {
         this.#maxValidationsPerJob = newVal;
     }
 
+    get matchEmptyFrames(): boolean {
+        return this.#matchEmptyFrames;
+    }
+
+    set matchEmptyFrames(newVal: boolean) {
+        this.#matchEmptyFrames = newVal;
+    }
+
     get descriptions(): Record<string, string> {
         const descriptions: Record<string, string> = Object.keys(this.#descriptions).reduce((acc, key) => {
             const camelCaseKey = _.camelCase(key);
@@ -226,6 +236,7 @@ export default class QualitySettings {
             target_metric: this.#targetMetric,
             target_metric_threshold: this.#targetMetricThreshold,
             max_validations_per_job: this.#maxValidationsPerJob,
+            match_empty_frames: this.#matchEmptyFrames,
         };
 
         return result;

diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts
@@ -258,6 +258,7 @@ export interface SerializedQualitySettingsData {
     object_visibility_threshold?: number;
     panoptic_comparison?: boolean;
     compare_attributes?: boolean;
+    match_empty_frames?: boolean;
     descriptions?: Record<string, string>;
 }
 

diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx
@@ -223,6 +223,7 @@ function QualityControlPage(): JSX.Element {
                 settings.lowOverlapThreshold = values.lowOverlapThreshold / 100;
                 settings.iouThreshold = values.iouThreshold / 100;
                 settings.compareAttributes = values.compareAttributes;
+                settings.matchEmptyFrames = values.matchEmptyFrames;
 
                 settings.oksSigma = values.oksSigma / 100;
                 settings.pointSizeBase = values.pointSizeBase;

diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx
@@ -34,6 +34,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
         lowOverlapThreshold: settings.lowOverlapThreshold * 100,
         iouThreshold: settings.iouThreshold * 100,
         compareAttributes: settings.compareAttributes,
+        matchEmptyFrames: settings.matchEmptyFrames,
 
         oksSigma: settings.oksSigma * 100,
         pointSizeBase: settings.pointSizeBase,
@@ -79,6 +80,8 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
         <>
             {makeTooltipFragment('Target metric', targetMetricDescription)}
             {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)}
+            {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)}
+            {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)}
         </>,
     );
 
@@ -181,6 +184,30 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
                     </Form.Item>
                 </Col>
             </Row>
+            <Row>
+                <Col span={12}>
+                    <Form.Item
+                        name='compareAttributes'
+                        valuePropName='checked'
+                        rules={[{ required: true }]}
+                    >
+                        <Checkbox>
+                            <Text className='cvat-text-color'>Compare attributes</Text>
+                        </Checkbox>
+                    </Form.Item>
+                </Col>
+                <Col span={12}>
+                    <Form.Item
+                        name='matchEmptyFrames'
+                        valuePropName='checked'
+                        rules={[{ required: true }]}
+                    >
+                        <Checkbox>
+                            <Text className='cvat-text-color'>Match empty frames</Text>
+                        </Checkbox>
+                    </Form.Item>
+                </Col>
+            </Row>
             <Divider />
             <Row className='cvat-quality-settings-title'>
                 <Text strong>

diff --git a/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py b/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-11-05 14:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("quality_control", "0004_qualitysettings_point_size_base"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="qualitysettings",
+            name="match_empty_frames",
+            field=models.BooleanField(default=False),
+        ),
+    ]
diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py
@@ -234,6 +234,8 @@ class QualitySettings(models.Model):
 
     compare_attributes = models.BooleanField()
 
+    match_empty_frames = models.BooleanField(default=False)
+
     target_metric = models.CharField(
         max_length=32,
         choices=QualityTargetMetricType.choices(),

diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py
@@ -217,6 +217,13 @@ class ComparisonParameters(_Serializable):
     panoptic_comparison: bool = True
     "Use only the visible part of the masks and polygons in comparisons"
 
+    match_empty_frames: bool = False
+    """
+    Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy,
+    will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead.
+    This will also add virtual annotations to empty frames in the comparison results.
+    """
+
     def _value_serializer(self, v):
         if isinstance(v, dm.AnnotationType):
             return str(v.name)
@@ -232,11 +239,11 @@ def from_dict(cls, d: dict):
 @define(kw_only=True)
 class ConfusionMatrix(_Serializable):
     labels: List[str]
-    rows: np.array
-    precision: np.array
-    recall: np.array
-    accuracy: np.array
-    jaccard_index: Optional[np.array]
+    rows: np.ndarray
+    precision: np.ndarray
+    recall: np.ndarray
+    accuracy: np.ndarray
+    jaccard_index: Optional[np.ndarray]
 
     @property
     def axes(self):
@@ -1972,8 +1979,18 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
             gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX
             confusion_matrix[ds_label_idx, gt_label_idx] += 1
 
+        if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations:
+            # Add virtual annotations for empty frames
+            valid_labels_count = 1
+            total_labels_count = 1
+
+            valid_shapes_count = 1
+            total_shapes_count = 1
+            ds_shapes_count = 1
+            gt_shapes_count = 1
+
         self._frame_results[frame_id] = ComparisonReportFrameSummary(
-            annotations=self._generate_annotations_summary(
+            annotations=self._generate_frame_annotations_summary(
                 confusion_matrix, confusion_matrix_labels
             ),
             annotation_components=ComparisonReportAnnotationComponentsSummary(
@@ -2015,9 +2032,8 @@ def _make_zero_confusion_matrix(self) -> Tuple[List[str], np.ndarray, Dict[int,
 
         return label_names, confusion_matrix, label_id_idx_map
 
-    @classmethod
-    def _generate_annotations_summary(
-        cls, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str]
+    def _compute_annotations_summary(
+        self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str]
     ) -> ComparisonReportAnnotationsSummary:
         matched_ann_counts = np.diag(confusion_matrix)
         ds_ann_counts = np.sum(confusion_matrix, axis=1)
@@ -2037,10 +2053,10 @@ def _generate_annotations_summary(
         ) / (total_annotations_count or 1)
 
         valid_annotations_count = np.sum(matched_ann_counts)
-        missing_annotations_count = np.sum(confusion_matrix[cls._UNMATCHED_IDX, :])
-        extra_annotations_count = np.sum(confusion_matrix[:, cls._UNMATCHED_IDX])
-        ds_annotations_count = np.sum(ds_ann_counts[: cls._UNMATCHED_IDX])
-        gt_annotations_count = np.sum(gt_ann_counts[: cls._UNMATCHED_IDX])
+        missing_annotations_count = np.sum(confusion_matrix[self._UNMATCHED_IDX, :])
+        extra_annotations_count = np.sum(confusion_matrix[:, self._UNMATCHED_IDX])
+        ds_annotations_count = np.sum(ds_ann_counts[: self._UNMATCHED_IDX])
+        gt_annotations_count = np.sum(gt_ann_counts[: self._UNMATCHED_IDX])
 
         return ComparisonReportAnnotationsSummary(
             valid_count=valid_annotations_count,
@@ -2059,12 +2075,24 @@ def _generate_annotations_summary(
             ),
         )
 
-    def generate_report(self) -> ComparisonReport:
-        self._find_gt_conflicts()
+    def _generate_frame_annotations_summary(
+        self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str]
+    ) -> ComparisonReportAnnotationsSummary:
+        summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels)
+
+        if self.settings.match_empty_frames and summary.total_count == 0:
+            # Add virtual annotations for empty frames
+            summary.valid_count = 1
+            summary.total_count = 1
+            summary.ds_count = 1
+            summary.gt_count = 1
 
+        return summary
+
+    def _generate_dataset_annotations_summary(
+        self, frame_summaries: Dict[int, ComparisonReportFrameSummary]
+    ) -> Tuple[ComparisonReportAnnotationsSummary, ComparisonReportAnnotationComponentsSummary]:
         # accumulate stats
-        intersection_frames = []
-        conflicts = []
         annotation_components = ComparisonReportAnnotationComponentsSummary(
             shape=ComparisonReportAnnotationShapeSummary(
                 valid_count=0,
@@ -2082,19 +2110,52 @@ def generate_report(self) -> ComparisonReport:
             ),
         )
         mean_ious = []
+        empty_frame_count = 0
         confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix()
 
-        for frame_id, frame_result in self._frame_results.items():
-            intersection_frames.append(frame_id)
-            conflicts += frame_result.conflicts
+        for frame_result in frame_summaries.values():
             confusion_matrix += frame_result.annotations.confusion_matrix.rows
 
+            if not np.any(frame_result.annotations.confusion_matrix.rows):
+                empty_frame_count += 1
+
             if annotation_components is None:
                 annotation_components = deepcopy(frame_result.annotation_components)
             else:
                 annotation_components.accumulate(frame_result.annotation_components)
+
             mean_ious.append(frame_result.annotation_components.shape.mean_iou)
 
+        annotation_summary = self._compute_annotations_summary(
+            confusion_matrix, confusion_matrix_labels
+        )
+
+        if self.settings.match_empty_frames and empty_frame_count:
+            # Add virtual annotations for empty frames,
+            # they are not included in the confusion matrix
+            annotation_summary.valid_count += empty_frame_count
+            annotation_summary.total_count += empty_frame_count
+            annotation_summary.ds_count += empty_frame_count
+            annotation_summary.gt_count += empty_frame_count
+
+        # Cannot be computed in accumulate()
+        annotation_components.shape.mean_iou = np.mean(mean_ious)
+
+        return annotation_summary, annotation_components
+
+    def generate_report(self) -> ComparisonReport:
+        self._find_gt_conflicts()
+
+        intersection_frames = []
+        conflicts = []
+        for frame_id, frame_result in self._frame_results.items():
+            intersection_frames.append(frame_id)
+            conflicts += frame_result.conflicts
+
+        annotation_summary, annotations_component_summary = (
+            self._generate_dataset_annotations_summary(self._frame_results)
+        )
+
         return ComparisonReport(
             parameters=self.settings,
             comparison_summary=ComparisonReportComparisonSummary(
@@ -2110,25 +2171,8 @@ def generate_report(self) -> ComparisonReport:
                     [c for c in conflicts if c.severity == AnnotationConflictSeverity.ERROR]
                 ),
                 conflicts_by_type=Counter(c.type for c in conflicts),
-                annotations=self._generate_annotations_summary(
-                    confusion_matrix, confusion_matrix_labels
-                ),
-                annotation_components=ComparisonReportAnnotationComponentsSummary(
-                    shape=ComparisonReportAnnotationShapeSummary(
-                        valid_count=annotation_components.shape.valid_count,
-                        missing_count=annotation_components.shape.missing_count,
-                        extra_count=annotation_components.shape.extra_count,
-                        total_count=annotation_components.shape.total_count,
-                        ds_count=annotation_components.shape.ds_count,
-                        gt_count=annotation_components.shape.gt_count,
-                        mean_iou=np.mean(mean_ious),
-                    ),
-                    label=ComparisonReportAnnotationLabelSummary(
-                        valid_count=annotation_components.label.valid_count,
-                        invalid_count=annotation_components.label.invalid_count,
-                        total_count=annotation_components.label.total_count,
-                    ),
-                ),
+                annotations=annotation_summary,
+                annotation_components=annotations_component_summary,
             ),
             frame_results=self._frame_results,
         )

diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py
@@ -92,13 +92,15 @@ class Meta:
             "object_visibility_threshold",
             "panoptic_comparison",
             "compare_attributes",
+            "match_empty_frames",
         )
         read_only_fields = (
             "id",
             "task_id",
         )
 
         extra_kwargs = {k: {"required": False} for k in fields}
+        extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False)
 
         for field_name, help_text in {
             "target_metric": "The primary metric used for quality estimation",
@@ -164,6 +166,12 @@ class Meta:
                 Use only the visible part of the masks and polygons in comparisons
             """,
             "compare_attributes": "Enables or disables annotation attribute comparison",
+            "match_empty_frames": """
+                Count empty frames as matching. This affects target metrics like accuracy in cases
+                there are no annotations. If disabled, frames without annotations
+                are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
+                This will also add virtual annotations to empty frames in the comparison results.
+            """,
         }.items():
             extra_kwargs.setdefault(field_name, {}).setdefault(
                 "help_text", textwrap.dedent(help_text.lstrip("\n"))

diff --git a/cvat/schema.yml b/cvat/schema.yml
@@ -9729,6 +9729,14 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
+        match_empty_frames:
+          type: boolean
+          default: false
+          description: |
+            Count empty frames as matching. This affects target metrics like accuracy in cases
+            there are no annotations. If disabled, frames without annotations
+            are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
+            This will also add virtual annotations to empty frames in the comparison results.
     PatchedTaskValidationLayoutWriteRequest:
       type: object
       properties:
@@ -10236,6 +10244,14 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
+        match_empty_frames:
+          type: boolean
+          default: false
+          description: |
+            Count empty frames as matching. This affects target metrics like accuracy in cases
+            there are no annotations. If disabled, frames without annotations
+            are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
+            This will also add virtual annotations to empty frames in the comparison results.
     RegisterSerializerEx:
       type: object
       properties: