Add a quality setting for avoiding using bbox in point group matching (…

…#8634)

In some cases point groups do not represent a single object or a
bounding box attached to the group does not represent the object. A new
quality check option is added for such cases. The new option allows to
control whether to use the bbox for point group comparisons or not.

- Added a new quality setting to enable or disable using of group bbox
  for point group comparisons
- Improved OKS sigma description

changelog.d/20241101_140759_mzhiltso_compare_point_groups_in_image_space.md

@@ -0,0 +1,4 @@
+### Added
+
+- A quality setting to compare point groups without using bbox
+  (<https://github.com/cvat-ai/cvat/pull/8634>)

cvat-core/src/quality-settings.ts

@@ -14,6 +14,11 @@ export enum TargetMetric {
     RECALL = 'recall',
 }
 
+export enum PointSizeBase {
+    IMAGE_SIZE = 'image_size',
+    GROUP_BBOX_SIZE = 'group_bbox_size',
+}
+
 export default class QualitySettings {
     #id: number;
     #targetMetric: TargetMetric;
@@ -22,6 +27,7 @@ export default class QualitySettings {
     #task: number;
     #iouThreshold: number;
     #oksSigma: number;
+    #pointSizeBase: PointSizeBase;
     #lineThickness: number;
     #lowOverlapThreshold: number;
     #orientedLines: boolean;
@@ -42,6 +48,7 @@ export default class QualitySettings {
         this.#maxValidationsPerJob = initialData.max_validations_per_job;
         this.#iouThreshold = initialData.iou_threshold;
         this.#oksSigma = initialData.oks_sigma;
+        this.#pointSizeBase = initialData.point_size_base as PointSizeBase;
         this.#lineThickness = initialData.line_thickness;
         this.#lowOverlapThreshold = initialData.low_overlap_threshold;
         this.#orientedLines = initialData.compare_line_orientation;
@@ -79,6 +86,14 @@ export default class QualitySettings {
         this.#oksSigma = newVal;
     }
 
+    get pointSizeBase(): PointSizeBase {
+        return this.#pointSizeBase;
+    }
+
+    set pointSizeBase(newVal: PointSizeBase) {
+        this.#pointSizeBase = newVal;
+    }
+
     get lineThickness(): number {
         return this.#lineThickness;
     }
@@ -197,6 +212,7 @@ export default class QualitySettings {
         const result: SerializedQualitySettingsData = {
             iou_threshold: this.#iouThreshold,
             oks_sigma: this.#oksSigma,
+            point_size_base: this.#pointSizeBase,
             line_thickness: this.#lineThickness,
             low_overlap_threshold: this.#lowOverlapThreshold,
             compare_line_orientation: this.#orientedLines,

cvat-core/src/server-response-types.ts

@@ -247,6 +247,7 @@ export interface SerializedQualitySettingsData {
     max_validations_per_job?: number;
     iou_threshold?: number;
     oks_sigma?: number;
+    point_size_base?: string;
     line_thickness?: number;
     low_overlap_threshold?: number;
     compare_line_orientation?: boolean;

cvat-ui/src/components/quality-control/quality-control-page.tsx

@@ -225,6 +225,7 @@ function QualityControlPage(): JSX.Element {
                 settings.compareAttributes = values.compareAttributes;
 
                 settings.oksSigma = values.oksSigma / 100;
+                settings.pointSizeBase = values.pointSizeBase;
 
                 settings.lineThickness = values.lineThickness / 100;
                 settings.lineOrientationThreshold = values.lineOrientationThreshold / 100;

cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx

@@ -14,6 +14,7 @@ import Button from 'antd/lib/button';
 import Select from 'antd/lib/select';
 import CVATTooltip from 'components/common/cvat-tooltip';
 import { QualitySettings, TargetMetric } from 'cvat-core-wrapper';
+import { PointSizeBase } from 'cvat-core/src/quality-settings';
 
 interface Props {
     form: FormInstance;
@@ -35,6 +36,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
         compareAttributes: settings.compareAttributes,
 
         oksSigma: settings.oksSigma * 100,
+        pointSizeBase: settings.pointSizeBase,
 
         lineThickness: settings.lineThickness * 100,
         lineOrientationThreshold: settings.lineOrientationThreshold * 100,
@@ -50,7 +52,13 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
 
     const targetMetricDescription = `${settings.descriptions.targetMetric
         .replaceAll(/\* [a-z` -]+[A-Z]+/g, '')
-        .replaceAll(/\n/g, '')}.`;
+        .replaceAll(/\n/g, '')
+    }`;
+
+    const pointSizeBaseDescription = `${settings.descriptions.pointSizeBase
+        .substring(0, settings.descriptions.pointSizeBase.indexOf('\n\n\n'))
+        .replaceAll(/\n/g, ' ')
+    }`;
 
     const makeTooltipFragment = (metric: string, description: string): JSX.Element => (
         <div>
@@ -89,6 +97,10 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
         makeTooltipFragment('Object Keypoint Similarity (OKS)', settings.descriptions.oksSigma),
     );
 
+    const pointTooltip = makeTooltip(
+        makeTooltipFragment('Point size base', pointSizeBaseDescription),
+    );
+
     const linesTooltip = makeTooltip(
         <>
             {makeTooltipFragment('Line thickness', settings.descriptions.lineThickness)}
@@ -249,6 +261,38 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
                 </Col>
             </Row>
             <Divider />
+            <Row className='cvat-quality-settings-title'>
+                <Text strong>
+                    Point Comparison
+                </Text>
+                <CVATTooltip title={pointTooltip} className='cvat-analytics-tooltip' overlayStyle={{ maxWidth: '500px' }}>
+                    <QuestionCircleOutlined
+                        style={{ opacity: 0.5 }}
+                    />
+                </CVATTooltip>
+            </Row>
+            <Row>
+                <Col span={12}>
+                    <Form.Item
+                        name='pointSizeBase'
+                        label='Point size base'
+                        rules={[{ required: true }]}
+                    >
+                        <Select
+                            style={{ width: '70%' }}
+                            virtual={false}
+                        >
+                            <Select.Option value={PointSizeBase.IMAGE_SIZE}>
+                                Image size
+                            </Select.Option>
+                            <Select.Option value={PointSizeBase.GROUP_BBOX_SIZE}>
+                                Group bbox size
+                            </Select.Option>
+                        </Select>
+                    </Form.Item>
+                </Col>
+            </Row>
+            <Divider />
             <Row className='cvat-quality-settings-title'>
                 <Text strong>
                     Line Comparison

cvat/apps/quality_control/migrations/0004_qualitysettings_point_size_base.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.15 on 2024-11-06 15:39
+
+from django.db import migrations, models
+
+import cvat.apps.quality_control.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("quality_control", "0003_qualityreport_assignee_last_updated_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="qualitysettings",
+            name="point_size_base",
+            field=models.CharField(
+                choices=[("image_size", "IMAGE_SIZE"), ("group_bbox_size", "GROUP_BBOX_SIZE")],
+                default=cvat.apps.quality_control.models.PointSizeBase["GROUP_BBOX_SIZE"],
+                max_length=32,
+            ),
+        ),
+    ]

cvat/apps/quality_control/models.py

@@ -196,6 +196,18 @@ def clean(self) -> None:
             raise ValidationError(f"Unexpected type value '{self.type}'")
 
 
+class PointSizeBase(str, Enum):
+    IMAGE_SIZE = "image_size"
+    GROUP_BBOX_SIZE = "group_bbox_size"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @classmethod
+    def choices(cls):
+        return tuple((x.value, x.name) for x in cls)
+
+
 class QualitySettings(models.Model):
     task = models.OneToOneField(Task, on_delete=models.CASCADE, related_name="quality_settings")
 
@@ -205,6 +217,10 @@ class QualitySettings(models.Model):
 
     low_overlap_threshold = models.FloatField()
 
+    point_size_base = models.CharField(
+        max_length=32, choices=PointSizeBase.choices(), default=PointSizeBase.GROUP_BBOX_SIZE
+    )
+
     compare_line_orientation = models.BooleanField()
     line_orientation_threshold = models.FloatField()
 

cvat/apps/quality_control/quality_reports.py

@@ -187,6 +187,9 @@ class ComparisonParameters(_Serializable):
     oks_sigma: float = 0.09
     "Like IoU threshold, but for points, % of the bbox area to match a pair of points"
 
+    point_size_base: models.PointSizeBase = models.PointSizeBase.GROUP_BBOX_SIZE
+    "Determines how to obtain the object size for point comparisons"
+
     line_thickness: float = 0.01
     "Thickness of polylines, relatively to the (image area) ^ 0.5"
 
@@ -955,6 +958,7 @@ def __init__(
         # https://cocodataset.org/#keypoints-eval
         # https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py#L523
         oks_sigma: float = 0.09,
+        point_size_base: models.PointSizeBase = models.PointSizeBase.GROUP_BBOX_SIZE,
         compare_line_orientation: bool = False,
         line_torso_radius: float = 0.01,
         panoptic_comparison: bool = False,
@@ -968,6 +972,9 @@ def __init__(
         self.oks_sigma = oks_sigma
         "% of the shape area"
 
+        self.point_size_base = point_size_base
+        "Compare point groups using the group bbox size or the image size"
+
         self.compare_line_orientation = compare_line_orientation
         "Whether lines are oriented or not"
 
@@ -1293,13 +1300,20 @@ def _distance(a: dm.Points, b: dm.Points) -> float:
             else:
                 # Complex case: multiple points, grouped points, points with a bbox
                 # Try to align points and then return the metric
-                # match them in their bbox space
 
-                if dm.ops.bbox_iou(a_bbox, b_bbox) <= 0:
-                    return 0
+                if self.point_size_base == models.PointSizeBase.IMAGE_SIZE:
+                    scale = img_h * img_w
+                elif self.point_size_base == models.PointSizeBase.GROUP_BBOX_SIZE:
+                    # match points in their bbox space
 
-                bbox = dm.ops.mean_bbox([a_bbox, b_bbox])
-                scale = bbox[2] * bbox[3]
+                    if dm.ops.bbox_iou(a_bbox, b_bbox) <= 0:
+                        # this early exit may not work for points forming an axis-aligned line
+                        return 0
+
+                    bbox = dm.ops.mean_bbox([a_bbox, b_bbox])
+                    scale = bbox[2] * bbox[3]
+                else:
+                    assert False, f"Unknown point size base {self.point_size_base}"
 
                 a_points = np.reshape(a.points, (-1, 2))
                 b_points = np.reshape(b.points, (-1, 2))
@@ -1525,6 +1539,7 @@ def __init__(self, categories: dm.CategoriesInfo, *, settings: ComparisonParamet
             panoptic_comparison=settings.panoptic_comparison,
             iou_threshold=settings.iou_threshold,
             oks_sigma=settings.oks_sigma,
+            point_size_base=settings.point_size_base,
             line_torso_radius=settings.line_thickness,
             compare_line_orientation=False,  # should not be taken from outside, handled differently
         )

cvat/apps/quality_control/serializers.py

@@ -81,6 +81,7 @@ class Meta:
             "max_validations_per_job",
             "iou_threshold",
             "oks_sigma",
+            "point_size_base",
             "line_thickness",
             "low_overlap_threshold",
             "compare_line_orientation",
@@ -115,10 +116,26 @@ class Meta:
             """,
             "oks_sigma": """
                 Like IoU threshold, but for points.
-                The percent of the bbox area, used as the radius of the circle around the GT point,
-                where the checked point is expected to be.
+                The percent of the bbox side, used as the radius of the circle around the GT point,
+                where the checked point is expected to be. For boxes with different width and
+                height, the "side" is computed as a geometric mean of the width and height.
                 Read more: https://cocodataset.org/#keypoints-eval
             """,
+            "point_size_base": """
+                When comparing point annotations (including both separate points and point groups),
+                the OKS sigma parameter defines matching area for each GT point based to the
+                object size. The point size base parameter allows to configure how to determine
+                the object size.
+                If {image_size}, the image size is used. Useful if each point
+                annotation represents a separate object or boxes grouped with points do not
+                represent object boundaries.
+                If {group_bbox_size}, the object size is based on
+                the point group bbox size. Useful if each point group represents an object
+                or there is a bbox grouped with points, representing the object size.
+            """.format(
+                image_size=models.PointSizeBase.IMAGE_SIZE,
+                group_bbox_size=models.PointSizeBase.GROUP_BBOX_SIZE,
+            ),
             "line_thickness": """
                 Thickness of polylines, relatively to the (image area) ^ 0.5.
                 The distance to the boundary around the GT line,

cvat/schema.yml

@@ -9658,9 +9658,28 @@ components:
           format: double
           description: |
             Like IoU threshold, but for points.
-            The percent of the bbox area, used as the radius of the circle around the GT point,
-            where the checked point is expected to be.
+            The percent of the bbox side, used as the radius of the circle around the GT point,
+            where the checked point is expected to be. For boxes with different width and
+            height, the "side" is computed as a geometric mean of the width and height.
             Read more: https://cocodataset.org/#keypoints-eval
+        point_size_base:
+          allOf:
+          - $ref: '#/components/schemas/PointSizeBaseEnum'
+          description: |-
+            When comparing point annotations (including both separate points and point groups),
+            the OKS sigma parameter defines matching area for each GT point based to the
+            object size. The point size base parameter allows to configure how to determine
+            the object size.
+            If image_size, the image size is used. Useful if each point
+            annotation represents a separate object or boxes grouped with points do not
+            represent object boundaries.
+            If group_bbox_size, the object size is based on
+            the point group bbox size. Useful if each point group represents an object
+            or there is a bbox grouped with points, representing the object size.
+
+
+            * `image_size` - IMAGE_SIZE
+            * `group_bbox_size` - GROUP_BBOX_SIZE
         line_thickness:
           type: number
           format: double
@@ -9854,6 +9873,14 @@ components:
       - GIT_INTEGRATION
       - MODELS
       - PREDICT
+    PointSizeBaseEnum:
+      enum:
+      - image_size
+      - group_bbox_size
+      type: string
+      description: |-
+        * `image_size` - IMAGE_SIZE
+        * `group_bbox_size` - GROUP_BBOX_SIZE
     ProjectFileRequest:
       type: object
       properties:
@@ -10138,9 +10165,28 @@ components:
           format: double
           description: |
             Like IoU threshold, but for points.
-            The percent of the bbox area, used as the radius of the circle around the GT point,
-            where the checked point is expected to be.
+            The percent of the bbox side, used as the radius of the circle around the GT point,
+            where the checked point is expected to be. For boxes with different width and
+            height, the "side" is computed as a geometric mean of the width and height.
             Read more: https://cocodataset.org/#keypoints-eval
+        point_size_base:
+          allOf:
+          - $ref: '#/components/schemas/PointSizeBaseEnum'
+          description: |-
+            When comparing point annotations (including both separate points and point groups),
+            the OKS sigma parameter defines matching area for each GT point based to the
+            object size. The point size base parameter allows to configure how to determine
+            the object size.
+            If image_size, the image size is used. Useful if each point
+            annotation represents a separate object or boxes grouped with points do not
+            represent object boundaries.
+            If group_bbox_size, the object size is based on
+            the point group bbox size. Useful if each point group represents an object
+            or there is a bbox grouped with points, representing the object size.
+
+
+            * `image_size` - IMAGE_SIZE
+            * `group_bbox_size` - GROUP_BBOX_SIZE
         line_thickness:
           type: number
           format: double