[GSoC'24] Consensus annotation

.vscode/launch.json

@@ -360,6 +360,28 @@
             },
             "console": "internalConsole"
         },
+        {
+            "name": "server: RQ - consensus",
+            "type": "debugpy",
+            "request": "launch",
+            "stopOnEntry": false,
+            "justMyCode": false,
+            "python": "${command:python.interpreterPath}",
+            "program": "${workspaceRoot}/manage.py",
+            "args": [
+                "rqworker",
+                "consensus",
+                "--worker-class",
+                "cvat.rqworker.SimpleWorker"
+            ],
+            "django": true,
+            "cwd": "${workspaceFolder}",
+            "env": {
+                "DJANGO_LOG_SERVER_HOST": "localhost",
+                "DJANGO_LOG_SERVER_PORT": "8282"
+            },
+            "console": "internalConsole"
+        },
         {
             "name": "server: migrate",
             "type": "debugpy",
@@ -537,7 +559,8 @@
                 "server: RQ - scheduler",
                 "server: RQ - quality reports",
                 "server: RQ - analytics reports",
-                "server: RQ - cleaning"
+                "server: RQ - cleaning",
+                "server: RQ - consensus",
             ]
         }
     ]

cvat-core/src/api-implementation.ts

@@ -31,8 +31,11 @@ import Organization, { Invitation } from './organization';
 import Webhook from './webhook';
 import { ArgumentError } from './exceptions';
 import {
-    AnalyticsReportFilter, QualityConflictsFilter, QualityReportsFilter,
+    AnalyticsReportFilter, ConflictsFilter, QualityReportsFilter,
     QualitySettingsFilter, SerializedAsset,
+    ConsensusReportsFilter,
+    AssigneeConsensusReportsFilter,
+    ConsensusSettingsFilter,
 } from './server-response-types';
 import QualityReport from './quality-report';
 import QualityConflict, { ConflictSeverity } from './quality-conflict';
@@ -44,11 +47,85 @@ import { convertDescriptions, getServerAPISchema } from './server-schema';
 import { JobType } from './enums';
 import { PaginatedResource } from './core-types';
 import CVATCore from '.';
+import ConsensusSettings from './consensus-settings';
+import ConsensusReport from './consensus-report';
+import AssigneeConsensusReport from './assignee-consensus-report';
+import ConsensusConflict from './consensus-conflict';
 
 function implementationMixin(func: Function, implementation: Function): void {
     Object.assign(func, { implementation });
 }
 
+type ConflictType = ConsensusConflict | QualityConflict;
+
+function mergeConflicts<T extends ConflictType>(conflicts: T[]): T[] {
+    const frames = Array.from(new Set(conflicts.map((conflict) => conflict.frame)))
+        .sort((a, b) => a - b);
+
+    const mergedConflicts: T[] = [];
+
+    for (const frame of frames) {
+        const frameConflicts = conflicts.filter((conflict) => conflict.frame === frame);
+        const conflictsByObject: Record<string, T[]> = {};
+
+        frameConflicts.forEach((qualityConflict: T) => {
+            const { type, serverID } = qualityConflict.annotationConflicts[0];
+            const firstObjID = `${type}_${serverID}`;
+            conflictsByObject[firstObjID] = conflictsByObject[firstObjID] || [];
+            conflictsByObject[firstObjID].push(qualityConflict);
+        });
+
+        for (const objectConflicts of Object.values(conflictsByObject)) {
+            if (objectConflicts.length === 1) {
+                mergedConflicts.push(objectConflicts[0]);
+            } else {
+                const [firstConflict] = objectConflicts;
+                let mainObjectConflict: T;
+
+                if (firstConflict instanceof QualityConflict) {
+                    mainObjectConflict = objectConflicts.find(
+                        (conflict) => (conflict as QualityConflict).severity === ConflictSeverity.ERROR,
+                    ) || firstConflict;
+                } else {
+                    mainObjectConflict = firstConflict;
+                }
+                const descriptionList: string[] = [mainObjectConflict.description];
+
+                for (const objectConflict of objectConflicts) {
+                    if (objectConflict !== mainObjectConflict) {
+                        descriptionList.push(objectConflict.description);
+
+                        for (const annotationConflict of objectConflict.annotationConflicts) {
+                            if (!mainObjectConflict.annotationConflicts.find((_annotationConflict) => (
+                                _annotationConflict.serverID === annotationConflict.serverID &&
+                                _annotationConflict.type === annotationConflict.type))
+                            ) {
+                                mainObjectConflict.annotationConflicts.push(annotationConflict);
+                            }
+                        }
+                    }
+                }
+
+                const description = descriptionList.join(', ');
+                const visibleConflict = new Proxy(mainObjectConflict, {
+                    get(target, prop) {
+                        if (prop === 'description') {
+                            return description;
+                        }
+
+                        const val = Reflect.get(target, prop);
+                        return typeof val === 'function' ? (...args: any[]) => val.apply(target, args) : val;
+                    },
+                });
+
+                mergedConflicts.push(visibleConflict);
+            }
+        }
+    }
+
+    return mergedConflicts;
+}
+
 export default function implementAPI(cvat: CVATCore): CVATCore {
     implementationMixin(cvat.plugins.list, PluginRegistry.list);
     implementationMixin(cvat.plugins.register, PluginRegistry.register.bind(cvat));
@@ -434,7 +511,7 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
         );
         return reports;
     });
-    implementationMixin(cvat.analytics.quality.conflicts, async (filter: QualityConflictsFilter) => {
+    implementationMixin(cvat.analytics.quality.conflicts, async (filter: ConflictsFilter) => {
         checkFilter(filter, {
             reportID: isInteger,
         });
@@ -443,72 +520,7 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
 
         const conflictsData = await serverProxy.analytics.quality.conflicts(params);
         const conflicts = conflictsData.map((conflict) => new QualityConflict({ ...conflict }));
-        const frames = Array.from(new Set(conflicts.map((conflict) => conflict.frame)))
-            .sort((a, b) => a - b);
-
-        // each QualityConflict may have several AnnotationConflicts bound
-        // at the same time, many quality conflicts may refer
-        // to the same labeled object (e.g. mismatch label, low overlap)
-        // the code below unites quality conflicts bound to the same object into one QualityConflict object
-        const mergedConflicts: QualityConflict[] = [];
-
-        for (const frame of frames) {
-            const frameConflicts = conflicts.filter((conflict) => conflict.frame === frame);
-            const conflictsByObject: Record<string, QualityConflict[]> = {};
-
-            frameConflicts.forEach((qualityConflict: QualityConflict) => {
-                const { type, serverID } = qualityConflict.annotationConflicts[0];
-                const firstObjID = `${type}_${serverID}`;
-                conflictsByObject[firstObjID] = conflictsByObject[firstObjID] || [];
-                conflictsByObject[firstObjID].push(qualityConflict);
-            });
-
-            for (const objectConflicts of Object.values(conflictsByObject)) {
-                if (objectConflicts.length === 1) {
-                    // only one quality conflict refers to the object on current frame
-                    mergedConflicts.push(objectConflicts[0]);
-                } else {
-                    const mainObjectConflict = objectConflicts
-                        .find((conflict) => conflict.severity === ConflictSeverity.ERROR) || objectConflicts[0];
-                    const descriptionList: string[] = [mainObjectConflict.description];
-
-                    for (const objectConflict of objectConflicts) {
-                        if (objectConflict !== mainObjectConflict) {
-                            descriptionList.push(objectConflict.description);
-
-                            for (const annotationConflict of objectConflict.annotationConflicts) {
-                                if (!mainObjectConflict.annotationConflicts.find((_annotationConflict) => (
-                                    _annotationConflict.serverID === annotationConflict.serverID &&
-                                    _annotationConflict.type === annotationConflict.type))
-                                ) {
-                                    mainObjectConflict.annotationConflicts.push(annotationConflict);
-                                }
-                            }
-                        }
-                    }
-
-                    // decorate the original conflict to avoid changing it
-                    const description = descriptionList.join(', ');
-                    const visibleConflict = new Proxy(mainObjectConflict, {
-                        get(target, prop) {
-                            if (prop === 'description') {
-                                return description;
-                            }
-
-                            // By default, it looks like Reflect.get(target, prop, receiver)
-                            // which has a different value of `this`. It doesn't allow to
-                            // work with methods / properties that use private members.
-                            const val = Reflect.get(target, prop);
-                            return typeof val === 'function' ? (...args: any[]) => val.apply(target, args) : val;
-                        },
-                    });
-
-                    mergedConflicts.push(visibleConflict);
-                }
-            }
-        }
-
-        return mergedConflicts;
+        return mergeConflicts(conflicts);
     });
     implementationMixin(cvat.analytics.quality.settings.get, async (filter: QualitySettingsFilter) => {
         checkFilter(filter, {
@@ -523,6 +535,58 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
 
         return new QualitySettings({ ...settings, descriptions });
     });
+    implementationMixin(cvat.consensus.reports, async (filter: ConsensusReportsFilter) => {
+        checkFilter(filter, {
+            page: isInteger,
+            pageSize: isPageSize,
+            projectID: isInteger,
+            taskID: isInteger,
+            jobID: isInteger,
+            filter: isString,
+            search: isString,
+            target: isString,
+            sort: isString,
+        });
+
+        const params = fieldsToSnakeCase({ ...filter, sort: '-created_date' });
+
+        const reportsData = await serverProxy.consensus.reports(params);
+        const reports = Object.assign(
+            reportsData.map((report) => new ConsensusReport({ ...report })),
+            { count: reportsData.count },
+        );
+        return reports;
+    });
+    implementationMixin(cvat.consensus.assigneeReports, async (filter: AssigneeConsensusReportsFilter) => {
+        checkFilter(filter, {
+            page: isInteger,
+            pageSize: isPageSize,
+            taskID: isInteger,
+            filter: isString,
+            consensusReportID: isInteger,
+            search: isString,
+            sort: isString,
+        });
+
+        const params = fieldsToSnakeCase({ ...filter, sort: '-id' });
+
+        const reportsData = await serverProxy.consensus.assigneeReports(params);
+        const reports = Object.assign(
+            reportsData.map((report) => new AssigneeConsensusReport({ ...report })),
+            { count: reportsData.count },
+        );
+        return reports;
+    });
+    implementationMixin(cvat.consensus.settings.get, async (filter: ConsensusSettingsFilter) => {
+        checkFilter(filter, {
+            taskID: isInteger,
+        });
+
+        const params = fieldsToSnakeCase(filter);
+
+        const settings = await serverProxy.consensus.settings.get(params);
+        return new ConsensusSettings({ ...settings });
+    });
     implementationMixin(cvat.analytics.performance.reports, async (filter: AnalyticsReportFilter) => {
         checkFilter(filter, {
             jobID: isInteger,
@@ -538,6 +602,17 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
         const reportData = await serverProxy.analytics.performance.reports(params);
         return new AnalyticsReport(reportData);
     });
+    implementationMixin(cvat.consensus.conflicts, async (filter: ConflictsFilter) => {
+        checkFilter(filter, {
+            reportID: isInteger,
+        });
+
+        const params = fieldsToSnakeCase(filter);
+
+        const conflictsData = await serverProxy.consensus.conflicts(params);
+        const conflicts = conflictsData.map((conflict) => new ConsensusConflict({ ...conflict }));
+        return mergeConflicts(conflicts);
+    });
     implementationMixin(cvat.analytics.performance.calculate, async (
         body: Parameters<CVATCore['analytics']['performance']['calculate']>[0],
         onUpdate: Parameters<CVATCore['analytics']['performance']['calculate']>[1],

cvat-core/src/api.ts

@@ -401,6 +401,26 @@ function build(): CVATCore {
                 return result;
             },
         },
+        consensus: {
+            async assigneeReports(filter = {}) {
+                const result = await PluginRegistry.apiWrapper(cvat.consensus.assigneeReports, filter);
+                return result;
+            },
+            async reports(filter = {}) {
+                const result = await PluginRegistry.apiWrapper(cvat.consensus.reports, filter);
+                return result;
+            },
+            async conflicts(filter = {}) {
+                const result = await PluginRegistry.apiWrapper(cvat.consensus.conflicts, filter);
+                return result;
+            },
+            settings: {
+                async get(filter = {}) {
+                    const result = await PluginRegistry.apiWrapper(cvat.consensus.settings.get, filter);
+                    return result;
+                },
+            },
+        },
         classes: {
             User,
             Project: implementProject(Project),
@@ -452,6 +472,7 @@ function build(): CVATCore {
     cvat.organizations = Object.freeze(cvat.organizations);
     cvat.webhooks = Object.freeze(cvat.webhooks);
     cvat.analytics = Object.freeze(cvat.analytics);
+    cvat.consensus = Object.freeze(cvat.consensus);
     cvat.classes = Object.freeze(cvat.classes);
     cvat.utils = Object.freeze(cvat.utils);
 

cvat-core/src/assignee-consensus-report.ts

@@ -0,0 +1,53 @@
+// Copyright (C) 2024 CVAT.ai Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import { SerializedAssigneeConsensusReportData } from './server-response-types';
+import User from './user';
+
+export default class AssigneeConsensusReport {
+    #id: number;
+    #taskID: number;
+    #assignee: User;
+    #consensusScore: number;
+    #conflictCount: number;
+    #consensusReportID: number;
+
+    constructor(initialData: SerializedAssigneeConsensusReportData) {
+        this.#id = initialData.id;
+        this.#taskID = initialData.task_id;
+        this.#consensusScore = initialData.consensus_score;
+        this.#consensusReportID = initialData.consensus_report_id;
+        this.#conflictCount = initialData.conflict_count;
+
+        if (initialData.assignee) {
+            this.#assignee = new User(initialData.assignee);
+        } else {
+            this.#assignee = null;
+        }
+    }
+
+    get id(): number {
+        return this.#id;
+    }
+
+    get taskID(): number {
+        return this.#taskID;
+    }
+
+    get assignee(): User {
+        return this.#assignee;
+    }
+
+    get consensusScore(): number {
+        return this.#consensusScore;
+    }
+
+    get conflictCount(): number {
+        return this.#conflictCount;
+    }
+
+    get consensusReportID(): number {
+        return this.#consensusReportID;
+    }
+}