Skip to content

Commit 7b67f6b

Browse files
mdurrani808mdurrani808
authored
Add RKNN / Object Detection Pipeline (PhotonVision#1144)
Tested on Orange Pi 5 and Cool Pi 4B. Merge with parts of the OpenCV DNN PR. Adds support for YOLOv5s models for Rockchip CPUs with a NPU. Right now hard coded to a note model from alex_idk. Very much still incubating and largely untested.
1 parent e1f550a commit 7b67f6b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+830
-63
lines changed

‎.styleguide

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ modifiableFileExclude {
1818
\.dll$
1919
\.webp$
2020
\.ico$
21+
\.rknn$
2122
gradlew
2223
}
2324

‎build.gradle

+2
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,11 @@ ext {
3030
joglVersion = "2.4.0-rc-20200307"
3131
javalinVersion = "5.6.2"
3232
photonGlDriverLibVersion = "dev-v2023.1.0-9-g75fc678"
33+
rknnVersion = "dev-v2024.0.0-30-g001b5ec"
3334
frcYear = "2024"
3435
mrcalVersion = "dev-v2024.0.0-7-gc976aaa";
3536

37+
3638
pubVersion = versionString
3739
isDev = pubVersion.startsWith("dev")
3840

‎photon-client/index.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<!DOCTYPE html>
1+
<!doctype html>
22
<html lang="en">
33
<head>
44
<meta charset="UTF-8" />

‎photon-client/package-lock.json

+4-4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎photon-client/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,17 @@
2727
},
2828
"devDependencies": {
2929
"@rushstack/eslint-patch": "^1.3.2",
30-
"@vue/eslint-config-prettier": "^9.0.0",
31-
"@vue/eslint-config-typescript": "^12.0.0",
32-
"prettier": "^3.1.1",
3330
"@types/node": "^16.11.45",
3431
"@types/three": "^0.160.0",
3532
"@vitejs/plugin-vue2": "^2.3.1",
33+
"@vue/eslint-config-prettier": "^9.0.0",
34+
"@vue/eslint-config-typescript": "^12.0.0",
3635
"@vue/tsconfig": "^0.5.1",
3736
"deepmerge": "^4.3.1",
3837
"eslint": "^8.56.0",
3938
"eslint-plugin-vue": "^9.19.2",
4039
"npm-run-all": "^4.1.5",
40+
"prettier": "3.2.2",
4141
"sass": "~1.32",
4242
"sass-loader": "^13.3.2",
4343
"terser": "^5.14.2",

‎photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

+11-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { computed, ref } from "vue";
77
import PvIcon from "@/components/common/pv-icon.vue";
88
import PvInput from "@/components/common/pv-input.vue";
99
import { PipelineType } from "@/types/PipelineTypes";
10+
import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
1011
1112
const changeCurrentCameraIndex = (index: number) => {
1213
useCameraSettingsStore().setCurrentCameraIndex(index, true);
@@ -24,6 +25,8 @@ const changeCurrentCameraIndex = (index: number) => {
2425
case PipelineType.Aruco:
2526
pipelineType.value = WebsocketPipelineType.Aruco;
2627
break;
28+
case PipelineType.ObjectDetection:
29+
pipelineType.value = WebsocketPipelineType.ObjectDetection;
2730
}
2831
};
2932
@@ -154,6 +157,9 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
154157
{ name: "AprilTag", value: WebsocketPipelineType.AprilTag },
155158
{ name: "Aruco", value: WebsocketPipelineType.Aruco }
156159
];
160+
if (useSettingsStore().general.rknnSupported) {
161+
pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
162+
}
157163
158164
if (useCameraSettingsStore().isDriverMode) {
159165
pipelineTypes.push({ name: "Driver Mode", value: WebsocketPipelineType.DriverMode });
@@ -208,6 +214,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
208214
case PipelineType.Aruco:
209215
pipelineType.value = WebsocketPipelineType.Aruco;
210216
break;
217+
case PipelineType.ObjectDetection:
218+
pipelineType.value = WebsocketPipelineType.ObjectDetection;
219+
break;
211220
}
212221
});
213222
</script>
@@ -354,7 +363,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
354363
{ name: 'Reflective', value: WebsocketPipelineType.Reflective },
355364
{ name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
356365
{ name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
357-
{ name: 'Aruco', value: WebsocketPipelineType.Aruco }
366+
{ name: 'Aruco', value: WebsocketPipelineType.Aruco },
367+
{ name: 'Object Detection', value: WebsocketPipelineType.ObjectDetection }
358368
]"
359369
/>
360370
</v-card-text>

‎photon-client/src/components/dashboard/ConfigOptions.vue

+22-6
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import ThresholdTab from "@/components/dashboard/tabs/ThresholdTab.vue";
88
import ContoursTab from "@/components/dashboard/tabs/ContoursTab.vue";
99
import AprilTagTab from "@/components/dashboard/tabs/AprilTagTab.vue";
1010
import ArucoTab from "@/components/dashboard/tabs/ArucoTab.vue";
11+
import ObjectDetectionTab from "@/components/dashboard/tabs/ObjectDetectionTab.vue";
1112
import OutputTab from "@/components/dashboard/tabs/OutputTab.vue";
1213
import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue";
1314
import PnPTab from "@/components/dashboard/tabs/PnPTab.vue";
@@ -40,6 +41,10 @@ const allTabs = Object.freeze({
4041
tabName: "Aruco",
4142
component: ArucoTab
4243
},
44+
objectDetectionTab: {
45+
tabName: "Object Detection",
46+
component: ObjectDetectionTab
47+
},
4348
outputTab: {
4449
tabName: "Output",
4550
component: OutputTab
@@ -75,21 +80,29 @@ const getTabGroups = (): ConfigOption[][] => {
7580
allTabs.contoursTab,
7681
allTabs.apriltagTab,
7782
allTabs.arucoTab,
83+
allTabs.objectDetectionTab,
7884
allTabs.outputTab
7985
],
8086
[allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
8187
];
8288
} else if (lgAndDown) {
8389
return [
8490
[allTabs.inputTab],
85-
[allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
91+
[
92+
allTabs.thresholdTab,
93+
allTabs.contoursTab,
94+
allTabs.apriltagTab,
95+
allTabs.arucoTab,
96+
allTabs.objectDetectionTab,
97+
allTabs.outputTab
98+
],
8699
[allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
87100
];
88101
} else if (xl) {
89102
return [
90103
[allTabs.inputTab],
91104
[allTabs.thresholdTab],
92-
[allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
105+
[allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.objectDetectionTab, allTabs.outputTab],
93106
[allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
94107
];
95108
}
@@ -103,17 +116,20 @@ const tabGroups = computed<ConfigOption[][]>(() => {
103116
const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled;
104117
const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag;
105118
const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
119+
const isObjectDetection =
120+
useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection;
106121
107122
return getTabGroups()
108123
.map((tabGroup) =>
109124
tabGroup.filter(
110125
(tabConfig) =>
111126
!(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
112-
!((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
113-
!((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
114-
!((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
127+
!((!allow3d || isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
128+
!((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
129+
!((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
115130
!(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
116-
!(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco
131+
!(!isAruco && tabConfig.tabName === "Aruco") &&
132+
!(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out aruco unless we actually are doing Aruco
117133
)
118134
)
119135
.filter((it) => it.length); // Remove empty tab groups

‎photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<script setup lang="ts">
2+
import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
3+
import { PipelineType } from "@/types/PipelineTypes";
4+
import PvSlider from "@/components/common/pv-slider.vue";
5+
import { computed, getCurrentInstance } from "vue";
6+
import { useStateStore } from "@/stores/StateStore";
7+
8+
// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
9+
// Defer reference to store access method
10+
const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
11+
12+
const interactiveCols = computed(
13+
() =>
14+
(getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
15+
(!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
16+
)
17+
? 9
18+
: 8;
19+
</script>
20+
21+
<template>
22+
<div v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
23+
<pv-slider
24+
v-model="currentPipelineSettings.confidence"
25+
class="pt-2"
26+
:slider-cols="interactiveCols"
27+
label="Confidence"
28+
tooltip="The minimum confidence for a detection to be considered valid. Bigger numbers mean fewer but more probable detections are allowed through."
29+
:min="0"
30+
:max="1"
31+
:step="0.01"
32+
@input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
33+
/>
34+
</div>
35+
</template>

‎photon-client/src/components/dashboard/tabs/TargetsTab.vue

+16
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ const resetCurrentBuffer = () => {
4848
>
4949
Fiducial ID
5050
</th>
51+
<template v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
52+
<th class="text-center white--text">Class</th>
53+
<th class="text-center white--text">Confidence</th>
54+
</template>
5155
<template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
5256
<th class="text-center white--text">Pitch &theta;&deg;</th>
5357
<th class="text-center white--text">Yaw &theta;&deg;</th>
@@ -85,6 +89,18 @@ const resetCurrentBuffer = () => {
8589
>
8690
{{ target.fiducialId }}
8791
</td>
92+
<td
93+
v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
94+
class="text-center white--text"
95+
>
96+
{{ useStateStore().currentPipelineResults?.classNames[target.classId] }}
97+
</td>
98+
<td
99+
v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
100+
class="text-center white--text"
101+
>
102+
{{ target.confidence.toFixed(2) }}
103+
</td>
88104
<template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
89105
<td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
90106
<td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>

‎photon-client/src/stores/settings/GeneralSettingsStore.ts

+4-2
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ export const useSettingsStore = defineStore("settings", {
2727
gpuAcceleration: undefined,
2828
hardwareModel: undefined,
2929
hardwarePlatform: undefined,
30-
mrCalWorking: true
30+
mrCalWorking: true,
31+
rknnSupported: false
3132
},
3233
network: {
3334
ntServerAddress: "",
@@ -99,7 +100,8 @@ export const useSettingsStore = defineStore("settings", {
99100
hardwareModel: data.general.hardwareModel || undefined,
100101
hardwarePlatform: data.general.hardwarePlatform || undefined,
101102
gpuAcceleration: data.general.gpuAcceleration || undefined,
102-
mrCalWorking: data.general.mrCalWorking
103+
mrCalWorking: data.general.mrCalWorking,
104+
rknnSupported: data.general.rknnSupported
103105
};
104106
this.lighting = data.lighting;
105107
this.network = data.networkSettings;

‎photon-client/src/types/PhotonTrackingTypes.ts

+4
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ export interface PhotonTarget {
5454
ambiguity: number;
5555
// -1 if not set
5656
fiducialId: number;
57+
confidence: number;
58+
classId: number;
5759
// undefined if 3d isn't enabled
5860
pose?: Transform3d;
5961
}
@@ -70,4 +72,6 @@ export interface PipelineResult {
7072
targets: PhotonTarget[];
7173
// undefined if multitag failed or non-tag pipeline
7274
multitagResult?: MultitagResult;
75+
// Object detection class names -- empty if not doing object detection
76+
classNames: string[];
7377
}

‎photon-client/src/types/PipelineTypes.ts

+29-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ export enum PipelineType {
55
Reflective = 2,
66
ColoredShape = 3,
77
AprilTag = 4,
8-
Aruco = 5
8+
Aruco = 5,
9+
ObjectDetection = 6
910
}
1011

1112
export enum AprilTagFamily {
@@ -281,14 +282,39 @@ export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
281282
doSingleTargetAlways: false
282283
};
283284

285+
export interface ObjectDetectionPipelineSettings extends PipelineSettings {
286+
pipelineType: PipelineType.ObjectDetection;
287+
confidence: number;
288+
nms: number;
289+
box_thresh: number;
290+
}
291+
export type ConfigurableObjectDetectionPipelineSettings = Partial<
292+
Omit<ObjectDetectionPipelineSettings, "pipelineType">
293+
> &
294+
ConfigurablePipelineSettings;
295+
export const DefaultObjectDetectionPipelineSettings: ObjectDetectionPipelineSettings = {
296+
...DefaultPipelineSettings,
297+
pipelineType: PipelineType.ObjectDetection,
298+
cameraGain: 20,
299+
targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
300+
ledMode: true,
301+
outputShowMultipleTargets: false,
302+
cameraExposure: 6,
303+
confidence: 0.9,
304+
nms: 0.45,
305+
box_thresh: 0.25
306+
};
307+
284308
export type ActivePipelineSettings =
285309
| ReflectivePipelineSettings
286310
| ColoredShapePipelineSettings
287311
| AprilTagPipelineSettings
288-
| ArucoPipelineSettings;
312+
| ArucoPipelineSettings
313+
| ObjectDetectionPipelineSettings;
289314

290315
export type ActiveConfigurablePipelineSettings =
291316
| ConfigurableReflectivePipelineSettings
292317
| ConfigurableColoredShapePipelineSettings
293318
| ConfigurableAprilTagPipelineSettings
294-
| ConfigurableArucoPipelineSettings;
319+
| ConfigurableArucoPipelineSettings
320+
| ConfigurableObjectDetectionPipelineSettings;

‎photon-client/src/types/SettingTypes.ts

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ export interface GeneralSettings {
77
hardwareModel?: string;
88
hardwarePlatform?: string;
99
mrCalWorking: boolean;
10+
rknnSupported: boolean;
1011
}
1112

1213
export interface MetricData {

‎photon-client/src/types/WebsocketDataTypes.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -101,5 +101,6 @@ export enum WebsocketPipelineType {
101101
Reflective = 0,
102102
ColoredShape = 1,
103103
AprilTag = 2,
104-
Aruco = 3
104+
Aruco = 3,
105+
ObjectDetection = 4
105106
}

‎photon-client/vite.config.ts

+8-11
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,22 @@ export default defineConfig({
1010
plugins: [
1111
Vue2(),
1212
Components({
13-
resolvers: [
14-
VuetifyResolver()
15-
],
13+
resolvers: [VuetifyResolver()],
1614
dts: true,
1715
transformer: "vue2",
18-
types: [{
19-
from: "vue-router",
20-
names: ["RouterLink", "RouterView"]
21-
}],
16+
types: [
17+
{
18+
from: "vue-router",
19+
names: ["RouterLink", "RouterView"]
20+
}
21+
],
2222
version: 2.7
2323
})
2424
],
2525
css: {
2626
preprocessorOptions: {
2727
sass: {
28-
additionalData: [
29-
"@import \"@/assets/styles/variables.scss\"",
30-
""
31-
].join("\n")
28+
additionalData: ['@import "@/assets/styles/variables.scss"', ""].join("\n")
3229
}
3330
}
3431
},

0 commit comments

Comments
 (0)