Skip to content

Commit

Permalink
Merge pull request #11 from DDD-Community/feat/#3
Browse files Browse the repository at this point in the history 
[feat#3] 자세 추적 기능 고도화
  • Loading branch information
@lkhoony
lkhoony authored Jul 16, 2024
2 parents 2c215f8 + 098bd6e commit 3fc2a22
Show file tree
Hide file tree
Showing 5 changed files with 382 additions and 75 deletions.
165 changes: 96 additions & 69 deletions src/components/PoseDetector.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
import { useState, useEffect, useRef } from "react"
import { useState, useEffect, useRef, useCallback } from "react"
import { Camera } from "."
import type { pose } from "@/utils/detector"
import { detectSlope, detectTextNeck } from "@/utils/detector"
import { drawPose } from "@/utils/drawer"
import usePushNotification from "@/hooks/usePushNotification"
import { worker } from "@/utils/worker"

declare let ml5: any

const PoseDetector: React.FC = () => {
const [isScriptLoaded, setIsScriptLoaded] = useState<boolean>(false)
const [isScriptError, setIsScriptError] = useState<boolean>(false)
Expand All @@ -28,7 +26,7 @@ const PoseDetector: React.FC = () => {
const requestApi = (delay: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, delay))

const setup = async (): Promise<void> => {
ml5.bodyPose(
window.ml5.bodyPose(
"MoveNet",
{
modelType: "SINGLEPOSE_THUNDER",
Expand Down Expand Up @@ -63,45 +61,51 @@ const PoseDetector: React.FC = () => {

// webgl설정
const initializeBackend = async (): Promise<void> => {
await ml5.setBackend("webgl")
await window.ml5.setBackend("webgl")
}

const detect = (results: pose[]): void => {
resultRef.current = results
if (canvasRef.current) drawPose(results, canvasRef.current)
if (snapRef.current) {
const _slope = detectSlope(snapRef.current, results, mode === "snapshot")
const _isTextNeck = detectTextNeck(snapRef.current, results)
if (_slope !== null) setSlope(_slope)
if (_isTextNeck !== null) setIsTextNeck(_isTextNeck)

if (_isTextNeck) {
if (!textNeckStartTime || !textNeckStartTime.current) {
textNeckStartTime.current = Date.now()
// 거북목 자세 3초 유지 시, api 요청을 보내게 (콘솔 로그에서 확인)
} else if (Date.now() - textNeckStartTime.current >= 3000) {
if (!timer.current) {
timer.current = setInterval(() => {
requestApi(1000).then(() => console.log("api request"))
showNotification()
}, 2000)
const detect = useCallback(
(results: pose[]): void => {
resultRef.current = results
if (canvasRef.current) drawPose(results, canvasRef.current)
if (snapRef.current) {
const _slope = detectSlope(snapRef.current, results, mode === "snapshot")
const _isTextNeck = detectTextNeck(snapRef.current, results, mode === "snapshot")
if (_slope !== null) setSlope(_slope)
if (_isTextNeck !== null) setIsTextNeck(_isTextNeck)

if (_isTextNeck) {
if (!textNeckStartTime || !textNeckStartTime.current) {
textNeckStartTime.current = Date.now()
// 거북목 자세 3초 유지 시, api 요청을 보내게 (콘솔 로그에서 확인)
} else if (Date.now() - textNeckStartTime.current >= 3000) {
if (!timer.current) {
timer.current = setInterval(() => {
requestApi(1000).then(() => console.log("api request"))
showNotification()
}, 2000)
}
}
} else {
clearInterval(timer.current)
timer.current = null
textNeckStartTime.current = null
}
} else {
clearInterval(timer.current)
timer.current = null
textNeckStartTime.current = null
}
}
}
},
[mode, setSlope, setIsTextNeck, showNotification]
)

const detectStart = async (video: HTMLVideoElement): Promise<void> => {
worker.onmessage = ({ }: any) => {
if (modelRef.current) {
modelRef.current.detect(video, detect)
const detectStart = useCallback(
async (video: HTMLVideoElement): Promise<void> => {
worker.onmessage = ({ data }: any) => {
if (modelRef.current) {
modelRef.current.detect(video, detect)
}
}
}
}
},
[detect]
)

const getInitSnap = (): void => {
if (modelRef && modelRef.current) snapRef.current = resultRef.current
Expand All @@ -112,12 +116,32 @@ const PoseDetector: React.FC = () => {
getScript()
}, [])

useEffect(() => {
if (isModelLoaded) {
const video = document.querySelector("video")
if (video) {
detectStart(video)
}
}
}, [isModelLoaded, detectStart])

const initializePoseMonitoring = () => {
setIsTextNeck(null)
setSlope(null)
snapRef.current = null
}

const onChangeMode = (e: React.ChangeEvent<HTMLSelectElement>) => {
if (e.target.value) {
setMode(e.target.value)
initializePoseMonitoring()
}
}

const onCancelAutoPoseMonitoring = () => {
initializePoseMonitoring()
}

return (
<div>
{isScriptError ? (
Expand All @@ -141,53 +165,56 @@ const PoseDetector: React.FC = () => {
{isModelLoaded && (
<>
<div className="font-bold text-red-500">본 화면은 좌우가 반대로 보이고 있으니 주의하세요!</div>
<div className="w-1/2">
<select
className="w-full appearance-none rounded border border-gray-400 bg-white p-2"
onChange={onChangeMode}
>
<div>
<select className="rounded border border-gray-400 bg-white p-2" onChange={onChangeMode}>
<option value={"snapshot"}>스냅샷 모드 (올바른 자세 촬영 후, 해당 자세 기준으로 측정)</option>
<option value={"skeleton"}>스켈레톤 모드 (올바른 자세 제시 후, 해당 자세 기준으로 측정)</option>
<option value={"skeleton"}>자동 모드 (올바른 자세 기준으로 측정)</option>
</select>
</div>
{mode === "snapshot" && (
<>
<button
className="rounded bg-blue-500 px-4 py-2 font-bold text-white hover:bg-blue-700"
onClick={getInitSnap}
>
<div className="p-10 font-bold">
스냅샷 모드입니다. 올바른 자세를 하신 후에, 버튼을 눌러 촬영을 하면 해당 자세를 기준으로 부적절한
자세를 추적합니다!
</div>
<button className="rounded bg-blue-500 px-4 py-2 font-bold text-white" onClick={getInitSnap}>
올바른 자세를 촬영한 후 자세 측정 시작!
</button>
<div>
거북목 상태:&nbsp;
{isTextNeck === null
? "상태를 확인할 수 없습니다."
: isTextNeck
? "거북목 상태 입니다"
: "정상적인 자세 입니다"}
</div>
<div>어깨 기울기: {slope === null ? "상태를 확인할 수 없습니다." : slope}</div>
</>
)}
{mode === "skeleton" && (
<>
<button
className="rounded bg-blue-500 px-4 py-2 font-bold text-white hover:bg-blue-700"
onClick={getInitSnap}
>
기준 그림에 몸을 맞춘 후 측정 시작!
</button>
<div>
거북목 상태:&nbsp;
{isTextNeck === null
? "상태를 확인할 수 없습니다."
: isTextNeck
? "거북목 상태 입니다"
: "정상적인 자세 입니다"}
<div className="p-10 font-bold">자동 모드입니다. 자동으로 부적절한 자세를 추적합니다.</div>
<div className="flex gap-10">
<button
className="rounded bg-blue-500 px-4 py-2 font-bold text-white disabled:bg-gray-400"
onClick={getInitSnap}
// disabled={snapShoptPose.length > 0}
>
자세 모니터링 시작!
</button>
<button
className="rounded bg-red-500 px-4 py-2 font-bold text-white disabled:bg-gray-400"
onClick={onCancelAutoPoseMonitoring}
// disabled={snapShoptPose.length === 0}
>
자세 모니터링 취소
</button>
</div>
<div>어깨 기울기: {slope === null ? "상태를 확인할 수 없습니다." : slope}</div>
</>
)}

<div>
거북목 상태:&nbsp;
{isTextNeck === null ? (
"상태를 확인할 수 없습니다."
) : isTextNeck ? (
<span className="font-extrabold text-red-500">"거북목 상태 입니다"</span>
) : (
"정상적인 자세 입니다"
)}
</div>
<div>어깨 기울기: {slope === null ? "상태를 확인할 수 없습니다." : slope}</div>
</>
)}
</>
Expand Down
Loading

0 comments on commit 3fc2a22

Please sign in to comment.