File size: 5,176 Bytes
065d164 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 |
import { useState, useEffect, useRef } from 'react';
import { HandLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
import { drawLandmarks, analyzeHandGesture } from '../utils/handUtils';
const useHandDetection = (videoRef, canvasRef, isMobile) => {
const [handLandmarker, setHandLandmarker] = useState(null);
const [handDetected, setHandDetected] = useState(false);
const [isMouthOpen, setIsMouthOpen] = useState(false);
const [isLeftHand, setIsLeftHand] = useState(true);
const [thumbPosition, setThumbPosition] = useState({ x: 0, y: 0 });
const [isFirstLoad, setIsFirstLoad] = useState(true);
const requestRef = useRef(null);
const lastDetectionTimeRef = useRef(0);
const isComponentMounted = useRef(true);
// Initialize the HandLandmarker
useEffect(() => {
isComponentMounted.current = true;
const initializeHandLandmarker = async () => {
try {
const vision = await FilesetResolver.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
);
if (!isComponentMounted.current) return;
const landmarker = await HandLandmarker.createFromOptions(vision, {
baseOptions: {
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task",
delegate: "GPU"
},
runningMode: "VIDEO",
numHands: 1,
minHandDetectionConfidence: 0.5,
minHandPresenceConfidence: 0.5,
minTrackingConfidence: 0.5
});
if (!isComponentMounted.current) return;
setHandLandmarker(landmarker);
console.log("Hand landmarker initialized successfully");
// Set first load to false after initialization
setTimeout(() => {
if (isComponentMounted.current) {
setIsFirstLoad(false);
}
}, 3000);
} catch (error) {
console.error("Error initializing hand landmarker:", error);
}
};
initializeHandLandmarker();
return () => {
isComponentMounted.current = false;
if (requestRef.current) {
cancelAnimationFrame(requestRef.current);
requestRef.current = null;
}
};
}, []);
// Process video frames and detect hand gestures
useEffect(() => {
if (!handLandmarker || !videoRef.current || !canvasRef.current) return;
const video = videoRef.current;
const canvas = canvasRef.current;
const ctx = canvas.getContext('2d');
const detectHands = async (now) => {
if (!isComponentMounted.current) return;
if (video.readyState < 2) {
requestRef.current = requestAnimationFrame(detectHands);
return;
}
// Only run detection every 100ms for performance
if (now - lastDetectionTimeRef.current > 100) {
lastDetectionTimeRef.current = now;
// Process the frame with HandLandmarker
const results = handLandmarker.detectForVideo(video, now);
// Clear the canvas
ctx.clearRect(0, 0, canvas.width, canvas.height);
// Draw the video frame on the canvas, maintaining aspect ratio
const videoWidth = video.videoWidth;
const videoHeight = video.videoHeight;
// Calculate dimensions to maintain aspect ratio
let drawWidth = canvas.width;
let drawHeight = canvas.height;
let offsetX = 0;
let offsetY = 0;
// Center the video in the canvas
ctx.drawImage(video, offsetX, offsetY, drawWidth, drawHeight);
// Check if hands are detected
if (results.landmarks && results.landmarks.length > 0) {
const landmarks = results.landmarks[0];
setHandDetected(true);
// Draw hand landmarks
drawLandmarks(ctx, landmarks, canvas, isMobile);
// Analyze hand gesture
const { isOpen, isLeftHand: isLeft, thumbPosition: thumbPos } = analyzeHandGesture(landmarks);
// Update state with hand information
setIsLeftHand(isLeft);
setThumbPosition({
x: thumbPos.x * canvas.width,
y: thumbPos.y * canvas.height
});
// Update UI based on hand state
if (isOpen !== isMouthOpen) {
setIsMouthOpen(isOpen);
}
} else {
// No hands detected
setHandDetected(false);
if (isMouthOpen) {
setIsMouthOpen(false);
}
}
}
requestRef.current = requestAnimationFrame(detectHands);
};
requestRef.current = requestAnimationFrame(detectHands);
return () => {
if (requestRef.current) {
cancelAnimationFrame(requestRef.current);
requestRef.current = null;
}
};
}, [handLandmarker, isMouthOpen, isMobile, videoRef, canvasRef]);
return {
handDetected,
isMouthOpen,
isLeftHand,
thumbPosition,
isFirstLoad,
isComponentMounted
};
};
export default useHandDetection; |