face detect initial commit

package.json

@@ -19,6 +19,7 @@
     "@react-three/fiber": "^7.0.2",
     "@react-three/gltfjsx": "^4.2.3",
     "@speechly/speech-recognition-polyfill": "^1.0.0",
+    "@tensorflow-models/blazeface": "^0.0.7",
     "@tensorflow-models/handpose": "^0.0.7",
     "@tensorflow-models/speech-commands": "^0.5.4",
     "@tensorflow/tfjs": "^3.7.0",

src/common/ui/layouts/MainMiddleArea.tsx

@@ -2,6 +2,7 @@ import ThreeWorld from '@components/ThreeWorld';
 import { AnimationContext } from '@ctx/AnimationContext';
 import styles from '@layouts_style/MainMiddleArea.module.sass';
 import * as handpose from '@tensorflow-models/handpose';
+import * as blazeface from '@tensorflow-models/blazeface';
 import { drawFullHand, drawZooming, fullCalculation } from '@util/handPose';
 import React, { useContext, useEffect } from 'react';
 import Webcam from 'react-webcam';
@@ -14,6 +15,13 @@ type TProps = {
 const MainMiddleArea = ({ canvasRef, webcamRef }: TProps): JSX.Element => {
   const { setShouldRotate, setVideoHeight, setVideoWidth, setX, setY, setZoom } = useContext(AnimationContext);
   const showWebCam = true;
+  // let ctx;
+  let currentDateTime = new Date();
+  // const videoStartTime = new Date();
+  let needAttentionTimeSec = 0;
+  let needAttentionTimeMin = 0;
+  let noFaceTimeSec = 0;
+  let noFaceTimeMin = 0;
   const { containerHeight, containerWidth } = useContext(AnimationContext);
   const runHandpose = async () => {
     const net = await handpose.load();
@@ -23,6 +31,76 @@ const MainMiddleArea = ({ canvasRef, webcamRef }: TProps): JSX.Element => {
     }, 10); // 10
   };
 
+  const runFaceDetect = async () => {
+    const model = await blazeface.load();
+    setInterval(() => {
+      faceDetect(model);
+    }, 10);
+  };
+
+  const faceDetect = async (model: blazeface.BlazeFaceModel) => {
+    const videoReference = webcamRef?.current?.video as HTMLVideoElement;
+    const canvasReference = canvasRef?.current as HTMLCanvasElement;
+
+    if (
+      typeof webcamRef.current !== 'undefined' &&
+      webcamRef.current !== null &&
+      videoReference.readyState === 4 &&
+      model
+    ) {
+      const { videoWidth } = videoReference;
+      const { videoHeight } = videoReference;
+      videoReference.width = videoWidth;
+      videoReference.height = videoHeight;
+      canvasReference.width = videoWidth;
+      canvasReference.height = videoHeight;
+      setVideoWidth(videoWidth);
+      setVideoHeight(videoHeight);
+
+      // ctx = canvasReference.getContext('2d');
+      // ctx.fillStyle = "rgba(255, 0, 0, 0.5)";
+
+      const predictions = await model.estimateFaces(videoReference, false);
+      if (predictions.length > 0) {
+        // ctx.clearRect(0, 0, canvasReference.width, canvasReference.height);
+        for (let i = 0; i < predictions.length; i++) {
+          const rightEye = predictions[i].landmarks[0];
+          const leftEye = predictions[i].landmarks[1];
+          const nose = predictions[i].landmarks[2];
+
+          // ctx.fillStyle = "blue";
+          // ctx.fillRect(rightEye[0], rightEye[1], 10, 10);
+          // ctx.fillStyle = "red";
+          // ctx.fillRect(leftEye[0], leftEye[1], 10, 10);
+          // ctx.fillStyle = "green";
+          // ctx.fillRect(nose[0], nose[1], 20, 10);
+
+          const tempNewDate = new Date();
+          if (rightEye[0] > nose[0] || nose[0] > leftEye[0]) {
+            const durationInSec = ((tempNewDate.getTime() - currentDateTime.getTime()) / 1000);
+            needAttentionTimeSec += durationInSec;
+            if (needAttentionTimeSec > 60) {
+              needAttentionTimeMin += 1;
+              needAttentionTimeSec -= 60;
+            }
+          }
+          currentDateTime = tempNewDate;
+          // console.log('Need Attention -', needAttentionTimeMin, 'min ', Math.trunc(needAttentionTimeSec) , 'sec');
+        }
+      } else {
+        const tempNewDate = new Date();
+        const durationInSec = ((tempNewDate.getTime() - currentDateTime.getTime()) / 1000);
+        noFaceTimeSec += durationInSec;
+        if (noFaceTimeSec > 60) {
+          noFaceTimeMin += 1;
+          noFaceTimeSec -= 60;
+        }
+        // console.log('No Face -', noFaceTimeMin, 'min ', Math.trunc(noFaceTimeSec) , 'sec');
+      }
+      // console.log('No Attention-', needAttentionTimeMin, 'min ', Math.trunc(needAttentionTimeSec) , 'sec','-------------','No Face-', noFaceTimeMin, 'min ', Math.trunc(noFaceTimeSec) , 'sec');
+    }
+  };
+
   const detect = async (net: handpose.HandPose) => {
     const videoReference = webcamRef?.current?.video as HTMLVideoElement;
     const canvasReference = canvasRef?.current as HTMLCanvasElement;
@@ -69,6 +147,7 @@ const MainMiddleArea = ({ canvasRef, webcamRef }: TProps): JSX.Element => {
 
   useEffect(() => {
     runHandpose();
+    runFaceDetect();
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);