Merge branch 'KIQ-01' into 'master'

Added video player and resolved POST Error

See merge request !36

web-app/backend/knowledgeIqScoreCalculation.py

+import difflib
+
+colourNumbers = [ "5 4","5 6 7","5 6 7 8"]
+
+digitSpan     = [ "5 4","5 6 7","5 6 7 8"]
+
+def calKnowledgeIqActivityScore(activity, questionIndex, prediction):
+    questionIndex = int(questionIndex)
+    prediction = "".join(prediction).replace(".","")
+    cal_score = []
+    if (activity == "CN"):
+
+        if (colourNumbers[questionIndex-1] == prediction):
+            score = 100
+        else:
+            score = 0
+
+        temp = difflib.SequenceMatcher(None, colourNumbers[questionIndex-1], prediction)
+
+        print(temp.get_matching_blocks())
+        print('Similarity Score: ',temp.ratio())
+
+        cal_score = [score, temp.ratio()*100]
+
+    elif (activity == "DS"):
+
+        if (digitSpan[questionIndex-1] == prediction):
+            score = 100
+        else:
+            score = 0
+
+        temp = difflib.SequenceMatcher(None, colourNumbers[questionIndex-1], prediction)
+
+        print(temp.get_matching_blocks())
+        print('Similarity Score: ',temp.ratio())
+
+        cal_score = [score, temp.ratio()*100]
+
+    else:
+        cal_score = [0,0]
+        print('Score cannot be specified to the activity specified in the system!')
+
+    return  cal_score          
\ No newline at end of file

web-app/backend/server.py

@@ -11,6 +11,7 @@ import datetime
 import base64
 import io
 from preProcessor import transform_audio
+from knowledgeIqScoreCalculation import calKnowledgeIqActivityScore
 # Libraries required for model utilization
 import cv2 as cv
 import numpy as np
@@ -422,19 +423,22 @@ def predictKnowledgeIq():
     if request.method == 'POST':
         file = request.files.get('file')
         questionIndex = request.form.get('questionIndex')
+        activityName = request.form.get('activityName')
         print(questionIndex)
         if file is None or file.filename == "":
             return jsonify({'error: no file'})
 
         try:
-        
             file.save("./"+file.filename)
             prediction = transform_audio(file.filename)
             data = {'prediction': prediction}
+            score = calKnowledgeIqActivityScore(activityName, questionIndex, prediction)
             db.db['knowledgeIQScore'].insert_one({
-                "activityName": "Colour Numbers",
+                "activityName": activityName,
                 "questionIndex": questionIndex,
                 "transcription": prediction,
+                "accuracyScore": score[0],
+                "similarityScore": score[1]
             })
             return jsonify(data)
         except:

web-app/frontend/src/components/activities/ActivityContainer.jsx

@@ -4,6 +4,7 @@ import {
   PictureConcept,
   Arithmetic,
   ColourNumbers,
+  ColourNumbersController,
   DigitSpan,
   DigitSpanController,
   VideoPlayerScreen,
@@ -149,7 +150,7 @@ const ActivityContainer = () => {
                 <PairCancerlation nextActivity={nextActivityHandler} />
               )}
               {currentActivityNo === 6 && (
-                <ColourNumbers nextActivity={nextActivityHandler} />
+                <ColourNumbersController nextActivity={nextActivityHandler} />
               )}
               {currentActivityNo === 7 && (
                 <DigitSpanController nextActivity={nextActivityHandler} />

web-app/frontend/src/components/activities/knowledgeIqEval/colourNumbers/ColourNumbers.jsx

@@ -5,7 +5,7 @@ import Box from "@mui/material/Box";
 import "./ColourNumbers.css";
 import { lightBlue, yellow } from "@mui/material/colors";
 import { createTheme, ThemeProvider, styled } from "@mui/material/styles";
-import { RecordingHandler } from "../../reasoningIqEval/recorder/Recorder";
+import { RecordingHandler } from "../recorder/Recorder";
 import Timer from "../../reasoningIqEval/timer/Timer";
 const Item = styled(Paper)(({ theme }) => ({
   textAlign: "center",
@@ -18,7 +18,7 @@ const Item = styled(Paper)(({ theme }) => ({
 
 
 
-const ColourNumbers = ({ nextActivity }) => {
+const ColourNumbers = ({ GoNext, isAllCompleted }) => {
   const [allCompleted, setAllCompleted] = useState(false);
 const [activityIndex, setActivityIndex] = useState(1);
 
@@ -26,13 +26,15 @@ const switchActivityHandler = () => {
   let activityNo = activityIndex + 1;
   setActivityIndex(activityNo);
 // switch question
-console.log('switch');
+  console.log('switch');
 
 };
 
 useEffect(() => {
+  setTimeout(() => {
+  }, 5000);
   console.log('rec');
-  RecordingHandler(`CN.wav`, activityIndex);
+  RecordingHandler(`CN.wav`, activityIndex, 'CN');
 }, [activityIndex]);
 
   return (
@@ -66,6 +68,7 @@ useEffect(() => {
           </Box>
         </Grid>
       </Grid>
+
       {!allCompleted && <Timer switchActivity={switchActivityHandler} />}
 
     </div>

web-app/frontend/src/components/activities/knowledgeIqEval/colourNumbers/colourNumbersController.jsx

+import React, { useState, useEffect } from "react";
+import { ColourNumbers } from "../../..";
+
+import VideoPlayerScreen from "./videoPlayer";
+
+const ColourNumbersController = () => {
+  const activityCount = 4;
+  const [currentActivityNo, setCurrentActivityNo] = useState(1);
+  const [isAudioCompleted, setIsAudioCompleted] = useState(false);
+  const [isAllCompleted, setIsAllCompleted] = useState(false);
+
+  const nextActivityHandler = () => {
+    if (currentActivityNo < activityCount - 1) {
+      setCurrentActivityNo(currentActivityNo + 1);
+    } else {
+      setIsAllCompleted(true);
+    }
+
+    setIsAudioCompleted(false);
+  };
+
+  const StartRecording = (state) => {
+    setTimeout(() => {
+      setIsAudioCompleted(true);
+    }, 1000);
+  };
+
+  return (
+    <>
+      <div className="w-full h-full">
+        <div>
+          {!isAudioCompleted ? (
+            <VideoPlayerScreen
+              currentActivtyIndex={currentActivityNo}
+              changeScreen={StartRecording}
+            />
+          ) : (
+            <>
+              <ColourNumbers
+                GoNext={nextActivityHandler}
+                isAllCompleted={isAllCompleted}
+                currentActivityNo={currentActivityNo}
+              />
+            </>
+          )}
+        </div>
+      </div>
+    </>
+  );
+};
+
+export default ColourNumbersController;

web-app/frontend/src/components/activities/knowledgeIqEval/colourNumbers/videoPlayer.css

+.activity-container-angry-grid {
+    display: grid;
+  
+    grid-template-rows: 1fr 1fr 1fr;
+    grid-template-columns: 1fr 1fr 1fr 1fr 1fr;
+  
+    gap: 0px;
+    height: 100%;
+  
+    padding: 20px;
+  }
+
+  #activity-container-item-0 {
+    grid-row-start: 1;
+    grid-column-start: 1;
+  
+    grid-row-end: 4;
+    grid-column-end: 2;
+  
+    margin-right: 10px;
+  
+    border: solid gainsboro;
+    border-radius: 5px;
+    padding: 5px;
+  }
\ No newline at end of file

web-app/frontend/src/components/activities/knowledgeIqEval/colourNumbers/videoPlayer.jsx

+import React, { useState, useEffect } from "react";
+import toast, { Toaster } from "react-hot-toast";
+import { ContainerCard } from "../../../../components/index";
+import Video from "./videos";
+
+const VideoPlayerScreen = ({ currentActivtyIndex, changeScreen }) => {
+  return (
+    <>
+      <div className="student-page-angry-grid">
+        <div id="student-page-item-0">
+          <ContainerCard>
+            <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
+              <div className="lg:text-center">
+                <h2 className="text-base text-[#3d59c1] font-semibold tracking-wide uppercase mt-4">
+                  ICAAT
+                </h2>
+              </div>
+              <div className="mt-10 sm:mt-0">
+                <Video currentActivtyIndex={currentActivtyIndex} changeScreen={changeScreen} />
+              </div>
+            </div>
+          </ContainerCard>
+        </div>
+      </div>
+    </>
+  );
+};
+
+export default VideoPlayerScreen;

web-app/frontend/src/components/activities/knowledgeIqEval/colourNumbers/videos.jsx

+import React, { useState, useEffect, useRef } from "react";
+import Vid4 from "../../../../assets/video/vid6.mp4";
+
+const Video = ({ currentActivtyIndex, changeScreen }) => {
+  const [videoLink, setVideoLink] = useState("");
+  const vidRef = useRef();
+
+  useEffect(() => {
+    switch (currentActivtyIndex) {
+      case 1:
+        setVideoLink(Vid4);
+        break;
+      case 2:
+        setVideoLink(Vid4);
+        break;
+      case 3:
+        setVideoLink(Vid4);
+        break;
+      case 4:
+        setVideoLink(Vid4);
+        break;
+      case 5:
+        setVideoLink("/Videos/vid1.mp4");
+        break;
+      default:
+    }
+    const timer = setTimeout(() => {
+      vidRef.current.play();
+    }, 3000);
+    return () => clearTimeout(timer);
+  }, []);
+
+  const myCallback = () => {
+    changeScreen(false);
+  }
+
+  return (
+    <div className="flex items-center">
+      {videoLink && (
+        <>
+          <video ref={vidRef} className="w-full h-full p-2" controls onEnded={() => myCallback()}>
+            <source src={videoLink} type="video/mp4" />
+          </video>
+        </>
+      )}
+    </div>
+  );
+};
+
+export default Video;

web-app/frontend/src/components/activities/knowledgeIqEval/digitSpan/DigitSpan.jsx

@@ -8,82 +8,22 @@ import VolumeUpIcon from "@mui/icons-material/VolumeUp";
 import VolumeOffIcon from "@mui/icons-material/VolumeOff";
 import Picture from "../../../../assets/digitspan.jpg";
 import Button from "@mui/material/Button";
-// import Instruction1 from '../../../../assets/audio/audio.aac';
-// import Instruction2 from '../../../../assets/audio/audio2.aac';
 import Timer from "../../reasoningIqEval/timer/Timer";
 import { RecordingHandler } from "../recorder/Recorder";
 import "./DigitSpan.css";
 
-const DigitSpan = ({ GoNext, isAllCompleted }) => {
-  const [buttonClicked, setButtonClicked] = useState(1);
-  const [audio, setAudio] = useState("Instruction1");
-  const [allCompleted, setAllCompleted] = useState(false);
+const DigitSpan = ({ GoNext, isAllCompleted, currentActivityNo }) => {
   const [activityIndex, setActivityIndex] = useState(1);
 
-  // const switchActivityHandler = ({switchActivity}) => {
-  //   let activityNo = activityIndex + 1;
-  //   setActivityIndex(activityNo);
-  //   // switch question
-  //   console.log("switch");
-  //   // if(activityNo < 4){
-
-  //   // }
-  //   // else{
-  //   //   setAllCompleted(true);
-  //   // }
-  // };
-  // const audioFiles =[{source: Instruction1},
-  //                   {source: Instruction2}];
-
-  // const audioFiles2 =[Instruction1,Instruction2]
-
-  // const playAudio = () => {
-  //     const audioPromise = this.audio.play()
-  //     if (audioPromise !== undefined) {
-  //       audioPromise
-  //         .then(_ => {
-  //           // autoplay started
-  //           console.log('Audio Playing')
-  //         })
-  //         .catch(err => {
-  //           // catch dom exception
-  //           console.info(err)
-  //         })
-  //     }
-  //   }
   useEffect(() => {
     console.log("rec");
-    RecordingHandler(`DS.wav`, activityIndex);
-  }, [activityIndex]);
-
-  // const switchAudio = () => {
+    RecordingHandler(`DS.wav`, currentActivityNo, 'DS');
+  }, [currentActivityNo]);
 
-  //   if (audio < audioFiles.length - 1) {
-  //     setActivityIndex(activityIndex+1)
-
-  //     this.setState({
-  //       audio: audioFiles2[activityIndex]
-  //     });
-  //     //restart playlist
-  //   } else {
-  //     console.log('ERROR');
-  //   }
-  // }
-
-  // useEffect(() => {
-  //   console.log('rec');
-  //   RecordingHandler(`CN.wav`, activityIndex);
-  // }, [activityIndex]);
 
   return (
     <div className="container">
-      {/* <div class="flex flex-wrap justify-center">
-        <img
-          alt=""
-          class="max-w-sm h-auto shadow-lg"
-          src={Picture}
-        />
-      </div> */}
+
       {isAllCompleted && (
         <div className="w-4/6 h-4/6 m-auto">
           {" "}
@@ -100,30 +40,6 @@ const DigitSpan = ({ GoNext, isAllCompleted }) => {
         justifyContent="center"
         alignItems="center"
       >
-        {/* {buttonClicked === 1 && (
-          <Avatar sx={{ bgcolor: green[500], width: 100, height: 100}}
-          onClick={()=> {setButtonClicked(2)}}>
-          <MicSharpIcon sx={{ fontSize: 60 }}/>
-        </Avatar>
-        )}
-      {buttonClicked === 2 && (
-          <Avatar sx={{ bgcolor: red[500], width: 100, height: 100 }}
-          onClick={()=> {setButtonClicked(1)}}>
-          <MicOffSharpIcon sx={{ fontSize: 60 }}/>
-        </Avatar>
-        )} */}
-        {/* {buttonClicked === 1 && (
-          <Avatar sx={{ bgcolor: green[500], width: 100, height: 100}}
-          onClick={()=> {}}>
-          <VolumeUpIcon sx={{ fontSize: 60 }}/>
-        </Avatar>
-        )}
-      {buttonClicked === 2 && (
-          <Avatar sx={{ bgcolor: red[500], width: 100, height: 100 }}
-          onClick={()=> {setButtonClicked(1)}}>
-          <VolumeOffIcon sx={{ fontSize: 60 }}/>
-        </Avatar>
-        )}   */}
         {!isAllCompleted && <Timer switchActivity={GoNext} />}
       </Stack>
       {/* <Button

web-app/frontend/src/components/activities/knowledgeIqEval/digitSpan/digitSpanController.jsx

@@ -39,6 +39,7 @@ const DigitSpanController = () => {
               <DigitSpan
                 GoNext={nextActivityHandler}
                 isAllCompleted={isAllCompleted}
+                currentActivityNo={currentActivityNo}
               />
             </>
           )}

web-app/frontend/src/components/activities/knowledgeIqEval/recorder/Recorder.js

@@ -3,7 +3,7 @@ import axios from "axios";
 import baseURL from "../../../../config/api";
 import API from "../../../../config/api";
 
-export const RecordingHandler = async (fileName, questionIndex) => {
+export const RecordingHandler = async (fileName, questionIndex, activityName) => {
   let stream = await navigator.mediaDevices.getUserMedia({
     video: false,
     audio: true,
@@ -21,23 +21,29 @@ export const RecordingHandler = async (fileName, questionIndex) => {
 
   await recorder.stop(function () {
     let blob = recorder.blob;
-    processRecording(blob, fileName, questionIndex);
+    processRecording(blob, fileName, questionIndex, activityName);
   });
   stream.getTracks().forEach(function (track) {
     track.stop();
   });
 };
 
-const processRecording = (blob, fileName, questionIndex) => {
+const processRecording = (blob, fileName, questionIndex, activityName) => {
   let recordedFile = new File([blob], fileName);
-  uploadRecording(recordedFile, fileName, questionIndex);
+  uploadRecording(recordedFile, fileName, questionIndex, activityName);
 };
 
-const uploadRecording = async (file, fileName, questionIndex) => {
+const uploadRecording = async (file, fileName, questionIndex, activityName) => {
   let data = new FormData();
+  let candidateID = 0;
+  if (localStorage) {
+    candidateID = localStorage.getItem("candidateID");
+  }
   console.log(questionIndex);
   data.append("file", file, fileName);
+  data.append("activityName", activityName)
   data.append("questionIndex", questionIndex);
+  data.append("candidateID", candidateID);
 
   const config = {
     headers: {

web-app/frontend/src/components/index.js

@@ -12,3 +12,4 @@ export { default as EthicalClearenceScreen } from "./activities/ethicalClearence
 export { default as VideoPlayerScreen } from "./activities/videoPlayer";
 export { default as AudioPlayerScreen } from "./activities/audioPlayerScreen";
 export { default as DigitSpanController } from "./activities/knowledgeIqEval/digitSpan/digitSpanController";
+export { default as ColourNumbersController } from "./activities/knowledgeIqEval/colourNumbers/colourNumbersController";

web-app/frontend/src/utility/audio.jsx

 import React, { useState, useEffect, useRef } from "react";
-import A1 from "../Audio/A1.mp3";
-import A2 from "../Audio/A2.mp3";
+import D1 from "../Audio/DS1.mp3";
+import D2 from "../Audio/DS2.mp3";
+import D3 from "../Audio/DS3.mp3";
 
 const Audio = ({ currentActivtyIndex, changeScreen }) => {
   const myAudio = useRef();
@@ -10,19 +11,19 @@ const Audio = ({ currentActivtyIndex, changeScreen }) => {
   useEffect(() => {
     switch (currentActivtyIndex) {
       case 1:
-        setAudio(A1);
+        setAudio(D1);
         break;
       case 2:
-        setAudio(A2);
+        setAudio(D2);
         break;
       case 3:
-        setAudio(A1);
+        setAudio(D3);
         break;
       case 4:
-        setAudio(A2);
+        setAudio(D1);
         break;
       case 5:
-        setAudio(A1);
+        setAudio(D2);
         break;
       default:
     }