Merge branch 'knowledgeIq' into 'master'

Knowledge iq

See merge request !25

web-app/backend/server.py

@@ -381,6 +381,32 @@ def mentalChromScores():
         'status': 'MentalCrom scores are added to the system successfully!',
     })
 
+# Knowledge IQ evaluation
+@app.route('/predictKnowledgeIq', methods = ['POST','GET'])
+@cross_origin()
+def predictKnowledgeIq():
+    if request.method == 'POST':
+        file = request.files.get('file')
+        questionIndex = request.form.get('questionIndex')
+        print(questionIndex)
+        if file is None or file.filename == "":
+            return jsonify({'error: no file'})
+
+        try:
+        
+            file.save("./"+file.filename)
+            prediction = transform_audio(file.filename)
+            data = {'prediction': prediction}
+            db.db['knowledgeIQScore'].insert_one({
+                "activityName": "Colour Numbers",
+                "questionIndex": questionIndex,
+                "transcription": prediction,
+            })
+            return jsonify(data)
+        except:
+            return jsonify({'error: Error during pipeline execution'})
+    return jsonify({'result: test'})
+
 
 # @app.route("/testDB")
 # def test():

web-app/frontend/src/components/activities/knowledgeIqEval/colourNumbers/ColourNumbers.jsx

-import React from "react";
+import React, {useState, useEffect} from "react";
 import Grid from "@mui/material/Grid";
 import Paper from "@mui/material/Paper";
 import Box from "@mui/material/Box";
 import "./ColourNumbers.css";
 import { lightBlue, yellow } from "@mui/material/colors";
 import { createTheme, ThemeProvider, styled } from "@mui/material/styles";
-
+import { RecordingHandler } from "../../reasoningIqEval/recorder/Recorder";
+import Timer from "../../reasoningIqEval/timer/Timer";
 const Item = styled(Paper)(({ theme }) => ({
   textAlign: "center",
   color: theme.palette.text.primary,
@@ -15,7 +16,25 @@ const Item = styled(Paper)(({ theme }) => ({
   fontWeight: "Bold",
 }));
 
+
+
 const ColourNumbers = ({ nextActivity }) => {
+  const [allCompleted, setAllCompleted] = useState(false);
+const [activityIndex, setActivityIndex] = useState(1);
+
+const switchActivityHandler = () => {
+  let activityNo = activityIndex + 1;
+  setActivityIndex(activityNo);
+// switch question
+console.log('switch');
+
+};
+
+useEffect(() => {
+  console.log('rec');
+  RecordingHandler(`CN.wav`, activityIndex);
+}, [activityIndex]);
+
   return (
     <div className="new">
       <Grid container>
@@ -47,6 +66,8 @@ const ColourNumbers = ({ nextActivity }) => {
           </Box>
         </Grid>
       </Grid>
+      {!allCompleted && <Timer switchActivity={switchActivityHandler} />}
+
     </div>
   );
 };

web-app/frontend/src/components/activities/knowledgeIqEval/digitSpan/DigitSpan.jsx

-import React, { useState } from "react";
+import React, { useState, useEffect } from "react";
 import { green, red } from '@mui/material/colors';
 import Avatar from '@mui/material/Avatar';
 import Stack from '@mui/material/Stack';
 import MicSharpIcon from '@mui/icons-material/MicSharp';
 import MicOffSharpIcon from '@mui/icons-material/MicOffSharp';
+import VolumeUpIcon from '@mui/icons-material/VolumeUp';
+import VolumeOffIcon from '@mui/icons-material/VolumeOff';
+import Picture from '../../../../assets/digitspan.jpg';
 import Button from '@mui/material/Button';
+import Instruction1 from '../../../../assets/audio/audio.aac';
+import Instruction2 from '../../../../assets/audio/audio2.aac';
+import Timer from "../../reasoningIqEval/timer/Timer";
+import {RecordingHandler} from "../recorder/Recorder";
 import "./DigitSpan.css";
 
 
-
 const DigitSpan = ({ nextActivity }) => {
 
-    const [buttonClicked, setButtonClicked] = useState(1);
+  const [buttonClicked, setButtonClicked] = useState(1);
+  const [audio, setAudio] = useState(Instruction1);
+  const [allCompleted, setAllCompleted] = useState(false);
+  const [activityIndex, setActivityIndex] = useState(1);
+
+  const switchActivityHandler = () => {
+    let activityNo = activityIndex + 1;
+    setActivityIndex(activityNo);
+    // switch question
+    console.log('switch');
+    // if(activityNo < 4){
+      
+    // }
+    // else{
+    //   setAllCompleted(true);
+    // }
+  };
+  // const audioFiles =[{source: Instruction1},
+  //                   {source: Instruction2}];
+
+  // const audioFiles2 =[Instruction1,Instruction2]
+
+  // const playAudio = () => {
+  //     const audioPromise = this.audio.play()
+  //     if (audioPromise !== undefined) {
+  //       audioPromise
+  //         .then(_ => {
+  //           // autoplay started
+  //           console.log('Audio Playing')
+  //         })
+  //         .catch(err => {
+  //           // catch dom exception
+  //           console.info(err)
+  //         })
+  //     }
+  //   }
+    useEffect(() => {
+      console.log('rec');
+      RecordingHandler(`DS.wav`, activityIndex);
+    }, [activityIndex]);
+
+
+    // const switchAudio = () => {
+     
+    //   if (audio < audioFiles.length - 1) {
+    //     setActivityIndex(activityIndex+1)
+
+    //     this.setState({
+    //       audio: audioFiles2[activityIndex]
+    //     });
+    //     //restart playlist
+    //   } else {
+    //     console.log('ERROR');
+    //   }
+    // }
+
+    // useEffect(() => {
+    //   console.log('rec');
+    //   RecordingHandler(`CN.wav`, activityIndex);
+    // }, [activityIndex]);
 
   return (
     <div className="container">
+      {/* <div class="flex flex-wrap justify-center">
+        <img
+          alt=""
+          class="max-w-sm h-auto shadow-lg"
+          src={Picture}
+        />
+      </div> */}
+      {allCompleted && (
+        <div className="w-4/6 h-4/6 m-auto">
+          {" "}
+          <img
+            className="m-auto"
+            src="https://i.ibb.co/sJWJZsD/completed.png"
+            alt="pattern"
+          />
+        </div>
+      )}
       <Stack direction="row" spacing={2} justifyContent="center" alignItems="center">
-      {buttonClicked === 1 && (
+      {/* {buttonClicked === 1 && (
           <Avatar sx={{ bgcolor: green[500], width: 100, height: 100}}
           onClick={()=> {setButtonClicked(2)}}>
           <MicSharpIcon sx={{ fontSize: 60 }}/>
@@ -27,7 +109,20 @@ const DigitSpan = ({ nextActivity }) => {
           onClick={()=> {setButtonClicked(1)}}>
           <MicOffSharpIcon sx={{ fontSize: 60 }}/>
         </Avatar>
+        )} */}
+      {/* {buttonClicked === 1 && (
+          <Avatar sx={{ bgcolor: green[500], width: 100, height: 100}}
+          onClick={()=> {}}>
+          <VolumeUpIcon sx={{ fontSize: 60 }}/>
+        </Avatar>
         )}
+      {buttonClicked === 2 && (
+          <Avatar sx={{ bgcolor: red[500], width: 100, height: 100 }}
+          onClick={()=> {setButtonClicked(1)}}>
+          <VolumeOffIcon sx={{ fontSize: 60 }}/>
+        </Avatar>
+        )}   */}
+        {!allCompleted && <Timer switchActivity={switchActivityHandler} />}
     </Stack>
       {/* <Button
         onClick={() => {

web-app/frontend/src/components/activities/knowledgeIqEval/recorder/Recorder.js

+import * as RecordRTC from "recordrtc";
+import axios from "axios";
+import baseURL from "../../../../config/api";
+import API from "../../../../config/api";
+
+export const RecordingHandler = async (fileName, questionIndex) => {
+  let stream = await navigator.mediaDevices.getUserMedia({
+    video: false,
+    audio: true,
+  });
+  let recorder = new RecordRTC.StereoAudioRecorder(stream, {
+    type: "audio",
+    mimeType: "audio/wav",
+    numberOfAudioChannels: 1,
+    sampleRate: 44100,
+  });
+  recorder.record();
+
+  const sleep = (m) => new Promise((r) => setTimeout(r, m));
+  await sleep(15000);
+
+  await recorder.stop(function () {
+    let blob = recorder.blob;
+    processRecording(blob, fileName, questionIndex);
+  });
+  stream.getTracks().forEach(function (track) {
+    track.stop();
+  });
+};
+
+const processRecording = (blob, fileName, questionIndex) => {
+  let recordedFile = new File([blob], fileName);
+  uploadRecording(recordedFile, fileName, questionIndex);
+};
+
+const uploadRecording = async (file, fileName, questionIndex) => {
+  let data = new FormData();
+  console.log(questionIndex);
+  data.append("file", file, fileName);
+  data.append("questionIndex", questionIndex);
+
+  const config = {
+    headers: {
+      "content-type": "multipart/form-data",
+      "Access-Control-Allow-Origin": "*",
+    },
+  };
+
+  await API.post(`predictKnowledgeIq`, data, config)
+    .then((res) => {
+      console.log(res, "DONE" + new Date().toISOString);
+    })
+    .catch((err) => {
+      console.log(err, "ERROR" + new Date().toISOString);
+    });
+};