Merge branch 'master' of http://gitlab.sliit.lk/2022-073/2022-073

web-app/backend/controler.py

 import cv2 as cv
 import numpy as np
 from trainAnn import headPose,headDistence,facialExperssions,blinkCount,detector
-
+import os
 from keras.models import load_model
 from sklearn.preprocessing import OneHotEncoder
 
@@ -76,4 +76,7 @@ def captureImage(link,frameSkip):
     
 
 if __name__=="__main__":
-    print("Controller working")
+    directory = os.getcwd()
+    destination =  directory + r"\uploads\att.mp4"
+    scoreLst=captureImage(destination,10)
+    print(scoreLst)

web-app/backend/server.py

@@ -371,7 +371,9 @@ def predictShapePattern():
     return jsonify(response)
 
 # Reasoning IQ evaluation
-@app.route('/predictPictureConcept', methods = ['POST','GET'])
+
+
+@app.route('/predictPictureConcept', methods=['POST', 'GET'])
 @cross_origin()
 def predictPictureConcept():
     if request.method == 'POST':
@@ -383,7 +385,7 @@ def predictPictureConcept():
             return jsonify({'error: no file'})
 
         try:
-        
+
             file.save("./"+file.filename)
             prediction = transform_audio(file.filename)
             data = {'prediction': prediction}
@@ -398,7 +400,8 @@ def predictPictureConcept():
             return jsonify({'error: Error during pipeline execution'})
     return jsonify({'result: test'})
 
-@app.route('/predictArithmetic', methods = ['POST','GET'])
+
+@app.route('/predictArithmetic', methods=['POST', 'GET'])
 @cross_origin()
 def predictArithmetic():
     if request.method == 'POST':
@@ -410,7 +413,7 @@ def predictArithmetic():
             return jsonify({'error: no file'})
 
         try:
-        
+
             file.save("./"+file.filename)
             prediction = transform_audio(file.filename)
             data = {'prediction': prediction}
@@ -425,9 +428,11 @@ def predictArithmetic():
             return jsonify({'error: Error during pipeline execution'})
     return jsonify({'result: test'})
 
+
 @app.route('/mentalChromScores', methods=['POST'])
 @cross_origin()
 def mentalChromScores():
+
     body = request.json
     activityName = body['activityName']
     score = body['score']
@@ -435,6 +440,8 @@ def mentalChromScores():
     predResult = body['predResult']
     createdTime = body['createdTime']
     totalTime = body['totalTime']
+    candidateID = body['candidateID']
+    answerImage = body['answerImage']
     # db.users.insert_one({
     db.db['mentalChronomrtryScore'].insert_one({
         "activityName": activityName,
@@ -443,13 +450,17 @@ def mentalChromScores():
         "predResult": predResult,
         "createdTime": createdTime,
         "totalTime": totalTime,
+        "candidateID": candidateID,
+        "answerImage": answerImage
     })
     return jsonify({
         'status': 'MentalCrom scores are added to the system successfully!',
     })
 
 # Knowledge IQ evaluation
-@app.route('/predictKnowledgeIq', methods = ['POST','GET'])
+
+
+@app.route('/predictKnowledgeIq', methods=['POST', 'GET'])
 @cross_origin()
 def predictKnowledgeIq():
     if request.method == 'POST':
@@ -464,7 +475,8 @@ def predictKnowledgeIq():
             file.save("./"+file.filename)
             prediction = transform_audio(file.filename)
             data = {'prediction': prediction}
-            score = calKnowledgeIqActivityScore(activityName, questionIndex, prediction)
+            score = calKnowledgeIqActivityScore(
+                activityName, questionIndex, prediction)
             db.db['knowledgeIQScore'].insert_one({
                 "activityName": activityName,
                 "questionIndex": questionIndex,

web-app/backend/trainAnn.py

@@ -70,7 +70,7 @@ def headDistence(frame3,detector):
         # Finding distance
         f = 650
         d = (W * f) / w
-        cvzone.putTextRect(frame3, f'Depth: {int(d)}cm',(facedis[10][0] - 100, facedis[10][1] - 50),scale=2)
+        # cvzone.putTextRect(frame3, f'Depth: {int(d)}cm',(facedis[10][0] - 100, facedis[10][1] - 50),scale=2)
         return frame3,str(d),True
     return frame3,False,False
 
@@ -87,8 +87,8 @@ def blinkCount(frame2,detector):
     frame2, faces = detector.findFaceMesh(frame2, draw=False)
     if faces:
         face = faces[0]
-        for id in idList:
-            cv.circle(frame2, face[id], 2, (255, 0, 255), cv.FILLED)
+        # for id in idList:
+        #     cv.circle(frame2, face[id], 2, (255, 0, 255), cv.FILLED)
 
         leftUp = face[159]
         leftDown = face[23]
@@ -98,8 +98,8 @@ def blinkCount(frame2,detector):
         lengthVerticl, _ = detector.findDistance(leftUp, leftDown)
         lengthHorz, _ = detector.findDistance(leftLEnd, leftREnd)
 
-        cv.line(frame2, leftUp, leftDown, (0, 200, 0), 2)
-        cv.line(frame2, leftLEnd, leftREnd, (0, 200, 0), 2)
+        # cv.line(frame2, leftUp, leftDown, (0, 200, 0), 2)
+        # cv.line(frame2, leftLEnd, leftREnd, (0, 200, 0), 2)
 
         ratio = int((lengthVerticl / lengthHorz) * 100)
         ratioList.append(ratio)
@@ -119,8 +119,8 @@ def blinkCount(frame2,detector):
                 counter = 0
 
         # writer3.writerow([blinkCounter])
-        cvzone.putTextRect(frame2, f'Blink Time: {blinkTime}', (20, 50), 1, 2, (255, 255, 255), (255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
-        imgPlot = plotY.update(ratioAvg, color=(255, 0, 0))
+        # cvzone.putTextRect(frame2, f'Blink Time: {blinkTime}', (20, 50), 1, 2, (255, 255, 255), (255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
+        # imgPlot = plotY.update(ratioAvg, color=(255, 0, 0))
 
     return frame2,blinkTime
         
@@ -143,9 +143,9 @@ def headPose(image,imgCount=0):
         for face_landmarks in results.multi_face_landmarks:
             for idx, lm in enumerate(face_landmarks.landmark):
                 if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:
-                    if idx == 1:
-                        nose_2d = (lm.x * img_w, lm.y * img_h)
-                        nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
+                    # if idx == 1:
+                    #     nose_2d = (lm.x * img_w, lm.y * img_h)
+                    #     nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
 
                     x, y = int(lm.x * img_w), int(lm.y * img_h)
 
@@ -185,44 +185,44 @@ def headPose(image,imgCount=0):
             z = angles[2] * 360
 
             # See where the user's head tilting
-            if y < -10:
-                text = "Looking Left"
-            elif y > 10:
-                text = "Looking Right"
-            elif x < -10:
-                text = "Looking Down"
-            elif x > 10:
-                text = "Looking Up"
-            else:
-                text = "Forward"
+            # if y < -10:
+            #     text = "Looking Left"
+            # elif y > 10:
+            #     text = "Looking Right"
+            # elif x < -10:
+            #     text = "Looking Down"
+            # elif x > 10:
+            #     text = "Looking Up"
+            # else:
+            #     text = "Forward"
             
             # Display the nose direction
-            nose_3d_projection, jacobian = cv.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix)
+            # nose_3d_projection, jacobian = cv.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix)
 
-            p1 = (int(nose_2d[0]), int(nose_2d[1]))
-            p2 = (int(nose_2d[0] + y * 10), int(nose_2d[1] - x * 10))
+            # p1 = (int(nose_2d[0]), int(nose_2d[1]))
+            # p2 = (int(nose_2d[0] + y * 10), int(nose_2d[1] - x * 10))
 
-            cv.line(image, p1, p2, (255, 0, 0), 3)
+            # cv.line(image, p1, p2, (255, 0, 0), 3)
 
             # Add the text on the image
             # writer.writerow([str(np.round(x, 2)), str(np.round(y, 2)), str(np.round(z, 2)), text])
-            text=str(imgCount+1)+"  "+text
+            # text=str(imgCount+1)+"  "+text
             
-            cvzone.putTextRect(image, text, (20, 50), 1, 2, (255, 255, 255),
-                               (255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
-            cv.putText(image, "x: " + str(np.round(x, 2)), (500, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
-            cv.putText(image, "y: " + str(np.round(y, 2)), (500, 100), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
-            cv.putText(image, "z: " + str(np.round(z, 2)), (500, 150), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
+            # cvzone.putTextRect(image, text, (20, 50), 1, 2, (255, 255, 255),
+            #                    (255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
+            # cv.putText(image, "x: " + str(np.round(x, 2)), (500, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
+            # cv.putText(image, "y: " + str(np.round(y, 2)), (500, 100), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
+            # cv.putText(image, "z: " + str(np.round(z, 2)), (500, 150), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
             
         end = time.time()
-        totalTime = end - start
+        # totalTime = end - start
 
-        try:
-            fps = 1 / totalTime
-        except:
-            fps =0
+        # try:
+        #     fps = 1 / totalTime
+        # except:
+        #     fps =0
         # print("FPS: ", fps)
-        cv.putText(image, f'FPS: {int(fps)}', (20, 450), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2)
+        # cv.putText(image, f'FPS: {int(fps)}', (20, 450), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2)
 
         mp_drawing.draw_landmarks(
             image=image,
@@ -255,10 +255,10 @@ def facialExperssions(image):
 
                 img_arr = np.asarray(image2).reshape((224, 224, 3))
                 pre=model.predict(np.array([img_arr]))
-                predictions =np.argmax(pre,axis=1)
+                # predictions =np.argmax(pre,axis=1)
                 # print(expressions[predictions[0]])
-                cv.putText(image,expressions[predictions[0]], (x-padding, y-padding-20), cv.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 255), 2,cv.LINE_4,)
-                cv.rectangle(image, (x-padding, y-padding), (x + w+padding, y + h+padding), (255, 255, 255), thickness = 2)
+                # cv.putText(image,expressions[predictions[0]], (x-padding, y-padding-20), cv.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 255), 2,cv.LINE_4,)
+                # cv.rectangle(image, (x-padding, y-padding), (x + w+padding, y + h+padding), (255, 255, 255), thickness = 2)
                 # cv.imshow("image",image)
             return image,pre,True
     except:

web-app/frontend/src/components/activities/mentalChronometryEval/coding/CodingContainer.jsx

@@ -20,18 +20,19 @@ const CodingContainer = () => {
   const [activityCompleted, setActivityCompleted] = useState(false);
   const [finalScore, setFinalScore] = useState(0);
   const [letterPatternANswers, setLetterPatternANswers] = useState([]);
+  const [answerImage] = useState([]);
   const [actualResult, setActualResul] = useState([
     "square",
     // "circle",
-    "triangle",
+    // "triangle",
     // "circle",
     "circle",
     // "triangle",
-    "circle",
+    // "circle",
     // "square",
     "circle",
     // "circle",
-    "square",
+    // "square",
     // "triangle",
     "triangle",
   ]);
@@ -40,15 +41,15 @@ const CodingContainer = () => {
   const [drawingPad, setDrawingPad] = useState([
     "c",
     // "s",
-    "c",
+    // "c",
     // "r",
     "s",
     // "c",
-    "r",
+    // "r",
     // "c",
     "c",
     // "s",
-    "c",
+    // "c",
     // "s",
     "s",
   ]);
@@ -64,6 +65,10 @@ const CodingContainer = () => {
         "Content-Type": "application/json",
       },
     };
+    let candidateID = null;
+    if (localStorage) {
+      candidateID = localStorage.getItem("candidateID");
+    }
     try {
       const payload = {
         activityName: activityName,
@@ -72,6 +77,8 @@ const CodingContainer = () => {
         predResult: letterPatternANswers,
         totalTime: time,
         createdTime: Date(),
+        candidateID: candidateID,
+        answerImage: answerImage,
       };
 
       await API.post("mentalChromScores", JSON.stringify(payload), headerConfig)
@@ -139,6 +146,10 @@ const CodingContainer = () => {
       .toDataURL("image/png")
       .replace("data:image/png;base64,", "");
 
+    answerImage.push(
+      sigCanvas.current.getTrimmedCanvas().toDataURL("image/png")
+    );
+
     let message = { image: image };
     sigCanvas.current.clear();
     await API.post("predicthanddrwnshapes", JSON.stringify(message))
@@ -153,15 +164,15 @@ const CodingContainer = () => {
   var codingURLS = [
     "https://i.ibb.co/9cwf5x2/image.png",
     // "https://i.ibb.co/JFNZwCz/1.png",
-    "https://i.ibb.co/ykdhHM6/2.png",
+    // "https://i.ibb.co/ykdhHM6/2.png",
     // "https://i.ibb.co/KL9JDkB/3.png",
     "https://i.ibb.co/9Y13xQ9/6.png",
     // "https://i.ibb.co/b5B7js8/7.png",
-    "https://i.ibb.co/2SzzMCz/8.png",
+    // "https://i.ibb.co/2SzzMCz/8.png",
     // "https://i.ibb.co/tc7dPJ0/9.png",
     "https://i.ibb.co/3kSQzMM/13.png",
     // "https://i.ibb.co/HPPMv94/14.png",
-    "https://i.ibb.co/KLbgtTx/19.png",
+    // "https://i.ibb.co/KLbgtTx/19.png",
     // "https://i.ibb.co/kKBgmB2/18.png",
     "https://i.ibb.co/93DNN1q/17.png",
   ];

web-app/frontend/src/components/activities/mentalChronometryEval/letterPattern/LetterPattern.jsx

-import React from "react";
+import React, { useEffect } from "react";
 import LetterPatternContainer from "./LetterPatternContainer";
 
 const LetterPattern = ({ nextActivity }) => {
+  // useEffect(() => {
+  //   var candidateID = null;
+  //   if (localStorage) {
+  //     candidateID = localStorage.getItem("candidateID");
+  //   }
+
+  //   alert(candidateID);
+  // });
+
   return (
     <div className="h-full  ">
       <LetterPatternContainer />

web-app/frontend/src/components/activities/mentalChronometryEval/letterPattern/LetterPatternContainer.jsx

@@ -16,7 +16,9 @@ const LetterPatternContainer = ({ nextActivity }) => {
   const [activityCompleted, setActivityCompleted] = useState(false);
   const [finalScore, setFinalScore] = useState(0);
   const [letterPatternANswers] = useState([]);
-  const [actualResult] = useState(["B", "R", "U", "Y", "B", "P", "D"]);
+  const [answerImage] = useState([]);
+  // const [actualResult] = useState(["B", "R", "U", "Y", "B", "P", "D"]);
+  const [actualResult] = useState(["B", "U", "B", "D"]);
   const [isCancelled, setIsCancelled] = useState(false);
 
   var finalResult = 0;
@@ -30,6 +32,11 @@ const LetterPatternContainer = ({ nextActivity }) => {
         "Content-Type": "application/json",
       },
     };
+    let candidateID = null;
+    if (localStorage) {
+      candidateID = localStorage.getItem("candidateID");
+    }
+
     try {
       const payload = {
         activityName: activityName,
@@ -38,6 +45,8 @@ const LetterPatternContainer = ({ nextActivity }) => {
         predResult: letterPatternANswers,
         totalTime: time,
         createdTime: Date(),
+        candidateID: candidateID,
+        answerImage: answerImage,
       };
 
       await API.post("mentalChromScores", JSON.stringify(payload), headerConfig)
@@ -105,6 +114,10 @@ const LetterPatternContainer = ({ nextActivity }) => {
       .toDataURL("image/png")
       .replace("data:image/png;base64,", "");
 
+    answerImage.push(
+      sigCanvas.current.getTrimmedCanvas().toDataURL("image/png")
+    );
+
     let message = { image: image };
     sigCanvas.current.clear();
     await API.post("predictletters", JSON.stringify(message))
@@ -116,13 +129,23 @@ const LetterPatternContainer = ({ nextActivity }) => {
         console.log(err);
       });
   };
+  // var patternURLS = [
+  //   "https://i.ibb.co/XV9RV1M/letter-Pattern-1.png",
+  //   "https://i.ibb.co/4Y3WdDW/letter-Pattern-2.png",
+  //   "https://i.ibb.co/D4VGM7J/letter-Pattern-3.png",
+  //   "https://i.ibb.co/xsVbwyw/letter-Pattern-4.png",
+  //   "https://i.ibb.co/c3wVB5Z/letter-Pattern-5.png",
+  //   "https://i.ibb.co/1K0YCvz/letter-Pattern-6.png",
+  //   "https://i.ibb.co/wYqYDjb/letter-Pattern-7.png",
+  // ];
+
   var patternURLS = [
     "https://i.ibb.co/XV9RV1M/letter-Pattern-1.png",
-    "https://i.ibb.co/4Y3WdDW/letter-Pattern-2.png",
+    // "https://i.ibb.co/4Y3WdDW/letter-Pattern-2.png",
     "https://i.ibb.co/D4VGM7J/letter-Pattern-3.png",
-    "https://i.ibb.co/xsVbwyw/letter-Pattern-4.png",
+    // "https://i.ibb.co/xsVbwyw/letter-Pattern-4.png",
     "https://i.ibb.co/c3wVB5Z/letter-Pattern-5.png",
-    "https://i.ibb.co/1K0YCvz/letter-Pattern-6.png",
+    // "https://i.ibb.co/1K0YCvz/letter-Pattern-6.png",
     "https://i.ibb.co/wYqYDjb/letter-Pattern-7.png",
   ];
 

web-app/frontend/src/components/activities/mentalChronometryEval/pairCancerlation/PairCancerlationContainer.jsx

@@ -7,7 +7,6 @@ import PCB2 from "./img/PCB2.png";
 import PCB3 from "./img/PCB3.png";
 import PCB4 from "./img/PCB4.png";
 import PCA1 from "./img/PCA1.png";
-import PCA2 from "./img/PCA2.png";
 import PCA3 from "./img/PCA3.png";
 import PCA4 from "./img/PCA4.png";
 // import { PCA1, PCB1 } from "./img/";
@@ -16,19 +15,20 @@ import axios from "axios";
 const PairCancerlationContainer = () => {
   const [src, selectFile] = useState(null);
   const [activityCompleted, setActivityCompleted] = useState(false);
-  const [Pair, setPair] = useState([PCA1, PCA2, PCA3, PCA4]);
-  const [picture, setPicture] = useState([PCB1, PCB2, PCB3, PCB4]);
-  const [finalScore, setFinalScore] = useState([PCB1, PCB2, PCB3, PCB4]);
+  const [Pair, setPair] = useState([PCA1, PCA3, PCA4]);
+  const [picture, setPicture] = useState([PCB1, PCB3, PCB4]);
+  const [finalScore, setFinalScore] = useState(0);
   const [predResult] = useState([]);
+  const [finalResultInPair, setFinalResultInPair] = useState(null);
 
   const [actualResult, setActualResult] = useState([
     ["star", "circle"],
-    ["triangle", "square"],
     ["triangle", "star"],
     ["star", "star"],
   ]);
   let results = 0;
   const [activityIndex, setActivityIndex] = useState(0);
+  const [answerImage] = useState([]);
   const handleFileChange = (e) => {
     selectFile(URL.createObjectURL(e.target.files[0]));
   };
@@ -50,14 +50,20 @@ const PairCancerlationContainer = () => {
         "Content-Type": "application/json",
       },
     };
+    let candidateID = null;
+    if (localStorage) {
+      candidateID = localStorage.getItem("candidateID");
+    }
     try {
       const payload = {
         activityName: activityName,
-        score: finalResult,
+        score: result,
         actualResult: actualResult,
-        predResult: result,
+        predResult: predResult,
         totalTime: time,
         createdTime: Date(),
+        candidateID: candidateID,
+        answerImage: answerImage,
       };
 
       await API.post("mentalChromScores", JSON.stringify(payload), headerConfig)
@@ -109,7 +115,7 @@ const PairCancerlationContainer = () => {
   });
 
   const getCroppedImg = async () => {
-    if (activityIndex === 3) {
+    if (activityIndex === 2) {
       setActivityCompleted(true);
       setIsCancelled(true);
       setTimeout(() => {
@@ -140,8 +146,10 @@ const PairCancerlationContainer = () => {
 
     let imagetoPred = base64.replace("data:image/png;base64,", "");
 
+    answerImage.push(canvas.toDataURL("image/png"));
+
     // console.log(base64);
-    if (activityIndex < 3) {
+    if (activityIndex < 2) {
       setActivityIndex(activityIndex + 1);
     }
 
@@ -156,15 +164,18 @@ const PairCancerlationContainer = () => {
         console.log(err);
       });
   };
+
   let res = 0;
   let count = 1;
   const generateScore = () => {
     for (let i = 0; i < actualResult.length; i++) {
       if (JSON.stringify(actualResult[i]) === JSON.stringify(predResult[i])) {
         res += 1;
+        alert(res);
         // setFinalScore(result);
       }
     }
+
     // finalResult = result;
     setFinalScore(res);
     console.log(res);