Merge branch 'master' into ethicalClearence_page

web-app/backend/app.py

@@ -11,15 +11,16 @@ import io
 from PIL import Image
 import cv2 as cv
 import numpy as np
-from trainAnn import headPose,headDistence,facialExperssions,blinkCount,detector
+from trainAnn import headPose, headDistence, facialExperssions, blinkCount, detector
 from keras.models import load_model
 from sklearn.preprocessing import OneHotEncoder
 
-#Attention classes
-attClass={0:'Low Attention',1:'Mid Attention',2:'High Attention'}
+# Attention classes
+attClass = {0: 'Low Attention', 1: 'Mid Attention', 2: 'High Attention'}
 
 UPLOAD_FOLDER = 'C:/SLIIT/Final Research implementation/Backend/new model 2.0/Main Project code/uploads'
-ALLOWED_EXTENSIONS = set(['mkv','webm','mp4', 'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
+ALLOWED_EXTENSIONS = set(
+    ['mkv', 'webm', 'mp4', 'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
 
 x = datetime.datetime.now()
 
@@ -28,69 +29,68 @@ app = Flask(__name__)
 app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
 CORS(app)
 
-def captureImage(link,frameSkip):
+
+def captureImage(link, frameSkip):
     global lastAttention
 
     model_new = load_model("attScore.h5")
 
     encoder2 = OneHotEncoder()
-    encoder2.fit_transform(np.array([0,1,2]).reshape(-1, 1))
-     
-    allFrames=0
-    finalList=[0,0,0]
+    encoder2.fit_transform(np.array([0, 1, 2]).reshape(-1, 1))
 
+    allFrames = 0
+    finalList = [0, 0, 0]
 
     cap = cv.VideoCapture(link)
     while cap.isOpened():
-        
-        success,image = cap.read()
+
+        success, image = cap.read()
         if success:
-          if allFrames%frameSkip==0:
-            
-            headPoseImg,landMarks,state1=headPose(image.copy())
-            headDisImg,headDis,state2=headDistence(image.copy(),detector)
-            expImg,expression,state3=facialExperssions(image.copy())
-            blinkCountImg,blinkTime=blinkCount(image.copy(),detector)
-            
-    
-            if state1  and state2 and state3:
-                arr=[list(map(float,landMarks))+[float(headDis)]+expression.tolist()[0]]
-                preClass = np.zeros([3], dtype = int)
-                pre=list(model_new.predict(arr)[0])
-                preClass[pre.index(max(pre))]=1
-                
-
-                
-                final=encoder2.inverse_transform([preClass])
-                finalList[final[0][0]]+=1
-
-            headPoseImg = cv.resize(headPoseImg, (640, 380))
-            headDisImg = cv.resize(headDisImg, (640, 380))
-            blinkCountImg=cv.resize(blinkCountImg, (640, 380))
-            expImg=cv.resize(expImg, (640, 380))
-            image=cv.resize(image,(640, 380))
-
-            # imgPlot = plotY.update(lastAttention, color=(255, 0, 0))
-   
-            # hor = np.hstack((headPoseImg, headDisImg))
-            # hor2 = np.hstack((blinkCountImg, expImg))
-            # ver = np.vstack((hor, hor2))
-
-
-            # finaliImage = cv.putText(headPoseImg," ".join(list(map(lambda x:str(round(x,2)),finalList)))+" "+attClass[final[0][0]], (20,80), cv.FONT_HERSHEY_SIMPLEX, 0.7,(0,255,0), 2, cv.LINE_AA)
-            # cv.imshow("ImageStack", finaliImage)
+            if allFrames % frameSkip == 0:
+
+                headPoseImg, landMarks, state1 = headPose(image.copy())
+                headDisImg, headDis, state2 = headDistence(
+                    image.copy(), detector)
+                expImg, expression, state3 = facialExperssions(image.copy())
+                blinkCountImg, blinkTime = blinkCount(image.copy(), detector)
+
+                if state1 and state2 and state3:
+                    arr = [list(map(float, landMarks)) +
+                           [float(headDis)]+expression.tolist()[0]]
+                    preClass = np.zeros([3], dtype=int)
+                    pre = list(model_new.predict(arr)[0])
+                    preClass[pre.index(max(pre))] = 1
+
+                    final = encoder2.inverse_transform([preClass])
+                    finalList[final[0][0]] += 1
+
+                headPoseImg = cv.resize(headPoseImg, (640, 380))
+                headDisImg = cv.resize(headDisImg, (640, 380))
+                blinkCountImg = cv.resize(blinkCountImg, (640, 380))
+                expImg = cv.resize(expImg, (640, 380))
+                image = cv.resize(image, (640, 380))
+
+                # imgPlot = plotY.update(lastAttention, color=(255, 0, 0))
+
+                # hor = np.hstack((headPoseImg, headDisImg))
+                # hor2 = np.hstack((blinkCountImg, expImg))
+                # ver = np.vstack((hor, hor2))
+
+                # finaliImage = cv.putText(headPoseImg," ".join(list(map(lambda x:str(round(x,2)),finalList)))+" "+attClass[final[0][0]], (20,80), cv.FONT_HERSHEY_SIMPLEX, 0.7,(0,255,0), 2, cv.LINE_AA)
+                # cv.imshow("ImageStack", finaliImage)
         else:
             break
 
         if cv.waitKey(5) & 0xFF == 27:
             break
-        allFrames+=1
+        allFrames += 1
 
         # if allFrames==1000:
         #     break
 
-    cap.release() 
-    return list(map(lambda x:round(x/sum(finalList),4),finalList)) 
+    cap.release()
+    return list(map(lambda x: round(x/sum(finalList), 4), finalList))
+
 
 # create our model
 IMG_SHAPE = (224, 224, 3)
@@ -129,22 +129,25 @@ model = get_model()
 
 # function predict is called at each request
 
-#for prediction
+# for prediction
+
+
 @app.route('/upload', methods=['POST'])
 def predictAttention():
     print(request)
-    target=os.path.join(UPLOAD_FOLDER,'test_vids')
+    target = os.path.join(UPLOAD_FOLDER, 'test_vids')
     if not os.path.isdir(target):
         os.mkdir(target)
-    file = request.files['file'] 
+    file = request.files['file']
     filename = secure_filename("Testing_vid")
-    destination="/".join([target, filename])
+    destination = "/".join([target, filename])
     file.save(destination)
-    session['uploadFilePath']=destination
-    scoreLst=captureImage(destination,10)
-    print("Prediction",scoreLst)
+    session['uploadFilePath'] = destination
+    scoreLst = captureImage(destination, 10)
+    print("Prediction", scoreLst)
     return scoreLst
 
+
 @app.route("/predictletters", methods=["POST"])
 def predict():
 
@@ -244,9 +247,86 @@ def predictHandDrawnShapes():
     return jsonify(response)
 
 
+@app.route("/predictShapePattern", methods=["POST"])
+def predictShapePattern():
+    handDrawnShapeClassificationmodel = load_model(
+        'shapeClassification1.h5', compile=False)
+    # get the data from the request and put ir under the right format
+    req = request.get_json(force=True)
+    # print(req)
+    encoded = req["image"]
+    decoded = base64.b64decode(encoded)
+    print(decoded)
+
+    with open("imgt.png", "wb") as fh:
+        fh.write(base64.decodebytes(decoded))
+
+
+# print(decoded)
+
+    pil_image = Image.open(io.BytesIO(decoded)).resize(
+        (224, 224), Image.LANCZOS).convert("RGB")
+
+    image = pil_image
+
+    image.save("imgtest1.png")
+
+    img = cv.imread('imgtest1.png')
+
+    h, w, channels = img.shape
+
+    half = w//2
+
+    left_part = img[:, :half]
+
+    right_part = img[:, half:]
+
+    half2 = h//2
+
+    top = img[:half2, :]
+    bottom = img[half2:, :]
+
+    right_part = cv.resize(right_part, (224, 224))
+    left_part = cv.resize(left_part, (224, 224))
+
+    cv.imwrite('right.jpg', right_part)
+    cv.imwrite('left.jpg', left_part)
+
+
+# image = decoded  # Image.open('img.png')
+    imgArray = ["left.jpg", "right.jpg"]
+    for img in imgArray:
+        data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
+        image = Image.open(img)
+        size = (224, 224)
+        image_array = np.asarray(image)
+        image_array = cv.resize(image_array, (224, 224))
+        normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
+        data[0] = normalized_image_array
+
+        word_dict = {0: 'circle', 1: 'triangle', 2: 'star', 3: 'square'}
+
+        prediction = word_dict[np.argmax(
+            handDrawnShapeClassificationmodel.predict(data))]
+
+        prediction2 = model.predict(data)
+        response = {"prediction": prediction}
+
+        print(prediction)
+# print(prediction)
+# response = jsonify({"prediction": prediction})
+# response.headers.add('Access-Control-Allow-Origin', '*')
+# return response
+
+    # response = {"prediction": prediction}
+
+    # print(prediction2)
+    return jsonify(response)
+
+
 # Running app
 if __name__ == "__main__":
     app.secret_key = os.urandom(24)
-    app.run(debug=True,use_reloader=False)
+    app.run(debug=True, use_reloader=False)
 
-flask_cors.CORS(app, expose_headers='Authorization')
+flask_cors.CORS(app, expose_headers='Authorization')
\ No newline at end of file

web-app/frontend/src/components/activities/mentalChronometryEval/coding/CodingContainer.jsx

@@ -222,7 +222,7 @@ const CodingContainer = () => {
               <div className="grid grid-cols-2 justify-items-center m-3">
                 <div>
                   {" "}
-                  <DeleteForeverIcon
+                  {/* <DeleteForeverIcon
                     className="mr-8"
                     sx={{ fontSize: 40, color: "black" }}
                     onClick={() => {
@@ -231,16 +231,25 @@ const CodingContainer = () => {
                     style={{ float: "right" }}
                   >
                     Clear
-                  </DeleteForeverIcon>
+                  </DeleteForeverIcon> */}
                 </div>
                 <div>
                   <div className="row-start-2 row-span-2 ...">
                     {" "}
                     <button
                       onClick={increase}
-                      className="ml-10 bg-transparent hover:bg-blue-500 text-blue-700 font-semibold hover:text-white py-2 px-4 border border-blue-500 hover:border-transparent rounded "
+                      // className="ml-10 bg-transparent hover:bg-blue-500 text-blue-700 font-semibold hover:text-white py-2 px-4 border border-blue-500 hover:border-transparent rounded "
                     >
-                      Next
+                      <img
+                        style={{
+                          // border: "2px solid black",
+                          // borderRadius: "50px",
+                          width: "250px",
+                          height: "80px",
+                        }}
+                        src="https://i.ibb.co/Hz254BT/Screenshot-2022-10-02-093444.png"
+                        alt="pattern"
+                      />
                     </button>
                   </div>
                 </div>

web-app/frontend/src/components/activities/mentalChronometryEval/letterPattern/LetterPatternContainer.jsx

@@ -152,7 +152,7 @@ const LetterPatternContainer = ({ nextActivity }) => {
               <div className="grid grid-cols-2 justify-items-center m-3">
                 <div>
                   {" "}
-                  <DeleteForeverIcon
+                  {/* <DeleteForeverIcon
                     className="mr-8"
                     sx={{ fontSize: 40, color: "black" }}
                     onClick={() => {
@@ -161,16 +161,25 @@ const LetterPatternContainer = ({ nextActivity }) => {
                     style={{ float: "right" }}
                   >
                     Clear
-                  </DeleteForeverIcon>
+                  </DeleteForeverIcon> */}
                 </div>
                 <div>
                   <div className="row-start-2 row-span-2 ...">
                     {" "}
                     <button
                       onClick={increase}
-                      className="ml-10 bg-transparent hover:bg-blue-500 text-blue-700 font-semibold hover:text-white py-2 px-4 border border-blue-500 hover:border-transparent rounded "
+                      // className="ml-10 bg-transparent hover:bg-blue-500 text-blue-700 font-semibold hover:text-white py-2 px-4 border border-blue-500 hover:border-transparent rounded "
                     >
-                      Next
+                      <img
+                        style={{
+                          // border: "2px solid black",
+                          // borderRadius: "50px",
+                          width: "250px",
+                          height: "80px",
+                        }}
+                        src="https://i.ibb.co/Hz254BT/Screenshot-2022-10-02-093444.png"
+                        alt="pattern"
+                      />
                     </button>
                   </div>
                 </div>

web-app/frontend/src/components/activities/mentalChronometryEval/pairCancerlation/PairCancerlationContainer.jsx

-import React, { useState, useRef, useEffect } from "react";
+import React, { useState, useEffect } from "react";
 import ReactCrop from "react-image-crop";
 import "react-image-crop/dist/ReactCrop.css";
+import PCB1 from "./img/pair-Cancerlation1.png";
+import PCA1 from "./img/PCA1.png";
+import PCB2 from "./img/PCB2.png";
+// import { PCA1, PCB1 } from "./img/";
+import axios from "axios";
+import { height, width } from "@mui/system";
 
 const PairCancerlationContainer = () => {
-  const test = () => {
-    console.log(crop);
+  const [src, selectFile] = useState(null);
+  const [activityCompleted, setActivityCompleted] = useState(false);
+  const handleFileChange = (e) => {
+    selectFile(URL.createObjectURL(e.target.files[0]));
   };
+  const [time, setTime] = useState({
+    seconds: 0,
+    minutes: 0,
+    hours: 0,
+  });
+  const [isCancelled, setIsCancelled] = useState(false);
+  useEffect(() => {
+    const calculateTime = () => {
+      setTimeout(() => {
+        let nSeconds = time.seconds;
+        let nMinutes = time.minutes;
+        let nHours = time.hours;
 
-  const [crop, setCrop] = useState("");
+        nSeconds++;
+
+        if (nSeconds > 59) {
+          nMinutes++;
+          nSeconds = 0;
+        }
+        if (nMinutes > 59) {
+          nHours++;
+          nMinutes = 0;
+        }
+        if (nHours > 24) {
+          nHours = 0;
+        }
+
+        !isCancelled &&
+          setTime({ seconds: nSeconds, minutes: nMinutes, hours: nHours });
+      }, 1000);
+    };
+    calculateTime();
+  });
+  const [setImage] = useState(null);
+  const [crop, setCrop] = useState({
+    aspect: 16 / 9,
+    unit: "px",
+    width: 120,
+    height: 60,
+    x: 25,
+    y: 25,
+  });
+
+  const getCroppedImg = async () => {
+    const canvas = document.createElement("canvas");
+    const image = document.getElementById("result");
+    const ctx = canvas.getContext("2d");
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+    console.log(
+      image.width,
+      image.height,
+      image.naturalWidth,
+      image.naturalHeight
+    );
+
+    const imageWidthRatio = image.naturalWidth / image.width;
+    const imageHeightRatio = image.naturalHeight / image.height;
+
+    // console.log(crop);
+    ctx.drawImage(
+      image,
+      crop.x * imageWidthRatio,
+      crop.y * imageHeightRatio,
+      crop.width * imageWidthRatio,
+      crop.height * imageHeightRatio,
+      0,
+      0,
+      crop.width,
+      crop.height
+    );
+
+    console.log(crop.width);
+    console.log(crop.height);
+    const base64 = canvas.toDataURL("image/png");
+
+    let imagetoPred = base64.replace("data:image/png;base64,", "");
+
+    console.log(imagetoPred);
+
+    let message = { image: imagetoPred };
+
+    await axios
+      .post(
+        "http://127.0.0.1:5000/predictShapePattern",
+        JSON.stringify(message)
+      )
+      .then((res) => {
+        console.log(res.data.prediction);
+      })
+      .catch((err) => {
+        console.log(err);
+      });
+
+    // console.log(base64);
+  };
   return (
-    <div>
-      <ReactCrop
-        crop={crop}
-        onChange={(c) => {
-          setCrop(c);
-          test();
-        }}
-      >
-        <img
-          style={{
-            // border: "2px solid black",
-            // borderRadius: "50px",
-            width: "100%",
-            height: "100%",
-          }}
-          src={"https://i.ibb.co/tmtT6hM/pair-Cancerlation1.png"}
-          alt="pattern"
-        />
-      </ReactCrop>
+    // <div className="container">
+    //   <div className="row">
+    //     {/* <div className="col-6">
+    //       <input type="file" accept="image/*" onChange={handleFileChange} />
+    //     </div> */}
+
+    //     <div>
+    //       <ReactCrop crop={crop} onChange={setCrop}>
+    //         <img id="result" src={PCB2} alt="description" onLoad={setImage} />
+    //       </ReactCrop>
+    //       <button className="text-black" onClick={getCroppedImg}>
+    //         Crop image
+    //       </button>
+    //     </div>
+
+    //     {/* <canvas
+    //       id="canvas"
+    //       width={crop.width}
+    //       height={crop.height}
+    //       style={{
+    //         border: "1px solid black",
+    //         objectFit: "contain",
+    //       }}
+    //     ></canvas> */}
+    //   </div>
+    // </div>
+    <div className="justify-items-center h-full">
+      {!activityCompleted && (
+        <div className=" h-full">
+          <div className="grid grid-cols-1 h-1/4 ">
+            {" "}
+            <div>
+              {" "}
+              <img
+                id="result"
+                src={PCA1}
+                alt="description"
+                onLoad={setImage}
+              />{" "}
+            </div>
+          </div>
+          <div className="grid grid-cols-1 h-fit m-10">
+            <div className=" w-fit h-fit m-auto border-solid border-2 border-sky-500">
+              {" "}
+              <ReactCrop crop={crop} onChange={setCrop} s>
+                {" "}
+                <img
+                  id="result"
+                  src={PCB1}
+                  style={{
+                    width: "100%",
+                    height: "100%",
+                  }}
+                  alt="description"
+                  onLoad={setImage}
+                />{" "}
+              </ReactCrop>{" "}
+            </div>
+          </div>
+          <div className="h-full w-full bg-red">
+            <div className="grid grid-cols-2 justify-items-center m-3">
+              <div>
+                {" "}
+                {/* <DeleteForeverIcon
+                  className="mr-8"
+                  sx={{ fontSize: 40, color: "black" }}
+                  onClick={() => {
+                    sigCanvas.current.clear();
+                  }}
+                  style={{ float: "right" }}
+                >
+                  Clear
+                </DeleteForeverIcon> */}
+              </div>
+              <div>
+                <div className="row-start-2 row-span-2 ...">
+                  {" "}
+                  <button
+                    onClick={getCroppedImg}
+                    // className="ml-10 bg-transparent hover:bg-blue-500 text-blue-700 font-semibold hover:text-white py-2 px-4 border border-blue-500 hover:border-transparent rounded "
+                  >
+                    <img
+                      style={{
+                        // border: "2px solid black",
+                        // borderRadius: "50px",
+                        width: "250px",
+                        height: "80px",
+                      }}
+                      src="https://i.ibb.co/Hz254BT/Screenshot-2022-10-02-093444.png"
+                      alt="pattern"
+                    />
+                  </button>
+                </div>
+
+                {false && (
+                  <canvas
+                    id="canvas"
+                    width={crop.width}
+                    height={crop.height}
+                    style={{
+                      border: "1px solid black",
+                      objectFit: "contain",
+                    }}
+                  />
+                )}
+              </div>
+            </div>
+          </div>
+        </div>
+      )}
+
+      {activityCompleted && (
+        <div className="grid grid-cols-1 m-4">
+          <div className="grid justify-items-center">
+            <img
+              style={{
+                // border: "2px solid black",
+                // borderRadius: "50px",
+                width: "250px",
+                height: "250px",
+              }}
+              src="https://i.ibb.co/sJWJZsD/completed.png"
+              alt="pattern"
+            />
+            <h1 className="text-black">
+              {/* Final Score {finalScore && finalScore}/{actualResul.length} */}
+            </h1>
+            <h1 className="text-black">
+              Total Time {time.hours < 10 ? "0" + time.hours : time.hours} :{" "}
+              {time.minutes < 10 ? "0" + time.minutes : time.minutes} :{" "}
+              {time.seconds < 10 ? "0" + time.seconds : time.seconds}
+            </h1>
+          </div>
+        </div>
+      )}
     </div>
   );
 };
-
 export default PairCancerlationContainer;

web-app/frontend/src/components/activities/reasoningIqEval/pictureConcept/PictureConcept.jsx

 import React, { useState, useRef } from "react";
+import { useEffect } from "react";
 import { ImageRow } from "../../../index";
 import { RecordingHandler } from "../recorder/Recorder";
 import Timer from "../timer/Timer";
@@ -89,8 +90,50 @@ const PictureConcept = ({ nextActivity }) => {
     "https://i.ibb.co/rQS963q/tiger-PNG23230.png",
   ];
 
+  const [time, setTime] = useState({
+    seconds: 0,
+  });
+  const [isCancelled, setIsCancelled] = useState(false);
+
+  useEffect(() => {
+    const calculateTime = () => {
+      setTimeout(() => {
+        let nSeconds = time.seconds;
+        let nMinutes = time.minutes;
+        let nHours = time.hours;
+
+        nSeconds++;
+
+        // if (nSeconds > 59) {
+        //   nMinutes++;
+        //   nSeconds = 0;
+        // }
+        // if (nMinutes > 59) {
+        //   nHours++;
+        //   nMinutes = 0;
+        // }
+        // if (nHours > 24) {
+        //   nHours = 0;
+        // }
+
+        !isCancelled &&
+          setTime({ seconds: nSeconds, minutes: nMinutes, hours: nHours });
+      }, 1000);
+    };
+    calculateTime();
+    console.log(time);
+    if (time.seconds === 100) {
+      setImg(img + 4);
+      testFunction();
+      time.seconds = 0;
+    }
+  });
+
   const testFunction = () => {
+    console.log("Hi");
     var urlCount = 2;
+
+    console.log(time);
     if (activityIndex === 3 || activityIndex === 4) {
       urlCount = 3;
     }