Skip to content

2
2022-073
Commit 627ab296 authored by NilanDMeegoda's avatar NilanDMeegoda
Browse files
Options

Merge branch 'master' of http://gitlab.sliit.lk/2022-073/2022-073

parents 3ae60f45 6c637833
Hide whitespace changes
Inline Side-by-side
Showing with 139 additions and 70 deletions
web-app/backend/controler.py
View file @ 627ab296
import cv2 as cv
import numpy as np
from trainAnn import headPose,headDistence,facialExperssions,blinkCount,detector
import os
from keras.models import load_model
from sklearn.preprocessing import OneHotEncoder
......@@ -76,4 +76,7 @@ def captureImage(link,frameSkip):
if __name__=="__main__":
print("Controller working")
directory = os.getcwd()
destination = directory + r"\uploads\att.mp4"
scoreLst=captureImage(destination,10)
print(scoreLst)
web-app/backend/server.py
View file @ 627ab296
......@@ -371,7 +371,9 @@ def predictShapePattern():
return jsonify(response)
# Reasoning IQ evaluation
@app.route('/predictPictureConcept', methods = ['POST','GET'])
@app.route('/predictPictureConcept', methods=['POST', 'GET'])
@cross_origin()
def predictPictureConcept():
if request.method == 'POST':
......@@ -383,7 +385,7 @@ def predictPictureConcept():
return jsonify({'error: no file'})
try:
file.save("./"+file.filename)
prediction = transform_audio(file.filename)
data = {'prediction': prediction}
......@@ -398,7 +400,8 @@ def predictPictureConcept():
return jsonify({'error: Error during pipeline execution'})
return jsonify({'result: test'})
@app.route('/predictArithmetic', methods = ['POST','GET'])
@app.route('/predictArithmetic', methods=['POST', 'GET'])
@cross_origin()
def predictArithmetic():
if request.method == 'POST':
......@@ -410,7 +413,7 @@ def predictArithmetic():
return jsonify({'error: no file'})
try:
file.save("./"+file.filename)
prediction = transform_audio(file.filename)
data = {'prediction': prediction}
......@@ -425,9 +428,11 @@ def predictArithmetic():
return jsonify({'error: Error during pipeline execution'})
return jsonify({'result: test'})
@app.route('/mentalChromScores', methods=['POST'])
@cross_origin()
def mentalChromScores():
body = request.json
activityName = body['activityName']
score = body['score']
......@@ -435,6 +440,8 @@ def mentalChromScores():
predResult = body['predResult']
createdTime = body['createdTime']
totalTime = body['totalTime']
candidateID = body['candidateID']
answerImage = body['answerImage']
# db.users.insert_one({
db.db['mentalChronomrtryScore'].insert_one({
"activityName": activityName,
......@@ -443,13 +450,17 @@ def mentalChromScores():
"predResult": predResult,
"createdTime": createdTime,
"totalTime": totalTime,
"candidateID": candidateID,
"answerImage": answerImage
})
return jsonify({
'status': 'MentalCrom scores are added to the system successfully!',
})
# Knowledge IQ evaluation
@app.route('/predictKnowledgeIq', methods = ['POST','GET'])
@app.route('/predictKnowledgeIq', methods=['POST', 'GET'])
@cross_origin()
def predictKnowledgeIq():
if request.method == 'POST':
......@@ -464,7 +475,8 @@ def predictKnowledgeIq():
file.save("./"+file.filename)
prediction = transform_audio(file.filename)
data = {'prediction': prediction}
score = calKnowledgeIqActivityScore(activityName, questionIndex, prediction)
score = calKnowledgeIqActivityScore(
activityName, questionIndex, prediction)
db.db['knowledgeIQScore'].insert_one({
"activityName": activityName,
"questionIndex": questionIndex,
......
web-app/backend/trainAnn.py
View file @ 627ab296
......@@ -70,7 +70,7 @@ def headDistence(frame3,detector):
# Finding distance
f = 650
d = (W * f) / w
cvzone.putTextRect(frame3, f'Depth: {int(d)}cm',(facedis[10][0] - 100, facedis[10][1] - 50),scale=2)
# cvzone.putTextRect(frame3, f'Depth: {int(d)}cm',(facedis[10][0] - 100, facedis[10][1] - 50),scale=2)
return frame3,str(d),True
return frame3,False,False
......@@ -87,8 +87,8 @@ def blinkCount(frame2,detector):
frame2, faces = detector.findFaceMesh(frame2, draw=False)
if faces:
face = faces[0]
for id in idList:
cv.circle(frame2, face[id], 2, (255, 0, 255), cv.FILLED)
# for id in idList:
# cv.circle(frame2, face[id], 2, (255, 0, 255), cv.FILLED)
leftUp = face[159]
leftDown = face[23]
......@@ -98,8 +98,8 @@ def blinkCount(frame2,detector):
lengthVerticl, _ = detector.findDistance(leftUp, leftDown)
lengthHorz, _ = detector.findDistance(leftLEnd, leftREnd)
cv.line(frame2, leftUp, leftDown, (0, 200, 0), 2)
cv.line(frame2, leftLEnd, leftREnd, (0, 200, 0), 2)
# cv.line(frame2, leftUp, leftDown, (0, 200, 0), 2)
# cv.line(frame2, leftLEnd, leftREnd, (0, 200, 0), 2)
ratio = int((lengthVerticl / lengthHorz) * 100)
ratioList.append(ratio)
......@@ -119,8 +119,8 @@ def blinkCount(frame2,detector):
counter = 0
# writer3.writerow([blinkCounter])
cvzone.putTextRect(frame2, f'Blink Time: {blinkTime}', (20, 50), 1, 2, (255, 255, 255), (255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
imgPlot = plotY.update(ratioAvg, color=(255, 0, 0))
# cvzone.putTextRect(frame2, f'Blink Time: {blinkTime}', (20, 50), 1, 2, (255, 255, 255), (255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
# imgPlot = plotY.update(ratioAvg, color=(255, 0, 0))
return frame2,blinkTime
......@@ -143,9 +143,9 @@ def headPose(image,imgCount=0):
for face_landmarks in results.multi_face_landmarks:
for idx, lm in enumerate(face_landmarks.landmark):
if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:
if idx == 1:
nose_2d = (lm.x * img_w, lm.y * img_h)
nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
# if idx == 1:
# nose_2d = (lm.x * img_w, lm.y * img_h)
# nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
x, y = int(lm.x * img_w), int(lm.y * img_h)
......@@ -185,44 +185,44 @@ def headPose(image,imgCount=0):
z = angles[2] * 360
# See where the user's head tilting
if y < -10:
text = "Looking Left"
elif y > 10:
text = "Looking Right"
elif x < -10:
text = "Looking Down"
elif x > 10:
text = "Looking Up"
else:
text = "Forward"
# if y < -10:
# text = "Looking Left"
# elif y > 10:
# text = "Looking Right"
# elif x < -10:
# text = "Looking Down"
# elif x > 10:
# text = "Looking Up"
# else:
# text = "Forward"
# Display the nose direction
nose_3d_projection, jacobian = cv.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix)
# nose_3d_projection, jacobian = cv.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix)
p1 = (int(nose_2d[0]), int(nose_2d[1]))
p2 = (int(nose_2d[0] + y * 10), int(nose_2d[1] - x * 10))
# p1 = (int(nose_2d[0]), int(nose_2d[1]))
# p2 = (int(nose_2d[0] + y * 10), int(nose_2d[1] - x * 10))
cv.line(image, p1, p2, (255, 0, 0), 3)
# cv.line(image, p1, p2, (255, 0, 0), 3)
# Add the text on the image
# writer.writerow([str(np.round(x, 2)), str(np.round(y, 2)), str(np.round(z, 2)), text])
text=str(imgCount+1)+" "+text
# text=str(imgCount+1)+" "+text
cvzone.putTextRect(image, text, (20, 50), 1, 2, (255, 255, 255),
(255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
cv.putText(image, "x: " + str(np.round(x, 2)), (500, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv.putText(image, "y: " + str(np.round(y, 2)), (500, 100), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv.putText(image, "z: " + str(np.round(z, 2)), (500, 150), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
# cvzone.putTextRect(image, text, (20, 50), 1, 2, (255, 255, 255),
# (255, 0, 0), cv.FONT_HERSHEY_SIMPLEX)
# cv.putText(image, "x: " + str(np.round(x, 2)), (500, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
# cv.putText(image, "y: " + str(np.round(y, 2)), (500, 100), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
# cv.putText(image, "z: " + str(np.round(z, 2)), (500, 150), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
end = time.time()
totalTime = end - start
# totalTime = end - start
try:
fps = 1 / totalTime
except:
fps =0
# try:
# fps = 1 / totalTime
# except:
# fps =0
# print("FPS: ", fps)
cv.putText(image, f'FPS: {int(fps)}', (20, 450), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2)
# cv.putText(image, f'FPS: {int(fps)}', (20, 450), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2)
mp_drawing.draw_landmarks(
image=image,
......@@ -255,10 +255,10 @@ def facialExperssions(image):
img_arr = np.asarray(image2).reshape((224, 224, 3))
pre=model.predict(np.array([img_arr]))
predictions =np.argmax(pre,axis=1)
# predictions =np.argmax(pre,axis=1)
# print(expressions[predictions[0]])
cv.putText(image,expressions[predictions[0]], (x-padding, y-padding-20), cv.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 255), 2,cv.LINE_4,)
cv.rectangle(image, (x-padding, y-padding), (x + w+padding, y + h+padding), (255, 255, 255), thickness = 2)
# cv.putText(image,expressions[predictions[0]], (x-padding, y-padding-20), cv.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 255), 2,cv.LINE_4,)
# cv.rectangle(image, (x-padding, y-padding), (x + w+padding, y + h+padding), (255, 255, 255), thickness = 2)
# cv.imshow("image",image)
return image,pre,True
except:
......
web-app/frontend/src/components/activities/mentalChronometryEval/coding/CodingContainer.jsx
View file @ 627ab296
......@@ -20,18 +20,19 @@ const CodingContainer = () => {
const [activityCompleted, setActivityCompleted] = useState(false);
const [finalScore, setFinalScore] = useState(0);
const [letterPatternANswers, setLetterPatternANswers] = useState([]);
const [answerImage] = useState([]);
const [actualResult, setActualResul] = useState([
"square",
// "circle",
"triangle",
// "triangle",
// "circle",
"circle",
// "triangle",
"circle",
// "circle",
// "square",
"circle",
// "circle",
"square",
// "square",
// "triangle",
"triangle",
]);
......@@ -40,15 +41,15 @@ const CodingContainer = () => {
const [drawingPad, setDrawingPad] = useState([
"c",
// "s",
"c",
// "c",
// "r",
"s",
// "c",
"r",
// "r",
// "c",
"c",
// "s",
"c",
// "c",
// "s",
"s",
]);
......@@ -64,6 +65,10 @@ const CodingContainer = () => {
"Content-Type": "application/json",
},
};
let candidateID = null;
if (localStorage) {
candidateID = localStorage.getItem("candidateID");
}
try {
const payload = {
activityName: activityName,
......@@ -72,6 +77,8 @@ const CodingContainer = () => {
predResult: letterPatternANswers,
totalTime: time,
createdTime: Date(),
candidateID: candidateID,
answerImage: answerImage,
};
await API.post("mentalChromScores", JSON.stringify(payload), headerConfig)
......@@ -139,6 +146,10 @@ const CodingContainer = () => {
.toDataURL("image/png")
.replace("data:image/png;base64,", "");
answerImage.push(
sigCanvas.current.getTrimmedCanvas().toDataURL("image/png")
);
let message = { image: image };
sigCanvas.current.clear();
await API.post("predicthanddrwnshapes", JSON.stringify(message))
......@@ -153,15 +164,15 @@ const CodingContainer = () => {
var codingURLS = [
"https://i.ibb.co/9cwf5x2/image.png",
// "https://i.ibb.co/JFNZwCz/1.png",
"https://i.ibb.co/ykdhHM6/2.png",
// "https://i.ibb.co/ykdhHM6/2.png",
// "https://i.ibb.co/KL9JDkB/3.png",
"https://i.ibb.co/9Y13xQ9/6.png",
// "https://i.ibb.co/b5B7js8/7.png",
"https://i.ibb.co/2SzzMCz/8.png",
// "https://i.ibb.co/2SzzMCz/8.png",
// "https://i.ibb.co/tc7dPJ0/9.png",
"https://i.ibb.co/3kSQzMM/13.png",
// "https://i.ibb.co/HPPMv94/14.png",
"https://i.ibb.co/KLbgtTx/19.png",
// "https://i.ibb.co/KLbgtTx/19.png",
// "https://i.ibb.co/kKBgmB2/18.png",
"https://i.ibb.co/93DNN1q/17.png",
];
......
web-app/frontend/src/components/activities/mentalChronometryEval/letterPattern/LetterPattern.jsx
View file @ 627ab296
import React from "react";
import React, { useEffect } from "react";
import LetterPatternContainer from "./LetterPatternContainer";
const LetterPattern = ({ nextActivity }) => {
// useEffect(() => {
// var candidateID = null;
// if (localStorage) {
// candidateID = localStorage.getItem("candidateID");
// }
// alert(candidateID);
// });
return (
<div className="h-full ">
<LetterPatternContainer />
......
web-app/frontend/src/components/activities/mentalChronometryEval/letterPattern/LetterPatternContainer.jsx
View file @ 627ab296
......@@ -16,7 +16,9 @@ const LetterPatternContainer = ({ nextActivity }) => {
const [activityCompleted, setActivityCompleted] = useState(false);
const [finalScore, setFinalScore] = useState(0);
const [letterPatternANswers] = useState([]);
const [actualResult] = useState(["B", "R", "U", "Y", "B", "P", "D"]);
const [answerImage] = useState([]);
// const [actualResult] = useState(["B", "R", "U", "Y", "B", "P", "D"]);
const [actualResult] = useState(["B", "U", "B", "D"]);
const [isCancelled, setIsCancelled] = useState(false);
var finalResult = 0;
......@@ -30,6 +32,11 @@ const LetterPatternContainer = ({ nextActivity }) => {
"Content-Type": "application/json",
},
};
let candidateID = null;
if (localStorage) {
candidateID = localStorage.getItem("candidateID");
}
try {
const payload = {
activityName: activityName,
......@@ -38,6 +45,8 @@ const LetterPatternContainer = ({ nextActivity }) => {
predResult: letterPatternANswers,
totalTime: time,
createdTime: Date(),
candidateID: candidateID,
answerImage: answerImage,
};
await API.post("mentalChromScores", JSON.stringify(payload), headerConfig)
......@@ -105,6 +114,10 @@ const LetterPatternContainer = ({ nextActivity }) => {
.toDataURL("image/png")
.replace("data:image/png;base64,", "");
answerImage.push(
sigCanvas.current.getTrimmedCanvas().toDataURL("image/png")
);
let message = { image: image };
sigCanvas.current.clear();
await API.post("predictletters", JSON.stringify(message))
......@@ -116,13 +129,23 @@ const LetterPatternContainer = ({ nextActivity }) => {
console.log(err);
});
};
// var patternURLS = [
// "https://i.ibb.co/XV9RV1M/letter-Pattern-1.png",
// "https://i.ibb.co/4Y3WdDW/letter-Pattern-2.png",
// "https://i.ibb.co/D4VGM7J/letter-Pattern-3.png",
// "https://i.ibb.co/xsVbwyw/letter-Pattern-4.png",
// "https://i.ibb.co/c3wVB5Z/letter-Pattern-5.png",
// "https://i.ibb.co/1K0YCvz/letter-Pattern-6.png",
// "https://i.ibb.co/wYqYDjb/letter-Pattern-7.png",
// ];
var patternURLS = [
"https://i.ibb.co/XV9RV1M/letter-Pattern-1.png",
"https://i.ibb.co/4Y3WdDW/letter-Pattern-2.png",
// "https://i.ibb.co/4Y3WdDW/letter-Pattern-2.png",
"https://i.ibb.co/D4VGM7J/letter-Pattern-3.png",
"https://i.ibb.co/xsVbwyw/letter-Pattern-4.png",
// "https://i.ibb.co/xsVbwyw/letter-Pattern-4.png",
"https://i.ibb.co/c3wVB5Z/letter-Pattern-5.png",
"https://i.ibb.co/1K0YCvz/letter-Pattern-6.png",
// "https://i.ibb.co/1K0YCvz/letter-Pattern-6.png",
"https://i.ibb.co/wYqYDjb/letter-Pattern-7.png",
];
......
web-app/frontend/src/components/activities/mentalChronometryEval/pairCancerlation/PairCancerlationContainer.jsx
View file @ 627ab296
......@@ -7,7 +7,6 @@ import PCB2 from "./img/PCB2.png";
import PCB3 from "./img/PCB3.png";
import PCB4 from "./img/PCB4.png";
import PCA1 from "./img/PCA1.png";
import PCA2 from "./img/PCA2.png";
import PCA3 from "./img/PCA3.png";
import PCA4 from "./img/PCA4.png";
// import { PCA1, PCB1 } from "./img/";
......@@ -16,19 +15,20 @@ import axios from "axios";
const PairCancerlationContainer = () => {
const [src, selectFile] = useState(null);
const [activityCompleted, setActivityCompleted] = useState(false);
const [Pair, setPair] = useState([PCA1, PCA2, PCA3, PCA4]);
const [picture, setPicture] = useState([PCB1, PCB2, PCB3, PCB4]);
const [finalScore, setFinalScore] = useState([PCB1, PCB2, PCB3, PCB4]);
const [Pair, setPair] = useState([PCA1, PCA3, PCA4]);
const [picture, setPicture] = useState([PCB1, PCB3, PCB4]);
const [finalScore, setFinalScore] = useState(0);
const [predResult] = useState([]);
const [finalResultInPair, setFinalResultInPair] = useState(null);
const [actualResult, setActualResult] = useState([
["star", "circle"],
["triangle", "square"],
["triangle", "star"],
["star", "star"],
]);
let results = 0;
const [activityIndex, setActivityIndex] = useState(0);
const [answerImage] = useState([]);
const handleFileChange = (e) => {
selectFile(URL.createObjectURL(e.target.files[0]));
};
......@@ -50,14 +50,20 @@ const PairCancerlationContainer = () => {
"Content-Type": "application/json",
},
};
let candidateID = null;
if (localStorage) {
candidateID = localStorage.getItem("candidateID");
}
try {
const payload = {
activityName: activityName,
score: finalResult,
score: result,
actualResult: actualResult,
predResult: result,
predResult: predResult,
totalTime: time,
createdTime: Date(),
candidateID: candidateID,
answerImage: answerImage,
};
await API.post("mentalChromScores", JSON.stringify(payload), headerConfig)
......@@ -109,7 +115,7 @@ const PairCancerlationContainer = () => {
});
const getCroppedImg = async () => {
if (activityIndex === 3) {
if (activityIndex === 2) {
setActivityCompleted(true);
setIsCancelled(true);
setTimeout(() => {
......@@ -140,8 +146,10 @@ const PairCancerlationContainer = () => {
let imagetoPred = base64.replace("data:image/png;base64,", "");
answerImage.push(canvas.toDataURL("image/png"));
// console.log(base64);
if (activityIndex < 3) {
if (activityIndex < 2) {
setActivityIndex(activityIndex + 1);
}
......@@ -156,15 +164,18 @@ const PairCancerlationContainer = () => {
console.log(err);
});
};
let res = 0;
let count = 1;
const generateScore = () => {
for (let i = 0; i < actualResult.length; i++) {
if (JSON.stringify(actualResult[i]) === JSON.stringify(predResult[i])) {
res += 1;
alert(res);
// setFinalScore(result);
}
}
// finalResult = result;
setFinalScore(res);
console.log(res);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment