Skip to content

2
2020-101
Commit a48f6270 authored by I.K Seneviratne's avatar I.K Seneviratne
Browse files
Options

Merge branch 'monitoring_student_behavior_IT17138000' into 'QA_RELEASE' 

Monitoring student behavior it17138000

See merge request !33
parents 7c8b8d71 723f804a
Hide whitespace changes
Inline Side-by-side
Showing with 297 additions and 137 deletions
FirstApp/admin.py
View file @ a48f6270
...@@ -14,4 +14,10 @@ admin.site.register(LectureVideo) ...@@ -14,4 +14,10 @@ admin.site.register(LectureVideo)
admin.site.register(LectureActivity) admin.site.register(LectureActivity)
admin.site.register(LectureGazeEstimation) admin.site.register(LectureGazeEstimation)
admin.site.register(Admin) admin.site.register(Admin)
admin.site.register(AdminCredentialDetails) admin.site.register(AdminCredentialDetails)
\ No newline at end of file admin.site.register(LectureActivityFrameRecognitions)
admin.site.register(LectureActivityFrameGroupings)
admin.site.register(LectureEmotionFrameRecognitions)
admin.site.register(LectureEmotionFrameGroupings)
admin.site.register(LectureGazeFrameRecognitions)
admin.site.register(LectureGazeFrameGroupings)
\ No newline at end of file
FirstApp/api.py
View file @ a48f6270
...@@ -375,17 +375,19 @@ class LectureEmotionProcess(APIView): ...@@ -375,17 +375,19 @@ class LectureEmotionProcess(APIView):
def get(self, request): def get(self, request):
video_name = request.query_params.get('lecture_video_name') video_name = request.query_params.get('lecture_video_name')
video_id = request.query_params.get('lecture_video_id') # video_id = request.query_params.get('lecture_video_id')
int_video_id = int(request.query_params.get('lecture_video_id'))
percentages = ed.detect_emotion(video_name) percentages = ed.detect_emotion(video_name)
percentages.calcPercentages() percentages.calcPercentages()
self.save_emotion_report(video_id, percentages) self.save_emotion_report(int_video_id, percentages)
return Response({"response": True}) return Response({"response": True})
def post(self, request): def post(self, request):
pass pass
def save_emotion_report(self, lec_video_id, percentages): def save_emotion_report(self, lec_video_id, percentages):
lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id) # lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
lec_video = LectureVideo.objects.filter(id=lec_video_id)
lec_video_serializer = LectureVideoSerializer(lec_video, many=True) lec_video_serializer = LectureVideoSerializer(lec_video, many=True)
lec_video_data = lec_video_serializer.data[0] lec_video_data = lec_video_serializer.data[0]
last_lec_emotion = LectureEmotionReport.objects.order_by('lecture_emotion_id').last() last_lec_emotion = LectureEmotionReport.objects.order_by('lecture_emotion_id').last()
...@@ -500,7 +502,8 @@ class ProcessLectureGazeEstimation(APIView): ...@@ -500,7 +502,8 @@ class ProcessLectureGazeEstimation(APIView):
pass pass
def estimate_gaze(self, lec_video_id, percentages): def estimate_gaze(self, lec_video_id, percentages):
lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id) # lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
lec_video = LectureVideo.objects.filter(id=lec_video_id)
last_lec_gaze = LectureGazeEstimation.objects.order_by('lecture_gaze_id').last() last_lec_gaze = LectureGazeEstimation.objects.order_by('lecture_gaze_id').last()
lec_video_serializer = LectureVideoSerializer(lec_video, many=True) lec_video_serializer = LectureVideoSerializer(lec_video, many=True)
lec_video_data = lec_video_serializer.data[0] lec_video_data = lec_video_serializer.data[0]
...@@ -921,65 +924,65 @@ class GetLectureEmotionSummary(APIView): ...@@ -921,65 +924,65 @@ class GetLectureEmotionSummary(APIView):
"emotion_labels": class_labels "emotion_labels": class_labels
}) })
# else: else:
#
# frame_landmarks = [] frame_landmarks = []
#
# # retrieve frame landmarks from db # retrieve frame landmarks from db
# lec_video_frame_landmarks = LectureVideoFrameLandmarks.objects.filter( lec_video_frame_landmarks = LectureVideoFrameLandmarks.objects.filter(
# lecture_video_id__video_name=video_name) lecture_video_id__video_name=video_name)
# lec_video_frame_landmarks_ser = LectureVideoFrameLandmarksSerializer(lec_video_frame_landmarks, many=True) lec_video_frame_landmarks_ser = LectureVideoFrameLandmarksSerializer(lec_video_frame_landmarks, many=True)
# lec_video_frame_landmarks_data = lec_video_frame_landmarks_ser.data[0] lec_video_frame_landmarks_data = lec_video_frame_landmarks_ser.data[0]
#
# retrieved_frame_landmarks = lec_video_frame_landmarks_data["frame_landmarks"] retrieved_frame_landmarks = lec_video_frame_landmarks_data["frame_landmarks"]
#
# # creating a new list to display in the frontend # creating a new list to display in the frontend
# for landmark in retrieved_frame_landmarks: for landmark in retrieved_frame_landmarks:
# frame_landmarks.append(int(landmark['landmark'])) frame_landmarks.append(int(landmark['landmark']))
#
#
# l, frame_group_dict = ve.getFrameLandmarks(video_name, "Emotion") l, frame_group_dict = ve.getFrameLandmarks(video_name, "Emotion")
# frame_group_percentages, emotion_labels = ed.emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict) frame_group_percentages, emotion_labels = ed.emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict)
#
#
#
# # save the frame group details into db (temp method) # save the frame group details into db (temp method)
#
# last_lec_emotion_frame_grouping = LectureEmotionFrameGroupings.objects.order_by('lecture_emotion_frame_groupings_id').last() last_lec_emotion_frame_grouping = LectureEmotionFrameGroupings.objects.order_by('lecture_emotion_frame_groupings_id').last()
# new_lecture_emotion_frame_grouping_id = "LEFG00001" if (last_lec_emotion_frame_grouping is None) else \ new_lecture_emotion_frame_grouping_id = "LEFG00001" if (last_lec_emotion_frame_grouping is None) else \
# ig.generate_new_id(last_lec_emotion_frame_grouping.lecture_emotion_frame_groupings_id) ig.generate_new_id(last_lec_emotion_frame_grouping.lecture_emotion_frame_groupings_id)
#
# # retrieve the lecture emotion id # retrieve the lecture emotion id
# lec_emotion = LectureEmotionReport.objects.filter(lecture_video_id__video_name=video_name) lec_emotion = LectureEmotionReport.objects.filter(lecture_video_id__video_name=video_name)
# lec_emotion_ser = LectureEmotionSerializer(lec_emotion, many=True) lec_emotion_ser = LectureEmotionSerializer(lec_emotion, many=True)
# lec_emotion_id = lec_emotion_ser.data[0]['id'] lec_emotion_id = lec_emotion_ser.data[0]['id']
#
# # create the frame group details # create the frame group details
# frame_group_details = [] frame_group_details = []
#
# for key in frame_group_percentages.keys(): for key in frame_group_percentages.keys():
# # create an object of type 'LectureActivityFrameGroupDetails' # create an object of type 'LectureActivityFrameGroupDetails'
# lec_emotion_frame_group_details = LectureEmotionFrameGroupDetails() lec_emotion_frame_group_details = LectureEmotionFrameGroupDetails()
# lec_emotion_frame_group_details.frame_group = key lec_emotion_frame_group_details.frame_group = key
# lec_emotion_frame_group_details.frame_group_percentages = frame_group_percentages[key] lec_emotion_frame_group_details.frame_group_percentages = frame_group_percentages[key]
#
# frame_group_details.append(lec_emotion_frame_group_details) frame_group_details.append(lec_emotion_frame_group_details)
#
#
# new_lec_emotion_frame_groupings = LectureEmotionFrameGroupings() new_lec_emotion_frame_groupings = LectureEmotionFrameGroupings()
# new_lec_emotion_frame_groupings.lecture_emotion_frame_groupings_id = new_lecture_emotion_frame_grouping_id new_lec_emotion_frame_groupings.lecture_emotion_frame_groupings_id = new_lecture_emotion_frame_grouping_id
# new_lec_emotion_frame_groupings.lecture_emotion_id_id = lec_emotion_id new_lec_emotion_frame_groupings.lecture_emotion_id_id = lec_emotion_id
# new_lec_emotion_frame_groupings.frame_group_details = frame_group_details new_lec_emotion_frame_groupings.frame_group_details = frame_group_details
#
# # save # save
# new_lec_emotion_frame_groupings.save() new_lec_emotion_frame_groupings.save()
#
#
# return Response({ return Response({
# "frame_landmarks": frame_landmarks, "frame_landmarks": frame_landmarks,
# "frame_group_percentages": frame_group_percentages, "frame_group_percentages": frame_group_percentages,
# "emotion_labels": emotion_labels "emotion_labels": emotion_labels
# }) })
# this API will retrieve lecture gaze summary # this API will retrieve lecture gaze summary
......
FirstApp/emotion_detector.py
View file @ a48f6270
...@@ -52,6 +52,8 @@ def emotion_recognition(classifier, face_classifier, image): ...@@ -52,6 +52,8 @@ def emotion_recognition(classifier, face_classifier, image):
roi_gray = gray[y:y + h, x:x + w] roi_gray = gray[y:y + h, x:x + w]
roi_gray = cv2.resize(roi_gray, (48, 48), interpolation=cv2.INTER_AREA) roi_gray = cv2.resize(roi_gray, (48, 48), interpolation=cv2.INTER_AREA)
# rect,face,image = face_detector(frame) # rect,face,image = face_detector(frame)
# draw a rectangle
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
if np.sum([roi_gray]) != 0: if np.sum([roi_gray]) != 0:
roi = roi_gray.astype('float') / 255.0 roi = roi_gray.astype('float') / 255.0
...@@ -63,6 +65,9 @@ def emotion_recognition(classifier, face_classifier, image): ...@@ -63,6 +65,9 @@ def emotion_recognition(classifier, face_classifier, image):
preds = classifier.predict(roi)[0] preds = classifier.predict(roi)[0]
label = class_labels[preds.argmax()] label = class_labels[preds.argmax()]
# put the emotion label
cv2.putText(image, label, (x, y), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 0), 3)
return label return label
...@@ -79,6 +84,7 @@ def detect_emotion(video): ...@@ -79,6 +84,7 @@ def detect_emotion(video):
face_classifier = cv2.CascadeClassifier(os.path.join(BASE_DIR, 'FirstApp\classifiers\haarcascade_frontalface_default.xml')) face_classifier = cv2.CascadeClassifier(os.path.join(BASE_DIR, 'FirstApp\classifiers\haarcascade_frontalface_default.xml'))
classifier_path = os.path.join(BASE_DIR, 'FirstApp\classifiers\Emotion_little_vgg.h5') classifier_path = os.path.join(BASE_DIR, 'FirstApp\classifiers\Emotion_little_vgg.h5')
classifier = load_model(classifier_path) classifier = load_model(classifier_path)
EMOTION_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\emotion")
meta_data = VideoMeta() meta_data = VideoMeta()
class_labels = ['Angry', 'Happy', 'Neutral', 'Sad', 'Surprise'] class_labels = ['Angry', 'Happy', 'Neutral', 'Sad', 'Surprise']
...@@ -99,6 +105,20 @@ def detect_emotion(video): ...@@ -99,6 +105,20 @@ def detect_emotion(video):
# for testing purposes # for testing purposes
print('starting the emotion recognition process') print('starting the emotion recognition process')
# get width and height of the video frames
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
# get the video frame size
size = (frame_width, frame_height)
# this is the annotated video path
ANNOTATED_VIDEO_PATH = os.path.join(EMOTION_DIR, video)
# initiailizing the video writer
vid_cod = cv2.VideoWriter_fourcc(*'XVID')
output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, size)
while (count_frames < frame_count): while (count_frames < frame_count):
# Grab a single frame of video # Grab a single frame of video
ret, frame = cap.read() ret, frame = cap.read()
...@@ -135,6 +155,9 @@ def detect_emotion(video): ...@@ -135,6 +155,9 @@ def detect_emotion(video):
# for testing purposes # for testing purposes
print('emotion frame count: ', count_frames) print('emotion frame count: ', count_frames)
# write the video frame to the video writer
output.write(frame)
count_frames += 1 count_frames += 1
# setting up the counted values # setting up the counted values
...@@ -146,8 +169,13 @@ def detect_emotion(video): ...@@ -146,8 +169,13 @@ def detect_emotion(video):
meta_data.surprise_count = count_surprise meta_data.surprise_count = count_surprise
cap.release() cap.release()
output.release()
cv2.destroyAllWindows() cv2.destroyAllWindows()
# after saving the video, save the changes to static content
p = os.popen("python manage.py collectstatic", "w")
p.write("yes")
# for testing purposes # for testing purposes
print('ending the emotion recognition process') print('ending the emotion recognition process')
...@@ -198,6 +226,8 @@ def get_frame_emotion_recognition(video_name): ...@@ -198,6 +226,8 @@ def get_frame_emotion_recognition(video_name):
# for testing purposes # for testing purposes
print('starting the emotion frame recognition process') print('starting the emotion frame recognition process')
# looping through the frames # looping through the frames
while (frame_count < no_of_frames): while (frame_count < no_of_frames):
...@@ -216,18 +246,19 @@ def get_frame_emotion_recognition(video_name): ...@@ -216,18 +246,19 @@ def get_frame_emotion_recognition(video_name):
surprise_count = 0 surprise_count = 0
# get the detections # get the detections
detections = ar.person_detection(image, net) detections, persons = ar.person_detection(image, net)
# to count the extracted detections for a frame # to count the extracted detections for a frame
detection_count = 0 detection_count = 0
# if there are detections # if there are detections
if (len(detections) > 0): if (len(detections) > 0):
# loop through the detections # loop through the detections
for detection in detections: for person in persons:
label = emotion_recognition(classifier, face_classifier, detection) label = emotion_recognition(classifier, face_classifier, person)
# checking for the label # checking for the label
if label == class_labels[0]: if label == class_labels[0]:
...@@ -422,17 +453,17 @@ def emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict): ...@@ -422,17 +453,17 @@ def emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict):
neutral_count = 0 neutral_count = 0
detection_count = 0 detection_count = 0
detections = ar.person_detection(image, net) detections, persons = ar.person_detection(image, net)
# if there are detections # if there are detections
if (len(detections) > 0): if (len(detections) > 0):
# looping through the detections in each frame # looping through the detections in each frame
for detection in detections: for person in persons:
# run the model and get the emotion label # run the model and get the emotion label
label = emotion_recognition(classifier, face_classifier, detection) label = emotion_recognition(classifier, face_classifier, person)
# increment the count based on the label # increment the count based on the label
if label == class_labels[0]: if label == class_labels[0]:
......
FirstApp/logic/activity_recognition.py
View file @ a48f6270
...@@ -38,7 +38,8 @@ def activity_recognition(video_path): ...@@ -38,7 +38,8 @@ def activity_recognition(video_path):
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_path)) VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_path))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
ACTIVITY_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\activity") ACTIVITY_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\activity")
# files required for person detection # files required for person detection
...@@ -55,7 +56,9 @@ def activity_recognition(video_path): ...@@ -55,7 +56,9 @@ def activity_recognition(video_path):
np.set_printoptions(suppress=True) np.set_printoptions(suppress=True)
# define the student activity labels # define the student activity labels
class_labels = ['Phone checking', 'Listening', 'Note taking'] # class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# load the model # load the model
model = tensorflow.keras.models.load_model(CLASSIFIER_DIR) model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
...@@ -80,14 +83,21 @@ def activity_recognition(video_path): ...@@ -80,14 +83,21 @@ def activity_recognition(video_path):
# for testing purposes # for testing purposes
print('starting the activity recognition process') print('starting the activity recognition process')
# this is the annotated video path
ANNOTATED_VIDEO_PATH = os.path.join(ACTIVITY_DIR, video_path)
# initiailizing the video writer
vid_cod = cv2.VideoWriter_fourcc(*'XVID')
output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, size)
# looping through the frames # looping through the frames
while (frame_count < no_of_frames): while (frame_count < no_of_frames):
ret, image = video.read() ret, image = video.read()
image = cv2.resize(image, size) # image = cv2.resize(image, size)
# perform person detection on the extracted image # perform person detection on the extracted image
detections = person_detection(image, net) detections, persons = person_detection(image, net)
# this is for testing purposes # this is for testing purposes
print('frame count: ', frame_count) print('frame count: ', frame_count)
...@@ -102,13 +112,26 @@ def activity_recognition(video_path): ...@@ -102,13 +112,26 @@ def activity_recognition(video_path):
# initialize the detection count # initialize the detection count
detection_count = 0 detection_count = 0
# to iterate each person
no_of_persons = 0
# looping through the person detections of the frame # looping through the person detections of the frame
for detection in detections: for detection in detections:
detection = cv2.resize(detection, size) # get the coordinates for the detection
startX = detection['startX']
startY = detection['startY']
endX = detection['endX']
endY = detection['endY']
# detection = cv2.resize(detection, size)
# draw the coordinates of the persons' identified
cv2.rectangle(image, (startX, startY), (endX, endY), (0, 255, 0), 5)
image_array = np.asarray(detection) image_array = np.asarray(persons[no_of_persons])
normalized_image_array = (detection.astype(np.float32) / 127.0) - 1 image_array_resized = cv2.resize(image_array, size)
# normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
# Load the image into the array # Load the image into the array
data[0] = normalized_image_array data[0] = normalized_image_array
...@@ -120,18 +143,36 @@ def activity_recognition(video_path): ...@@ -120,18 +143,36 @@ def activity_recognition(video_path):
# counting the detections under each label # counting the detections under each label
if (label == class_labels[0]): if (label == class_labels[0]):
label = "Phone checking"
phone_checking_count += 1 phone_checking_count += 1
elif (label == class_labels[1]): elif (label == class_labels[1]):
listening_count += 1 listening_count += 1
elif (label == class_labels[2]): elif (label == class_labels[2]):
label = "Writing"
note_taking_count += 1 note_taking_count += 1
# vertical_pos = startY + int(endY / 2)
vertical_pos = int(endY / 2)
# put the identified label above the detected person
# cv2.putText(image, label, (startX, vertical_pos), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0), 2)
cv2.putText(image, label, (startX, vertical_pos), cv2.FONT_HERSHEY_COMPLEX, 4, (0, 255, 0), 10)
# increment the no.of persons
no_of_persons += 1
# increment the detection count # increment the detection count
detection_count += 1 detection_count += 1
# increment the frame count # increment the frame count
frame_count += 1 frame_count += 1
# resize the image
image = cv2.resize(image, (224, 224))
# write the frame to the video writer
output.write(image)
# calculating the percentages for each label # calculating the percentages for each label
phone_perct = float(phone_checking_count / total_detections) * 100 if total_detections > 0 else 0 phone_perct = float(phone_checking_count / total_detections) * 100 if total_detections > 0 else 0
...@@ -143,6 +184,10 @@ def activity_recognition(video_path): ...@@ -143,6 +184,10 @@ def activity_recognition(video_path):
percentages["writing_perct"] = note_perct percentages["writing_perct"] = note_perct
percentages["listening_perct"] = listening_perct percentages["listening_perct"] = listening_perct
# after saving the video, save the changes to static content
p = os.popen("python manage.py collectstatic", "w")
p.write("yes")
# for testing purposes # for testing purposes
print('activity recognition process is over') print('activity recognition process is over')
...@@ -163,6 +208,7 @@ def person_detection(image, net): ...@@ -163,6 +208,7 @@ def person_detection(image, net):
# set the threshold balue # set the threshold balue
threshold = 0.2 threshold = 0.2
detected_person = [] detected_person = []
persons = []
# initialize the list of class labels MobileNet SSD was trained to # initialize the list of class labels MobileNet SSD was trained to
# detect, then generate a set of bounding box colors for each class # detect, then generate a set of bounding box colors for each class
...@@ -211,14 +257,22 @@ def person_detection(image, net): ...@@ -211,14 +257,22 @@ def person_detection(image, net):
startX = 0 if startX < 0 else startX startX = 0 if startX < 0 else startX
startY = 0 if startY < 0 else startY startY = 0 if startY < 0 else startY
# extract the person # this dictionary will contain the bounding box coordinates
coordinates = {}
person = image[startY:startY + endY, startX:startX + endX] person = image[startY:startY + endY, startX:startX + endX]
detected_person.append(person) coordinates['startX'] = startX
coordinates['startY'] = startY
coordinates['endX'] = endX
coordinates['endY'] = endY
persons.append(person)
detected_person.append(coordinates)
person_count += 1 person_count += 1
# return the detection person list # return the detection person list
return detected_person return detected_person, persons
# this method will recognize the activity for each frame # this method will recognize the activity for each frame
...@@ -233,7 +287,8 @@ def get_frame_activity_recognition(video_name): ...@@ -233,7 +287,8 @@ def get_frame_activity_recognition(video_name):
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name)) VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
# files required for person detection # files required for person detection
config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt") config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
...@@ -247,7 +302,9 @@ def get_frame_activity_recognition(video_name): ...@@ -247,7 +302,9 @@ def get_frame_activity_recognition(video_name):
np.set_printoptions(suppress=True) np.set_printoptions(suppress=True)
# class labels # class labels
class_labels = ['Phone checking', 'Listening', 'Note taking'] # class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# load the activity recogntion model # load the activity recogntion model
model = tensorflow.keras.models.load_model(CLASSIFIER_DIR) model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
...@@ -295,19 +352,27 @@ def get_frame_activity_recognition(video_name): ...@@ -295,19 +352,27 @@ def get_frame_activity_recognition(video_name):
detection_count = 0 detection_count = 0
detected_percentages = [] detected_percentages = []
detections = person_detection(image, net) detections, persons = person_detection(image, net)
# if there are detections # if there are detections
if (len(detections) > 0): if (len(detections) > 0):
no_of_persons = 0
# loop through each detection in the frame # loop through each detection in the frame
for detection in detections: for detection in detections:
detection = cv2.resize(detection, size) # get the coordinates for the detection
startX = detection['startX']
startY = detection['startY']
endX = detection['endX']
endY = detection['endY']
image_array = np.asarray(detection) image_array = np.asarray(persons[no_of_persons])
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1 image_array_resized = cv2.resize(image_array, size)
# normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
# Load the image into the array # Load the image into the array
data[0] = normalized_image_array data[0] = normalized_image_array
...@@ -427,10 +492,21 @@ def get_student_activity_summary_for_period(activities): ...@@ -427,10 +492,21 @@ def get_student_activity_summary_for_period(activities):
def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict): def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
EXTRACTED_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\activity\\{}".format(video_name)) EXTRACTED_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\activity\\{}".format(video_name))
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5") # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
# files required for person detection
config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
model_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.caffemodel")
# load our serialized person detection model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(config_file, model_file)
np.set_printoptions(suppress=True) np.set_printoptions(suppress=True)
...@@ -443,11 +519,16 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict): ...@@ -443,11 +519,16 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32) data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
size = (224, 224) size = (224, 224)
# initializing the count variables # class labels
# class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# iteration
video = cv2.VideoCapture(VIDEO_DIR)
no_of_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
frame_count = 0 frame_count = 0
# class labels
class_labels = ['Phone checking', 'Listening', 'Note taking']
# get the frame differences for each frame group # get the frame differences for each frame group
frame_group_diff = {} frame_group_diff = {}
...@@ -463,9 +544,8 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict): ...@@ -463,9 +544,8 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
frame_group_diff[key] = diff if diff > 0 else 1 frame_group_diff[key] = diff if diff > 0 else 1
# looping through the frames # looping through the frames
for frame in os.listdir(EXTRACTED_DIR): # for frame in os.listdir(EXTRACTED_DIR):
# getting the frame folder while (frame_count < no_of_frames):
FRAME_FOLDER = os.path.join(EXTRACTED_DIR, frame)
# initializing the variables # initializing the variables
phone_count = 0 phone_count = 0
...@@ -473,57 +553,55 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict): ...@@ -473,57 +553,55 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
listen_count = 0 listen_count = 0
detection_count = 0 detection_count = 0
# looping through the detections in each frame ret, image = video.read()
for detections in os.listdir(FRAME_FOLDER):
# checking whether the image contains only one person detections, persons = person_detection(image, net)
if "frame" not in detections:
# get the label for this image
IMAGE_PATH = os.path.join(FRAME_FOLDER, detections)
image = cv2.imread(IMAGE_PATH)
image = cv2.resize(image, size) # looping through the detections in each frame
for person in persons:
image_array = np.asarray(image) image = cv2.resize(person, size)
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
# Load the image into the array image_array = np.asarray(image)
data[0] = normalized_image_array normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
# run the inference # Load the image into the array
prediction = model.predict(data) data[0] = normalized_image_array
# get the predicted label # run the inference
label = class_labels[prediction.argmax()] prediction = model.predict(data)
# increment the count based on the label # get the predicted label
if label == class_labels[0]: label = class_labels[prediction.argmax()]
phone_count += 1
elif label == class_labels[1]:
listen_count += 1
elif label == class_labels[2]:
note_count += 1
# increment the detection count # increment the count based on the label
detection_count += 1 if label == class_labels[0]:
phone_count += 1
elif label == class_labels[1]:
listen_count += 1
elif label == class_labels[2]:
note_count += 1
# increment the detection count
detection_count += 1
# finding the time landmark that the current frame is in # finding the time landmark that the current frame is in
for i in frame_landmarks: for i in frame_landmarks:
index = frame_landmarks.index(i) index = frame_landmarks.index(i)
j = index + 1 j = index + 1
# checking whether the next index is within the range # checking whether the next index is within the range
if j < len(frame_landmarks): if j < len(frame_landmarks):
next_value = frame_landmarks[j] next_value = frame_landmarks[j]
# checking the correct time landmark range # checking the correct time landmark range
if (frame_count >= i) & (frame_count < next_value): if (frame_count >= i) & (frame_count < next_value):
frame_name = "{}-{}".format(i, next_value) frame_name = "{}-{}".format(i, next_value)
frame_group_dict[frame_name]['phone_count'] += phone_count frame_group_dict[frame_name]['phone_count'] += phone_count
frame_group_dict[frame_name]['listen_count'] += listen_count frame_group_dict[frame_name]['listen_count'] += listen_count
frame_group_dict[frame_name]['note_count'] += note_count frame_group_dict[frame_name]['note_count'] += note_count
frame_group_dict[frame_name]['detection_count'] += detection_count frame_group_dict[frame_name]['detection_count'] += detection_count
# increment the frame count # increment the frame count
frame_count += 1 frame_count += 1
......
FirstApp/logic/head_gaze_estimation.py
View file @ a48f6270
...@@ -171,6 +171,7 @@ def process_gaze_estimation(video_path): ...@@ -171,6 +171,7 @@ def process_gaze_estimation(video_path):
ret, img = cap.read() ret, img = cap.read()
size = img.shape size = img.shape
font = cv2.FONT_HERSHEY_SIMPLEX
# 3D model points. # 3D model points.
model_points = np.array([ model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip (0.0, 0.0, 0.0), # Nose tip
...@@ -211,6 +212,18 @@ def process_gaze_estimation(video_path): ...@@ -211,6 +212,18 @@ def process_gaze_estimation(video_path):
# for testing purposes # for testing purposes
print('starting the gaze estimation process') print('starting the gaze estimation process')
# get the frame sizes
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
frame_size = (frame_width, frame_height)
# this is the annotated video path
ANNOTATED_VIDEO_PATH = os.path.join(GAZE_DIR, video_path)
# initiailizing the video writer
vid_cod = cv2.VideoWriter_fourcc(*'XVID')
output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, frame_size)
# iterate the video frames # iterate the video frames
while True: while True:
ret, img = cap.read() ret, img = cap.read()
...@@ -285,14 +298,19 @@ def process_gaze_estimation(video_path): ...@@ -285,14 +298,19 @@ def process_gaze_estimation(video_path):
# checking for vertical and horizontal directions # checking for vertical and horizontal directions
if isLookingDown & isLookingRight: if isLookingDown & isLookingRight:
cv2.putText(img, 'looking down and right', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_down_right_count += 1 head_down_right_count += 1
elif isLookingDown & isLookingLeft: elif isLookingDown & isLookingLeft:
cv2.putText(img, 'looking down and left', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_down_left_count += 1 head_down_left_count += 1
elif isLookingUp & isLookingRight: elif isLookingUp & isLookingRight:
cv2.putText(img, 'looking up and right', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_up_right_count += 1 head_up_right_count += 1
elif isLookingUp & isLookingLeft: elif isLookingUp & isLookingLeft:
cv2.putText(img, 'looking up and left', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_up_left_count += 1 head_up_left_count += 1
elif isLookingFront: elif isLookingFront:
cv2.putText(img, 'looking front', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_front_count += 1 head_front_count += 1
...@@ -304,6 +322,9 @@ def process_gaze_estimation(video_path): ...@@ -304,6 +322,9 @@ def process_gaze_estimation(video_path):
# for testing purposes # for testing purposes
print('gaze estimation count: ', frame_count) print('gaze estimation count: ', frame_count)
# write to the video writer
output.write(img)
# increment the frame count # increment the frame count
frame_count += 1 frame_count += 1
...@@ -330,6 +351,12 @@ def process_gaze_estimation(video_path): ...@@ -330,6 +351,12 @@ def process_gaze_estimation(video_path):
cv2.destroyAllWindows() cv2.destroyAllWindows()
cap.release() cap.release()
output.release()
# after saving the video, save the changes to static content
p = os.popen("python manage.py collectstatic", "w")
p.write("yes")
# for testing purposes # for testing purposes
print('ending the gaze estimation process') print('ending the gaze estimation process')
...@@ -538,6 +565,7 @@ def get_lecture_gaze_estimation_for_frames(video_name): ...@@ -538,6 +565,7 @@ def get_lecture_gaze_estimation_for_frames(video_name):
# for testing purposes # for testing purposes
print('ending the gaze estimation for frames process') print('ending the gaze estimation for frames process')
......
FirstApp/templates/FirstApp/activity.html
View file @ a48f6270
...@@ -241,7 +241,8 @@ ...@@ -241,7 +241,8 @@
//to handle the 'integrate' modal //to handle the 'integrate' modal
$('#integrate_activity').click(function () { $('#integrate_activity').click(function () {
//define the student video src //define the student video src
let video_src = "{% static '' %}FirstApp/videos/" + global_video_name; {#let video_src = "{% static '' %}FirstApp/videos/" + global_video_name;#}
let video_src = "{% static '' %}FirstApp/activity/" + global_video_name;
//assign the video src //assign the video src
$('#student_video').attr('src', video_src); $('#student_video').attr('src', video_src);
......
FirstApp/templates/FirstApp/emotion.html
View file @ a48f6270
...@@ -143,7 +143,8 @@ ...@@ -143,7 +143,8 @@
$('#video_name').text(video.video_name); $('#video_name').text(video.video_name);
$('#video_duration').text(video.video_length); $('#video_duration').text(video.video_length);
$('#video_date').text(video.date); $('#video_date').text(video.date);
global_lecture_video_id = video.lecture_video_id; {#global_lecture_video_id = video.lecture_video_id;#}
global_lecture_video_id = video.id;
global_video_name = video.video_name; global_video_name = video.video_name;
......
FirstApp/templates/FirstApp/gaze.html
View file @ a48f6270
...@@ -142,7 +142,8 @@ ...@@ -142,7 +142,8 @@
$('#video_name').text(video.video_name); $('#video_name').text(video.video_name);
$('#video_duration').text(video.video_length); $('#video_duration').text(video.video_length);
$('#video_date').text(video.date); $('#video_date').text(video.date);
global_lecture_video_id = video.lecture_video_id; {#global_lecture_video_id = video.lecture_video_id;#}
global_lecture_video_id = video.id;
global_video_name = video.video_name; global_video_name = video.video_name;
......
FirstApp/templates/FirstApp/video_results.html
View file @ a48f6270
...@@ -40,7 +40,7 @@ ...@@ -40,7 +40,7 @@
$(document).ready(function () { $(document).ready(function () {
let folder = ''; let folder = '';
$('#activity_loader').attr('hidden', false); {#$('#activity_loader').attr('hidden', false);#}
{#$('#emotion_loader').attr('hidden', false);#} {#$('#emotion_loader').attr('hidden', false);#}
{#$('#gaze_loader').attr('hidden', false);#} {#$('#gaze_loader').attr('hidden', false);#}
...@@ -298,7 +298,7 @@ ...@@ -298,7 +298,7 @@
} }
//this is a test function (delete later) //this is a test function (delete later)
/*
let interval = setInterval(() => { let interval = setInterval(() => {
{#let url = 'http://127.0.0.1:8000/get-random_number';#} {#let url = 'http://127.0.0.1:8000/get-random_number';#}
...@@ -355,6 +355,7 @@ ...@@ -355,6 +355,7 @@
} }
*/
}); });
......
FirstApp/views.py
View file @ a48f6270
...@@ -189,6 +189,8 @@ def video_result(request): ...@@ -189,6 +189,8 @@ def video_result(request):
data = serializer.data data = serializer.data
print('data length: ', len(data))
# iterate through the existing lecture videos for the lecturer # iterate through the existing lecture videos for the lecturer
for video in data: for video in data:
video_id = video['id'] video_id = video['id']
...@@ -197,6 +199,8 @@ def video_result(request): ...@@ -197,6 +199,8 @@ def video_result(request):
# check whether the video id exist in the Activity Recognition table # check whether the video id exist in the Activity Recognition table
lec_activity = LectureActivity.objects.filter(lecture_video_id_id=video_id).exists() lec_activity = LectureActivity.objects.filter(lecture_video_id_id=video_id).exists()
print('lecture activity existence: ', lec_activity)
if lec_activity == False: if lec_activity == False:
to_do_lecture_list.append({ to_do_lecture_list.append({
"lecturer": lecturer, "lecturer": lecturer,
...@@ -227,11 +231,15 @@ def video_result(request): ...@@ -227,11 +231,15 @@ def video_result(request):
# loop through the to-do lecture list # loop through the to-do lecture list
for item in to_do_lecture_list: for item in to_do_lecture_list:
isDate = item['date'] == str(day_timetable['date']) isDate = item['date'] == str(day_timetable['date'])
print('item date: ', item['date'])
print('timetable date: ', str(day_timetable['date']))
# isLecturer = item['lecturer'] == # isLecturer = item['lecturer'] ==
# check for the particular lecture on the day # check for the particular lecture on the day
if isDate: if isDate:
slots = day_timetable['time_slots'] slots = day_timetable['time_slots']
# loop through the slots # loop through the slots
for slot in slots: for slot in slots:
# check for the lecturer and subject # check for the lecturer and subject
...@@ -260,6 +268,8 @@ def video_result(request): ...@@ -260,6 +268,8 @@ def video_result(request):
print('what is wrong?: ', exc) print('what is wrong?: ', exc)
return redirect('/500') return redirect('/500')
print('due lectures: ', due_lecture_list)
return render(request, "FirstApp/video_results.html", return render(request, "FirstApp/video_results.html",
{"lecturer": lecturer, "due_lectures": due_lecture_list}) {"lecturer": lecturer, "due_lectures": due_lecture_list})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment