Skip to content

2
2020-101
Commit a48f6270 authored by I.K Seneviratne's avatar I.K Seneviratne
Browse files
Options

Merge branch 'monitoring_student_behavior_IT17138000' into 'QA_RELEASE' 

Monitoring student behavior it17138000

See merge request !33
parents 7c8b8d71 723f804a
Hide whitespace changes
Inline Side-by-side
Showing with 297 additions and 137 deletions
FirstApp/admin.py
View file @ a48f6270
......@@ -14,4 +14,10 @@ admin.site.register(LectureVideo)
admin.site.register(LectureActivity)
admin.site.register(LectureGazeEstimation)
admin.site.register(Admin)
admin.site.register(AdminCredentialDetails)
\ No newline at end of file
admin.site.register(AdminCredentialDetails)
admin.site.register(LectureActivityFrameRecognitions)
admin.site.register(LectureActivityFrameGroupings)
admin.site.register(LectureEmotionFrameRecognitions)
admin.site.register(LectureEmotionFrameGroupings)
admin.site.register(LectureGazeFrameRecognitions)
admin.site.register(LectureGazeFrameGroupings)
\ No newline at end of file
FirstApp/api.py
View file @ a48f6270
......@@ -375,17 +375,19 @@ class LectureEmotionProcess(APIView):
def get(self, request):
video_name = request.query_params.get('lecture_video_name')
video_id = request.query_params.get('lecture_video_id')
# video_id = request.query_params.get('lecture_video_id')
int_video_id = int(request.query_params.get('lecture_video_id'))
percentages = ed.detect_emotion(video_name)
percentages.calcPercentages()
self.save_emotion_report(video_id, percentages)
self.save_emotion_report(int_video_id, percentages)
return Response({"response": True})
def post(self, request):
pass
def save_emotion_report(self, lec_video_id, percentages):
lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
# lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
lec_video = LectureVideo.objects.filter(id=lec_video_id)
lec_video_serializer = LectureVideoSerializer(lec_video, many=True)
lec_video_data = lec_video_serializer.data[0]
last_lec_emotion = LectureEmotionReport.objects.order_by('lecture_emotion_id').last()
......@@ -500,7 +502,8 @@ class ProcessLectureGazeEstimation(APIView):
pass
def estimate_gaze(self, lec_video_id, percentages):
lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
# lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
lec_video = LectureVideo.objects.filter(id=lec_video_id)
last_lec_gaze = LectureGazeEstimation.objects.order_by('lecture_gaze_id').last()
lec_video_serializer = LectureVideoSerializer(lec_video, many=True)
lec_video_data = lec_video_serializer.data[0]
......@@ -921,65 +924,65 @@ class GetLectureEmotionSummary(APIView):
"emotion_labels": class_labels
})
# else:
#
# frame_landmarks = []
#
# # retrieve frame landmarks from db
# lec_video_frame_landmarks = LectureVideoFrameLandmarks.objects.filter(
# lecture_video_id__video_name=video_name)
# lec_video_frame_landmarks_ser = LectureVideoFrameLandmarksSerializer(lec_video_frame_landmarks, many=True)
# lec_video_frame_landmarks_data = lec_video_frame_landmarks_ser.data[0]
#
# retrieved_frame_landmarks = lec_video_frame_landmarks_data["frame_landmarks"]
#
# # creating a new list to display in the frontend
# for landmark in retrieved_frame_landmarks:
# frame_landmarks.append(int(landmark['landmark']))
#
#
# l, frame_group_dict = ve.getFrameLandmarks(video_name, "Emotion")
# frame_group_percentages, emotion_labels = ed.emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict)
#
#
#
# # save the frame group details into db (temp method)
#
# last_lec_emotion_frame_grouping = LectureEmotionFrameGroupings.objects.order_by('lecture_emotion_frame_groupings_id').last()
# new_lecture_emotion_frame_grouping_id = "LEFG00001" if (last_lec_emotion_frame_grouping is None) else \
# ig.generate_new_id(last_lec_emotion_frame_grouping.lecture_emotion_frame_groupings_id)
#
# # retrieve the lecture emotion id
# lec_emotion = LectureEmotionReport.objects.filter(lecture_video_id__video_name=video_name)
# lec_emotion_ser = LectureEmotionSerializer(lec_emotion, many=True)
# lec_emotion_id = lec_emotion_ser.data[0]['id']
#
# # create the frame group details
# frame_group_details = []
#
# for key in frame_group_percentages.keys():
# # create an object of type 'LectureActivityFrameGroupDetails'
# lec_emotion_frame_group_details = LectureEmotionFrameGroupDetails()
# lec_emotion_frame_group_details.frame_group = key
# lec_emotion_frame_group_details.frame_group_percentages = frame_group_percentages[key]
#
# frame_group_details.append(lec_emotion_frame_group_details)
#
#
# new_lec_emotion_frame_groupings = LectureEmotionFrameGroupings()
# new_lec_emotion_frame_groupings.lecture_emotion_frame_groupings_id = new_lecture_emotion_frame_grouping_id
# new_lec_emotion_frame_groupings.lecture_emotion_id_id = lec_emotion_id
# new_lec_emotion_frame_groupings.frame_group_details = frame_group_details
#
# # save
# new_lec_emotion_frame_groupings.save()
#
#
# return Response({
# "frame_landmarks": frame_landmarks,
# "frame_group_percentages": frame_group_percentages,
# "emotion_labels": emotion_labels
# })
else:
frame_landmarks = []
# retrieve frame landmarks from db
lec_video_frame_landmarks = LectureVideoFrameLandmarks.objects.filter(
lecture_video_id__video_name=video_name)
lec_video_frame_landmarks_ser = LectureVideoFrameLandmarksSerializer(lec_video_frame_landmarks, many=True)
lec_video_frame_landmarks_data = lec_video_frame_landmarks_ser.data[0]
retrieved_frame_landmarks = lec_video_frame_landmarks_data["frame_landmarks"]
# creating a new list to display in the frontend
for landmark in retrieved_frame_landmarks:
frame_landmarks.append(int(landmark['landmark']))
l, frame_group_dict = ve.getFrameLandmarks(video_name, "Emotion")
frame_group_percentages, emotion_labels = ed.emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict)
# save the frame group details into db (temp method)
last_lec_emotion_frame_grouping = LectureEmotionFrameGroupings.objects.order_by('lecture_emotion_frame_groupings_id').last()
new_lecture_emotion_frame_grouping_id = "LEFG00001" if (last_lec_emotion_frame_grouping is None) else \
ig.generate_new_id(last_lec_emotion_frame_grouping.lecture_emotion_frame_groupings_id)
# retrieve the lecture emotion id
lec_emotion = LectureEmotionReport.objects.filter(lecture_video_id__video_name=video_name)
lec_emotion_ser = LectureEmotionSerializer(lec_emotion, many=True)
lec_emotion_id = lec_emotion_ser.data[0]['id']
# create the frame group details
frame_group_details = []
for key in frame_group_percentages.keys():
# create an object of type 'LectureActivityFrameGroupDetails'
lec_emotion_frame_group_details = LectureEmotionFrameGroupDetails()
lec_emotion_frame_group_details.frame_group = key
lec_emotion_frame_group_details.frame_group_percentages = frame_group_percentages[key]
frame_group_details.append(lec_emotion_frame_group_details)
new_lec_emotion_frame_groupings = LectureEmotionFrameGroupings()
new_lec_emotion_frame_groupings.lecture_emotion_frame_groupings_id = new_lecture_emotion_frame_grouping_id
new_lec_emotion_frame_groupings.lecture_emotion_id_id = lec_emotion_id
new_lec_emotion_frame_groupings.frame_group_details = frame_group_details
# save
new_lec_emotion_frame_groupings.save()
return Response({
"frame_landmarks": frame_landmarks,
"frame_group_percentages": frame_group_percentages,
"emotion_labels": emotion_labels
})
# this API will retrieve lecture gaze summary
......
FirstApp/emotion_detector.py
View file @ a48f6270
......@@ -52,6 +52,8 @@ def emotion_recognition(classifier, face_classifier, image):
roi_gray = gray[y:y + h, x:x + w]
roi_gray = cv2.resize(roi_gray, (48, 48), interpolation=cv2.INTER_AREA)
# rect,face,image = face_detector(frame)
# draw a rectangle
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
if np.sum([roi_gray]) != 0:
roi = roi_gray.astype('float') / 255.0
......@@ -63,6 +65,9 @@ def emotion_recognition(classifier, face_classifier, image):
preds = classifier.predict(roi)[0]
label = class_labels[preds.argmax()]
# put the emotion label
cv2.putText(image, label, (x, y), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 0), 3)
return label
......@@ -79,6 +84,7 @@ def detect_emotion(video):
face_classifier = cv2.CascadeClassifier(os.path.join(BASE_DIR, 'FirstApp\classifiers\haarcascade_frontalface_default.xml'))
classifier_path = os.path.join(BASE_DIR, 'FirstApp\classifiers\Emotion_little_vgg.h5')
classifier = load_model(classifier_path)
EMOTION_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\emotion")
meta_data = VideoMeta()
class_labels = ['Angry', 'Happy', 'Neutral', 'Sad', 'Surprise']
......@@ -99,6 +105,20 @@ def detect_emotion(video):
# for testing purposes
print('starting the emotion recognition process')
# get width and height of the video frames
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
# get the video frame size
size = (frame_width, frame_height)
# this is the annotated video path
ANNOTATED_VIDEO_PATH = os.path.join(EMOTION_DIR, video)
# initiailizing the video writer
vid_cod = cv2.VideoWriter_fourcc(*'XVID')
output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, size)
while (count_frames < frame_count):
# Grab a single frame of video
ret, frame = cap.read()
......@@ -135,6 +155,9 @@ def detect_emotion(video):
# for testing purposes
print('emotion frame count: ', count_frames)
# write the video frame to the video writer
output.write(frame)
count_frames += 1
# setting up the counted values
......@@ -146,8 +169,13 @@ def detect_emotion(video):
meta_data.surprise_count = count_surprise
cap.release()
output.release()
cv2.destroyAllWindows()
# after saving the video, save the changes to static content
p = os.popen("python manage.py collectstatic", "w")
p.write("yes")
# for testing purposes
print('ending the emotion recognition process')
......@@ -198,6 +226,8 @@ def get_frame_emotion_recognition(video_name):
# for testing purposes
print('starting the emotion frame recognition process')
# looping through the frames
while (frame_count < no_of_frames):
......@@ -216,18 +246,19 @@ def get_frame_emotion_recognition(video_name):
surprise_count = 0
# get the detections
detections = ar.person_detection(image, net)
detections, persons = ar.person_detection(image, net)
# to count the extracted detections for a frame
detection_count = 0
# if there are detections
if (len(detections) > 0):
# loop through the detections
for detection in detections:
for person in persons:
label = emotion_recognition(classifier, face_classifier, detection)
label = emotion_recognition(classifier, face_classifier, person)
# checking for the label
if label == class_labels[0]:
......@@ -422,17 +453,17 @@ def emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict):
neutral_count = 0
detection_count = 0
detections = ar.person_detection(image, net)
detections, persons = ar.person_detection(image, net)
# if there are detections
if (len(detections) > 0):
# looping through the detections in each frame
for detection in detections:
for person in persons:
# run the model and get the emotion label
label = emotion_recognition(classifier, face_classifier, detection)
label = emotion_recognition(classifier, face_classifier, person)
# increment the count based on the label
if label == class_labels[0]:
......
FirstApp/logic/activity_recognition.py
View file @ a48f6270
......@@ -38,7 +38,8 @@ def activity_recognition(video_path):
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_path))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
ACTIVITY_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\activity")
# files required for person detection
......@@ -55,7 +56,9 @@ def activity_recognition(video_path):
np.set_printoptions(suppress=True)
# define the student activity labels
class_labels = ['Phone checking', 'Listening', 'Note taking']
# class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# load the model
model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
......@@ -80,14 +83,21 @@ def activity_recognition(video_path):
# for testing purposes
print('starting the activity recognition process')
# this is the annotated video path
ANNOTATED_VIDEO_PATH = os.path.join(ACTIVITY_DIR, video_path)
# initiailizing the video writer
vid_cod = cv2.VideoWriter_fourcc(*'XVID')
output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, size)
# looping through the frames
while (frame_count < no_of_frames):
ret, image = video.read()
image = cv2.resize(image, size)
# image = cv2.resize(image, size)
# perform person detection on the extracted image
detections = person_detection(image, net)
detections, persons = person_detection(image, net)
# this is for testing purposes
print('frame count: ', frame_count)
......@@ -102,13 +112,26 @@ def activity_recognition(video_path):
# initialize the detection count
detection_count = 0
# to iterate each person
no_of_persons = 0
# looping through the person detections of the frame
for detection in detections:
detection = cv2.resize(detection, size)
# get the coordinates for the detection
startX = detection['startX']
startY = detection['startY']
endX = detection['endX']
endY = detection['endY']
# detection = cv2.resize(detection, size)
# draw the coordinates of the persons' identified
cv2.rectangle(image, (startX, startY), (endX, endY), (0, 255, 0), 5)
image_array = np.asarray(detection)
normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
image_array = np.asarray(persons[no_of_persons])
image_array_resized = cv2.resize(image_array, size)
# normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
# Load the image into the array
data[0] = normalized_image_array
......@@ -120,18 +143,36 @@ def activity_recognition(video_path):
# counting the detections under each label
if (label == class_labels[0]):
label = "Phone checking"
phone_checking_count += 1
elif (label == class_labels[1]):
listening_count += 1
elif (label == class_labels[2]):
label = "Writing"
note_taking_count += 1
# vertical_pos = startY + int(endY / 2)
vertical_pos = int(endY / 2)
# put the identified label above the detected person
# cv2.putText(image, label, (startX, vertical_pos), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0), 2)
cv2.putText(image, label, (startX, vertical_pos), cv2.FONT_HERSHEY_COMPLEX, 4, (0, 255, 0), 10)
# increment the no.of persons
no_of_persons += 1
# increment the detection count
detection_count += 1
# increment the frame count
frame_count += 1
# resize the image
image = cv2.resize(image, (224, 224))
# write the frame to the video writer
output.write(image)
# calculating the percentages for each label
phone_perct = float(phone_checking_count / total_detections) * 100 if total_detections > 0 else 0
......@@ -143,6 +184,10 @@ def activity_recognition(video_path):
percentages["writing_perct"] = note_perct
percentages["listening_perct"] = listening_perct
# after saving the video, save the changes to static content
p = os.popen("python manage.py collectstatic", "w")
p.write("yes")
# for testing purposes
print('activity recognition process is over')
......@@ -163,6 +208,7 @@ def person_detection(image, net):
# set the threshold balue
threshold = 0.2
detected_person = []
persons = []
# initialize the list of class labels MobileNet SSD was trained to
# detect, then generate a set of bounding box colors for each class
......@@ -211,14 +257,22 @@ def person_detection(image, net):
startX = 0 if startX < 0 else startX
startY = 0 if startY < 0 else startY
# extract the person
# this dictionary will contain the bounding box coordinates
coordinates = {}
person = image[startY:startY + endY, startX:startX + endX]
detected_person.append(person)
coordinates['startX'] = startX
coordinates['startY'] = startY
coordinates['endX'] = endX
coordinates['endY'] = endY
persons.append(person)
detected_person.append(coordinates)
person_count += 1
# return the detection person list
return detected_person
return detected_person, persons
# this method will recognize the activity for each frame
......@@ -233,7 +287,8 @@ def get_frame_activity_recognition(video_name):
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
# files required for person detection
config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
......@@ -247,7 +302,9 @@ def get_frame_activity_recognition(video_name):
np.set_printoptions(suppress=True)
# class labels
class_labels = ['Phone checking', 'Listening', 'Note taking']
# class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# load the activity recogntion model
model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
......@@ -295,19 +352,27 @@ def get_frame_activity_recognition(video_name):
detection_count = 0
detected_percentages = []
detections = person_detection(image, net)
detections, persons = person_detection(image, net)
# if there are detections
if (len(detections) > 0):
no_of_persons = 0
# loop through each detection in the frame
for detection in detections:
detection = cv2.resize(detection, size)
# get the coordinates for the detection
startX = detection['startX']
startY = detection['startY']
endX = detection['endX']
endY = detection['endY']
image_array = np.asarray(detection)
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
image_array = np.asarray(persons[no_of_persons])
image_array_resized = cv2.resize(image_array, size)
# normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
# Load the image into the array
data[0] = normalized_image_array
......@@ -427,10 +492,21 @@ def get_student_activity_summary_for_period(activities):
def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
EXTRACTED_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\activity\\{}".format(video_name))
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
# files required for person detection
config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
model_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.caffemodel")
# load our serialized person detection model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(config_file, model_file)
np.set_printoptions(suppress=True)
......@@ -443,11 +519,16 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
size = (224, 224)
# initializing the count variables
# class labels
# class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# iteration
video = cv2.VideoCapture(VIDEO_DIR)
no_of_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
frame_count = 0
# class labels
class_labels = ['Phone checking', 'Listening', 'Note taking']
# get the frame differences for each frame group
frame_group_diff = {}
......@@ -463,9 +544,8 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
frame_group_diff[key] = diff if diff > 0 else 1
# looping through the frames
for frame in os.listdir(EXTRACTED_DIR):
# getting the frame folder
FRAME_FOLDER = os.path.join(EXTRACTED_DIR, frame)
# for frame in os.listdir(EXTRACTED_DIR):
while (frame_count < no_of_frames):
# initializing the variables
phone_count = 0
......@@ -473,57 +553,55 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
listen_count = 0
detection_count = 0
# looping through the detections in each frame
for detections in os.listdir(FRAME_FOLDER):
ret, image = video.read()
# checking whether the image contains only one person
if "frame" not in detections:
# get the label for this image
IMAGE_PATH = os.path.join(FRAME_FOLDER, detections)
image = cv2.imread(IMAGE_PATH)
detections, persons = person_detection(image, net)
image = cv2.resize(image, size)
# looping through the detections in each frame
for person in persons:
image_array = np.asarray(image)
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
image = cv2.resize(person, size)
# Load the image into the array
data[0] = normalized_image_array
image_array = np.asarray(image)
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
# run the inference
prediction = model.predict(data)
# Load the image into the array
data[0] = normalized_image_array
# get the predicted label
label = class_labels[prediction.argmax()]
# run the inference
prediction = model.predict(data)
# increment the count based on the label
if label == class_labels[0]:
phone_count += 1
elif label == class_labels[1]:
listen_count += 1
elif label == class_labels[2]:
note_count += 1
# get the predicted label
label = class_labels[prediction.argmax()]
# increment the detection count
detection_count += 1
# increment the count based on the label
if label == class_labels[0]:
phone_count += 1
elif label == class_labels[1]:
listen_count += 1
elif label == class_labels[2]:
note_count += 1
# increment the detection count
detection_count += 1
# finding the time landmark that the current frame is in
for i in frame_landmarks:
index = frame_landmarks.index(i)
j = index + 1
# finding the time landmark that the current frame is in
for i in frame_landmarks:
index = frame_landmarks.index(i)
j = index + 1
# checking whether the next index is within the range
if j < len(frame_landmarks):
next_value = frame_landmarks[j]
# checking whether the next index is within the range
if j < len(frame_landmarks):
next_value = frame_landmarks[j]
# checking the correct time landmark range
if (frame_count >= i) & (frame_count < next_value):
frame_name = "{}-{}".format(i, next_value)
# checking the correct time landmark range
if (frame_count >= i) & (frame_count < next_value):
frame_name = "{}-{}".format(i, next_value)
frame_group_dict[frame_name]['phone_count'] += phone_count
frame_group_dict[frame_name]['listen_count'] += listen_count
frame_group_dict[frame_name]['note_count'] += note_count
frame_group_dict[frame_name]['detection_count'] += detection_count
frame_group_dict[frame_name]['phone_count'] += phone_count
frame_group_dict[frame_name]['listen_count'] += listen_count
frame_group_dict[frame_name]['note_count'] += note_count
frame_group_dict[frame_name]['detection_count'] += detection_count
# increment the frame count
frame_count += 1
......
FirstApp/logic/head_gaze_estimation.py
View file @ a48f6270
......@@ -171,6 +171,7 @@ def process_gaze_estimation(video_path):
ret, img = cap.read()
size = img.shape
font = cv2.FONT_HERSHEY_SIMPLEX
# 3D model points.
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
......@@ -211,6 +212,18 @@ def process_gaze_estimation(video_path):
# for testing purposes
print('starting the gaze estimation process')
# get the frame sizes
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
frame_size = (frame_width, frame_height)
# this is the annotated video path
ANNOTATED_VIDEO_PATH = os.path.join(GAZE_DIR, video_path)
# initiailizing the video writer
vid_cod = cv2.VideoWriter_fourcc(*'XVID')
output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, frame_size)
# iterate the video frames
while True:
ret, img = cap.read()
......@@ -285,14 +298,19 @@ def process_gaze_estimation(video_path):
# checking for vertical and horizontal directions
if isLookingDown & isLookingRight:
cv2.putText(img, 'looking down and right', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_down_right_count += 1
elif isLookingDown & isLookingLeft:
cv2.putText(img, 'looking down and left', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_down_left_count += 1
elif isLookingUp & isLookingRight:
cv2.putText(img, 'looking up and right', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_up_right_count += 1
elif isLookingUp & isLookingLeft:
cv2.putText(img, 'looking up and left', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_up_left_count += 1
elif isLookingFront:
cv2.putText(img, 'looking front', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
head_front_count += 1
......@@ -304,6 +322,9 @@ def process_gaze_estimation(video_path):
# for testing purposes
print('gaze estimation count: ', frame_count)
# write to the video writer
output.write(img)
# increment the frame count
frame_count += 1
......@@ -330,6 +351,12 @@ def process_gaze_estimation(video_path):
cv2.destroyAllWindows()
cap.release()
output.release()
# after saving the video, save the changes to static content
p = os.popen("python manage.py collectstatic", "w")
p.write("yes")
# for testing purposes
print('ending the gaze estimation process')
......@@ -538,6 +565,7 @@ def get_lecture_gaze_estimation_for_frames(video_name):
# for testing purposes
print('ending the gaze estimation for frames process')
......
FirstApp/templates/FirstApp/activity.html
View file @ a48f6270
......@@ -241,7 +241,8 @@
//to handle the 'integrate' modal
$('#integrate_activity').click(function () {
//define the student video src
let video_src = "{% static '' %}FirstApp/videos/" + global_video_name;
{#let video_src = "{% static '' %}FirstApp/videos/" + global_video_name;#}
let video_src = "{% static '' %}FirstApp/activity/" + global_video_name;
//assign the video src
$('#student_video').attr('src', video_src);
......
FirstApp/templates/FirstApp/emotion.html
View file @ a48f6270
......@@ -143,7 +143,8 @@
$('#video_name').text(video.video_name);
$('#video_duration').text(video.video_length);
$('#video_date').text(video.date);
global_lecture_video_id = video.lecture_video_id;
{#global_lecture_video_id = video.lecture_video_id;#}
global_lecture_video_id = video.id;
global_video_name = video.video_name;
......
FirstApp/templates/FirstApp/gaze.html
View file @ a48f6270
......@@ -142,7 +142,8 @@
$('#video_name').text(video.video_name);
$('#video_duration').text(video.video_length);
$('#video_date').text(video.date);
global_lecture_video_id = video.lecture_video_id;
{#global_lecture_video_id = video.lecture_video_id;#}
global_lecture_video_id = video.id;
global_video_name = video.video_name;
......
FirstApp/templates/FirstApp/video_results.html
View file @ a48f6270
......@@ -40,7 +40,7 @@
$(document).ready(function () {
let folder = '';
$('#activity_loader').attr('hidden', false);
{#$('#activity_loader').attr('hidden', false);#}
{#$('#emotion_loader').attr('hidden', false);#}
{#$('#gaze_loader').attr('hidden', false);#}
......@@ -298,7 +298,7 @@
}
//this is a test function (delete later)
/*
let interval = setInterval(() => {
{#let url = 'http://127.0.0.1:8000/get-random_number';#}
......@@ -355,6 +355,7 @@
}
*/
});
......
FirstApp/views.py
View file @ a48f6270
......@@ -189,6 +189,8 @@ def video_result(request):
data = serializer.data
print('data length: ', len(data))
# iterate through the existing lecture videos for the lecturer
for video in data:
video_id = video['id']
......@@ -197,6 +199,8 @@ def video_result(request):
# check whether the video id exist in the Activity Recognition table
lec_activity = LectureActivity.objects.filter(lecture_video_id_id=video_id).exists()
print('lecture activity existence: ', lec_activity)
if lec_activity == False:
to_do_lecture_list.append({
"lecturer": lecturer,
......@@ -227,11 +231,15 @@ def video_result(request):
# loop through the to-do lecture list
for item in to_do_lecture_list:
isDate = item['date'] == str(day_timetable['date'])
print('item date: ', item['date'])
print('timetable date: ', str(day_timetable['date']))
# isLecturer = item['lecturer'] ==
# check for the particular lecture on the day
if isDate:
slots = day_timetable['time_slots']
# loop through the slots
for slot in slots:
# check for the lecturer and subject
......@@ -260,6 +268,8 @@ def video_result(request):
print('what is wrong?: ', exc)
return redirect('/500')
print('due lectures: ', due_lecture_list)
return render(request, "FirstApp/video_results.html",
{"lecturer": lecturer, "due_lectures": due_lecture_list})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment