Merge branch 'monitoring_student_behavior_IT17138000' into 'QA_RELEASE'

Monitoring student behavior it17138000

See merge request !33

FirstApp/admin.py

@@ -15,3 +15,9 @@ admin.site.register(LectureActivity)
 admin.site.register(LectureGazeEstimation)
 admin.site.register(Admin)
 admin.site.register(AdminCredentialDetails)
+admin.site.register(LectureActivityFrameRecognitions)
+admin.site.register(LectureActivityFrameGroupings)
+admin.site.register(LectureEmotionFrameRecognitions)
+admin.site.register(LectureEmotionFrameGroupings)
+admin.site.register(LectureGazeFrameRecognitions)
+admin.site.register(LectureGazeFrameGroupings)
\ No newline at end of file

FirstApp/api.py

@@ -375,17 +375,19 @@ class LectureEmotionProcess(APIView):
 
     def get(self, request):
         video_name = request.query_params.get('lecture_video_name')
-        video_id = request.query_params.get('lecture_video_id')
+        # video_id = request.query_params.get('lecture_video_id')
+        int_video_id = int(request.query_params.get('lecture_video_id'))
         percentages = ed.detect_emotion(video_name)
         percentages.calcPercentages()
-        self.save_emotion_report(video_id, percentages)
+        self.save_emotion_report(int_video_id, percentages)
         return Response({"response": True})
 
     def post(self, request):
         pass
 
     def save_emotion_report(self, lec_video_id, percentages):
-        lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
+        # lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
+        lec_video = LectureVideo.objects.filter(id=lec_video_id)
         lec_video_serializer = LectureVideoSerializer(lec_video, many=True)
         lec_video_data = lec_video_serializer.data[0]
         last_lec_emotion = LectureEmotionReport.objects.order_by('lecture_emotion_id').last()
@@ -500,7 +502,8 @@ class ProcessLectureGazeEstimation(APIView):
         pass
 
     def estimate_gaze(self, lec_video_id, percentages):
-        lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
+        # lec_video = LectureVideo.objects.filter(lecture_video_id=lec_video_id)
+        lec_video = LectureVideo.objects.filter(id=lec_video_id)
         last_lec_gaze = LectureGazeEstimation.objects.order_by('lecture_gaze_id').last()
         lec_video_serializer = LectureVideoSerializer(lec_video, many=True)
         lec_video_data = lec_video_serializer.data[0]
@@ -921,65 +924,65 @@ class GetLectureEmotionSummary(APIView):
                 "emotion_labels": class_labels
             })
 
-        # else:
-        #
-        #     frame_landmarks = []
-        #
-        #     # retrieve frame landmarks from db
-        #     lec_video_frame_landmarks = LectureVideoFrameLandmarks.objects.filter(
-        #         lecture_video_id__video_name=video_name)
-        #     lec_video_frame_landmarks_ser = LectureVideoFrameLandmarksSerializer(lec_video_frame_landmarks, many=True)
-        #     lec_video_frame_landmarks_data = lec_video_frame_landmarks_ser.data[0]
-        #
-        #     retrieved_frame_landmarks = lec_video_frame_landmarks_data["frame_landmarks"]
-        #
-        #     # creating a new list to display in the frontend
-        #     for landmark in retrieved_frame_landmarks:
-        #         frame_landmarks.append(int(landmark['landmark']))
-        #
-        #
-        #     l, frame_group_dict = ve.getFrameLandmarks(video_name, "Emotion")
-        #     frame_group_percentages, emotion_labels = ed.emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict)
-        #
-        #
-        #
-        #     # save the frame group details into db (temp method)
-        #
-        #     last_lec_emotion_frame_grouping = LectureEmotionFrameGroupings.objects.order_by('lecture_emotion_frame_groupings_id').last()
-        #     new_lecture_emotion_frame_grouping_id = "LEFG00001" if (last_lec_emotion_frame_grouping is None) else \
-        #         ig.generate_new_id(last_lec_emotion_frame_grouping.lecture_emotion_frame_groupings_id)
-        #
-        #     # retrieve the lecture emotion id
-        #     lec_emotion = LectureEmotionReport.objects.filter(lecture_video_id__video_name=video_name)
-        #     lec_emotion_ser = LectureEmotionSerializer(lec_emotion, many=True)
-        #     lec_emotion_id = lec_emotion_ser.data[0]['id']
-        #
-        #     # create the frame group details
-        #     frame_group_details = []
-        #
-        #     for key in frame_group_percentages.keys():
-        #         # create an object of type 'LectureActivityFrameGroupDetails'
-        #         lec_emotion_frame_group_details = LectureEmotionFrameGroupDetails()
-        #         lec_emotion_frame_group_details.frame_group = key
-        #         lec_emotion_frame_group_details.frame_group_percentages = frame_group_percentages[key]
-        #
-        #         frame_group_details.append(lec_emotion_frame_group_details)
-        #
-        #
-        #     new_lec_emotion_frame_groupings = LectureEmotionFrameGroupings()
-        #     new_lec_emotion_frame_groupings.lecture_emotion_frame_groupings_id = new_lecture_emotion_frame_grouping_id
-        #     new_lec_emotion_frame_groupings.lecture_emotion_id_id = lec_emotion_id
-        #     new_lec_emotion_frame_groupings.frame_group_details = frame_group_details
-        #
-        #     # save
-        #     new_lec_emotion_frame_groupings.save()
-        #
-        #
-        #     return Response({
-        #         "frame_landmarks": frame_landmarks,
-        #         "frame_group_percentages": frame_group_percentages,
-        #         "emotion_labels": emotion_labels
-        #     })
+        else:
+
+            frame_landmarks = []
+
+            # retrieve frame landmarks from db
+            lec_video_frame_landmarks = LectureVideoFrameLandmarks.objects.filter(
+                lecture_video_id__video_name=video_name)
+            lec_video_frame_landmarks_ser = LectureVideoFrameLandmarksSerializer(lec_video_frame_landmarks, many=True)
+            lec_video_frame_landmarks_data = lec_video_frame_landmarks_ser.data[0]
+
+            retrieved_frame_landmarks = lec_video_frame_landmarks_data["frame_landmarks"]
+
+            # creating a new list to display in the frontend
+            for landmark in retrieved_frame_landmarks:
+                frame_landmarks.append(int(landmark['landmark']))
+
+
+            l, frame_group_dict = ve.getFrameLandmarks(video_name, "Emotion")
+            frame_group_percentages, emotion_labels = ed.emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict)
+
+
+
+            # save the frame group details into db (temp method)
+
+            last_lec_emotion_frame_grouping = LectureEmotionFrameGroupings.objects.order_by('lecture_emotion_frame_groupings_id').last()
+            new_lecture_emotion_frame_grouping_id = "LEFG00001" if (last_lec_emotion_frame_grouping is None) else \
+                ig.generate_new_id(last_lec_emotion_frame_grouping.lecture_emotion_frame_groupings_id)
+
+            # retrieve the lecture emotion id
+            lec_emotion = LectureEmotionReport.objects.filter(lecture_video_id__video_name=video_name)
+            lec_emotion_ser = LectureEmotionSerializer(lec_emotion, many=True)
+            lec_emotion_id = lec_emotion_ser.data[0]['id']
+
+            # create the frame group details
+            frame_group_details = []
+
+            for key in frame_group_percentages.keys():
+                # create an object of type 'LectureActivityFrameGroupDetails'
+                lec_emotion_frame_group_details = LectureEmotionFrameGroupDetails()
+                lec_emotion_frame_group_details.frame_group = key
+                lec_emotion_frame_group_details.frame_group_percentages = frame_group_percentages[key]
+
+                frame_group_details.append(lec_emotion_frame_group_details)
+
+
+            new_lec_emotion_frame_groupings = LectureEmotionFrameGroupings()
+            new_lec_emotion_frame_groupings.lecture_emotion_frame_groupings_id = new_lecture_emotion_frame_grouping_id
+            new_lec_emotion_frame_groupings.lecture_emotion_id_id = lec_emotion_id
+            new_lec_emotion_frame_groupings.frame_group_details = frame_group_details
+
+            # save
+            new_lec_emotion_frame_groupings.save()
+
+
+            return Response({
+                "frame_landmarks": frame_landmarks,
+                "frame_group_percentages": frame_group_percentages,
+                "emotion_labels": emotion_labels
+            })
 
 
 # this API will retrieve lecture gaze summary

FirstApp/emotion_detector.py

@@ -52,6 +52,8 @@ def emotion_recognition(classifier, face_classifier, image):
         roi_gray = gray[y:y + h, x:x + w]
         roi_gray = cv2.resize(roi_gray, (48, 48), interpolation=cv2.INTER_AREA)
         # rect,face,image = face_detector(frame)
+        # draw a rectangle
+        cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
 
         if np.sum([roi_gray]) != 0:
             roi = roi_gray.astype('float') / 255.0
@@ -63,6 +65,9 @@ def emotion_recognition(classifier, face_classifier, image):
             preds = classifier.predict(roi)[0]
             label = class_labels[preds.argmax()]
 
+            # put the emotion label
+            cv2.putText(image, label, (x, y), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 0), 3)
+
     return label
 
 
@@ -79,6 +84,7 @@ def detect_emotion(video):
     face_classifier = cv2.CascadeClassifier(os.path.join(BASE_DIR, 'FirstApp\classifiers\haarcascade_frontalface_default.xml'))
     classifier_path = os.path.join(BASE_DIR, 'FirstApp\classifiers\Emotion_little_vgg.h5')
     classifier = load_model(classifier_path)
+    EMOTION_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\emotion")
     meta_data = VideoMeta()
 
     class_labels = ['Angry', 'Happy', 'Neutral', 'Sad', 'Surprise']
@@ -99,6 +105,20 @@ def detect_emotion(video):
     # for testing purposes
     print('starting the emotion recognition process')
 
+    # get width and height of the video frames
+    frame_width = int(cap.get(3))
+    frame_height = int(cap.get(4))
+
+    # get the video frame size
+    size = (frame_width, frame_height)
+
+    # this is the annotated video path
+    ANNOTATED_VIDEO_PATH = os.path.join(EMOTION_DIR, video)
+
+    # initiailizing the video writer
+    vid_cod = cv2.VideoWriter_fourcc(*'XVID')
+    output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, size)
+
     while (count_frames < frame_count):
         # Grab a single frame of video
         ret, frame = cap.read()
@@ -135,6 +155,9 @@ def detect_emotion(video):
         # for testing purposes
         print('emotion frame count: ', count_frames)
 
+        # write the video frame to the video writer
+        output.write(frame)
+
         count_frames += 1
 
     # setting up the counted values
@@ -146,8 +169,13 @@ def detect_emotion(video):
     meta_data.surprise_count = count_surprise
 
     cap.release()
+    output.release()
     cv2.destroyAllWindows()
 
+    # after saving the video, save the changes to static content
+    p = os.popen("python manage.py collectstatic", "w")
+    p.write("yes")
+
     # for testing purposes
     print('ending the emotion recognition process')
 
@@ -198,6 +226,8 @@ def get_frame_emotion_recognition(video_name):
     # for testing purposes
     print('starting the emotion frame recognition process')
 
+
+
     # looping through the frames
     while (frame_count < no_of_frames):
 
@@ -216,18 +246,19 @@ def get_frame_emotion_recognition(video_name):
         surprise_count = 0
 
         # get the detections
-        detections = ar.person_detection(image, net)
+        detections, persons = ar.person_detection(image, net)
 
         # to count the extracted detections for a frame
         detection_count = 0
 
         # if there are detections
         if (len(detections) > 0):
+
             # loop through the detections
-            for detection in detections:
+            for person in persons:
 
 
-                label = emotion_recognition(classifier, face_classifier, detection)
+                label = emotion_recognition(classifier, face_classifier, person)
 
                 # checking for the label
                 if label == class_labels[0]:
@@ -422,17 +453,17 @@ def emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict):
         neutral_count = 0
         detection_count = 0
 
-        detections = ar.person_detection(image, net)
+        detections, persons = ar.person_detection(image, net)
 
         # if there are detections
         if (len(detections) > 0):
 
             # looping through the detections in each frame
-            for detection in detections:
+            for person in persons:
 
 
                 # run the model and get the emotion label
-                label = emotion_recognition(classifier, face_classifier, detection)
+                label = emotion_recognition(classifier, face_classifier, person)
 
                 # increment the count based on the label
                 if label == class_labels[0]:

FirstApp/logic/head_gaze_estimation.py

@@ -171,6 +171,7 @@ def process_gaze_estimation(video_path):
 
     ret, img = cap.read()
     size = img.shape
+    font = cv2.FONT_HERSHEY_SIMPLEX
     # 3D model points.
     model_points = np.array([
         (0.0, 0.0, 0.0),  # Nose tip
@@ -211,6 +212,18 @@ def process_gaze_estimation(video_path):
     # for testing purposes
     print('starting the gaze estimation process')
 
+    # get the frame sizes
+    frame_width = int(cap.get(3))
+    frame_height = int(cap.get(4))
+
+    frame_size = (frame_width, frame_height)
+    # this is the annotated video path
+    ANNOTATED_VIDEO_PATH = os.path.join(GAZE_DIR, video_path)
+
+    # initiailizing the video writer
+    vid_cod = cv2.VideoWriter_fourcc(*'XVID')
+    output = cv2.VideoWriter(ANNOTATED_VIDEO_PATH, vid_cod, 30.0, frame_size)
+
     # iterate the video frames
     while True:
         ret, img = cap.read()
@@ -285,14 +298,19 @@ def process_gaze_estimation(video_path):
 
                 # checking for vertical and horizontal directions
                 if isLookingDown & isLookingRight:
+                    cv2.putText(img, 'looking down and right', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
                     head_down_right_count += 1
                 elif isLookingDown & isLookingLeft:
+                    cv2.putText(img, 'looking down and left', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
                     head_down_left_count += 1
                 elif isLookingUp & isLookingRight:
+                    cv2.putText(img, 'looking up and right', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
                     head_up_right_count += 1
                 elif isLookingUp & isLookingLeft:
+                    cv2.putText(img, 'looking up and left', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
                     head_up_left_count += 1
                 elif isLookingFront:
+                    cv2.putText(img, 'looking front', (facebox[0], facebox[1]), font, 2, (255, 255, 128), 3)
                     head_front_count += 1
 
 
@@ -304,6 +322,9 @@ def process_gaze_estimation(video_path):
             # for testing purposes
             print('gaze estimation count: ', frame_count)
 
+            # write to the video writer
+            output.write(img)
+
             # increment the frame count
             frame_count += 1
 
@@ -330,6 +351,12 @@ def process_gaze_estimation(video_path):
 
     cv2.destroyAllWindows()
     cap.release()
+    output.release()
+
+
+    # after saving the video, save the changes to static content
+    p = os.popen("python manage.py collectstatic", "w")
+    p.write("yes")
 
     # for testing purposes
     print('ending the gaze estimation process')
@@ -538,6 +565,7 @@ def get_lecture_gaze_estimation_for_frames(video_name):
 
 
 
+
     # for testing purposes
     print('ending the gaze estimation for frames process')
 

FirstApp/templates/FirstApp/activity.html

@@ -241,7 +241,8 @@
             //to handle the 'integrate' modal
             $('#integrate_activity').click(function () {
                 //define the student video src
-                let video_src = "{% static '' %}FirstApp/videos/" + global_video_name;
+                {#let video_src = "{% static '' %}FirstApp/videos/" + global_video_name;#}
+                let video_src = "{% static '' %}FirstApp/activity/" + global_video_name;
 
                 //assign the video src
                 $('#student_video').attr('src', video_src);

FirstApp/templates/FirstApp/emotion.html

@@ -143,7 +143,8 @@
                 $('#video_name').text(video.video_name);
                 $('#video_duration').text(video.video_length);
                 $('#video_date').text(video.date);
-                global_lecture_video_id = video.lecture_video_id;
+                {#global_lecture_video_id = video.lecture_video_id;#}
+                global_lecture_video_id = video.id;
                 global_video_name = video.video_name;
 
 

FirstApp/templates/FirstApp/gaze.html

@@ -142,7 +142,8 @@
                 $('#video_name').text(video.video_name);
                 $('#video_duration').text(video.video_length);
                 $('#video_date').text(video.date);
-                global_lecture_video_id = video.lecture_video_id;
+                {#global_lecture_video_id = video.lecture_video_id;#}
+                global_lecture_video_id = video.id;
                 global_video_name = video.video_name;
 
 

FirstApp/templates/FirstApp/video_results.html

@@ -40,7 +40,7 @@
         $(document).ready(function () {
             let folder = '';
 
-            $('#activity_loader').attr('hidden', false);
+            {#$('#activity_loader').attr('hidden', false);#}
             {#$('#emotion_loader').attr('hidden', false);#}
             {#$('#gaze_loader').attr('hidden', false);#}
 
@@ -298,7 +298,7 @@
             }
 
             //this is a test function (delete later)
-
+            /*
             let interval = setInterval(() => {
 
                 {#let url = 'http://127.0.0.1:8000/get-random_number';#}
@@ -355,6 +355,7 @@
 
 
             }
+            */
 
 
         });

FirstApp/views.py

@@ -189,6 +189,8 @@ def video_result(request):
 
         data = serializer.data
 
+        print('data length: ', len(data))
+
         # iterate through the existing lecture videos for the lecturer
         for video in data:
             video_id = video['id']
@@ -197,6 +199,8 @@ def video_result(request):
             # check whether the video id exist in the Activity Recognition table
             lec_activity = LectureActivity.objects.filter(lecture_video_id_id=video_id).exists()
 
+            print('lecture activity existence: ', lec_activity)
+
             if lec_activity == False:
                 to_do_lecture_list.append({
                     "lecturer": lecturer,
@@ -227,11 +231,15 @@ def video_result(request):
                 # loop through the to-do lecture list
                 for item in to_do_lecture_list:
                     isDate = item['date'] == str(day_timetable['date'])
+
+                    print('item date: ', item['date'])
+                    print('timetable date: ', str(day_timetable['date']))
                     # isLecturer = item['lecturer'] ==
                     # check for the particular lecture on the day
                     if isDate:
                         slots = day_timetable['time_slots']
 
+
                         # loop through the slots
                         for slot in slots:
                             # check for the lecturer and subject
@@ -260,6 +268,8 @@ def video_result(request):
         print('what is wrong?: ', exc)
         return redirect('/500')
 
+    print('due lectures: ', due_lecture_list)
+
     return render(request, "FirstApp/video_results.html",
                   {"lecturer": lecturer, "due_lectures": due_lecture_list})