Committing the modification in some files.

FirstApp/emotion_detector.py

@@ -216,18 +216,19 @@ def get_frame_emotion_recognition(video_name):
         surprise_count = 0
 
         # get the detections
-        detections = ar.person_detection(image, net)
+        detections, persons = ar.person_detection(image, net)
 
         # to count the extracted detections for a frame
         detection_count = 0
 
         # if there are detections
         if (len(detections) > 0):
+
             # loop through the detections
-            for detection in detections:
+            for person in persons:
 
 
-                label = emotion_recognition(classifier, face_classifier, detection)
+                label = emotion_recognition(classifier, face_classifier, person)
 
                 # checking for the label
                 if label == class_labels[0]:
@@ -422,17 +423,17 @@ def emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict):
         neutral_count = 0
         detection_count = 0
 
-        detections = ar.person_detection(image, net)
+        detections, persons = ar.person_detection(image, net)
 
         # if there are detections
         if (len(detections) > 0):
 
             # looping through the detections in each frame
-            for detection in detections:
+            for person in persons:
 
 
                 # run the model and get the emotion label
-                label = emotion_recognition(classifier, face_classifier, detection)
+                label = emotion_recognition(classifier, face_classifier, person)
 
                 # increment the count based on the label
                 if label == class_labels[0]:

FirstApp/logic/activity_recognition.py

@@ -38,8 +38,9 @@ def activity_recognition(video_path):
     VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_path))
     # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
     # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
-    CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
-    ACTIVITY_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\activity")
+    # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
+    CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
+    # ACTIVITY_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\activity")
 
     # files required for person detection
     config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
@@ -55,7 +56,9 @@ def activity_recognition(video_path):
     np.set_printoptions(suppress=True)
 
     # define the student activity labels
-    class_labels = ['Phone checking', 'Listening', 'Note taking']
+    # class_labels = ['Phone checking', 'Listening', 'Note taking']
+    class_labels = ['Phone checki...', 'Listening', 'Note taking']
+
 
     # load the model
     model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
@@ -81,13 +84,17 @@ def activity_recognition(video_path):
     # for testing purposes
     print('starting the activity recognition process')
 
+    # initiailizing the video writer
+    vid_cod = cv2.VideoWriter_fourcc(*'XVID')
+    output = cv2.VideoWriter("videos/cam_video.mp4", vid_cod, 30.0, size)
+
     # looping through the frames
     while (frame_count < no_of_frames):
         ret, image = video.read()
 
-        image = cv2.resize(image, size)
+        # image = cv2.resize(image, size)
         # perform person detection on the extracted image
-        detections = person_detection(image, net)
+        detections, persons = person_detection(image, net)
 
         # this is for testing purposes
         print('frame count: ', frame_count)
@@ -102,13 +109,26 @@ def activity_recognition(video_path):
             # initialize the detection count
             detection_count = 0
 
+            # to iterate each person
+            no_of_persons = 0
+
             # looping through the person detections of the frame
             for detection in detections:
 
-                detection = cv2.resize(detection, size)
+                # get the coordinates for the detection
+                startX = detection['startX']
+                startY = detection['startY']
+                endX = detection['endX']
+                endY = detection['endY']
+
+                # detection = cv2.resize(detection, size)
+                # draw the coordinates of the persons' identified
+                cv2.rectangle(image, (startX, startY), (endX, endY), (0, 255, 0), 5)
 
-                image_array = np.asarray(detection)
-                normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
+                image_array = np.asarray(persons[no_of_persons])
+                image_array_resized = cv2.resize(image_array, size)
+                # normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
+                normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
 
                 # Load the image into the array
                 data[0] = normalized_image_array
@@ -132,6 +152,9 @@ def activity_recognition(video_path):
         # increment the frame count
         frame_count += 1
 
+        # write the frame to the video writer
+        output.write(image)
+
 
     # calculating the percentages for each label
     phone_perct = float(phone_checking_count / total_detections) * 100 if total_detections > 0 else 0
@@ -163,6 +186,7 @@ def person_detection(image, net):
     # set the threshold balue
     threshold = 0.2
     detected_person = []
+    persons = []
 
     # initialize the list of class labels MobileNet SSD was trained to
     # detect, then generate a set of bounding box colors for each class
@@ -211,14 +235,22 @@ def person_detection(image, net):
                 startX = 0 if startX < 0 else startX
                 startY = 0 if startY < 0 else startY
 
-                # extract the person
+                # this dictionary will contain the bounding box coordinates
+                coordinates = {}
+
                 person = image[startY:startY + endY, startX:startX + endX]
-                detected_person.append(person)
+                coordinates['startX'] = startX
+                coordinates['startY'] = startY
+                coordinates['endX'] = endX
+                coordinates['endY'] = endY
+
+                persons.append(person)
+                detected_person.append(coordinates)
 
                 person_count += 1
 
     # return the detection person list
-    return detected_person
+    return detected_person, persons
 
 
 # this method will recognize the activity for each frame
@@ -233,7 +265,8 @@ def get_frame_activity_recognition(video_name):
     VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
     # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
     # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
-    CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
+    # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
+    CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
 
     # files required for person detection
     config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
@@ -247,7 +280,9 @@ def get_frame_activity_recognition(video_name):
     np.set_printoptions(suppress=True)
 
     # class labels
-    class_labels = ['Phone checking', 'Listening', 'Note taking']
+    # class_labels = ['Phone checking', 'Listening', 'Note taking']
+    class_labels = ['Phone checki...', 'Listening', 'Note taking']
+
 
     # load the activity recogntion model
     model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
@@ -295,19 +330,27 @@ def get_frame_activity_recognition(video_name):
         detection_count = 0
         detected_percentages = []
 
-        detections = person_detection(image, net)
+        detections, persons = person_detection(image, net)
 
 
         # if there are detections
         if (len(detections) > 0):
 
+            no_of_persons = 0
+
             # loop through each detection in the frame
             for detection in detections:
 
-                detection = cv2.resize(detection, size)
+                # get the coordinates for the detection
+                startX = detection['startX']
+                startY = detection['startY']
+                endX = detection['endX']
+                endY = detection['endY']
 
-                image_array = np.asarray(detection)
-                normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
+                image_array = np.asarray(persons[no_of_persons])
+                image_array_resized = cv2.resize(image_array, size)
+                # normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
+                normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
 
                 # Load the image into the array
                 data[0] = normalized_image_array
@@ -427,10 +470,21 @@ def get_student_activity_summary_for_period(activities):
 def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
     BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     EXTRACTED_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\activity\\{}".format(video_name))
+    VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
     # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
     # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
     # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
-    CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5")
+    # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5")
+    CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
+
+    # files required for person detection
+    config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
+    model_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.caffemodel")
+
+    # load our serialized person detection model from disk
+    print("[INFO] loading model...")
+    net = cv2.dnn.readNetFromCaffe(config_file, model_file)
+
 
     np.set_printoptions(suppress=True)
 
@@ -443,11 +497,16 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
     data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
     size = (224, 224)
 
-    # initializing the count variables
+    # class labels
+    # class_labels = ['Phone checking', 'Listening', 'Note taking']
+    class_labels = ['Phone checki...', 'Listening', 'Note taking']
+
+
+    # iteration
+    video = cv2.VideoCapture(VIDEO_DIR)
+    no_of_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
     frame_count = 0
 
-    # class labels
-    class_labels = ['Phone checking', 'Listening', 'Note taking']
 
     # get the frame differences for each frame group
     frame_group_diff = {}
@@ -463,9 +522,8 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
         frame_group_diff[key] = diff if diff > 0 else 1
 
     # looping through the frames
-    for frame in os.listdir(EXTRACTED_DIR):
-        # getting the frame folder
-        FRAME_FOLDER = os.path.join(EXTRACTED_DIR, frame)
+    # for frame in os.listdir(EXTRACTED_DIR):
+    while (frame_count < no_of_frames):
 
         # initializing the variables
         phone_count = 0
@@ -473,57 +531,55 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
         listen_count = 0
         detection_count = 0
 
-        # looping through the detections in each frame
-        for detections in os.listdir(FRAME_FOLDER):
+        ret, image = video.read()
 
-            # checking whether the image contains only one person
-            if "frame" not in detections:
-                # get the label for this image
-                IMAGE_PATH = os.path.join(FRAME_FOLDER, detections)
-                image = cv2.imread(IMAGE_PATH)
+        detections, persons = person_detection(image, net)
 
-                image = cv2.resize(image, size)
+        # looping through the detections in each frame
+        for person in persons:
 
-                image_array = np.asarray(image)
-                normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
+            image = cv2.resize(person, size)
 
-                # Load the image into the array
-                data[0] = normalized_image_array
+            image_array = np.asarray(image)
+            normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
 
-                # run the inference
-                prediction = model.predict(data)
+            # Load the image into the array
+            data[0] = normalized_image_array
 
-                # get the predicted label
-                label = class_labels[prediction.argmax()]
+            # run the inference
+            prediction = model.predict(data)
 
-                # increment the count based on the label
-                if label == class_labels[0]:
-                    phone_count += 1
-                elif label == class_labels[1]:
-                    listen_count += 1
-                elif label == class_labels[2]:
-                    note_count += 1
+            # get the predicted label
+            label = class_labels[prediction.argmax()]
 
-                # increment the detection count
-                detection_count += 1
+            # increment the count based on the label
+            if label == class_labels[0]:
+                phone_count += 1
+            elif label == class_labels[1]:
+                listen_count += 1
+            elif label == class_labels[2]:
+                note_count += 1
+
+            # increment the detection count
+            detection_count += 1
 
-                # finding the time landmark that the current frame is in
-                for i in frame_landmarks:
-                    index = frame_landmarks.index(i)
-                    j = index + 1
+            # finding the time landmark that the current frame is in
+            for i in frame_landmarks:
+                index = frame_landmarks.index(i)
+                j = index + 1
 
-                    # checking whether the next index is within the range
-                    if j < len(frame_landmarks):
-                        next_value = frame_landmarks[j]
+                # checking whether the next index is within the range
+                if j < len(frame_landmarks):
+                    next_value = frame_landmarks[j]
 
-                        # checking the correct time landmark range
-                        if (frame_count >= i) & (frame_count < next_value):
-                            frame_name = "{}-{}".format(i, next_value)
+                    # checking the correct time landmark range
+                    if (frame_count >= i) & (frame_count < next_value):
+                        frame_name = "{}-{}".format(i, next_value)
 
-                            frame_group_dict[frame_name]['phone_count'] += phone_count
-                            frame_group_dict[frame_name]['listen_count'] += listen_count
-                            frame_group_dict[frame_name]['note_count'] += note_count
-                            frame_group_dict[frame_name]['detection_count'] += detection_count
+                        frame_group_dict[frame_name]['phone_count'] += phone_count
+                        frame_group_dict[frame_name]['listen_count'] += listen_count
+                        frame_group_dict[frame_name]['note_count'] += note_count
+                        frame_group_dict[frame_name]['detection_count'] += detection_count
 
         # increment the frame count
         frame_count += 1

FirstApp/templates/FirstApp/video_results.html

@@ -40,7 +40,7 @@
         $(document).ready(function () {
             let folder = '';
 
-            $('#activity_loader').attr('hidden', false);
+            {#$('#activity_loader').attr('hidden', false);#}
             {#$('#emotion_loader').attr('hidden', false);#}
             {#$('#gaze_loader').attr('hidden', false);#}
 
@@ -298,7 +298,7 @@
             }
 
             //this is a test function (delete later)
-
+            /*
             let interval = setInterval(() => {
 
                 {#let url = 'http://127.0.0.1:8000/get-random_number';#}
@@ -355,6 +355,7 @@
 
 
             }
+            */
 
 
         });