Skip to content

2
2020-101
Commit 9fa0f329 authored by I.K Seneviratne's avatar I.K Seneviratne
Browse files
Options

Committing the modification in some files.

parent 505c9017
Hide whitespace changes
Inline Side-by-side
Showing with 129 additions and 71 deletions
FirstApp/emotion_detector.py
View file @ 9fa0f329
......@@ -216,18 +216,19 @@ def get_frame_emotion_recognition(video_name):
surprise_count = 0
# get the detections
detections = ar.person_detection(image, net)
detections, persons = ar.person_detection(image, net)
# to count the extracted detections for a frame
detection_count = 0
# if there are detections
if (len(detections) > 0):
# loop through the detections
for detection in detections:
for person in persons:
label = emotion_recognition(classifier, face_classifier, detection)
label = emotion_recognition(classifier, face_classifier, person)
# checking for the label
if label == class_labels[0]:
......@@ -422,17 +423,17 @@ def emotion_frame_groupings(video_name, frame_landmarks, frame_group_dict):
neutral_count = 0
detection_count = 0
detections = ar.person_detection(image, net)
detections, persons = ar.person_detection(image, net)
# if there are detections
if (len(detections) > 0):
# looping through the detections in each frame
for detection in detections:
for person in persons:
# run the model and get the emotion label
label = emotion_recognition(classifier, face_classifier, detection)
label = emotion_recognition(classifier, face_classifier, person)
# increment the count based on the label
if label == class_labels[0]:
......
FirstApp/logic/activity_recognition.py
View file @ 9fa0f329
......@@ -38,8 +38,9 @@ def activity_recognition(video_path):
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_path))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
ACTIVITY_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\activity")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
# ACTIVITY_DIR = os.path.join(BASE_DIR, "static\\FirstApp\\activity")
# files required for person detection
config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
......@@ -55,7 +56,9 @@ def activity_recognition(video_path):
np.set_printoptions(suppress=True)
# define the student activity labels
class_labels = ['Phone checking', 'Listening', 'Note taking']
# class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# load the model
model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
......@@ -81,13 +84,17 @@ def activity_recognition(video_path):
# for testing purposes
print('starting the activity recognition process')
# initiailizing the video writer
vid_cod = cv2.VideoWriter_fourcc(*'XVID')
output = cv2.VideoWriter("videos/cam_video.mp4", vid_cod, 30.0, size)
# looping through the frames
while (frame_count < no_of_frames):
ret, image = video.read()
image = cv2.resize(image, size)
# image = cv2.resize(image, size)
# perform person detection on the extracted image
detections = person_detection(image, net)
detections, persons = person_detection(image, net)
# this is for testing purposes
print('frame count: ', frame_count)
......@@ -102,13 +109,26 @@ def activity_recognition(video_path):
# initialize the detection count
detection_count = 0
# to iterate each person
no_of_persons = 0
# looping through the person detections of the frame
for detection in detections:
detection = cv2.resize(detection, size)
# get the coordinates for the detection
startX = detection['startX']
startY = detection['startY']
endX = detection['endX']
endY = detection['endY']
# detection = cv2.resize(detection, size)
# draw the coordinates of the persons' identified
cv2.rectangle(image, (startX, startY), (endX, endY), (0, 255, 0), 5)
image_array = np.asarray(detection)
normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
image_array = np.asarray(persons[no_of_persons])
image_array_resized = cv2.resize(image_array, size)
# normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
# Load the image into the array
data[0] = normalized_image_array
......@@ -132,6 +152,9 @@ def activity_recognition(video_path):
# increment the frame count
frame_count += 1
# write the frame to the video writer
output.write(image)
# calculating the percentages for each label
phone_perct = float(phone_checking_count / total_detections) * 100 if total_detections > 0 else 0
......@@ -163,6 +186,7 @@ def person_detection(image, net):
# set the threshold balue
threshold = 0.2
detected_person = []
persons = []
# initialize the list of class labels MobileNet SSD was trained to
# detect, then generate a set of bounding box colors for each class
......@@ -211,14 +235,22 @@ def person_detection(image, net):
startX = 0 if startX < 0 else startX
startY = 0 if startY < 0 else startY
# extract the person
# this dictionary will contain the bounding box coordinates
coordinates = {}
person = image[startY:startY + endY, startX:startX + endX]
detected_person.append(person)
coordinates['startX'] = startX
coordinates['startY'] = startY
coordinates['endX'] = endX
coordinates['endY'] = endY
persons.append(person)
detected_person.append(coordinates)
person_count += 1
# return the detection person list
return detected_person
return detected_person, persons
# this method will recognize the activity for each frame
......@@ -233,7 +265,8 @@ def get_frame_activity_recognition(video_name):
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
# files required for person detection
config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
......@@ -247,7 +280,9 @@ def get_frame_activity_recognition(video_name):
np.set_printoptions(suppress=True)
# class labels
class_labels = ['Phone checking', 'Listening', 'Note taking']
# class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# load the activity recogntion model
model = tensorflow.keras.models.load_model(CLASSIFIER_DIR)
......@@ -295,19 +330,27 @@ def get_frame_activity_recognition(video_name):
detection_count = 0
detected_percentages = []
detections = person_detection(image, net)
detections, persons = person_detection(image, net)
# if there are detections
if (len(detections) > 0):
no_of_persons = 0
# loop through each detection in the frame
for detection in detections:
detection = cv2.resize(detection, size)
# get the coordinates for the detection
startX = detection['startX']
startY = detection['startY']
endX = detection['endX']
endY = detection['endY']
image_array = np.asarray(detection)
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
image_array = np.asarray(persons[no_of_persons])
image_array_resized = cv2.resize(image_array, size)
# normalized_image_array = (detection.astype(np.float32) / 127.0) - 1
normalized_image_array = (image_array_resized.astype(np.float32) / 127.0) - 1
# Load the image into the array
data[0] = normalized_image_array
......@@ -427,10 +470,21 @@ def get_student_activity_summary_for_period(activities):
def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
EXTRACTED_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\activity\\{}".format(video_name))
VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\videos\\{}".format(video_name))
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_04.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5")
# CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_05.h5")
CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_06.h5")
# files required for person detection
config_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.prototxt.txt")
model_file = os.path.join(BASE_DIR, "FirstApp\\classifiers\\MobileNetSSD_deploy.caffemodel")
# load our serialized person detection model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(config_file, model_file)
np.set_printoptions(suppress=True)
......@@ -443,11 +497,16 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
size = (224, 224)
# initializing the count variables
# class labels
# class_labels = ['Phone checking', 'Listening', 'Note taking']
class_labels = ['Phone checki...', 'Listening', 'Note taking']
# iteration
video = cv2.VideoCapture(VIDEO_DIR)
no_of_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
frame_count = 0
# class labels
class_labels = ['Phone checking', 'Listening', 'Note taking']
# get the frame differences for each frame group
frame_group_diff = {}
......@@ -463,9 +522,8 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
frame_group_diff[key] = diff if diff > 0 else 1
# looping through the frames
for frame in os.listdir(EXTRACTED_DIR):
# getting the frame folder
FRAME_FOLDER = os.path.join(EXTRACTED_DIR, frame)
# for frame in os.listdir(EXTRACTED_DIR):
while (frame_count < no_of_frames):
# initializing the variables
phone_count = 0
......@@ -473,57 +531,55 @@ def activity_frame_groupings(video_name, frame_landmarks, frame_group_dict):
listen_count = 0
detection_count = 0
# looping through the detections in each frame
for detections in os.listdir(FRAME_FOLDER):
ret, image = video.read()
# checking whether the image contains only one person
if "frame" not in detections:
# get the label for this image
IMAGE_PATH = os.path.join(FRAME_FOLDER, detections)
image = cv2.imread(IMAGE_PATH)
detections, persons = person_detection(image, net)
image = cv2.resize(image, size)
# looping through the detections in each frame
for person in persons:
image_array = np.asarray(image)
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
image = cv2.resize(person, size)
# Load the image into the array
data[0] = normalized_image_array
image_array = np.asarray(image)
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
# run the inference
prediction = model.predict(data)
# Load the image into the array
data[0] = normalized_image_array
# get the predicted label
label = class_labels[prediction.argmax()]
# run the inference
prediction = model.predict(data)
# increment the count based on the label
if label == class_labels[0]:
phone_count += 1
elif label == class_labels[1]:
listen_count += 1
elif label == class_labels[2]:
note_count += 1
# get the predicted label
label = class_labels[prediction.argmax()]
# increment the detection count
detection_count += 1
# increment the count based on the label
if label == class_labels[0]:
phone_count += 1
elif label == class_labels[1]:
listen_count += 1
elif label == class_labels[2]:
note_count += 1
# increment the detection count
detection_count += 1
# finding the time landmark that the current frame is in
for i in frame_landmarks:
index = frame_landmarks.index(i)
j = index + 1
# finding the time landmark that the current frame is in
for i in frame_landmarks:
index = frame_landmarks.index(i)
j = index + 1
# checking whether the next index is within the range
if j < len(frame_landmarks):
next_value = frame_landmarks[j]
# checking whether the next index is within the range
if j < len(frame_landmarks):
next_value = frame_landmarks[j]
# checking the correct time landmark range
if (frame_count >= i) & (frame_count < next_value):
frame_name = "{}-{}".format(i, next_value)
# checking the correct time landmark range
if (frame_count >= i) & (frame_count < next_value):
frame_name = "{}-{}".format(i, next_value)
frame_group_dict[frame_name]['phone_count'] += phone_count
frame_group_dict[frame_name]['listen_count'] += listen_count
frame_group_dict[frame_name]['note_count'] += note_count
frame_group_dict[frame_name]['detection_count'] += detection_count
frame_group_dict[frame_name]['phone_count'] += phone_count
frame_group_dict[frame_name]['listen_count'] += listen_count
frame_group_dict[frame_name]['note_count'] += note_count
frame_group_dict[frame_name]['detection_count'] += detection_count
# increment the frame count
frame_count += 1
......
FirstApp/templates/FirstApp/video_results.html
View file @ 9fa0f329
......@@ -40,7 +40,7 @@
$(document).ready(function () {
let folder = '';
$('#activity_loader').attr('hidden', false);
{#$('#activity_loader').attr('hidden', false);#}
{#$('#emotion_loader').attr('hidden', false);#}
{#$('#gaze_loader').attr('hidden', false);#}
......@@ -298,7 +298,7 @@
}
//this is a test function (delete later)
/*
let interval = setInterval(() => {
{#let url = 'http://127.0.0.1:8000/get-random_number';#}
......@@ -355,6 +355,7 @@
}
*/
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment