Merge branch 'db_and_monitoring' into 'QA_RELEASE'

Db and monitoring

See merge request !15

MonitorLecturerApp/api.py

@@ -210,3 +210,44 @@ class LecturerAudioSummaryPeriodAPI(APIView):
         })
 
 
+# this section is for student and lecturer behavior integration
+class StudentLecturerIntegratedAPI(APIView):
+
+    def get(self, request):
+        video_name = request.query_params.get('video_name')
+
+        # finding the existence of Lecture activity frame recognition record
+        isExist = LecturerActivityFrameRecognitions.objects.filter(
+            lecturer_meta_id__lecturer_video_id__lecture_video_name=video_name).exists()
+
+        if (isExist):
+            lecture_activity_frame_recognitions = LecturerActivityFrameRecognitions.objects.filter(
+                lecturer_meta_id__lecturer_video_id__lecture_video_name=video_name)
+            lecture_activity_frame_recognitions_ser = LecturerActivityFrameRecognitionsSerializer(
+                lecture_activity_frame_recognitions, many=True)
+            lecture_activity_frame_recognitions_data = lecture_activity_frame_recognitions_ser.data[0]
+
+            frame_detections = lecture_activity_frame_recognitions_data['frame_recognition_details']
+            fps = lecture_activity_frame_recognitions_data['fps']
+            int_fps = int(fps)
+
+            return Response({
+                "frame_recognitions": frame_detections,
+                "fps": fps
+            })
+
+        else:
+
+            # frame_recognitions = classroom_activity.get_lecturer_activity_for_frames(video_name)
+            frame_recognitions, fps = classroom_activity.save_frame_recognition(video_name)
+
+            int_fps = int(fps)
+
+            # print('frame recognitions: ', frame_recognitions)
+
+            return Response({
+                "frame_recognitions": frame_recognitions,
+                "fps": fps
+            })
+
+

MonitorLecturerApp/logic/classroom_activity.py

@@ -5,6 +5,13 @@ import numpy as np
 import cv2
 import os
 
+from FirstApp.logic.custom_sorter import custom_object_sorter
+from FirstApp.logic.id_generator import generate_new_id
+from MonitorLecturerApp.models import LecturerVideoMetaData, LecturerActivityFrameRecognitions, \
+    LecturerActivityFrameRecognitionDetails
+from MonitorLecturerApp.serializers import LecturerVideoMetaDataSerializer
+
+
 def activity_recognition(video_name):
     BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     CLASSIFIER_DIR = os.path.join(BASE_DIR, "MonitorLecturerApp\\models")
@@ -108,3 +115,162 @@ def activity_recognition(video_name):
 
 
 
+# this method will calculated lecturer activity for frames
+def get_lecturer_activity_for_frames(video_name):
+    BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    VIDEO_DIR = os.path.join(BASE_DIR, "assets\\FirstApp\\lecturer_videos\\{}".format(video_name))
+    # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_02.h5")
+    # CLASSIFIER_DIR = os.path.join(BASE_DIR, "FirstApp\\classifiers\\student_activity_version_03.h5")
+    CLASSIFIER_DIR = os.path.join(BASE_DIR, "MonitorLecturerApp\\models")
+    CLASSIFIER_PATH = os.path.join(CLASSIFIER_DIR, "keras_model_updated.h5")
+
+
+    # load our serialized persosn detection model from disk
+    print("[INFO] loading model...")
+
+
+    np.set_printoptions(suppress=True)
+
+
+    class_labels = ['Seated Teaching', 'Teaching by Standing', 'Teaching by Walking']
+
+    model = tensorflow.keras.models.load_model(CLASSIFIER_PATH)
+    model.compile(optimizer='adam',
+                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+                  metrics=['accuracy'])
+
+    data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
+    size = (224, 224)
+
+    # iteration
+    video = cv2.VideoCapture(VIDEO_DIR)
+    no_of_frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
+    fps = video.get(cv2.CAP_PROP_FPS)
+
+    print('fps: ', fps)
+    frame_count = 0
+
+    # frame activity recognitions
+    frame_activity_recognitions = []
+
+    # for testing purposes
+    print('starting the frame activity recognition process')
+
+    # looping through the frames
+    while (frame_count < no_of_frames):
+
+
+        # define the count variables for each frame
+        sitting_count = 0
+        standing_count = 0
+        walking_count = 0
+
+        ret, image = video.read()
+
+        # derive the frame name
+        frame_name = "frame-{}".format(frame_count)
+
+        frame_details = {}
+        frame_details['frame_name'] = frame_name
+
+
+        detection = cv2.resize(image, size)
+
+        image_array = np.asarray(detection)
+        normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
+
+        # Load the image into the array
+        data[0] = normalized_image_array
+
+        # run the inference
+        prediction = model.predict(data)
+
+        label = class_labels[prediction.argmax()]
+
+        # increment the relevant count, based on the label
+        if (label == class_labels[0]):
+            sitting_count += 1
+        elif (label == class_labels[1]):
+            standing_count += 1
+        elif (label == class_labels[2]):
+            walking_count += 1
+
+
+
+        print('current frame: ', frame_count)
+        # increment frame count
+        frame_count += 1
+
+
+        # calculating the percentages for the frame
+        sitting_perct = float(sitting_count) * 100
+        standing_perct = float(standing_count) * 100
+        walking_perct = float(walking_count) * 100
+
+        # adding the percentage values to the frame details
+        frame_details['sitting_perct'] = sitting_perct
+        frame_details['standing_perct'] = standing_perct
+        frame_details['walking_perct'] = walking_perct
+
+        # push to all the frame details
+        frame_activity_recognitions.append(frame_details)
+
+
+
+
+    # sort the recognitions based on the frame number
+    sorted_activity_frame_recognitions = custom_object_sorter(frame_activity_recognitions)
+
+    # for testing purposes
+    print('ending the frame activity recognition process')
+
+    # return the detected frame percentages
+    return sorted_activity_frame_recognitions, fps
+
+
+# this section will handle saving activity entities to the database
+def save_frame_recognition(video_name):
+
+    # for testing purposes
+    print('starting the saving activity frame recognition process')
+
+    # retrieve the lecture activity id
+    lec_activity = LecturerVideoMetaData.objects.filter(lecturer_video_id__lecture_video_name=video_name)
+    lec_activity_ser = LecturerVideoMetaDataSerializer(lec_activity, many=True)
+    lec_activity_data = lec_activity_ser.data[0]
+    lec_activity_id = lec_activity_data['id']
+
+    # create a new lecture activity frame detections id
+    last_lec_activity_frame_recognitions = LecturerActivityFrameRecognitions.objects.order_by(
+        'lecturer_activity_frame_recognition_id').last()
+    new_lecture_activity_frame_recognitions_id = "LLAFR00001" if (last_lec_activity_frame_recognitions is None) else \
+        generate_new_id(last_lec_activity_frame_recognitions.lecturer_activity_frame_recognition_id)
+
+    # calculate the frame detections
+    frame_detections, fps = get_lecturer_activity_for_frames(video_name)
+
+    frame_recognition_details = []
+
+    # save the new lecture activity frame recognitions
+    for detection in frame_detections:
+        lec_activity_frame_recognition_details = LecturerActivityFrameRecognitionDetails()
+        lec_activity_frame_recognition_details.frame_name = detection['frame_name']
+        lec_activity_frame_recognition_details.sitting_perct = detection['sitting_perct']
+        lec_activity_frame_recognition_details.standing_perct = detection['standing_perct']
+        lec_activity_frame_recognition_details.walking_perct = detection['walking_perct']
+
+        frame_recognition_details.append(lec_activity_frame_recognition_details)
+
+    lec_activity_frame_recognitions = LecturerActivityFrameRecognitions()
+    lec_activity_frame_recognitions.lecturer_activity_frame_recognition_id = new_lecture_activity_frame_recognitions_id
+    lec_activity_frame_recognitions.lecturer_meta_id_id = lec_activity_id
+    lec_activity_frame_recognitions.frame_recognition_details = frame_recognition_details
+    lec_activity_frame_recognitions.fps = float(fps)
+
+    lec_activity_frame_recognitions.save()
+
+    # for testing purposes
+    print('ending the saving activity frame recognition process')
+
+    # now return the frame detections
+    return frame_detections, fps

MonitorLecturerApp/migrations/0005_lectureractivityframerecognitions.py

+# Generated by Django 2.2.11 on 2020-10-25 10:09
+
+import MonitorLecturerApp.models
+from django.db import migrations, models
+import django.db.models.deletion
+import djongo.models.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('MonitorLecturerApp', '0004_lecturervideometadata_lecturer_video_id'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LecturerActivityFrameRecognitions',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('lecturer_activity_frame_recognition_id', models.CharField(max_length=15)),
+                ('frame_recognition_details', djongo.models.fields.ArrayField(model_container=MonitorLecturerApp.models.LecturerActivityFrameRecognitionDetails)),
+                ('lecturer_meta_id', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='MonitorLecturerApp.LecturerVideoMetaData')),
+            ],
+        ),
+    ]

MonitorLecturerApp/migrations/0006_lectureractivityframerecognitions_fps.py

+# Generated by Django 2.2.11 on 2020-10-25 10:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('MonitorLecturerApp', '0005_lectureractivityframerecognitions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='lectureractivityframerecognitions',
+            name='fps',
+            field=models.FloatField(default=30.0),
+        ),
+    ]

MonitorLecturerApp/models.py

@@ -87,3 +87,27 @@ class LecturerAudioText (models.Model):
     def __str__(self):
         return self.lecturer_audio_text_id
 
+
+
+# this abstract class will contain lecture activity frame recognition details
+class LecturerActivityFrameRecognitionDetails(models.Model):
+    frame_name = models.CharField(max_length=15)
+    sitting_perct = models.FloatField()
+    standing_perct = models.FloatField()
+    walking_perct = models.FloatField()
+
+    class Meta:
+        abstract = True
+
+
+# this class will contain lecture activity frame recognitions
+class LecturerActivityFrameRecognitions(models.Model):
+    lecturer_activity_frame_recognition_id = models.CharField(max_length=15)
+    lecturer_meta_id = models.ForeignKey(LecturerVideoMetaData, on_delete=models.CASCADE)
+    frame_recognition_details = models.ArrayField(LecturerActivityFrameRecognitionDetails)
+    fps = models.FloatField(default=30.0)
+
+    def __str__(self):
+        return self.lecturer_activity_frame_recognition_id
+
+

MonitorLecturerApp/serializers.py

@@ -2,7 +2,7 @@ from rest_framework import serializers
 
 from FirstApp.serializers import LecturerSerializer, SubjectSerializer
 from LectureSummarizingApp.models import LectureAudioSummary
-from .models import RegisterTeacher
+from .models import RegisterTeacher, LecturerActivityFrameRecognitions
 from .models import LecturerAudioText, LecturerVideoMetaData, LecturerVideo, LectureRecordedVideo
 
 
@@ -44,3 +44,35 @@ class LecturerVideoMetaDataSerializer(serializers.ModelSerializer):
     class Meta:
         model = LecturerVideoMetaData
         fields = '__all__'
+
+
+
+# lecture activity frame recognition serializer
+class LecturerActivityFrameRecognitionsSerializer(serializers.ModelSerializer):
+
+    lecturer_meta_id = LecturerVideoMetaDataSerializer()
+    frame_recognition_details = serializers.SerializerMethodField()
+
+    # this method will be used to serialize the 'frame_recogition_details' field
+    def get_frame_recognition_details(self, obj):
+
+        return_data = []
+
+        for frame_recognition in obj.frame_recognition_details:
+            recognition = {}
+
+            recognition["frame_name"] = frame_recognition.frame_name
+            recognition["sitting_perct"] = frame_recognition.sitting_perct
+            recognition["standing_perct"] = frame_recognition.standing_perct
+            recognition["walking_perct"] = frame_recognition.walking_perct
+
+            return_data.append(recognition)
+
+        # return the data
+        return return_data
+
+
+    class Meta:
+        model = LecturerActivityFrameRecognitions
+        fields = '__all__'
+

MonitorLecturerApp/templates/MonitorLecturerApp/index.html

@@ -336,23 +336,23 @@
         <div class="sidebar-heading">
 
         </div>
-
-        <!-- Nav Item - Pages Collapse Menu -->
-        <li class="nav-item">
-            <a class="nav-link collapsed" href="#" data-toggle="collapse" data-target="#collapsePages"
-               aria-expanded="true" aria-controls="collapsePages">
-                <i class="fas fa-fw fa-folder"></i>
-                <span>Pages</span>
-            </a>
-            <div id="collapsePages" class="collapse" aria-labelledby="headingPages" data-parent="#accordionSidebar">
-                <div class="bg-white py-2 collapse-inner rounded">
-                    <!--                    <h6 class="collapse-header">Login Screens:</h6>-->
-                    <a class="collapse-item" href="index.html">Dashboard</a>
-                    <a class="collapse-item" href="/lecturer/lecture-video">Video Page</a>
-
-                </div>
-            </div>
-        </li>
+{##}
+{#        <!-- Nav Item - Pages Collapse Menu -->#}
+{#        <li class="nav-item">#}
+{#            <a class="nav-link collapsed" href="#" data-toggle="collapse" data-target="#collapsePages"#}
+{#               aria-expanded="true" aria-controls="collapsePages">#}
+{#                <i class="fas fa-fw fa-folder"></i>#}
+{#                <span>Pages</span>#}
+{#            </a>#}
+{#            <div id="collapsePages" class="collapse" aria-labelledby="headingPages" data-parent="#accordionSidebar">#}
+{#                <div class="bg-white py-2 collapse-inner rounded">#}
+{#                    <!--                    <h6 class="collapse-header">Login Screens:</h6>-->#}
+{#                    <a class="collapse-item" href="index.html">Dashboard</a>#}
+{#                    <a class="collapse-item" href="/lecturer/lecture-video">Video Page</a>#}
+{##}
+{#                </div>#}
+{#            </div>#}
+{#        </li>#}
 
         <!-- Divider -->
         <hr class="sidebar-divider d-none d-md-block">

MonitorLecturerApp/templates/MonitorLecturerApp/lecVideo.html

-{% extends 'MonitorLecturerApp/template.html' %}
+{% extends 'FirstApp/template.html' %}
 <!DOCTYPE html>
 <html lang="en">
 <body id="page-top">

MonitorLecturerApp/urls.py

@@ -24,6 +24,9 @@ urlpatterns = [
     path('lecture-video', views.lecVideo),
     # path('Video', views.hello)
 
+    # delete this path later
+    path('test-frame-recognitions', views.testFrameRecognitions),
+
     ##### LECTURER ACTIVITY SECTION #####
     # API to retrieve activity recognition
     url(r'^activities/$', api.ActivityRecognitionAPI.as_view()),
@@ -31,6 +34,9 @@ urlpatterns = [
     # API to retrieve lecturer video meta data results
     url(r'^get-lecturer-video-results/$', api.GetLectureVideoResultsAPI.as_view()),
 
+    # API to retrieve lecturer video frame recognitions
+    url(r'^get-lecturer-video-frame-recognitions/$', api.StudentLecturerIntegratedAPI.as_view()),
+
     ##### END OF LECTURER ACTIVITY SECTION #####
 
 

MonitorLecturerApp/views.py

@@ -187,3 +187,6 @@ def lecVideo(request):
     # for  audioPath in audiopaths:
     #     audio = tAudio()
 
+
+def testFrameRecognitions(request):
+    return render(request, "MonitorLecturerApp/test_frame_recognitions.html")
\ No newline at end of file