Update SplitVideo.py, GetImageClass.py, GetVideoClass.py, PredictionFunction.py, Uploader.py files

GetImageClass.py

+# import json
+# from flask_cors import CORS
+#
+# from flask import Flask
+# from flask import request
+#
+# app = Flask(__name__)
+# cors = CORS(app)
+#
+# @app.route("/getImageClass")
+def getImageClass(image):
+    from keras.layers import Activation
+    import keras
+    import numpy as np
+    import tensorflow
+    from keras import backend as K
+    from keras.layers import Activation
+    from keras.layers import Conv2D, MaxPooling2D
+    from keras.layers import Dense, Dropout, Flatten
+    from keras.models import Sequential
+
+    K.clear_session()
+    tensorflow.reset_default_graph()
+
+    nb_train_samples = 112
+    nb_validation_samples = 20
+    epochs = 10
+    batch_size = 2
+    img_width, img_height = 350, 350
+
+    if K.image_data_format() == 'channels_first':
+        input_shape = (3, img_width, img_height)
+    else:
+        input_shape = (img_width, img_height, 3)
+
+    modelI = Sequential()
+    modelI.add(Conv2D(32, (2, 2), input_shape=input_shape))
+    modelI.add(Activation('relu'))
+    modelI.add(MaxPooling2D(pool_size=(2, 2)))
+
+    modelI.add(Conv2D(32, (2, 2)))
+    modelI.add(Activation('relu'))
+    modelI.add(MaxPooling2D(pool_size=(2, 2)))
+
+    modelI.add(Conv2D(64, (2, 2)))
+    modelI.add(Activation('relu'))
+    modelI.add(MaxPooling2D(pool_size=(2, 2)))
+
+    modelI.add(Flatten())
+    modelI.add(Dense(64))
+    modelI.add(Activation('relu'))
+    modelI.add(Dropout(0.5))
+    modelI.add(Dense(4))
+    modelI.add(Activation('softmax'))
+
+    modelI.compile(loss='sparse_categorical_crossentropy',
+                   optimizer='adam',
+                   metrics=['accuracy'])
+
+    modelI.load_weights('model_saved.h5')
+
+    sess = keras.backend.get_session()
+    # image = request.args.get('image', default = 1, type = str)
+    print("------------------")
+    print(image)
+    print("------------------")
+    img = tensorflow.read_file(image)
+    img = tensorflow.image.decode_jpeg(img, channels=3)
+    img.set_shape([None, None, 3])
+    img = tensorflow.image.resize_images(img, (350, 350))
+    img = img.eval(session=sess)  # convert to numpy array
+    img = np.expand_dims(img, 0)  # make 'batch' of 1
+
+    pred = modelI.predict(img)
+    # pred = labels["label_names"][np.argmax(pred)]
+    print(pred)
+    y = ["Adaraya", "Aubowan", "House", "Love"]
+    print(np.argmax(pred))
+    print(y[np.argmax(pred)])
+    return y[np.argmax(pred)]
+
+# getImageClass("labeledFrames/4_staticFrame4.jpg")
+
+# if __name__ == '__main__':
+#     app.run(debug=True, port=5001)

GetVideoClass.py

+def getVideoClass():
+    import numpy as np
+    from keras.applications.vgg16 import VGG16
+    from keras.layers import Dense, Dropout
+    from keras.models import Sequential
+    from keras.preprocessing import image
+    from scipy import stats as s
+    from glob import glob
+    from keras import backend as K
+    K.clear_session()
+    base_model = VGG16(weights='imagenet', include_top=False)
+
+    # defining the model architecture
+    model = Sequential()
+    model.add(Dense(1024, activation='relu', input_shape=(61440,)))
+    model.add(Dropout(0.5))
+    model.add(Dense(512, activation='relu'))
+    model.add(Dropout(0.5))
+    model.add(Dense(256, activation='relu'))
+    model.add(Dropout(0.5))
+    model.add(Dense(128, activation='relu'))
+    model.add(Dropout(0.5))
+    model.add(Dense(8, activation='softmax'))
+
+    # loading the trained weights
+    model.load_weights("weight.hdf5")
+
+    # compiling the model
+    model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])
+    images = glob("tempDynamic/*.png")
+    prediction_images = []
+    for i in range(len(images)):
+        print(images[i])
+        img = image.load_img(images[i], target_size=(480, 272, 3))
+        img = image.img_to_array(img)
+        img = img / 255
+        prediction_images.append(img)
+
+    predict = []
+    print(len(prediction_images))
+    # converting all the frames for a test video into numpy array
+    prediction_images = np.array(prediction_images)
+    print("-----------")
+    print(len(prediction_images))
+    print("-----------")
+    # extracting features using pre-trained model
+    ret = ""
+    if len(prediction_images) > 5:
+        prediction_images = base_model.predict(prediction_images)
+
+        # converting features in one dimensional array
+        print(prediction_images.shape)
+        # flattedZeroArr = np.zeros((58, 15, 8, 512))
+        # flattedZeroArr[:prediction_images.shape[0], :prediction_images.shape[1]] = prediction_images
+        prediction_images = prediction_images.reshape(prediction_images.shape[0], 15*8*512)
+        # predicting tags for each array
+        prediction = model.predict(prediction_images)
+        # appending the mode of predictions in predict list to assign the tag to the video
+        print("Vid pred")
+        print(s.mode(prediction)[0][0])
+        yVals = ["He/She","Hello","Here","Job","Me","Name","Teacher","You"]
+        ret = yVals[np.argmax(s.mode(prediction)[0][0])]
+    return ret
+
+# print(getVideoClass())

PredictionFunction.py

+import math
+import os
+import shutil
+
+import cv2
+import numpy as np
+from PIL import Image, ImageChops
+
+import GetVideoClass
+import GetImageClass
+
+##############################################################
+########## Start of image classification model load ##########
+##############################################################
+#
+# nb_train_samples = 112
+# nb_validation_samples = 20
+# epochs = 10
+# batch_size = 2
+# img_width, img_height = 350, 350
+#
+# if K.image_data_format() == 'channels_first':
+# 	input_shape = (3, img_width, img_height)
+# else:
+# 	input_shape = (img_width, img_height, 3)
+#
+# modelI = Sequential()
+# modelI.add(Conv2D(32, (2, 2), input_shape = input_shape))
+# modelI.add(Activation('relu'))
+# modelI.add(MaxPooling2D(pool_size =(2, 2)))
+#
+# modelI.add(Conv2D(32, (2, 2)))
+# modelI.add(Activation('relu'))
+# modelI.add(MaxPooling2D(pool_size =(2, 2)))
+#
+# modelI.add(Conv2D(64, (2, 2)))
+# modelI.add(Activation('relu'))
+# modelI.add(MaxPooling2D(pool_size =(2, 2)))
+#
+# modelI.add(Flatten())
+# modelI.add(Dense(64))
+# modelI.add(Activation('relu'))
+# modelI.add(Dropout(0.5))
+# modelI.add(Dense(4))
+# modelI.add(Activation('softmax'))
+#
+# modelI.compile(loss ='sparse_categorical_crossentropy',
+# 					optimizer ='adam',
+# 				metrics =['accuracy'])
+#
+# modelI.load_weights('model_saved.h5')
+#
+# sess = keras.backend.get_session()
+##############################################################
+########### End of image classification model load ###########
+##############################################################
+
+
+##############################################################
+########## Start of video classification model load ##########
+##############################################################
+
+# base_model = VGG16(weights='imagenet', include_top=False)
+#
+# #defining the model architecture
+# model = Sequential()
+# model.add(Dense(1024, activation='relu', input_shape=(61440,)))
+# model.add(Dropout(0.5))
+# model.add(Dense(512, activation='relu'))
+# model.add(Dropout(0.5))
+# model.add(Dense(256, activation='relu'))
+# model.add(Dropout(0.5))
+# model.add(Dense(128, activation='relu'))
+# model.add(Dropout(0.5))
+# model.add(Dense(8, activation='softmax'))
+#
+# # loading the trained weights
+# model.load_weights("weight.hdf5")
+#
+# # compiling the model
+# model.compile(loss='categorical_crossentropy',optimizer='Adam',metrics=['accuracy'])
+
+##############################################################
+########### End of video classification model load ###########
+##############################################################
+
+
+
+BLUR = 21
+CANNY_THRESH_1 = 0
+CANNY_THRESH_2 = 80
+MASK_DILATE_ITER = 50
+MASK_ERODE_ITER = 50
+MASK_COLOR = (1.0,1.0,1.0) #White mask
+
+class labeledFrame:
+    def __init__(self, type, file):
+        self.type = type
+        self.file = file
+
+labeledFrames = []
+
+def predict(filePath):
+    # Remove files in /frames
+    removeFilesInDir("frames")
+    removeFilesInDir("labeledFrames")
+
+    # Split frames
+    vidObj = cv2.VideoCapture(filePath)
+
+    count = 0
+
+    success = 1
+
+    while success:
+        success, image = vidObj.read()
+
+        if count % 2 == 0:
+            try:
+                cv2.imwrite("frames/frame%d.png" % count, image)
+                # Remove background
+                img = image
+                gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+                # fps = vidObj.get(cv2.CV_CAP_PROP_FPS)
+                # -- Edge detection -------------------------------------------------------------------
+                edges = cv2.Canny(gray, CANNY_THRESH_1, CANNY_THRESH_2)
+                edges = cv2.dilate(edges, None)
+                edges = cv2.erode(edges, None)
+                # -- Find contours in edges, sort by area ---------------------------------------------
+                contour_info = []
+                # _, contours, _ = cv2.findContours(edges, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
+                # Previously, for a previous version of cv2, this line was:
+                contours, _ = cv2.findContours(edges, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
+                # Thanks to notes from commenters, I've updated the code but left this note
+                for c in contours:
+                    contour_info.append((
+                        c,
+                        cv2.isContourConvex(c),
+                        cv2.contourArea(c),
+                    ))
+                contour_info = sorted(contour_info, key=lambda c: c[2], reverse=True)
+                max_contour = contour_info[0]
+
+                # -- Create empty mask, draw filled polygon on it corresponding to largest contour ----
+                # Mask is black, polygon is white
+                mask = np.zeros(edges.shape)
+                cv2.fillConvexPoly(mask, max_contour[0], (255))
+
+                # -- Smooth mask, then blur it --------------------------------------------------------
+                mask = cv2.dilate(mask, None, iterations=MASK_DILATE_ITER)
+                mask = cv2.erode(mask, None, iterations=MASK_ERODE_ITER)
+                mask = cv2.GaussianBlur(mask, (BLUR, BLUR), 0)
+                mask_stack = np.dstack([mask] * 3)  # Create 3-channel alpha mask
+
+                # -- Blend masked img into MASK_COLOR background --------------------------------------
+                mask_stack = mask_stack.astype('float32') / 255.0  # Use float matrices,
+                img = img.astype('float32') / 255.0  # for easy blending
+
+                masked = (mask_stack * img) + ((1 - mask_stack) * MASK_COLOR)  # Blend
+                masked = (masked * 255).astype('uint8')  # Convert back to 8-bit
+                cv2.imwrite("frames/frame%d.png" % count, masked)
+            except:
+                print("damaged image")
+
+        count += 1
+
+    prevFrameType = ""
+    # Split into static and dynamic
+    for i in range(int(count/2)):
+        imgFile = Image.open('frames/frame' + str((i * 2)) + '.png')
+        if i > 0:
+            prevImageFile = Image.open('frames/frame' + str(((i - 1) * 2)) + '.png')
+            chopsImage = ImageChops.difference(prevImageFile, imgFile)
+            chopsImage.save('temp/img' + str(i) + '.png')
+            frameEntropy = image_entropy(chopsImage)
+            # labeledFrame = {file: ""}
+            if frameEntropy > 3.5:
+                imgFile.save(str("labeledFrames/"+str(i)+"_dynamicFrame") + str(i) + '.png')
+                if i > 1:
+                    prevFrameType = "dynamic"
+                    # labeledFrame.file = str("labeledFrames/"+str(i)+"_dynamicFrame") + str(i) + '.png'
+                    # labeledFrame.type = "dynamic"
+                    labeledFrames.append(labeledFrame("dynamic", str("labeledFrames/"+str(i)+"_dynamicFrame") + str(i) + '.png'))
+                    # print(labeledFrame.type)
+            else:
+                imgFile.save(str("labeledFrames/"+str(i)+"_staticFrame") + str(i) + '.png')
+                if i > 1:
+                    prevFrameType = "static"
+                    labeledFrame.file = str("labeledFrames/"+str(i)+"_staticFrame") + str(i) + '.png'
+                    labeledFrame.type = "static"
+                    labeledFrames.append(labeledFrame("static", str("labeledFrames/"+str(i)+"_staticFrame") + str(i) + '.png'))
+                    # print(labeledFrame.type)
+
+    outputStr = ""
+    lastWord = ""
+    dynamicFrameSet = []
+    skipFrame = False
+    i = 0
+    print("Prediction loop starting ...")
+    for lFrame in labeledFrames:
+        if lFrame.type == "static":
+            word = getImageClass(lFrame.file)
+            if lastWord != word:
+                outputStr = outputStr + word + " "
+                lastWord = word
+        if lFrame.type == "dynamic":
+            dynamicFrameSet.append(lFrame.file)
+            if len(labeledFrames)>i+1:
+                if lFrame.type == labeledFrames[i+1].type:
+                    print("similar c"+str(i+3)+" "+str(len(labeledFrames)))
+                    if (i+3) > len(labeledFrames):
+                        print("skipped ======>>>>")
+                        skipFrame = False
+                    else:
+                        skipFrame = True
+                else:
+                    skipFrame = False
+                if not(skipFrame):
+                    x = 0
+                    for dynFrame in dynamicFrameSet:
+                        x += 1
+                        shutil.copy2(dynFrame, "tempDynamic/"+str(x)+".png")
+                    word = getVideoClass()
+                    removeFilesInDir("tempDynamic")
+                    outputStr = outputStr + word + " "
+                    dynamicFrameSet = []
+        i += 1
+    return outputStr
+
+def image_entropy(img):
+    """calculate the entropy of an image"""
+    histogram = img.histogram()
+    histogram_length = sum(histogram)
+    samples_probability = [float(h) / histogram_length for h in histogram]
+    return -sum([p * math.log(p, 2) for p in samples_probability if p != 0])
+
+def removeFilesInDir(folder):
+    for filename in os.listdir(folder):
+        # print(filename.split("_")[0])
+        file_path = os.path.join(folder, filename)
+        try:
+            if os.path.isfile(file_path) or os.path.islink(file_path):
+                os.unlink(file_path)
+            elif os.path.isdir(file_path):
+                shutil.rmtree(file_path)
+        except Exception as e:
+            print('Failed to delete %s. Reason: %s' % (file_path, e))
+
+def getImageClass(image):
+    return GetImageClass.getImageClass(image)
+    # return "house"
+    # URL = "http://localhost:5001/getImageClass"
+    # PARAMS = {'image':image}
+    # r = requests.get(url = URL, params = PARAMS)
+    # print(r)
+    # return r
+
+def getVideoClass():
+    return GetVideoClass.getVideoClass()
+    # return ""
+# sentence = predict("uploads/He_or_she_1.mp4")
+# print(sentence)
+# URL = "http://localhost:8081/sendSign"
+# PARAMS = {'sign':sentence}
+# requests.get(url = URL, params = PARAMS)

SplitVideo.py

+import cv2
+
+def FrameCapture(path):
+    vidObj = cv2.VideoCapture(path)
+
+    count = 0
+
+    success = 1
+
+    while success:
+        success, image = vidObj.read()
+
+        if count % 3 == 0:
+            cv2.imwrite("frames/frame%d.jpg" % count, image)
+        # fps = vidObj.get(cv2.CV_CAP_PROP_FPS)
+        # print(fps)
+
+        count += 1
+
+if __name__ == '__main__':
+    FrameCapture("video.mp4")
\ No newline at end of file

Uploader.py

+import os
+from datetime import datetime
+
+from flask import Flask, request
+
+import PredictionFunction
+
+uploadPath = "uploads"
+if not os.path.exists(uploadPath):
+    os.makedirs(uploadPath)
+    print("Path is created")
+
+app = Flask(__name__)
+
+@app.route('/uploader', methods=['GET', 'POST'])
+def uploader():
+    print(os.getcwd())
+    if request.method == 'POST':
+        f = request.files['file']
+        print(request.headers.get('lat'))
+        splits = f.filename.split("/")
+        today = datetime.now().strftime("%d-%m-%Y %H-%M-%S")
+        print(splits[len(splits)-1])
+        # os.remove(os.path.join(uploadPath, "temp.mp4"))
+        f.save(os.path.join(uploadPath, "temp.mp4"))
+        # cap = cv2.VideoCapture('uploads/temp.mp4')
+
+        # fourcc = cv2.VideoWriter_fourcc(*'XVID')
+        # out = cv2.VideoWriter('uploads/output.mp4', fourcc, 30, (272, 480))
+        #
+        # while True:
+        #     ret, frame = cap.read()
+        #     if ret == True:
+        #         b = cv2.resize(frame, (272, 480), fx=0, fy=0, interpolation=cv2.INTER_CUBIC)
+        #         out.write(b)
+        #     else:
+        #         break
+        #
+        # cap.release()
+        # out.release()
+        # cv2.destroyAllWindows()
+        # URL = "http://localhost:8081/sendSign"
+        # PARAMS = {'sign':sentence}
+        # try:
+        #     requests.get(url=URL, params=PARAMS)
+        # except:
+        #     print("exeption in send req")
+        return predict()
+        # return 'file uploaded successfully'
+
+def predict():
+    sentence = PredictionFunction.predict("uploads/Hello_1.mp4")
+    print(sentence)
+    return sentence
+
+if __name__ == '__main__':
+    app.run(debug=True)
\ No newline at end of file