all changes done

sign_language_backend/app.py

-from flask import Flask, request, jsonify
-import base64
+from flask import Flask, request
+from flask_socketio import SocketIO, emit, send
 import cv2
+import io
 import numpy as np
-import mediapipe as mp
+import base64
 import tensorflow as tf
-import logging
-
-app = Flask(__name__)
-
-# Load the model
-model = tf.keras.models.load_model('./finalModel.h5')
+from tensorflow.keras.models import load_model
+import mediapipe as mp
 
 mp_holistic = mp.solutions.holistic
-holistic = mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5)
+mp_drawing = mp.solutions.drawing_utils
+actions = (["haloo", "dannwa", "mama", "obata", "puluwan", "suba", "udaasanak"])
+
+# Load the saved model
+model = load_model(r'./finalModel.h5')
 
 sequence = []
 sentence = []
 predictions = []
 threshold = 0.7
-actions = ["haloo", "dannwa", "mama", "obata", "puluwan", "suba", "udaasanak"]
+count = 31
 
 def mediapipe_detection(image, model):
-    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-    image.flags.writeable = False
-    results = model.process(image)
-    image.flags.writeable = True
-    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB
+    image.flags.writeable = False                  # Image is no longer writeable
+    results = model.process(image)                 # Make prediction
+    image.flags.writeable = True                   # Image is now writeable
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR
     return image, results
 
+def draw_landmarks(image, results):
+    # Draw face connections
+    mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACEMESH_TESSELATION,
+                             mp_drawing.DrawingSpec(color=(70,110,10), thickness=1, circle_radius=1),
+                             mp_drawing.DrawingSpec(color=(70,256,121), thickness=1, circle_radius=1))
+    # Draw pose connections
+    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS,
+                             mp_drawing.DrawingSpec(color=(70,22,10), thickness=2, circle_radius=4),
+                             mp_drawing.DrawingSpec(color=(70,44,121), thickness=2, circle_radius=2))
+    # Draw left hand connections
+    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
+                             mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4),
+                             mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2))
+    # Draw right hand connections
+    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
+                             mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4),
+                             mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2))
+
 def extract_keypoints(results):
     pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)
     face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)
@@ -35,54 +54,81 @@ def extract_keypoints(results):
     rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)
     return np.concatenate([pose, face, lh, rh])
 
-@app.route('/endpoint', methods=['POST'])
-def process_frame():
-    try:
-        data = request.json
-        frame_data = data['frame']
+def pad_sequence(sequence, target_length):
+    if len(sequence) < target_length:
+        padding = np.zeros((target_length - len(sequence), sequence.shape[1]))
+        sequence = np.vstack((padding, sequence))
+    return sequence
 
-        # Fix incorrect padding for base64 string
-        frame_data += "=" * ((4 - len(frame_data) % 4) % 4)
+def PredictSign(frame):
+    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
+        global sequence
+        global sentence
+        global predictions
+        global threshold
+        global count
 
-        # Decode the base64 string to an image
-        decoded_data = base64.b64decode(frame_data)
-        np_data = np.frombuffer(decoded_data, dtype=np.uint8)
-        frame = cv2.imdecode(np_data, cv2.IMREAD_COLOR)
-        
-        if frame is None:
-            raise ValueError("Failed to decode image")
-        
-        # Process the frame with Mediapipe and your model
         image, results = mediapipe_detection(frame, holistic)
+        draw_landmarks(image, results)
         keypoints = extract_keypoints(results)
         sequence.append(keypoints)
-        sequence = sequence[-50:]
+        sequence = sequence[-50:]  # Keep the last 50 keypoints
 
         if len(sequence) == 50:
-            res = model.predict(np.expand_dims(sequence, axis=0))[0]
+            padded_sequence = pad_sequence(np.array(sequence), 50)
+            res = model.predict(np.expand_dims(padded_sequence, axis=0))[0]
             predictions.append(np.argmax(res))
-        
-            # Implement your prediction logic
-            if np.unique(predictions[-25:])[0] == np.argmax(res):
+            if np.unique(predictions[-3:])[0] == np.argmax(res):
                 if res[np.argmax(res)] > threshold:
                     if len(sentence) > 0:
                         if actions[np.argmax(res)] != sentence[-1]:
                             sentence.append(actions[np.argmax(res)])
+                            return sentence[-1]
                     else:
                         sentence.append(actions[np.argmax(res)])
+                        return sentence[-1]
+            else:
+                pass
+        else:
+            count -= 1
 
-            if len(sentence) > 5:
-                sentence = sentence[-5:]
-
-        # Print predictions and sentence to the console
-        print(f"Predicted: {actions[np.argmax(res)]}")
-        print(f"Sentence: {sentence}")
-        
-        return jsonify({'message': 'Frame processed', 'sentence': sentence}), 200
-    
-    except Exception as e:
-        logging.exception("Exception in /endpoint")
-        return jsonify({'message': 'Error processing frame', 'error': str(e)}), 500
+app = Flask(__name__)
+app.config['SECRET_KEY'] = 'mysecretkey'
+socketio = SocketIO(app, cors_allowed_origins='*')
+
+@app.route('/')
+def index():
+    return 'Running'
+
+@socketio.on('connect')
+def on_connect():
+    emit('me', request.sid)
+
+@socketio.on('disconnect')
+def on_disconnect():
+    send('callEnded', broadcast=True)
+
+@socketio.on('predictionVideo')
+def on_prediction_video(data):
+    global actions
+    img_bytes = base64.b64decode(data.split(',')[1])
+    np_arr = np.frombuffer(img_bytes, np.uint8)
+    img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
+    ans = PredictSign(img)
+    print(ans)
+    emit('predictionVideo', ans)
+
+@socketio.on('callUser')
+def on_call_user(data):
+    from_user = data['from']
+    userToCall = data['userToCall']
+    caller_name = data['name']
+    signal = data['signalData']
+    emit('callUser', {'from': from_user, 'name': caller_name, 'signal': signal}, room=userToCall)
+
+@socketio.on('answerCall')
+def on_answer_call(data):
+    emit('callAccepted', data['signal'], room=data['to'])
 
 if __name__ == '__main__':
-    app.run(debug=True, port=8000)
+    socketio.run(app, debug=True, port=5000)

sign_language_backend/requirements.txt

+Flask==3.0.0
+flask-socketio
+python-socketio==5.10.0
+opencv-python==4.9.0.80
+mediapipe==0.10.3
+scikit-learn==1.3.1
+matplotlib==3.8.3
+tensorflow==2.13.0