Merge branch 'SignLanguagePrediction' into 'master'

Sign language prediction

See merge request !1

sign_language_backend/.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or using Docker, you probably want to ignore it.
+Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+.env.*.local
+
+# virtualenv
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyderworkspace
+
+# Rope project settings
+.ropeproject
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+# Note: Comment this out if you have project-level dependencies in a `.python-version` file.
+.python-version
+
+# Pylint
+pylint.log
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+

sign_language_backend/app.py

+from flask import Flask, request
+from flask_socketio import SocketIO, emit, send
+import cv2
+import io
+import numpy as np
+import base64
+import tensorflow as tf
+from tensorflow.keras.models import load_model
+import mediapipe as mp
+
+mp_holistic = mp.solutions.holistic
+mp_drawing = mp.solutions.drawing_utils
+actions = (["haloo", "dannwa", "mama", "obata", "puluwan", "suba", "udaasanak"])
+
+# Load the saved model
+model = load_model(r'./finalModel.h5')
+
+sequence = []
+sentence = []
+predictions = []
+threshold = 0.7
+count = 31
+
+def mediapipe_detection(image, model):
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB
+    image.flags.writeable = False                  # Image is no longer writeable
+    results = model.process(image)                 # Make prediction
+    image.flags.writeable = True                   # Image is now writeable
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR
+    return image, results
+
+def draw_landmarks(image, results):
+    # Draw face connections
+    mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACEMESH_TESSELATION,
+                             mp_drawing.DrawingSpec(color=(70,110,10), thickness=1, circle_radius=1),
+                             mp_drawing.DrawingSpec(color=(70,256,121), thickness=1, circle_radius=1))
+    # Draw pose connections
+    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS,
+                             mp_drawing.DrawingSpec(color=(70,22,10), thickness=2, circle_radius=4),
+                             mp_drawing.DrawingSpec(color=(70,44,121), thickness=2, circle_radius=2))
+    # Draw left hand connections
+    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
+                             mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4),
+                             mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2))
+    # Draw right hand connections
+    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
+                             mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4),
+                             mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2))
+
+def extract_keypoints(results):
+    pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)
+    face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)
+    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)
+    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)
+    return np.concatenate([pose, face, lh, rh])
+
+def pad_sequence(sequence, target_length):
+    if len(sequence) < target_length:
+        padding = np.zeros((target_length - len(sequence), sequence.shape[1]))
+        sequence = np.vstack((padding, sequence))
+    return sequence
+
+def PredictSign(frame):
+    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
+        global sequence
+        global sentence
+        global predictions
+        global threshold
+        global count
+
+        image, results = mediapipe_detection(frame, holistic)
+        draw_landmarks(image, results)
+        keypoints = extract_keypoints(results)
+        sequence.append(keypoints)
+        sequence = sequence[-50:]  # Keep the last 50 keypoints
+
+        if len(sequence) == 50:
+            padded_sequence = pad_sequence(np.array(sequence), 50)
+            res = model.predict(np.expand_dims(padded_sequence, axis=0))[0]
+            predictions.append(np.argmax(res))
+            if np.unique(predictions[-3:])[0] == np.argmax(res):
+                if res[np.argmax(res)] > threshold:
+                    if len(sentence) > 0:
+                        if actions[np.argmax(res)] != sentence[-1]:
+                            sentence.append(actions[np.argmax(res)])
+                            return sentence[-1]
+                    else:
+                        sentence.append(actions[np.argmax(res)])
+                        return sentence[-1]
+            else:
+                pass
+        else:
+            count -= 1
+
+app = Flask(__name__)
+app.config['SECRET_KEY'] = 'mysecretkey'
+socketio = SocketIO(app, cors_allowed_origins='*')
+
+@app.route('/')
+def index():
+    return 'Running'
+
+@socketio.on('connect')
+def on_connect():
+    emit('me', request.sid)
+
+@socketio.on('disconnect')
+def on_disconnect():
+    send('callEnded', broadcast=True)
+
+@socketio.on('predictionVideo')
+def on_prediction_video(data):
+    global actions
+    img_bytes = base64.b64decode(data.split(',')[1])
+    np_arr = np.frombuffer(img_bytes, np.uint8)
+    img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
+    ans = PredictSign(img)
+    print(ans)
+    emit('predictionVideo', ans)
+
+@socketio.on('callUser')
+def on_call_user(data):
+    from_user = data['from']
+    userToCall = data['userToCall']
+    caller_name = data['name']
+    signal = data['signalData']
+    emit('callUser', {'from': from_user, 'name': caller_name, 'signal': signal}, room=userToCall)
+
+@socketio.on('answerCall')
+def on_answer_call(data):
+    emit('callAccepted', data['signal'], room=data['to'])
+
+if __name__ == '__main__':
+    socketio.run(app, debug=True, port=5000)

sign_language_backend/requirements.txt

+Flask==3.0.0
+flask-socketio
+python-socketio==5.10.0
+opencv-python==4.9.0.80
+mediapipe==0.10.3
+scikit-learn==1.3.1
+matplotlib==3.8.3
+tensorflow==2.13.0