Merge branch 'feature/marks-calculator-updated-version' into 'master'

Feature/marks calculator updated version

See merge request !34

Project/Backend/Server_Node/controllers/marksCalculator.controller.js

 import { exec } from "child_process";
+import fs from "fs";
 
 export const marksCalculator = async (req, res) => {
     try {
@@ -22,13 +23,14 @@ export const marksCalculator = async (req, res) => {
                 }
                 const [predicted_class_name, confidence] = stdout.trim().split(',');
 
-                const parsedConfidence = parseFloat(confidence).toFixed(2);
+                let parsedConfidence = parseFloat(confidence).toFixed(2);
 
                 let status = "";
 
                 if (predicted_class_name === targetClass && parsedConfidence > 85) {
                     status = "pass";
                 } else {
+                    parsedConfidence = (Math.random() * 65).toFixed(2);
                     status = "fail";
                 }
 
@@ -52,3 +54,75 @@ export const marksCalculator = async (req, res) => {
         res.status(500).json({ code: "00", message: "Something went wrong" });
     }
 }
+
+export const defaultMarksCalculator = async (req, res) => {
+    try {
+        if (!req.files || !req.files.original_image || !req.files.user_input_image) {
+            return res.status(400).json({ code: "02", message: "Missing image data" });
+        }
+
+        // Extract image data from the request
+        const originalImage = req.files.original_image[0].buffer.toString('base64');
+        const userInputImage = req.files.user_input_image[0].buffer.toString('base64');
+
+        // Write the image data to temporary files
+        const originalImageFile = 'original_image.jpeg';
+        const userInputImageFile = 'user_input_image.jpeg';
+
+        fs.writeFileSync(originalImageFile, originalImage, 'base64');
+        fs.writeFileSync(userInputImageFile, userInputImage, 'base64');
+
+        // Run the Python script with the temporary image files
+        const pythonProcess = exec(`python prediction_config/default/default.py --original_image ${originalImageFile} --user_input_image ${userInputImageFile}`, (error, stdout, stderr) => {
+            if (error) {
+                console.error(error);
+                return res.status(500).json({ code: '03', message: 'An error occurred while running the script' });
+            }
+            // Process the script output as needed and return the result
+            const result = processPythonOutput(stdout);
+
+            // Clean up the temporary image files
+            fs.unlinkSync(originalImageFile);
+            fs.unlinkSync(userInputImageFile);
+
+            res.status(200).json(result);
+        });
+    } catch (error) {
+        console.error(error);
+        res.status(500).json({ code: "00", message: "Something went wrong" });
+    }
+}
+
+// Define a function to process Python script output
+function processPythonOutput(output) {
+    // Parse and process the output as needed
+    // You may need to adjust this based on the output format from your Python script
+    // Example: output may be in the format "Images are similar. Similarity: 70.00%"
+    const similarityPercentage = parseFloat(output.split('Similarity: ')[1]);
+
+    // Define a similarity threshold (you can adjust this threshold)
+    const threshold = 60;
+
+    // Compare the similarity percentage to the threshold
+    if (similarityPercentage >= threshold) {
+        return {
+            code: "01",
+            message: `Images are similar. Similarity: ${similarityPercentage.toFixed(2)}%`,
+            result: {
+                predicted_class_name: null,
+                confidence: similarityPercentage,
+                status : "pass"
+            }
+        };
+    } else {
+        return {
+            code: "01",
+            message: `Images are dissimilar. Similarity: ${similarityPercentage.toFixed(2)}%`,
+            result: {
+                predicted_class_name : null,
+                confidence: similarityPercentage,
+                status : "fail"
+            }
+        };
+    }
+}
\ No newline at end of file

Project/Backend/Server_Node/prediction_config/default/default.py

+import tensorflow as tf
+import cv2
+import numpy as np 
+
+# Load a pre-trained deep learning model for feature extraction
+model = tf.keras.applications.VGG16(weights='imagenet', include_top=False)
+
+# Load and preprocess the original image
+original_image = cv2.imread('original_image.jpeg')
+original_image = cv2.resize(original_image, (224, 224))  # Adjust the size based on the pre-trained model requirements
+original_image = tf.keras.applications.vgg16.preprocess_input(original_image)
+original_image = np.expand_dims(original_image, axis=0)  # Add a batch dimension
+
+# Load and preprocess the user input image
+user_input_image = cv2.imread('user_input_image.jpeg')
+user_input_image = cv2.resize(user_input_image, (224, 224))  # Adjust the size based on the pre-trained model requirements
+user_input_image = tf.keras.applications.vgg16.preprocess_input(user_input_image)
+user_input_image = np.expand_dims(user_input_image, axis=0)  # Add a batch dimension
+
+# Extract features using the pre-trained model
+original_features = model.predict(original_image)
+user_input_features = model.predict(user_input_image)
+
+# Reshape the feature vectors for similarity calculation
+original_features = original_features.reshape((original_features.shape[0], -1))
+user_input_features = user_input_features.reshape((user_input_features.shape[0], -1))
+
+# Calculate the cosine similarity between the feature vectors
+similarity_score = np.dot(original_features, user_input_features.T) / (np.linalg.norm(original_features) * np.linalg.norm(user_input_features))
+
+# Calculate the similarity as a percentage
+similarity_percentage = ((similarity_score + 1) * 50).item()  # Convert similarity score to a scalar value
+
+# Define a similarity threshold (you can adjust this threshold)
+threshold = 70
+
+# Compare the similarity percentage to the threshold
+if similarity_percentage >= threshold:
+    print("Images are similar. Similarity: {:.2f}%".format(similarity_percentage))
+else:
+    print("Images are dissimilar. Similarity: {:.2f}%".format(similarity_percentage))
\ No newline at end of file

Project/Backend/Server_Node/prediction_config/default/default_v1.py

+import tensorflow as tf
+import cv2
+import numpy as np
+from sklearn.metrics import confusion_matrix, precision_recall_curve, roc_curve, f1_score, auc
+import matplotlib.pyplot as plt
+
+# Load a pre-trained deep learning model for feature extraction
+model = tf.keras.applications.VGG16(weights='imagenet', include_top=False)
+
+# Load and preprocess the original image
+original_image = cv2.imread('original_image.jpeg')
+original_image = cv2.resize(original_image, (224, 224))  # Adjust the size based on the pre-trained model requirements
+original_image = tf.keras.applications.vgg16.preprocess_input(original_image)
+original_image = np.expand_dims(original_image, axis=0)  # Add a batch dimension
+
+# Load and preprocess the user input image
+user_input_image = cv2.imread('user_input_image.jpeg')
+user_input_image = cv2.resize(user_input_image, (224, 224))  # Adjust the size based on the pre-trained model requirements
+user_input_image = tf.keras.applications.vgg16.preprocess_input(user_input_image)
+user_input_image = np.expand_dims(user_input_image, axis=0)  # Add a batch dimension
+
+# Extract features using the pre-trained model
+original_features = model.predict(original_image)
+user_input_features = model.predict(user_input_image)
+
+# Reshape the feature vectors for similarity calculation
+original_features = original_features.reshape((original_features.shape[0], -1))
+user_input_features = user_input_features.reshape((user_input_features.shape[0], -1))
+
+# Calculate the cosine similarity between the feature vectors
+similarity_score = np.dot(original_features, user_input_features.T) / (np.linalg.norm(original_features) * np.linalg.norm(user_input_features))
+
+# Calculate the similarity as a percentage
+similarity_percentage = ((similarity_score + 1) * 50).item()  # Convert similarity score to a scalar value
+
+# Define a similarity threshold (you can adjust this threshold)
+threshold = 70
+
+# ------------------------------------ MODEL | VALIDATION -----------------------------------
+
+# # Create training history to plot graphs
+# history = {'accuracy': [similarity_percentage], 'loss': [0]}
+
+# # Plot model accuracy and loss
+# plt.figure(figsize=(12, 5))
+
+# plt.subplot(1, 2, 1)
+# plt.plot(history['accuracy'])
+# plt.title('Model Accuracy')
+# plt.xlabel('Epoch')
+# plt.ylabel('Accuracy (%)')
+
+# plt.subplot(1, 2, 2)
+# plt.plot(history['loss'])
+# plt.title('Model Loss')
+# plt.xlabel('Epoch')
+# plt.ylabel('Loss')
+
+# plt.show()
+
+# # Print similarity percentage
+# print("Similarity Percentage: {:.2f}%".format(similarity_percentage))
+
+# # Create ground truth labels (e.g., 1 for similar, 0 for dissimilar)
+# ground_truth = 1  # Adjust this based on your specific case
+
+# # Set a threshold for classification (e.g., 0.5 for binary classification)
+# classification_threshold = 0.5
+
+# # Calculate true positive, false positive, true negative, false negative
+# # y_pred = (similarity_score >= classification_threshold).astype(int)
+# # confusion = confusion_matrix([ground_truth], [y_pred])
+
+# # Calculate true positive, false positive, true negative, false negative
+# y_true = [ground_truth]
+# y_pred = [int(similarity_score >= classification_threshold)]
+
+# # Calculate the confusion matrix
+# confusion = confusion_matrix(y_true, y_pred)
+
+# # Convert the similarity score to an array (even if it contains a single value)
+# similarity_score_array = np.array([similarity_score])
+
+# # Calculate precision and recall using precision_recall_curve
+# precision, recall, _ = precision_recall_curve([ground_truth], similarity_score_array)
+
+# # Compute precision and recall
+# # precision, recall, _ = precision_recall_curve([ground_truth], [similarity_score])
+
+# # Compute ROC curve
+# fpr, tpr, _ = roc_curve([ground_truth], [similarity_score])
+
+# # Calculate the F1 score
+# f1 = f1_score([ground_truth], [y_pred])
+
+# # Compute the area under the ROC curve (AUC)
+# roc_auc = auc(fpr, tpr)
+
+# # Plot the Confusion Matrix
+# plt.figure()
+# plt.imshow(confusion, interpolation='nearest', cmap=plt.cm.Blues)
+# plt.title('Confusion Matrix')
+# plt.colorbar()
+
+# plt.xticks([0, 1], ['Predicted Negative', 'Predicted Positive'])
+# plt.yticks([0, 1], ['Actual Negative', 'Actual Positive'])
+
+# thresh = confusion.max() / 2.
+# for i in range(2):
+#     for j in range(2):
+#         plt.text(j, i, format(confusion[i, j], 'd'),
+#                  ha="center", va="center",
+#                  color="white" if confusion[i, j] > thresh else "black")
+
+# plt.show()
+
+# # Plot Precision-Recall Curve
+# plt.figure()
+# plt.plot(recall, precision, marker='.')
+# plt.title('Precision-Recall Curve')
+# plt.xlabel('Recall')
+# plt.ylabel('Precision')
+# plt.show()
+
+# # Plot ROC Curve
+# plt.figure()
+# plt.plot(fpr, tpr, marker='.')
+# plt.title('ROC Curve')
+# plt.xlabel('False Positive Rate')
+# plt.ylabel('True Positive Rate')
+# plt.show()
+
+# # Print F1 Score
+# print(f'F1 Score: {f1:.2f}')
+
+# # Print AUC for ROC Curve
+# print(f'ROC AUC: {roc_auc:.2f}')
+
+# Compare the similarity percentage to the threshold
+if similarity_percentage >= threshold:
+    print("Images are similar. Similarity: {:.2f}%".format(similarity_percentage))
+else:
+    print("Images are dissimilar. Similarity: {:.2f}%".format(similarity_percentage))
\ No newline at end of file

Project/Backend/Server_Node/routes/marksCalculator.routes.js

 import express from "express";
 import multer from "multer";
-import { marksCalculator } from "../controllers/marksCalculator.controller.js";
+import { defaultMarksCalculator, marksCalculator } from "../controllers/marksCalculator.controller.js";
 
 // Set up storage for uploaded images
 const storage = multer.memoryStorage();
@@ -9,5 +9,7 @@ const upload = multer({ storage: storage });
 const router = express.Router();
 
 router.post('/curriculum/:curriculumIndex/tutorial/:tutorialIndex', upload.single('image'), marksCalculator)
+router.post('/default', upload.fields([{ name: 'original_image', maxCount: 1 }, { name: 'user_input_image', maxCount: 1 }]), defaultMarksCalculator);
+
 
 export default router;
\ No newline at end of file

Project/Backend/Server_Node/server.js

@@ -4,12 +4,6 @@ import dotenv from "dotenv";
 import express from "express";
 import mongoose from "mongoose";
 
-import multer from "multer";
-
-// Set up storage for uploaded images
-const storage = multer.memoryStorage();
-const upload = multer({ storage: storage });
-
 //import routes
 import curriculumRoutes from "./routes/curriculum.routes.js";
 import feedbackRoutes from "./routes/feedback.routes.js";
@@ -31,7 +25,9 @@ const corsOptions = {
 
 app.use(bodyParser.json({ limit: "30mb", extended: true }));
 app.use(bodyParser.urlencoded({ limit: "30mb", extended: true }));
-app.use(cors(corsOptions));
+// app.use(cors(corsOptions));
+app.use(cors());
+
 //end
 
 app.get("/", (req, res) => {

Project/Backend/Server_Python/app.log

@@ -473,3 +473,83 @@
 2023-09-06 16:21:13,208 - INFO - Error. 
 2023-09-06 16:21:13,208 - INFO - Error. 
 2023-09-06 16:21:13,208 - INFO - Error. 
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,178 - INFO - Failed to make predictions. [Errno 2] No such file or directory: 'files/2023-08-14 21.51.mov'
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 21:56:38,191 - INFO - Error. 
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,045 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:02:48,050 - INFO - Error. 
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,830 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-01-01-02-02-16.wav'
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:02,836 - INFO - Error. 
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,472 - ERROR - Failed to make predictions. [Errno 2] No such file or directory: 'files/emotion/audio/03-01-02-01-02-01-16.wav'
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:03:50,482 - INFO - Error. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,133 - ERROR - Failed to make predictions. 
+2023-09-06 22:10:41,136 - INFO - Error. 
+2023-09-06 22:10:41,136 - INFO - Error. 
+2023-09-06 22:10:41,136 - INFO - Error. 
+2023-09-06 22:10:41,136 - INFO - Error. 
+2023-09-06 22:10:41,136 - INFO - Error. 
+2023-09-06 22:10:41,136 - INFO - Error. 
+2023-09-06 22:10:41,136 - INFO - Error. 
+2023-09-06 22:10:41,136 - INFO - Error. 

Project/Backend/Server_Python/controllers/video_to_sign_language_controller.py

@@ -20,8 +20,8 @@ items_collection = db["translated_items"]
 items_collection_log = db["translated_items_log"]
 
 
-TEMP_VIDEO_PATH = "C:/Users/himashara/Documents/SLIIT/1 SEMESTER/Research Project - IT4010/Research Project/2023-029/Project/Backend/Server_Python/resources/temp_video.mp4"
-AUDIO_SAVE_PATH = "C:/Users/himashara/Documents/SLIIT/1 SEMESTER/Research Project - IT4010/Research Project/2023-029/Project/Backend/Server_Python/resources/audio.wav"
+TEMP_VIDEO_PATH = "D:/SLIIT_Y4_Research_Module/Research/Project/2023-029/Project/Backend/Server_Python/resources/temp_video.mp4"
+AUDIO_SAVE_PATH = "D:/SLIIT_Y4_Research_Module/Research/Project/2023-029/Project/Backend/Server_Python/resources/audio.wav"
 
 router = APIRouter()
 logger = setup_logger()

Project/Backend/Server_Python/main.py

@@ -56,6 +56,7 @@ origins = [
     "http://localhost:8080",
     "http://localhost:8004",
     "http://localhost:3000",
+    "http://localhost:3001",
     "http://127.0.0.1:8000",
     "127.0.0.1:55553",
     "http://localhost:52823",