Merge branch 'master' into it19954974/PriyankaPDMK

App/Backend/README.md

+
+<h1 align="center">Interactive, visual learning-based tool for hearing impaired children to improve language skills.</h3>
+
+
+## How to use the flask server for backend services
+
+1. Install required packages from requirements.txt
+2. Insert the path and the relevant logic to server.py
+3. Run server.py to start the flask server
+
+Server will be available at http://127.0.0.1:5000/
+

App/Backend/content_filtration.py

+from flask import request
+from pymongo import MongoClient
+
+def check_word_safety(word):
+    client = MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
+    db = client['word_filtration']
+    blacklist_col = db['blacklist']
+    whitelist_col = db['whitelist']
+    moderate_col = db['moderate']
+
+    word = word.lower()
+    # Check if the word exists in the blacklist
+    if blacklist_col.find_one({"word": word}):
+        return "unsafe"
+
+    # Check if the word exists in the whitelist
+    whitelist_entry = whitelist_col.find_one({"word": word})
+    if whitelist_entry:
+        sensitive_score = whitelist_entry.get("sensitivity_score")
+        print(sensitive_score)
+        if sensitive_score == 1:
+            return "safe"
+        else:
+            return "unsafe"
+
+    # If the word does not exist in the whitelist
+    moderate_col.insert_one({"word": word, "status": "not_verified"})
+    return "not_verified"

App/Backend/flip_card_content.py

+import pymongo
+
+def getFlipCardContent():
+    
+    client = pymongo.MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
+    db = client['word_card']
+    collection = db['card']
+
+    card_data = collection.find_one()
+
+    if card_data:
+        main_image_data = {
+            'isMainImage': True,
+            'image': card_data['image'],
+            'audio': card_data['audio']
+        }
+
+        other_images_data = [
+            {
+                'isMainImage': False,
+                'image': card['image'],
+                'audio': card['audio']
+            } for card in card_data['card_0']
+        ]
+
+        images_data = [main_image_data] + other_images_data
+
+        return images_data
+
+    return []

App/Backend/requirements.txt

+Flask==2.2.3
+numpy==1.24.2
+transformers
+torch
+pymongo
\ No newline at end of file

App/Backend/server.py

+import numpy as np
+from flask import Flask, request, jsonify, request
+from flask_cors import CORS
+#import pickle
+
+from word_card_game import wordGameData
+from word_generation import get_similar_words
+from flip_card_content import getFlipCardContent
+from content_filtration import check_word_safety
+
+
+
+app = Flask(__name__)
+
+@app.route('/api/word-game', methods=['GET'])
+def word_game_api():
+    w1 = request.args.get('w1')
+    w2 = request.args.get('w2')
+    w3 = request.args.get('w3')
+
+    if not all([w1, w2, w3]):
+        return jsonify({'error': 'All three words must be provided'}), 400
+
+    data = wordGameData(w1, w2, w3)
+    return jsonify(data)
+
+@app.route('/api/similar-words', methods=['GET'])
+def similar_words_api():
+    word = request.args.get('word')
+
+    if not word:
+        return jsonify({'error': 'A word must be provided'}), 400
+
+    similar_words = get_similar_words(word)
+    return jsonify({'similar_words': similar_words})
+
+@app.route('/api/flip-card-content')
+def flip_card_content():
+    data = getFlipCardContent()
+    return jsonify(data)
+
+@app.route('/api/images_data', methods=['GET'])
+def get_images_data():
+    images_data = wordGameData()
+    return jsonify(images_data)
+
+@app.route('/check_word', methods=['POST'])
+def check_word():
+    data = request.get_json()
+    word = data.get('word')
+    if not word:
+        return jsonify({"error": "No word provided"}), 400
+
+    result = check_word_safety(word)
+    return jsonify({"word": word, "status": result}), 200
+    
+if __name__ == '__main__':
+    CORS(app.run(host='0.0.0.0', port=5000, debug=True))
+    #app.run(host='0.0.0.0', port=5000, debug=True)
\ No newline at end of file

App/Backend/word_card_game.py

+import random
+import pymongo
+
+client = pymongo.MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
+db = client['word_card']
+collection = db['card']
+
+def get_words():
+    doc = collection.find_one()
+    if doc and 'card_0' in doc:
+        words = doc['card_0']
+        #print(words)
+        return words
+    else:
+        return []
+
+
+
+def wordGameData():
+    words_data = get_words()
+    main_word_data = random.choice(words_data)
+    card_words_data = random.sample(words_data, len(words_data))
+
+    def create_links(word_data):
+        return {'word': word_data['word'], 'image_link': word_data['image'], 'audio_link': word_data['audio']}
+
+    return {
+        'main_word': create_links(main_word_data),
+        'card_word1': create_links(card_words_data[0]),
+        'card_word2': create_links(card_words_data[1]),
+        'card_word3': create_links(card_words_data[2])
+    }

App/Backend/word_generation.py

+import torch
+from transformers import RobertaTokenizer, RobertaForMaskedLM
+import pymongo
+import random
+
+# Load the pretrained RoBERTa model and tokenizer
+tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+model = RobertaForMaskedLM.from_pretrained('roberta-base')
+
+def get_similar_words(input_word, top_k=3):
+    print(input_word)
+    #connect to mongoDB
+    client = pymongo.MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
+    db_0 = client['vocabulary']
+    collection_0 = db_0['object_expore']
+    cursor = collection_0.find()
+  
+
+    random_word = collection_0.aggregate([{'$sample': {'size': 1}}]).next()['object']
+    input_word = random_word.strip()
+    print('---------------------------------------------------------')
+    print('')
+    print("Input word = "+input_word)
+
+    # Create a masked sentence with the input word
+    masked_sentence = f"The {input_word} is related to the {tokenizer.mask_token}."
+    
+    # Tokenize the masked sentence
+    inputs = tokenizer(masked_sentence, return_tensors='pt')
+    
+    # Get the index of the mask token
+    mask_token_index = torch.where(inputs['input_ids'][0] == tokenizer.mask_token_id)[0].item()
+    
+    # Predict words for the mask token
+    with torch.no_grad():
+        output = model(**inputs)
+        predictions = output.logits[0, mask_token_index]
+    
+    # Get the top k predicted words
+    top_k_indices = torch.topk(predictions, top_k).indices.tolist()
+    related_words = [tokenizer.decode(idx).strip() for idx in top_k_indices]
+    
+    # Create the result array
+    result = []
+    for word in related_words:
+        image_url = f'https://fyp-word-images.s3.us-east-2.amazonaws.com/{word}.png'
+        audio_url = f'https://fyp-word-audio.s3.us-east-2.amazonaws.com/{word}.m4a'
+        result.append({
+            'word': word,
+            'image': image_url,
+            'audio': audio_url
+        })
+    
+    
+    #connect mongo
+    client = pymongo.MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
+    db_1 = client['word_card']
+    collection_1 = db_1['card']
+
+    document = {"card_0": result}
+    #print('---------------')
+    #print(document)
+
+    collection_1.delete_many({})
+    collection_1.insert_one(document)
+
+    return result

App/Frontend/.gitignore

+node_modules/
+.expo/
+dist/
+npm-debug.*
+*.jks
+*.p8
+*.p12
+*.key
+*.mobileprovision
+*.orig.*
+web-build/
+
+# macOS
+.DS_Store
+
+# Temporary files created by Metro to check the health of the file watcher
+.metro-health-check*

App/Frontend/App.js

+import React from 'react';
+import { createStackNavigator } from 'react-navigation-stack';
+import { createAppContainer } from 'react-navigation';
+import LoginScreen from './screens/LoginScreen';
+import HomeScreen from './screens/HomeScreen';
+import ObjectExploreScreen from './screens/ObjectExploreScreen';
+import WordGenerationScreen from './screens/WordGenerationScreen';
+import SpeechTherapyScreen from './screens/SpeechTherapyScreen';
+import ProgressMapScreen from './screens/ProgressMapScreen';
+import WordGameScreen from './screens/WordGameScreen';
+import PhoneticWordScreen from './screens/PhoneticWordsScreen';
+import FlipCardGameScreen from './screens/FlipCardGameScreen';
+import ContentFiltration from './screens/ContentFiltrationScreen';
+
+const MainStack = createStackNavigator(
+  {
+    Login: {
+      screen: LoginScreen,
+    },
+    Home: {
+      screen: HomeScreen,
+    },
+    ObjectExplore: {
+      screen: ObjectExploreScreen,
+    },
+    ContentGeneration: {
+      screen: WordGenerationScreen,
+    },
+    SpeechTherapy: {
+      screen: SpeechTherapyScreen,
+    },
+    ProgressMap: {
+      screen: ProgressMapScreen,
+    },
+    WordGame: {
+      screen: WordGameScreen,
+    },
+    PhoneticWord: {
+      screen: PhoneticWordScreen,
+    },
+    FlipCardGame: {
+      screen: FlipCardGameScreen,
+    },
+    ContentFiltration: {
+      screen: ContentFiltration,
+    },
+  },
+  {
+    initialRouteName: 'Login',
+    defaultNavigationOptions: {
+      headerShown: false,
+    },
+  }
+);
+
+const AppContainer = createAppContainer(MainStack);
+
+export default function App() {
+  return <AppContainer />;
+}

App/Frontend/ReadME.md

+## Install instructions
+npm install expo
+npm install react-native
+npm install react-navigation
+npm install @expo/vector-icons
+expo install react-native-gesture-handler react-native-reanimated
+expo install expo-screen-orientation
+expo install react-native-svg
+
+
+
+
+
+### Start app
+expo start
\ No newline at end of file

App/Frontend/app.json

+{
+  "expo": {
+    "name": "HearMe",
+    "slug": "HearMe",
+    "version": "1.0.0",
+    "orientation": "portrait",
+    "icon": "./assets/icon.png",
+    "userInterfaceStyle": "light",
+    "splash": {
+      "image": "./assets/splash.png",
+      "resizeMode": "contain",
+      "backgroundColor": "#ffffff"
+    },
+    "assetBundlePatterns": [
+      "**/*"
+    ],
+    "ios": {
+      "supportsTablet": true
+    },
+    "android": {
+      "adaptiveIcon": {
+        "foregroundImage": "./assets/adaptive-icon.png",
+        "backgroundColor": "#ffffff"
+      }
+    },
+    "web": {
+      "favicon": "./assets/favicon.png"
+    }
+  }
+}

App/Frontend/babel.config.js

+module.exports = function(api) {
+  api.cache(true);
+  return {
+    presets: ['babel-preset-expo'],
+    plugins: ['react-native-reanimated/plugin'],
+  };
+};

App/Frontend/package.json

+{
+  "name": "hearme",
+  "version": "1.0.0",
+  "main": "node_modules/expo/AppEntry.js",
+  "scripts": {
+    "start": "expo start",
+    "android": "expo start --android",
+    "ios": "expo start --ios",
+    "web": "expo start --web"
+  },
+  "dependencies": {
+    "@expo/vector-icons": "^13.0.0",
+    "@react-navigation/native": "^6.1.6",
+    "@react-navigation/stack": "^6.3.16",
+    "expo": "~48.0.9",
+    "expo-av": "^13.2.1",
+    "expo-screen-orientation": "~5.1.1",
+    "expo-status-bar": "~1.4.4",
+    "react": "18.2.0",
+    "react-native": "^0.71.6",
+    "react-native-draggable": "^3.3.0",
+    "react-native-draggable-flatlist": "^4.0.1",
+    "react-native-gesture-handler": "~2.9.0",
+    "react-native-orientation-locker": "^1.5.0",
+    "react-native-reanimated": "~2.14.4",
+    "react-native-safe-area-context": "4.5.0",
+    "react-native-screens": "~3.20.0",
+    "react-native-svg": "13.4.0",
+    "react-native-swiper": "^1.6.0",
+    "react-navigation": "^4.4.4",
+    "react-navigation-stack": "^2.10.4"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.20.0"
+  },
+  "private": true
+}

App/Frontend/screens/ContentFiltrationScreen.js

+import React, { useState } from 'react';
+import { StyleSheet, Text, View, TextInput, Button, Alert } from 'react-native';
+
+export default function ContentFiltration() {
+  const [word, setWord] = useState('');
+  const [response, setResponse] = useState('');
+
+  const checkWord = async () => {
+    try {
+      const response = await fetch('http://192.168.17.111:5000/check_word', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ word: word }),
+      });
+  
+      if (!response.ok) {
+        throw new Error('An error occurred while checking the word.');
+      }
+  
+      const result = await response.json();
+      setResponse(result.status);
+    } catch (error) {
+      Alert.alert('Error', error.message);
+    }
+  };
+  return (
+    <View style={styles.container}>
+      <TextInput
+        style={styles.input}
+        onChangeText={setWord}
+        value={word}
+        placeholder="Enter a word"
+      />
+      <Button title="Check" onPress={checkWord} />
+      {response ? <Text style={styles.response}>Status: {response}</Text> : null}
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    justifyContent: 'center',
+    paddingHorizontal: 16,
+  },
+  input: {
+    height: 40,
+    borderColor: 'gray',
+    borderWidth: 1,
+    marginBottom: 16,
+  },
+  response: {
+    marginTop: 16,
+  },
+});

App/Frontend/screens/FlipCardGameScreen.js

+import React, { useState, useEffect,useRef } from 'react';
+import { StyleSheet, View, TouchableOpacity, Image, Text, Modal, Button } from 'react-native';
+import { Audio } from 'expo-av';
+import { Asset } from 'expo-asset';
+
+const Card = ({ image, audio, onFlip, gameOver, correct, soundRef }) => {
+  const [isFlipped, setIsFlipped] = useState(false);
+  const [loadingAudio, setLoadingAudio] = useState(false);
+
+  useEffect(() => {
+    if (gameOver) {
+      setIsFlipped(true);
+    }
+  }, [gameOver]);
+  
+  const handleFlip = async () => {
+    if (!isFlipped && !gameOver && !loadingAudio) {
+      setIsFlipped(true);
+      if (audio && audio.uri) {
+        setLoadingAudio(true);
+        try {
+          await soundRef.current.unloadAsync();
+          await soundRef.current.loadAsync({ uri: audio.uri }, { shouldPlay: true });
+        } catch (error) {
+          console.log('Error loading audio:', error);
+        } finally {
+          setLoadingAudio(false);
+        }
+      } else {
+        console.log('audio.uri not set');
+      }
+      onFlip(image); // Pass the entire image object, not just the uri
+  
+      setTimeout(() => {
+        setIsFlipped(false); // Set isFlipped to false, not true
+      }, 800);
+    }
+  };
+  
+  return (
+    <TouchableOpacity
+      style={[styles.card, correct && gameOver ? styles.correctCard : null]}
+      onPress={handleFlip}
+    >
+      <Image source={isFlipped ? image : require('./assets/ST/owl.png')} style={styles.cardImage} />
+    </TouchableOpacity>
+  );
+};
+
+const MessageBox = ({ visible, onClose, onReset }) => {
+  const [messageBoxVisible, setMessageBoxVisible] = useState(visible);
+  const sound = React.useRef(new Audio.Sound());
+
+  useEffect(() => {
+    setMessageBoxVisible(visible);
+    if (visible) {
+      (async () => {
+        try {
+          await sound.current.unloadAsync();
+          const { sound: newSound } = await Audio.Sound.createAsync(
+            { uri: 'https://fyp-word-audio.s3.us-east-2.amazonaws.com/game_chime.m4a' }, // Replace with your audio file URL
+            { shouldPlay: true },
+          );
+          sound.current = newSound;
+        } catch (error) {
+          console.log('Error playing audio:', error);
+        }
+      })();
+      setTimeout(() => {
+        setMessageBoxVisible(false);
+        onReset();
+      }, 4000);
+    }
+  }, [visible]);
+
+  return (
+    <Modal animationType="slide" transparent={true} visible={messageBoxVisible}>
+      <View style={styles.modalContainer}>
+        <View style={styles.modalContent}>
+          <Text style={styles.modalText}>Awesome!!</Text>
+          <Button title="Close" onPress={onClose} />
+        </View>
+      </View>
+    </Modal>
+  );
+};
+
+export default function FlipCardGame() {
+  const [gameOver, setGameOver] = useState(false);
+  const [imagesData, setImagesData] = useState([]);
+  const soundRef = React.useRef(new Audio.Sound());
+  const mainImage = imagesData.find((img) => img.isMainImage);
+   
+  useEffect(() => {
+    const unloadAudio = async () => {
+      await soundRef.current.unloadAsync();
+    };
+
+    return () => {
+      soundRef.current.unloadAsync();
+    };
+  }, []);
+
+  useEffect(() => {
+    fetchImagesData();
+  }, []);
+
+  const resetGame = () => {
+    setGameOver(false);
+    fetchImagesData();
+  };
+
+  const fetchImagesData = async () => {
+    try {
+      const response = await fetch('http://192.168.17.111:5000/api/images_data');
+      const data = await response.json();
+      //console.log(data);
+  
+      const formattedData = [
+        { ...data.main_word, isMainImage: true, uri: data.main_word.audio_link },
+        { ...data.card_word1, isMainImage: false, uri: data.card_word1.audio_link },
+        { ...data.card_word2, isMainImage: false, uri: data.card_word2.audio_link },
+        { ...data.card_word3, isMainImage: false, uri: data.card_word3.audio_link },
+      ];
+
+      //console.log(formattedData);
+  
+      setImagesData(formattedData);
+    } catch (error) {
+      console.error('Error fetching images data:', error);
+    }
+  };
+
+  const handleCardFlip = (flippedImage) => {
+    if (flippedImage.uri === mainImage.image_link) {
+      setGameOver(true);
+    }
+  };
+
+  return (
+    
+    <View style={styles.container}>
+      <Image source={require('./assets/home/home_bg.png')} style={styles.backgroundImage} resizeMode="cover" />
+      {mainImage && <Image source={{ uri: mainImage.image_link }} style={styles.mainImage} />}
+      <View style={styles.cardsContainer}>
+        {imagesData
+          .filter((img) => !img.isMainImage) // Filter out the main image
+          .map((img, index) => (
+            <Card
+              key={index}
+              image={{ uri: img.image_link }}
+              audio={{ uri: img.audio_link }}
+              onFlip={handleCardFlip}
+              gameOver={gameOver}
+              correct={img.isMainImage}
+              soundRef={soundRef}
+            />
+          ))}
+      </View>
+      <MessageBox style={styles.messagebox}
+        visible={gameOver}
+        onClose={() => {
+          setGameOver(false);
+        }}
+        onReset={resetGame}
+      />
+    </View>
+  );
+
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    justifyContent: 'center',