Skip to content

2
22_23-J 18
Commit 8fa2389c authored by Lelkada L L P S M's avatar Lelkada L L P S M
Browse files
Options

Merge branch 'it19001708' into 'master' 

It19001708

See merge request !18
parents 6807c444 aa46d8d4
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 554 additions and 1796 deletions
App/Backend/flip_card_content.py 0 → 100644
View file @ 8fa2389c
import pymongo
def getFlipCardContent():
client = pymongo.MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
db = client['word_card']
collection = db['card']
card_data = collection.find_one()
if card_data:
main_image_data = {
'isMainImage': True,
'image': card_data['image'],
'audio': card_data['audio']
}
other_images_data = [
{
'isMainImage': False,
'image': card['image'],
'audio': card['audio']
} for card in card_data['card_0']
]
images_data = [main_image_data] + other_images_data
return images_data
return []
App/Backend/requirements.txt
View file @ 8fa2389c
Flask==2.2.3
numpy==1.24.2
transformers
torch
\ No newline at end of file
torch
pymongo
\ No newline at end of file
App/Backend/server.py
View file @ 8fa2389c
import numpy as np
from flask import Flask, request, jsonify, request
from flask_cors import CORS
#import pickle
from word_card_game import wordGameData
from word_generation import get_similar_words
from flask_cors import CORS
from flip_card_content import getFlipCardContent
app = Flask(__name__)
......@@ -50,6 +53,16 @@ def similar_words_api():
similar_words = get_similar_words(word)
return jsonify({'similar_words': similar_words})
@app.route('/api/flip-card-content')
def flip_card_content():
data = getFlipCardContent()
return jsonify(data)
@app.route('/api/images_data', methods=['GET'])
def get_images_data():
images_data = wordGameData()
return jsonify(images_data)
if __name__ == '__main__':
CORS(app.run(host='0.0.0.0', port=5000, debug=True))
#app.run(host='0.0.0.0', port=5000, debug=True)
\ No newline at end of file
App/Backend/word_card_game.py
View file @ 8fa2389c
import random
import pymongo
def wordGameData(w1, w2, w3):
words = [w1, w2, w3]
main_word = random.choice(words)
card_words = random.sample(words, len(words))
client = pymongo.MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
db = client['word_card']
collection = db['card']
card_word1, card_word2, card_word3 = card_words
def get_words():
doc = collection.find_one()
if doc and 'card_0' in doc:
words = doc['card_0']
#print(words)
return words
else:
return []
def create_links(word):
image_link = f'https://fyp-word-images.s3.us-east-2.amazonaws.com/{word}.png'
audio_link = f'https://fyp-word-audio.s3.us-east-2.amazonaws.com/{word}.m4a'
return {'word': word, 'image_link': image_link, 'audio_link': audio_link}
def wordGameData():
words_data = get_words()
main_word_data = random.choice(words_data)
card_words_data = random.sample(words_data, len(words_data))
def create_links(word_data):
return {'word': word_data['word'], 'image_link': word_data['image'], 'audio_link': word_data['audio']}
return {
'main_word': create_links(main_word),
'card_word1': create_links(card_word1),
'card_word2': create_links(card_word2),
'card_word3': create_links(card_word3)
}
\ No newline at end of file
'main_word': create_links(main_word_data),
'card_word1': create_links(card_words_data[0]),
'card_word2': create_links(card_words_data[1]),
'card_word3': create_links(card_words_data[2])
}
App/Backend/word_generation.py
View file @ 8fa2389c
import torch
from transformers import RobertaTokenizer, RobertaForMaskedLM
import pymongo
# Load the pretrained RoBERTa model and tokenizer
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
......@@ -35,4 +36,17 @@ def get_similar_words(input_word, top_k=3):
'audio': audio_url
})
#connect mongo
client = pymongo.MongoClient("mongodb+srv://hearme:hearme678@cluster0.kz66vdr.mongodb.net")
db = client['word_card']
collection = db['card']
document = {"card_0": result}
print('---------------')
print(document)
collection.delete_many({})
collection.insert_one(document)
return result
App/Frontend/package-lock.json
View file @ 8fa2389c
This source diff could not be displayed because it is too large. You can view the blob instead.
App/Frontend/package.json
View file @ 8fa2389c
......@@ -17,7 +17,7 @@
"expo-screen-orientation": "~5.1.1",
"expo-status-bar": "~1.4.4",
"react": "18.2.0",
"react-native": "^0.71.4",
"react-native": "^0.71.6",
"react-native-draggable": "^3.3.0",
"react-native-draggable-flatlist": "^4.0.1",
"react-native-gesture-handler": "~2.9.0",
......
App/Frontend/screens/FlipCardGameScreen.js
View file @ 8fa2389c
import React, { useState, useEffect } from 'react';
import { StyleSheet, View, TouchableOpacity, Image, Text, Modal, Button } from 'react-native';
import React, { useState, useEffect,useRef } from 'react';
import { StyleSheet, View, TouchableOpacity, Image, Text, Modal, Button } from 'react-native';
import { Audio } from 'expo-av';
import { Asset } from 'expo-asset';
const Card = ({ image, audio, onFlip, gameOver, correct }) => {
const [isFlipped, setIsFlipped] = useState(false);
const Card = ({ image, audio, onFlip, gameOver, correct, soundRef }) => {
const [isFlipped, setIsFlipped] = useState(false);
const [loadingAudio, setLoadingAudio] = useState(false);
useEffect(() => {
if (gameOver) {
setIsFlipped(true);
}
}, [gameOver]);
useEffect(() => {
if (gameOver) {
setIsFlipped(true);
const handleFlip = async () => {
if (!isFlipped && !gameOver && !loadingAudio) {
setIsFlipped(true);
if (audio && audio.uri) {
setLoadingAudio(true);
try {
await soundRef.current.unloadAsync();
await soundRef.current.loadAsync({ uri: audio.uri }, { shouldPlay: true });
} catch (error) {
console.log('Error loading audio:', error);
} finally {
setLoadingAudio(false);
}
} else {
console.log('audio.uri not set');
}
}, [gameOver]);
onFlip(image); // Pass the entire image object, not just the uri
const handleFlip = async () => {
if (!isFlipped && !gameOver) {
setIsFlipped(true);
await Audio.Sound.createAsync(audio, { shouldPlay: true });
onFlip(image);
setTimeout(() => {
setIsFlipped(false); // Set isFlipped to false, not true
}, 800);
}
};
setTimeout(() => {
setIsFlipped(true);
}, 800);
}
};
return (
<TouchableOpacity
style={[styles.card, correct && gameOver ? styles.correctCard : null]}
onPress={handleFlip}
>
<Image source={isFlipped ? image : require('./assets/ST/owl.png')} style={styles.cardImage} />
</TouchableOpacity>
);
};
return (
<TouchableOpacity
style={[styles.card, correct && gameOver ? styles.correctCard : null]}
onPress={handleFlip}
>
<Image source={isFlipped ? image : require('./assets/ST/owl.png')} style={styles.cardImage} />
</TouchableOpacity>
);
};
const MessageBox = ({ visible, onClose, onReset }) => {
const [messageBoxVisible, setMessageBoxVisible] = useState(visible);
const sound = React.useRef(new Audio.Sound());
useEffect(() => {
setMessageBoxVisible(visible);
if (visible) {
(async () => {
try {
await sound.current.unloadAsync();
const { sound: newSound } = await Audio.Sound.createAsync(
{ uri: 'https://fyp-word-audio.s3.us-east-2.amazonaws.com/game_chime.m4a' }, // Replace with your audio file URL
{ shouldPlay: true },
);
sound.current = newSound;
} catch (error) {
console.log('Error playing audio:', error);
}
})();
setTimeout(() => {
setMessageBoxVisible(false);
onReset();
}, 4000);
}
}, [visible]);
const MessageBox = ({ visible, onClose }) => {
return (
<Modal animationType="slide" transparent={true} visible={visible}>
<View style={styles.modalContainer}>
<View style={styles.modalContent}>
<Text style={styles.modalText}>Game Over</Text>
<Button title="Close" onPress={onClose} />
</View>
return (
<Modal animationType="slide" transparent={true} visible={messageBoxVisible}>
<View style={styles.modalContainer}>
<View style={styles.modalContent}>
<Text style={styles.modalText}>Awesome!!</Text>
<Button title="Close" onPress={onClose} />
</View>
</Modal>
);
};
</View>
</Modal>
);
};
export default function FlipCardGame() {
const [gameOver, setGameOver] = useState(false);
const [imagesData, setImagesData] = useState([]);
const soundRef = React.useRef(new Audio.Sound());
const mainImage = imagesData.find((img) => img.isMainImage);
useEffect(() => {
const unloadAudio = async () => {
await soundRef.current.unloadAsync();
};
return () => {
soundRef.current.unloadAsync();
};
}, []);
const images = [
{ image: require('./assets/CG/content/vegetable.png'), audio: require('./assets/CG/content/vegetable.m4a') },
{ image: require('./assets/CG/content/kitchen.png'), audio: require('./assets/CG/content/kitchen.m4a') },
{ image: require('./assets/CG/content/pot.png'), audio: require('./assets/CG/content/pot.m4a') },
];
useEffect(() => {
fetchImagesData();
}, []);
const mainImage = require('./assets/CG/content/kitchen.png');
const resetGame = () => {
setGameOver(false);
fetchImagesData();
};
const fetchImagesData = async () => {
try {
const response = await fetch('http://192.168.137.111:5000/api/images_data');
const data = await response.json();
//console.log(data);
const formattedData = [
{ ...data.main_word, isMainImage: true, uri: data.main_word.audio_link },
{ ...data.card_word1, isMainImage: false, uri: data.card_word1.audio_link },
{ ...data.card_word2, isMainImage: false, uri: data.card_word2.audio_link },
{ ...data.card_word3, isMainImage: false, uri: data.card_word3.audio_link },
];
const handleCardFlip = (image) => {
if (image === mainImage) {
//console.log(formattedData);
setImagesData(formattedData);
} catch (error) {
console.error('Error fetching images data:', error);
}
};
const handleCardFlip = (flippedImage) => {
if (flippedImage.uri === mainImage.image_link) {
setGameOver(true);
}
};
return (
<View style={styles.container}>
<Image source={mainImage} style={styles.mainImage} />
{mainImage && <Image source={{ uri: mainImage.image_link }} style={styles.mainImage} />}
<View style={styles.cardsContainer}>
{images.map((img, index) => (
<Card
key={index}
image={img.image}
audio={img.audio}
onFlip={handleCardFlip}
gameOver={gameOver}
correct={img.image === mainImage}
/>
))}
{imagesData
.filter((img) => !img.isMainImage) // Filter out the main image
.map((img, index) => (
<Card
key={index}
image={{ uri: img.image_link }}
audio={{ uri: img.audio_link }}
onFlip={handleCardFlip}
gameOver={gameOver}
correct={img.isMainImage}
soundRef={soundRef}
/>
))}
</View>
<MessageBox visible={gameOver} onClose={() => setGameOver(false)} />
<MessageBox style={styles.messagebox}
visible={gameOver}
onClose={() => {
setGameOver(false);
}}
onReset={resetGame}
/>
</View>
);
}
const styles = StyleSheet.create({
......@@ -90,13 +175,14 @@ const styles = StyleSheet.create({
backgroundColor: '#fff',
},
mainImage: {
width: '100%',
height: '40%',
resizeMode: 'contain',
width: '80%',
height: '35%',
resizeMode: 'cover',
},
cardsContainer: {
flexDirection: 'row',
justifyContent: 'space-around',
alignItems: 'center', // Add this line
width: '100%',
paddingTop: 20,
},
......@@ -116,4 +202,17 @@ const styles = StyleSheet.create({
borderColor: 'green',
borderWidth: 5,
},
modalContainer: {
flex: 1,
justifyContent: 'center',
alignItems: 'center',
margin:150,
},
modalContent: {
backgroundColor: '#fff',
padding: 20,
borderRadius: 10,
alignItems: 'center',
},
});
App/Frontend/screens/SpeechTherapyScreen.js
View file @ 8fa2389c
......@@ -65,6 +65,7 @@ const SpeechTherapyScreen = ({ navigation }) => {
}
};
}, []);
const updatePlaybackStatus = (status) => {
if (status.isLoaded) {
......
App/Frontend/screens/WordGenerationScreen.js
View file @ 8fa2389c
import React, { useState, useRef, useEffect } from "react";
import React, { useState, useRef, useEffect, useFocusEffect } from "react";
import {
View,
Text,
......@@ -9,32 +9,35 @@ import {
Dimensions,
ImageBackground ,
} from "react-native";
import { Audio } from "expo-av";
import { MaterialIcons } from '@expo/vector-icons';
import { Ionicons } from '@expo/vector-icons';
import FlipCardGame from "./FlipCardGameScreen";
const screenWidth = Dimensions.get("window").width;
const screenHeight = Dimensions.get("window").height;
const fetchCards = async () => {
try {
console.log("fetch cards 1")
const response = await fetch('http://192.168.216.111:5000/api/similar-words?word=school');
const response = await fetch('http://192.168.137.111:5000/api/similar-words?word=tomato');
const data = await response.json();
console.log(data)
//console.log(data)
return data.similar_words;
} catch (error) {
console.error('Error fetching cards:', error);
}
};
export default function ContentGenerationScreen({ navigation }) {
const [activeIndex, setActiveIndex] = useState(0);
const [cards, setCards] = useState([]);
const scrollViewRef = useRef(null);
const soundRef = useRef(new Audio.Sound());
//console.log(cards);
useEffect(() => {
useEffect(() => {
const loadCards = async () => {
const fetchedCards = await fetchCards();
setCards(fetchedCards);
......@@ -64,6 +67,7 @@ export default function ContentGenerationScreen({ navigation }) {
await soundRef.current.unloadAsync();
await soundRef.current.loadAsync({ uri: cards[newIndex].audio });
await soundRef.current.playAsync();
//console.log('Cards before navigating:', cards);
}
};
......@@ -71,6 +75,7 @@ export default function ContentGenerationScreen({ navigation }) {
await soundRef.current.unloadAsync();
await soundRef.current.loadAsync({ uri: cards[activeIndex].audio });
await soundRef.current.playAsync();
//console.log('Cards before navigating:', cards);
};
return (
......@@ -108,8 +113,8 @@ export default function ContentGenerationScreen({ navigation }) {
{index === cards.length - 1 && (
<TouchableOpacity
onPress={() => navigation.navigate('FlipCardGame')}
style={styles.nextButton}
onPress={() => navigation.navigate('FlipCardGame', cards )}
style={styles.nextButton}
>
<MaterialIcons name="keyboard-arrow-right" size={65} color="#FFCE6D" />
</TouchableOpacity>
......
App/Frontend/yarn.lock
View file @ 8fa2389c
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment