Skip to content

2
22_23-J 18
Commit f3d89352 authored by Lelkada L L P S M's avatar Lelkada L L P S M
Browse files
Options

word generation backend connection

parent dfd004de
Hide whitespace changes
Inline Side-by-side
Showing with 126 additions and 111 deletions
App/Backend/requirements.txt
View file @ f3d89352
Flask==2.2.3
numpy==1.24.2
transformers
torch
\ No newline at end of file
App/Backend/server.py
View file @ f3d89352
import numpy as np
from flask import Flask, request, jsonify, request
import pickle
#import pickle
from word_card_game import wordGameData
from word_generation import get_similar_words
from flask_cors import CORS
app = Flask(__name__)
......@@ -50,4 +51,5 @@ def similar_words_api():
return jsonify({'similar_words': similar_words})
if __name__ == '__main__':
app.run(port=5000, debug=True)
\ No newline at end of file
CORS(app.run(host='0.0.0.0', port=5000, debug=True))
#app.run(host='0.0.0.0', port=5000, debug=True)
\ No newline at end of file
App/Backend/word_generation.py
View file @ f3d89352
......@@ -24,4 +24,15 @@ def get_similar_words(input_word, top_k=3):
top_k_indices = torch.topk(predictions, top_k).indices.tolist()
related_words = [tokenizer.decode(idx).strip() for idx in top_k_indices]
return related_words
# Create the result array
result = []
for word in related_words:
image_url = f'https://fyp-word-images.s3.us-east-2.amazonaws.com/{word}.png'
audio_url = f'https://fyp-word-audio.s3.us-east-2.amazonaws.com/{word}.m4a'
result.append({
'word': word,
'image': image_url,
'audio': audio_url
})
return result
App/Frontend/screens/WordGenerationScreen.js
View file @ f3d89352
......@@ -12,32 +12,37 @@ import {
import { Audio } from "expo-av";
import { MaterialIcons } from '@expo/vector-icons';
import { Ionicons } from '@expo/vector-icons';
const cards = [
{
word: "kitchen",
image: require("./assets/CG/content/kitchen.png"),
audio: require("./assets/CG/content/kitchen.m4a"),
},
{
word: "pot",
image: require("./assets/CG/content/pot.png"),
audio: require("./assets/CG/content/pot.m4a"),
},
{
word: "vegetable",
image: require("./assets/CG/content/vegetable.png"),
audio: require("./assets/CG/content/vegetable.m4a"),
},
];
const screenWidth = Dimensions.get("window").width;
const screenHeight = Dimensions.get("window").height;
const fetchCards = async () => {
try {
console.log("fetch cards 1")
const response = await fetch('http://192.168.216.111:5000/api/similar-words?word=school');
const data = await response.json();
console.log(data)
return data.similar_words;
} catch (error) {
console.error('Error fetching cards:', error);
}
};
export default function ContentGenerationScreen({ navigation }) {
const [activeIndex, setActiveIndex] = useState(0);
const [cards, setCards] = useState([]);
const scrollViewRef = useRef(null);
const soundRef = useRef(new Audio.Sound());
useEffect(() => {
const loadCards = async () => {
const fetchedCards = await fetchCards();
setCards(fetchedCards);
};
loadCards();
}, []);
useEffect(() => {
const unloadAudio = async () => {
await soundRef.current.unloadAsync();
......@@ -50,7 +55,6 @@ export default function ContentGenerationScreen({ navigation }) {
};
}, []);
const handleScroll = async (event) => {
const newIndex = Math.round(
event.nativeEvent.contentOffset.x / screenWidth
......@@ -58,66 +62,64 @@ export default function ContentGenerationScreen({ navigation }) {
if (newIndex !== activeIndex) {
setActiveIndex(newIndex);
await soundRef.current.unloadAsync();
await soundRef.current.loadAsync(cards[newIndex].audio);
await soundRef.current.loadAsync({ uri: cards[newIndex].audio });
await soundRef.current.playAsync();
}
};
const handleAudioPress = async () => {
await soundRef.current.unloadAsync();
await soundRef.current.loadAsync(cards[activeIndex].audio);
await soundRef.current.loadAsync({ uri: cards[activeIndex].audio });
await soundRef.current.playAsync();
};
return (
<ImageBackground source={require('./assets/home/home_bg.png')} style={styles.backgroundImage}>
<Image
<Image
source={require("./assets/CG/monkey.png")}
style={styles.monkeyImage}
/>
<ScrollView
horizontal
pagingEnabled
showsHorizontalScrollIndicator={false}
onScroll={handleScroll}
ref={scrollViewRef}
>
{cards.map((card, index) => (
<View
key={index}
style={[
styles.card,
{ width: screenWidth * 1, height: screenHeight * 0.8 , marginTop: screenHeight * 0.15},
]}
>
<Image
source={require("./assets/CG/card_bg_2.png")}
style={styles.cardBackground}
/>
<Image source={card.image} style={styles.cardImage} />
<Text style={styles.cardText}>{card.word}</Text>
<ScrollView
horizontal
pagingEnabled
showsHorizontalScrollIndicator={false}
onScroll={handleScroll}
ref={scrollViewRef}
>
{cards.map((card, index) => (
<View
key={index}
style={[
styles.card,
{ width: screenWidth * 1, height: screenHeight * 0.8 , marginTop: screenHeight * 0.15},
]}
>
<Image
source={require("./assets/CG/card_bg_2.png")}
style={styles.cardBackground}
/>
<Image source={{ uri: card.image }} style={styles.cardImage} />
<Text style={styles.cardText}>{card.word}</Text>
<TouchableOpacity onPress={handleAudioPress} style={styles.audioButton}>
<Ionicons name="volume-high-outline" size={50} color="#FFCE6D" />
</TouchableOpacity>
<TouchableOpacity onPress={handleAudioPress} style={styles.audioButton}>
<Ionicons name="volume-high-outline" size={50} color="#FFCE6D" />
</TouchableOpacity>
{index === cards.length - 1 && (
<TouchableOpacity
onPress={() => navigation.navigate('FlipCardGame')}
style={styles.nextButton}
>
<MaterialIcons name="keyboard-arrow-right" size={65} color="#FFCE6D" />
</TouchableOpacity>
)}
</View>
))}
</ScrollView>
</ImageBackground>
);
{index === cards.length - 1 && (
<TouchableOpacity
onPress={() => navigation.navigate('FlipCardGame')}
style={styles.nextButton}
>
<MaterialIcons name="keyboard-arrow-right" size={65} color="#FFCE6D" />
</TouchableOpacity>
)}
</View>
))}
</ScrollView>
</ImageBackground>
);
}
const styles = StyleSheet.create({
......@@ -134,55 +136,53 @@ const styles = StyleSheet.create({
width: screenWidth * 0.6,
zIndex: 1,
},
card: {
justifyContent: "center",
alignItems: "center",
},
cardBackground: {
position: "absolute",
width: "100%",
height: "100%",
resizeMode: "cover",
},
cardImage: {
width: "70%",
height: "50%",
resizeMode: "contain",
},
cardText: {
fontSize: 55,
fontWeight: "bold",
marginTop: 20,
color:'#FFCE6D',
textShadowColor: 'rgba(0, 0, 0, 1)',
textShadowOffset: { width: -1, height: 1 },
textShadowRadius: 10,
},
audioButton: {
paddingHorizontal: 20,
paddingVertical: 10,
borderRadius: 5,
marginTop: 20,
backgroundColor: 'transparent'
},
audioButtonText: {
color: "#FFFFFF",
fontSize: 18,
},
nextButton: {
backgroundColor: "transparent",
position: 'absolute',
card: {
justifyContent: "center",
alignItems: "center",
},
cardBackground: {
position: "absolute",
width: "100%",
height: "100%",
resizeMode: "cover",
},
cardImage: {
width: "70%",
height: "50%",
resizeMode: "contain",
},
cardText: {
fontSize: 55,
fontWeight: "bold",
marginTop: 20,
color:'#FFCE6D',
textShadowColor: 'rgba(0, 0, 0, 1)',
textShadowOffset: { width: -1, height: 1 },
textShadowRadius: 10,
},
audioButton: {
paddingHorizontal: 20,
paddingVertical: 10,
borderRadius: 5,
marginTop: 20,
backgroundColor: 'transparent'
},
audioButtonText: {
color: "#FFFFFF",
fontSize: 18,
},
nextButton: {
backgroundColor: "transparent",
position: 'absolute',
bottom: 20,
right: 45,
justifyContent: 'center',
alignItems: 'center',
},
nextButtonText: {
color: "#000000",
fontSize: 18,
bottom: 10,
right: 10,
},
},
nextButtonText: {
color: "#000000",
fontSize: 18,
bottom: 10,
right: 10,
},
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment