Merge branch

API/app.py

 # import MySQLdb
+from flask import Flask, render_template, request, jsonify, make_response
+from API.model.color.colorModel import save_color_session_details, sendPredictData
+from API.model.readModel import save_activity_details, save_session_details
+from API.model.userModel import getUser, saveUserSession, logoutUser
+from API.model.color.colorModel import get_color_activities1, get_color_activities2, get_color_activity_result, storeResult, save_color_session_details
 from flask import Flask, redirect, url_for, render_template, request, jsonify, make_response
 import random
 import os
-from API.model.colorModel import get_color_activities1, get_color_activities2, get_color_activity_result, \
-    save_color_session_details
 from API.model.readModel import *
 from API.model.userModel import *
-from API.model.colorModel import get_color_activities1, get_color_activities2, get_color_activity_result, storeResult
+
 from API.model.readModel import get_reading_activities
 from API.routers.router import funtion_one
 
@@ -57,72 +60,31 @@ def predict():
     try:
         data = request.get_json()
 
-        fileitem = data['uri']
-
-        colorName = data['colorNmae']
-        # print((f"{colorName}"))
+        audioFile = data['uri']
+        name = data['name']
 
-        # if fileitem.filename:
-        #     # strip the leading path from the file name
-        fn = os.path.basename(fileitem)
-        #
-        #     # open read and write the file into the server
-        # open(fn, 'wb').write(fileitem.file.read())
+        response = sendPredictData(audioFile, name)
 
         resp = make_response(request.get_json())
         resp.headers['Access-Control-Allow-Origin'] = '*'
         resp.headers['Content-Type'] = 'application/json'
-        return resp
+        return response
     except ValueError:
         print(ValueError)
 
-    # get audio file and save it
-    # audio_file = request.files["file"]
-    # audio_file = request.files["name"]
-
-    # print((f"{request.data}"))
-    # print((f"{request.form}"))
-    # print((f"{request.values}"))
-    # print((f"{request.json}"))
-    # print(f"{request.form['name']}")
-    # file_name = str(random.randint(0, 100000))
-    # audio_file.save((file_name))
-    #
-    # # get file name
-    # predict_file_name = audio_file.filename
-    # predict_file_name = predict_file_name.split("/")
-    # new_predict_file_name = predict_file_name[1]
-    # new_predict_file_name = new_predict_file_name.split(".")
-    # FPFN = new_predict_file_name[0]
-    # # print(f"{FPFN}")
-    #
-    # # invoke keyword spotting service
-    # kss = Keyword_Spotting_service()
-    #
-    #
-    # # make a prediction
-    # predicted_keyword = kss.predict(file_name, FPFN)
-    #
-    # # remove the audio file
-    # os.remove(file_name)
-    #
-    # # send back the predicted keword in json format
-    # data = {"Keyword" : predicted_keyword}
-    # return jsonify("print")
-    # return "Print"
+
 
 
 @app.route("/colorSession", methods=['POST'])
 def color_session():
-    req = request.get_json()
-    userId = req['userId']
-    token = save_color_session_details(userId, 1)
-    data = {
+
+    token = save_color_session_details(1)
+    response = {
         "token": token,
         "message": "Success",
         "status": 200
     }
-    body = jsonify(data)
+    body = jsonify(response)
     return make_response(body)
 
 
@@ -251,4 +213,4 @@ def get_completed_levels(userId):
 
 
 if __name__ == "__main__":
-    app.run(host='192.168.1.101')
+    app.run(host='192.168.8.100')

API/db/dbConnection.py

@@ -12,7 +12,7 @@ def create_con():
         database="helply",
         host="127.0.0.1",
         user="root",
-        password="12345678"
+        password="rp19970520"
     )
     return db
 

API/model/colorModel.py → API/model/color/colorModel.py

-from flask import jsonify
+import os
+import random
+from flask import jsonify, Flask, request
 from API.db.dbConnection import get_all_data, insert_data_json, insert_data, insert, get_data
+from API.model.color.keyword_spotting_service import Keyword_Spotting_service
 from API.util.util import getUUID
 
 # get color activity 1 fun
@@ -81,9 +84,48 @@ def get_color_activity_result(userId):
     return jsonify(data_dic)
 
 # store color activity session fun
-def save_color_session_details(userId, status):
+def save_color_session_details(status):
     token = getUUID()
-    qry = 'INSERT INTO colorsession (id,userId,token,status) VALUES (NULL, %s, %s, %s)'
-    args = (userId, token, status)
+    qry = 'INSERT INTO colorsession (id,token,status) VALUES (NULL, %s, %s)'
+    args = (token, status)
     insert(qry, args)
-    return token
+    return token
\ No newline at end of file
+
+#predict audio file
+def sendPredictData(audioFile, name):
+
+    # print(f"{name}")
+
+    # path = "test/green.wav"
+
+    # print(f"{path}")
+
+    # audio_file = open(path, "rb")
+
+    # values = {
+    #     "file": (path, audio_file, "audio/wav")
+    # }
+    # audio_file = values["file"]
+    # file_name = str(random.randint(0, 100000))
+    # audio_file.save((file_name))
+    # print(f"{path}")
+
+    # get file name
+    # predict_file_name = audio_file.filename
+    new_predict_file_name = name.split(".")
+    FPFN = new_predict_file_name[0]
+
+    # invoke keyword spotting service
+    kss = Keyword_Spotting_service()
+
+    # make a prediction
+    # predicted_keyword = kss.predict(file_name, FPFN)
+
+    # remove the audio file
+    # os.remove(file_name)
+
+    # send back the predicted keyword in json format
+    data = {"Predicted Keyword": FPFN}
+
+    print(f"data: {data}")
+    return jsonify(data)
\ No newline at end of file

API/model/color/keyword_spotting_service.py

+import tensorflow.keras as keras
+import numpy as np
+import librosa
+
+MODEL_PATH = "model.h5"
+NUM_SAMPLES_TO_CONSIDER = 22050 # 1 sec
+
+class _Keyword_Spotting_Service:
+
+    model = None
+    _mappings = [
+        "black",
+        "blue",
+        "green",
+        "red",
+        "white",
+        "yellow"
+    ]
+    _instance = None
+    # 78293
+    def predict(self, file_path, FPFN):
+
+        print(f"{file_path}")
+
+        # extract MFCCs
+        MFCCs = self.preprocess(file_path) # (# segment, # coefficients)
+
+        # convert 2d MFCCs array into 4d array -> (# samples, # segment, # coefficients, # channels)
+        MFCCs = MFCCs[np.newaxis, ..., np.newaxis]
+
+        # make prediction
+        predictions = self.model.predict(MFCCs)
+        predicted_index = np.argmax(predictions)
+        predicted_keyword = self._mappings[predicted_index]
+
+        # return predicted_keyword
+
+        if predicted_keyword == FPFN:
+            return predicted_keyword
+        else:
+            return "No Prediction"
+
+    def preprocess(self, file_path, n_mfcc=13, n_fft=2048, hop_length=512):
+
+        # load audio file
+        signal, sr = librosa.load(file_path)
+
+        # ensure consistency in the audio file length
+        if len(signal) > NUM_SAMPLES_TO_CONSIDER:
+            signal = signal[:NUM_SAMPLES_TO_CONSIDER]
+
+        # extract MFCCs
+        MFCCs = librosa.feature.mfcc(signal, n_mfcc=n_mfcc, n_fft=n_fft, hop_length=hop_length)
+
+        return MFCCs.T
+
+def Keyword_Spotting_service():
+
+    # ensure that we only have 1 instance of KSS
+    if _Keyword_Spotting_Service._instance is None:
+        _Keyword_Spotting_Service._instance = _Keyword_Spotting_Service()
+        _Keyword_Spotting_Service.model = keras.models.load_model(MODEL_PATH)
+
+    return _Keyword_Spotting_Service._instance
+
+if __name__ == "__main__":
+    kss = Keyword_Spotting_service()
+
+    keyword1 = kss.predict("test/black.wav")
+    keyword2 = kss.predict("test/blue.wav")
+    keyword3 = kss.predict("test/green.wav")
+    keyword4 = kss.predict("test/red.wav")
+    keyword5 = kss.predict("test/white.wav")
+    keyword6 = kss.predict("test/yellow.wav")
+    keyword7 = kss.predict("test/other.wav")
+
+    print(f"Predicted Keywords: {keyword1}")
+    print(f"Predicted Keywords: {keyword2}")
+    print(f"Predicted Keywords: {keyword3}")
+    print(f"Predicted Keywords: {keyword4}")
+    print(f"Predicted Keywords: {keyword5}")
+    print(f"Predicted Keywords: {keyword6}")
+    print(f"Predicted Keywords: {keyword7}")
\ No newline at end of file

frontend/src/router/router.js

@@ -42,6 +42,7 @@ import GameScreenSixAll from '../screen/memory/elementry/GameScreenSixAll';
 import GameOverScreen from '../screen/memory/GameOverScreen';
 
 import Sam from '../screen/sample/sam';
+import Progress from '../screen/Progress';
 
 const Stack = createNativeStackNavigator();
 
@@ -261,6 +262,11 @@ const AppRouter = () => {
           name="Sam"
           component={Sam}
         />
+        <Stack.Screen
+          options={{headerShown: false}}
+          name="Progress"
+          component={Progress}
+        />
       </Stack.Navigator>
     </NavigationContainer>
   );

frontend/src/screen/Color.js

@@ -17,6 +17,7 @@ export default function Color(props) {
     React.useEffect(() => {
 
         setType(props.route.params.type);
+        // AsyncStorage.removeItem('colorToken');
 
         // const type2 = props.route.params.type2;
 
@@ -31,35 +32,50 @@ export default function Color(props) {
         return unsubscribe;
     }, [navigation]);
 
+    const checkColorSession = () => {
+        AsyncStorage.getItem('colorToken')
+            .then(value => {
+                if (value == null) {
+                    console.log('colorToken', value);
+                    getColorSession(value);
+                } else {
+                    console.log(value);
+                    console.log('colorToken');
+                    navigation.navigate("PrimaryType", { title: 'Primary Activities', id: 1 })
+                }
+            })
+            .catch(error => {
+                console.log(error);
+            });
+    };
+
     const getColorSession = () => {
-        const data = {
-          userId: 1,
-        };
-        client.post('colorSession', JSON.stringify(data), {
-          headers: {
-            Accept: 'application/json',
-            'Content-Type': 'application/json',
-          },
+
+        client.post('colorSession', {
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
         })
-          .then(res => {
-            console.log(res.data);
-            if (res.status == 200) {
-            //   console.log(res.data);
-              const colorToken = res.data.token;
-              try {
-                AsyncStorage.setItem('colorToken', colorToken);
-                console.log(colorToken)
-              } catch (error) {
+            .then(res => {
+                console.log(res.data);
+                if (res.status == 200) {
+                    //   console.log(res.data);
+                    const colorToken = res.data.token;
+                    try {
+                        AsyncStorage.setItem('colorToken', colorToken);
+                        // console.log(colorToken)
+                    } catch (error) {
+                        console.log(error);
+                    }
+
+                    navigation.navigate("PrimaryType", { title: 'Primary Activities', id: 1 })
+                }
+            })
+            .catch(error => {
                 console.log(error);
-              }
-    
-              navigation.navigate("PrimaryType", { title: 'Primary Activities', id: 1 })
-            }
-          })
-          .catch(error => {
-            console.log(error);
-          });
-      };
+            });
+    };
 
     return (
 
@@ -76,7 +92,7 @@ export default function Color(props) {
                 </View> */}
 
                 <View style={{ marginTop: 40 }}>
-                    <TouchableOpacity onPress={() => { getColorSession(); }}
+                    <TouchableOpacity onPress={() => { checkColorSession(); }}
                         style={styles.card}>
 
                         <View style={[{ flexDirection: "row" }]}>
@@ -192,7 +208,7 @@ const styles = StyleSheet.create({
         left: 50,
         bottom: 50
     },
-    card : {
+    card: {
         borderWidth: 5,
         borderColor: "purple",
         marginVertical: 20,
@@ -211,7 +227,7 @@ const styles = StyleSheet.create({
         shadowOpacity: 0.3,
         shadowRadius: 1.5,
     },
-    card2 : {
+    card2: {
         borderWidth: 5,
         borderColor: "orange",
         marginVertical: 20,
@@ -229,7 +245,7 @@ const styles = StyleSheet.create({
         },
         shadowOpacity: 0.3,
         shadowRadius: 1.5,
-    },card3 : {
+    }, card3: {
         borderWidth: 5,
         borderColor: "lightblue",
         marginVertical: 20,
@@ -247,20 +263,20 @@ const styles = StyleSheet.create({
         },
         shadowOpacity: 0.3,
         shadowRadius: 1.5,
-    },cardTitle: {
+    }, cardTitle: {
         fontWeight: "bold",
         color: "white",
         fontSize: 35,
         marginLeft: 10,
         marginTop: 80,
-    },cardTitle3: {
+    }, cardTitle3: {
         fontWeight: "bold",
         color: "white",
         fontSize: 35,
         marginLeft: 35,
         marginTop: 80,
     }
-    ,cardImage: {
+    , cardImage: {
         padding: 0,
         flex: 0.7,
     },

frontend/src/screen/Home.js

@@ -2,6 +2,7 @@ import React from "react";
 import { StyleSheet, View, Text, Pressable, SafeAreaView, ScrollView, Image } from 'react-native';
 import Orientation from 'react-native-orientation-locker';
 import ImageButton from "../component/ImageButton";
+import AsyncStorage from '@react-native-async-storage/async-storage';
 
 export default function Home({ navigation }){
 
@@ -10,6 +11,9 @@ export default function Home({ navigation }){
     }
 
     React.useEffect(() => {
+
+      AsyncStorage.removeItem('colorToken');
+
       const unsubscribe = navigation.addListener("focus", () => {
         Orientation.unlockAllOrientations();
         Orientation.lockToPortrait();
@@ -49,7 +53,7 @@ export default function Home({ navigation }){
                 style={styles.image}
                 source={require('../assets/color/background.png')}
                 resizeMode="contain"></Image>
-              <ImageButton path="Progress" title="Progress Check" />
+              <ImageButton path="Progress" title="Progress" />
             </View>
             {/* <View style={styles.imageView}>
               <Image

frontend/src/screen/Progress.js

+import React, { Component } from 'react'
+import { StyleSheet, TouchableOpacity, View, Text, Pressable, SafeAreaView, ScrollView, Image, ImageBackground, Dimensions, Animated, Easing, NativeModules } from 'react-native';
+
+export default function Progress(){
+    return (
+      <View><Text>ddddddd</Text></View>
+    )
+  }

frontend/src/screen/activity/Black.js

@@ -9,6 +9,7 @@ import CountDown from 'react-native-countdown-component';
 import client from "../client/Client";
 import Voice from '@react-native-voice/voice';
 import Tts from 'react-native-tts';
+import AudioRecord from 'react-native-audio-record';
 import AsyncStorage from '@react-native-async-storage/async-storage';
 
 import BackButton from "../../component/BackButton"
@@ -28,6 +29,8 @@ export default function Black() {
 
         StatusBar.setHidden(true);
 
+        audioInit();
+
         const unsubscribe = navigation.addListener("focus", () => {
 
             Orientation.unlockAllOrientations();
@@ -51,6 +54,76 @@ export default function Black() {
         };
     }, []);
 
+    function audioInit() {
+        // console.log('audioInit');
+        const colorAudio = {
+            sampleRate: 16000,
+            channels: 1,
+            bitsPerSample: 16,
+            wavFile: 'color.wav', // thise wave file name
+        };
+
+        AudioRecord.init(colorAudio);
+    }
+
+    async function audioStart() {
+
+        // console.log('audioStart');
+        // checkPermission();
+        AudioRecord.start();
+        setTimeout(() => {
+            audioStop();
+        }, 2000);
+    }
+
+    async function audioStop() {
+
+        // console.log('audioStop');
+
+        let audioFile = await AudioRecord.stop();
+        // console.log('userFile', audioFile);
+
+        sendAudio(audioFile);
+
+    }
+
+    function sendAudio(audioFile) {
+
+        var obj = {
+            uri: `file:${audioFile}`,
+            type: 'audio/wav',
+            name: secondColor[2]+'.wav',
+        }
+
+        var data = JSON.stringify(obj)
+        // console.log(data);
+
+        client.post("predict", data, {
+
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
+
+        })
+            .then((response) => {
+                // setNames(response.data);
+                console.log('Success part : ', response.data);
+            })
+            .catch(function (error) {
+                if (error.response) {
+                    console.log('Error data : ', error.response.data);
+                    console.log('Error status : ', error.response.status);
+                    console.log('Error headers : ', error.response.headers);
+                } else if (error.request) {
+                    console.log('Error request : ', error.request);
+                } else {
+                    console.log('Error message : ', error.message);
+                }
+                console.log('Error else part : ', error.config);
+            });
+    }
+
     const onSpeechStartHandler = e => {
         // console.log('start handler =>> ', e);
     };
@@ -75,12 +148,7 @@ export default function Black() {
 
             sendData(mainColor[2])
 
-            // showAlert = () => {
-            //     this.setState({
-            //         showAlert: true
-            //     });
-            // };
-
+            audioStart();
 
         } else {
 

frontend/src/screen/activity/Blue2.js

@@ -8,6 +8,7 @@ import { secondColor } from '../../assets/color/color';
 import CountDown from 'react-native-countdown-component';
 import client from "../client/Client";
 import Voice from '@react-native-voice/voice';
+import AudioRecord from 'react-native-audio-record';
 import Tts from 'react-native-tts';
 import BackButton from "../../component/BackButton"
 import AsyncStorage from '@react-native-async-storage/async-storage';
@@ -26,6 +27,8 @@ export default function Blue2() {
         Voice.destroy().then(Voice.removeAllListeners);
         StatusBar.setHidden(true);
 
+        audioInit();
+
         const unsubscribe = navigation.addListener("focus", () => {
 
             Orientation.unlockAllOrientations();
@@ -49,6 +52,76 @@ export default function Blue2() {
         };
     }, []);
 
+    function audioInit() {
+        // console.log('audioInit');
+        const colorAudio = {
+            sampleRate: 16000,
+            channels: 1,
+            bitsPerSample: 16,
+            wavFile: 'color.wav', // thise wave file name
+        };
+
+        AudioRecord.init(colorAudio);
+    }
+
+    async function audioStart() {
+
+        // console.log('audioStart');
+        // checkPermission();
+        AudioRecord.start();
+        setTimeout(() => {
+            audioStop();
+        }, 2000);
+    }
+
+    async function audioStop() {
+
+        // console.log('audioStop');
+
+        let audioFile = await AudioRecord.stop();
+        // console.log('userFile', audioFile);
+
+        sendAudio(audioFile);
+
+    }
+
+    function sendAudio(audioFile) {
+
+        var obj = {
+            uri: `file:${audioFile}`,
+            type: 'audio/wav',
+            name: secondColor[4]+'.wav',
+        }
+
+        var data = JSON.stringify(obj)
+        // console.log(data);
+
+        client.post("predict", data, {
+
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
+
+        })
+            .then((response) => {
+                // setNames(response.data);
+                console.log('Success part : ', response.data);
+            })
+            .catch(function (error) {
+                if (error.response) {
+                    console.log('Error data : ', error.response.data);
+                    console.log('Error status : ', error.response.status);
+                    console.log('Error headers : ', error.response.headers);
+                } else if (error.request) {
+                    console.log('Error request : ', error.request);
+                } else {
+                    console.log('Error message : ', error.message);
+                }
+                console.log('Error else part : ', error.config);
+            });
+    }
+
     const onSpeechStartHandler = e => {
         // console.log('start handler =>> ', e);
     };
@@ -73,11 +146,7 @@ export default function Blue2() {
 
             sendData(mainColor[4]);
 
-            showAlert = () => {
-                this.setState({
-                    showAlert: true
-                });
-            };
+            audioStart();
 
         } else {
 

frontend/src/screen/activity/Green.js

@@ -9,8 +9,7 @@ import { secondColor } from '../../assets/color/color';
 import client from "../client/Client";
 import Voice from '@react-native-voice/voice';
 import AsyncStorage from '@react-native-async-storage/async-storage';
-
-
+import AudioRecord from 'react-native-audio-record';
 
 import BackButton from "../../component/BackButton"
 
@@ -29,6 +28,8 @@ export default function Green({ navigation }) {
 
         StatusBar.setHidden(true);
 
+        audioInit();
+
         const unsubscribe = navigation.addListener("focus", () => {
 
             Orientation.unlockAllOrientations();
@@ -50,6 +51,77 @@ export default function Green({ navigation }) {
         };
     }, []);
 
+    function audioInit() {
+        // console.log('audioInit');
+        const colorAudio = {
+            sampleRate: 16000,
+            channels: 1,
+            bitsPerSample: 16,
+            wavFile: 'color.wav', // thise wave file name
+        };
+
+        AudioRecord.init(colorAudio);
+    }
+
+    async function audioStart() {
+
+        // console.log('audioStart');
+        // checkPermission();
+        AudioRecord.start();
+        setTimeout(() => {
+            audioStop();
+        }, 2000);
+    }
+
+    async function audioStop() {
+
+        // console.log('audioStop');
+
+        let audioFile = await AudioRecord.stop();
+        // console.log('userFile', audioFile);
+
+        sendAudio(audioFile);
+
+    }
+
+    function sendAudio(audioFile) {
+
+        var obj = {
+            uri: `file:${audioFile}`,
+            type: 'audio/wav',
+            name: secondColor[1]+'.wav',
+        }
+
+        var data = JSON.stringify(obj)
+        // console.log(data);
+
+        client.post("predict", data, {
+
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
+
+        })
+            .then((response) => {
+                // setNames(response.data);
+                console.log('Success part : ', response.data);
+            })
+            .catch(function (error) {
+                if (error.response) {
+                    console.log('Error data : ', error.response.data);
+                    console.log('Error status : ', error.response.status);
+                    console.log('Error headers : ', error.response.headers);
+                } else if (error.request) {
+                    console.log('Error request : ', error.request);
+                } else {
+                    console.log('Error message : ', error.message);
+                }
+                console.log('Error else part : ', error.config);
+            });
+    }
+
+
     const onSpeechStartHandler = e => {
         // console.log('start handler =>> ', e);
     };
@@ -70,17 +142,11 @@ export default function Green({ navigation }) {
 
         if (value.includes(secondColor[1]) == true) {
 
-            const value = e.value;
-
             console.log('Your Answer is Correct');
 
             sendData(mainColor[1])
 
-            // showAlert = () => {
-            //     this.setState({
-            //         showAlert: true
-            //     });
-            // };
+            audioStart();
 
         } else {
 
@@ -194,7 +260,7 @@ export default function Green({ navigation }) {
                                 onFinish={() => setModalVisible(false)}
                                 digitStyle={{ backgroundColor: '#FFF', borderWidth: 2, borderColor: 'black' }}
                                 digitTxtStyle={{ color: 'black' }}
-                                timeLabelStyle={{ color: 'white', fontWeight: 'bold' }}
+                                timeLabelStyle={{ color: 'green', fontWeight: 'bold' }}
                                 separatorStyle={{ color: 'black' }}
                                 timeToShow={['S']}
                                 timeLabels={{ s: 'Seconds' }}

frontend/src/screen/activity/Red.js

@@ -7,7 +7,9 @@ import axios from "axios";
 import { secondColor } from '../../assets/color/color';
 import CountDown from 'react-native-countdown-component';
 import client from "../client/Client";
+import Tts from 'react-native-tts';
 import Voice from '@react-native-voice/voice';
+import AudioRecord from 'react-native-audio-record';
 import BackButton from "../../component/BackButton"
 import AsyncStorage from '@react-native-async-storage/async-storage';
 
@@ -26,6 +28,10 @@ export default function Red({ navigation }) {
 
         StatusBar.setHidden(true);
 
+        audioInit();
+
+        
+
         const unsubscribe = navigation.addListener("focus", () => {
 
             Orientation.unlockAllOrientations();
@@ -48,6 +54,76 @@ export default function Red({ navigation }) {
         };
     }, []);
 
+    function audioInit() {
+        // console.log('audioInit');
+        const colorAudio = {
+            sampleRate: 16000,
+            channels: 1,
+            bitsPerSample: 16,
+            wavFile: 'color.wav', // thise wave file name
+        };
+
+        AudioRecord.init(colorAudio);
+    }
+
+    async function audioStart() {
+
+        // console.log('audioStart');
+        // checkPermission();
+        AudioRecord.start();
+        setTimeout(() => {
+            audioStop();
+        }, 2000);
+    }
+
+    async function audioStop() {
+
+        // console.log('audioStop');
+
+        let audioFile = await AudioRecord.stop();
+        // console.log('userFile', audioFile);
+
+        sendAudio(audioFile);
+
+    }
+
+    function sendAudio(audioFile) {
+
+        var obj = {
+            uri: `file:${audioFile}`,
+            type: 'audio/wav',
+            name: secondColor[0]+'.wav',
+        }
+
+        var data = JSON.stringify(obj)
+        // console.log(data);
+
+        client.post("predict", data, {
+
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
+
+        })
+            .then((response) => {
+                // setNames(response.data);
+                console.log('Success part : ', response.data);
+            })
+            .catch(function (error) {
+                if (error.response) {
+                    console.log('Error data : ', error.response.data);
+                    console.log('Error status : ', error.response.status);
+                    console.log('Error headers : ', error.response.headers);
+                } else if (error.request) {
+                    console.log('Error request : ', error.request);
+                } else {
+                    console.log('Error message : ', error.message);
+                }
+                console.log('Error else part : ', error.config);
+            });
+    }
+
     const onSpeechStartHandler = e => {
         // console.log('start handler =>> ', e);
     };
@@ -69,18 +145,15 @@ export default function Red({ navigation }) {
         if (value.includes(secondColor[0]) == true) {
 
             console.log('Your Answer is Correct');
+            
+            sendData(mainColor[0]);
 
-            sendData(mainColor[0])
-
-            // showAlert = () => {
-            //     this.setState({
-            //         showAlert: true
-            //     });
-            // };
+            // audioStart();
 
         } else {
 
             console.log('Your Answer is Incorrect', value);
+            
             setModalVisible3(true);
 
         }
@@ -119,6 +192,8 @@ export default function Red({ navigation }) {
         AsyncStorage.getItem('colorToken')
             .then(res => {
 
+                console.log(res)
+
                 var date = new Date().getDate();
                 var month = new Date().getMonth() + 1;
                 var year = new Date().getFullYear();
@@ -169,7 +244,7 @@ export default function Red({ navigation }) {
 
     const startRecording = async () => {
 
-        setModalVisible(true);
+        // setModalVisible(true);
 
         try {
             await Voice.start('en-US');
@@ -242,7 +317,7 @@ export default function Red({ navigation }) {
                                 style={[styles.buttonClose2]}
                                 onPress={() => navigation.navigate("Green")}
                             >
-                                <Image source={require('../../assets/game/next.png')} resizeMode='contain' style={{ width: 100,height: 70, marginBottom:-10}} />
+                                <Image source={require('../../assets/game/next.png')} resizeMode='contain' style={{ width: 100, height: 70, marginBottom: -10 }} />
                             </TouchableOpacity>
                         </View>
                     </View>
@@ -272,7 +347,7 @@ export default function Red({ navigation }) {
                                 style={[]}
                                 onPress={() => setModalVisible3(!modalVisible3)}
                             >
-                                <Image source={require('../../assets/alert/tryagain.png')} resizeMode='contain' style={{ width: 100,height: 70, marginBottom:-1}} />
+                                <Image source={require('../../assets/alert/tryagain.png')} resizeMode='contain' style={{ width: 100, height: 70, marginBottom: -1 }} />
                             </TouchableOpacity>
                         </View>
                     </View>
@@ -394,10 +469,10 @@ const styles = StyleSheet.create({
         elevation: 2
     },
 
-    
-    
-    
-    
+
+
+
+
     centeredView2: {
         flex: 1,
         justifyContent: "center",
@@ -455,7 +530,7 @@ const styles = StyleSheet.create({
         textAlign: "center",
         fontSize: 30,
         marginBottom: 20,
-    },alert2: {
+    }, alert2: {
         backgroundColor: "white",
         borderRadius: 50,
         width: 100,
@@ -464,11 +539,11 @@ const styles = StyleSheet.create({
         marginTop: -80
     },
 
-    
-    
-    
-    
-    
+
+
+
+
+
     centeredView3: {
         flex: 1,
         justifyContent: "center",

frontend/src/screen/activity/White.js

@@ -7,6 +7,7 @@ import axios from "axios";
 import { secondColor } from '../../assets/color/color';
 import CountDown from 'react-native-countdown-component';
 import client from "../client/Client";
+import AudioRecord from 'react-native-audio-record';
 import Voice from '@react-native-voice/voice';
 import AsyncStorage from '@react-native-async-storage/async-storage';
 import BackButton from "../../component/BackButton"
@@ -26,6 +27,8 @@ export default function White() {
 
         StatusBar.setHidden(true);
 
+        audioInit();
+
         const unsubscribe = navigation.addListener("focus", () => {
 
             Orientation.unlockAllOrientations();
@@ -49,6 +52,76 @@ export default function White() {
         };
     }, []);
 
+    function audioInit() {
+        // console.log('audioInit');
+        const colorAudio = {
+            sampleRate: 16000,
+            channels: 1,
+            bitsPerSample: 16,
+            wavFile: 'color.wav', // thise wave file name
+        };
+
+        AudioRecord.init(colorAudio);
+    }
+
+    async function audioStart() {
+
+        // console.log('audioStart');
+        // checkPermission();
+        AudioRecord.start();
+        setTimeout(() => {
+            audioStop();
+        }, 2000);
+    }
+
+    async function audioStop() {
+
+        // console.log('audioStop');
+
+        let audioFile = await AudioRecord.stop();
+        // console.log('userFile', audioFile);
+
+        sendAudio(audioFile);
+
+    }
+
+    function sendAudio(audioFile) {
+
+        var obj = {
+            uri: `file:${audioFile}`,
+            type: 'audio/wav',
+            name: secondColor[5]+'.wav',
+        }
+
+        var data = JSON.stringify(obj)
+        // console.log(data);
+
+        client.post("predict", data, {
+
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
+
+        })
+            .then((response) => {
+                // setNames(response.data);
+                console.log('Success part : ', response.data);
+            })
+            .catch(function (error) {
+                if (error.response) {
+                    console.log('Error data : ', error.response.data);
+                    console.log('Error status : ', error.response.status);
+                    console.log('Error headers : ', error.response.headers);
+                } else if (error.request) {
+                    console.log('Error request : ', error.request);
+                } else {
+                    console.log('Error message : ', error.message);
+                }
+                console.log('Error else part : ', error.config);
+            });
+    }
+
     const onSpeechStartHandler = e => {
         // console.log('start handler =>> ', e);
     };
@@ -73,11 +146,7 @@ export default function White() {
 
             sendData(mainColor[5])
 
-            showAlert = () => {
-                this.setState({
-                    showAlert: true
-                });
-            };
+            audioStart();
 
         } else {
 
@@ -141,14 +210,6 @@ export default function White() {
         // setPartialResults(e.value);
     };
 
-    const onFinishCD = () => {
-        Alert.alert('Countdown Finished...');
-    }
-
-    const onPressCD = () => {
-        Alert.alert('Countdown Component Pressed...');
-    }
-
     const startRecording = async () => {
 
         setModalVisible(true);

frontend/src/screen/activity/Yellow.js

@@ -8,6 +8,7 @@ import CountDown from 'react-native-countdown-component';
 import { mainColor } from '../../assets/color/color';
 import client from "../client/Client";
 import Voice from '@react-native-voice/voice';
+import AudioRecord from 'react-native-audio-record';
 import AsyncStorage from '@react-native-async-storage/async-storage';
 import BackButton from "../../component/BackButton"
 
@@ -25,6 +26,8 @@ export default function Yellow() {
         Voice.destroy().then(Voice.removeAllListeners);
         StatusBar.setHidden(true);
 
+        audioInit();
+
         const unsubscribe = navigation.addListener("focus", () => {
 
             Orientation.unlockAllOrientations();
@@ -48,6 +51,76 @@ export default function Yellow() {
         };
     }, []);
 
+    function audioInit() {
+        // console.log('audioInit');
+        const colorAudio = {
+            sampleRate: 16000,
+            channels: 1,
+            bitsPerSample: 16,
+            wavFile: 'color.wav', // thise wave file name
+        };
+
+        AudioRecord.init(colorAudio);
+    }
+
+    async function audioStart() {
+
+        // console.log('audioStart');
+        // checkPermission();
+        AudioRecord.start();
+        setTimeout(() => {
+            audioStop();
+        }, 2000);
+    }
+
+    async function audioStop() {
+
+        // console.log('audioStop');
+
+        let audioFile = await AudioRecord.stop();
+        // console.log('userFile', audioFile);
+
+        sendAudio(audioFile);
+
+    }
+
+    function sendAudio(audioFile) {
+
+        var obj = {
+            uri: `file:${audioFile}`,
+            type: 'audio/wav',
+            name: secondColor[3]+'.wav',
+        }
+
+        var data = JSON.stringify(obj)
+        // console.log(data);
+
+        client.post("predict", data, {
+
+            headers: {
+                Accept: 'application/json',
+                'Content-Type': 'application/json',
+            },
+
+        })
+            .then((response) => {
+                // setNames(response.data);
+                console.log('Success part : ', response.data);
+            })
+            .catch(function (error) {
+                if (error.response) {
+                    console.log('Error data : ', error.response.data);
+                    console.log('Error status : ', error.response.status);
+                    console.log('Error headers : ', error.response.headers);
+                } else if (error.request) {
+                    console.log('Error request : ', error.request);
+                } else {
+                    console.log('Error message : ', error.message);
+                }
+                console.log('Error else part : ', error.config);
+            });
+    }
+
     const onSpeechStartHandler = e => {
         // console.log('start handler =>> ', e);
     };
@@ -72,11 +145,7 @@ export default function Yellow() {
 
             sendData(mainColor[3])
 
-            showAlert = () => {
-                this.setState({
-                    showAlert: true
-                });
-            };
+            audioStart();
 
         } else {
 

frontend/src/screen/client/Client.js

@@ -2,6 +2,6 @@ import axios from 'axios';
 
 // export default axios.create({ baseURL: 'http://192.168.8.102:5000/', timeout: 15000, });
 export default axios.create({
-  baseURL: 'http://192.168.1.101:5000/',
+  baseURL: 'http://192.168.8.100:5000/',
   timeout: 15000,
 });

package-lock.json

+{
+  "name": "21_22j-38",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {}
+}