Skip to content

21_22-J 38
21_22-J 38
Commit b850fd89 authored by W.D.R.P. Sandeepa's avatar W.D.R.P. Sandeepa
Browse files
Options

change activity interface

parent 6b30a2aa
Hide whitespace changes
Inline Side-by-side
Showing with 229 additions and 85 deletions
.idea/21_22j-38.iml
View file @ b850fd89
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
<excludeFolder url="file://$MODULE_DIR$/API/venv" /> <excludeFolder url="file://$MODULE_DIR$/API/venv" />
<excludeFolder url="file://$MODULE_DIR$/backend/IT18257632/venv" /> <excludeFolder url="file://$MODULE_DIR$/backend/IT18257632/venv" />
</content> </content>
<orderEntry type="jdk" jdkName="Python 3.9" jdkType="Python SDK" /> <orderEntry type="jdk" jdkName="Python 3.9 (21_22j-38)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="sourceFolder" forTests="false" />
</component> </component>
</module> </module>
\ No newline at end of file
.idea/misc.xml
View file @ b850fd89
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" /> <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (21_22j-38)" project-jdk-type="Python SDK" />
</project> </project>
\ No newline at end of file
API/app.py
View file @ b850fd89
from flask import Flask, redirect, url_for, render_template, request, jsonify, make_response from flask import Flask, redirect, url_for, render_template, request, jsonify, make_response
import random import random
import os import os
from API.model.colorModel import get_color_activities1, get_color_activities2, get_color_activity_result
from API.model.readModel import get_reading_activities from API.keyword_spotting_service import Keyword_Spotting_service
from API.model.colorModel import get_color_activities1, get_color_activities2
from API.routers.router import funtion_one from API.routers.router import funtion_one
from backend.IT18218640.keyword_spotting_service import Keyword_Spotting_service
app = Flask(__name__) app = Flask(__name__)
# Initial Route # Initial Route
@app.route("/") @app.route("/")
def home(): def home():
return render_template('home.html') return render_template('home.html')
# Color Function Route (IT18218640) # Color Function Route (IT18218640)
# Get Color activities route
@app.route("/getColorActivities1") @app.route("/getColorActivities1")
def getColorActivities1(): def getColorActivities1():
return get_color_activities1() return get_color_activities1()
...@@ -24,63 +24,71 @@ def getColorActivities1(): ...@@ -24,63 +24,71 @@ def getColorActivities1():
def getColorActivities2(): def getColorActivities2():
return get_color_activities2() return get_color_activities2()
@app.route("/getColorActivitiesResult")
def getColorActivitiesResult():
# print("data_dic")
return get_color_activity_result()
@app.route("/predict", methods=["POST"]) @app.route("/predict", methods=["POST"])
def predict(): def predict():
# get audio file and save it
audio_file = request.files["file"]
print(f"{request.files['file']}") data = request.get_json()
# print(f"{request.files['name']}") print((f"{data}"))
###################################### fileitem = data['uri']
print(f"{request.form['name']}")
#####################################
# print(f"{request.json}")
file_name = str(random.randint(0, 100000)) # if fileitem.filename:
# audio_file.save((file_name)) # # strip the leading path from the file name
fn = os.path.basename(fileitem)
#
# # open read and write the file into the server
open(fn, 'wb').write(fileitem.file.read())
resp = make_response(request.get_json())
resp.headers['Access-Control-Allow-Origin'] = '*'
resp.headers['Content-Type'] = 'application/json'
return resp
# get file name # get audio file and save it
# audio_file = request.files["file"]
# audio_file = request.files["name"]
# print((f"{request.data}"))
# print((f"{request.form}"))
# print((f"{request.values}"))
# print((f"{request.json}"))
# print(f"{request.form['name']}")
# file_name = str(random.randint(0, 100000))
# audio_file.save((file_name))
#
# # get file name
# predict_file_name = audio_file.filename # predict_file_name = audio_file.filename
# predict_file_name = predict_file_name.split("/") # predict_file_name = predict_file_name.split("/")
# new_predict_file_name = predict_file_name[1] # new_predict_file_name = predict_file_name[1]
# new_predict_file_name = new_predict_file_name.split(".") # new_predict_file_name = new_predict_file_name.split(".")
# FPFN = new_predict_file_name[0] # FPFN = new_predict_file_name[0]
# print(f"{FPFN}") # # print(f"{FPFN}")
#
# # invoke keyword spotting service
# kss = Keyword_Spotting_service()
#
#
# # make a prediction
# predicted_keyword = kss.predict(file_name, FPFN)
#
# # remove the audio file
# os.remove(file_name)
#
# # send back the predicted keword in json format
# data = {"Keyword" : predicted_keyword}
# return jsonify("print")
# return "Print"
# invoke keyword spotting service
kss = Keyword_Spotting_service()
# make a prediction
predicted_keyword = kss.predict(file_name, request.form['name'])
# remove the audio file
os.remove(file_name)
# send back the predicted keword in json format
data = {"Keyword": predicted_keyword}
return jsonify(data)
# return "audio_file"
# Read Function Route (IT) # Read Function Route (IT)
@app.route("/testings") @app.route("/testings")
def checkothers(): def checkothers():
return "testing funtions" return "testing funtions"
# Color Function Route (IT18218640)
@app.route("/reading")
def getReadActivities():
return get_reading_activities()
@app.route("/ru") @app.route("/ru")
def abc(): def abc():
response_val = funtion_one() response_val = funtion_one()
...@@ -91,9 +99,7 @@ def abc(): ...@@ -91,9 +99,7 @@ def abc():
# return make_response(d, 200) # return make_response(d, 200)
return response_val return response_val
if __name__ == "__main__": if __name__ == "__main__":
app.run(host='192.168.8.101') app.run(host='192.168.8.102')
# app.run(host='192.168.8.100,port='5000', debug=True) #app.run(host='192.168.8.100,port='5000', debug=True)
# app.run(debug=True) # app.run(debug=True)
\ No newline at end of file
API/keyword_spotting_service.py 0 → 100644
View file @ b850fd89
import tensorflow.keras as keras
import numpy as np
import librosa
MODEL_PATH = "model.h5"
NUM_SAMPLES_TO_CONSIDER = 22050 # 1 sec
class _Keyword_Spotting_Service:
model = None
_mappings = [
"black",
"blue",
"green",
"red",
"white",
"yellow"
]
_instance = None
# 78293
def predict(self, file_path, FPFN):
print(f"{file_path}")
# extract MFCCs
MFCCs = self.preprocess(file_path) # (# segment, # coefficients)
# convert 2d MFCCs array into 4d array -> (# samples, # segment, # coefficients, # channels)
MFCCs = MFCCs[np.newaxis, ..., np.newaxis]
# make prediction
predictions = self.model.predict(MFCCs)
predicted_index = np.argmax(predictions)
predicted_keyword = self._mappings[predicted_index]
# return predicted_keyword
if predicted_keyword == FPFN:
return predicted_keyword
else:
return "No Prediction"
def preprocess(self, file_path, n_mfcc=13, n_fft=2048, hop_length=512):
# load audio file
signal, sr = librosa.load(file_path)
# ensure consistency in the audio file length
if len(signal) > NUM_SAMPLES_TO_CONSIDER:
signal = signal[:NUM_SAMPLES_TO_CONSIDER]
# extract MFCCs
MFCCs = librosa.feature.mfcc(signal, n_mfcc=n_mfcc, n_fft=n_fft, hop_length=hop_length)
return MFCCs.T
def Keyword_Spotting_service():
# ensure that we only have 1 instance of KSS
if _Keyword_Spotting_Service._instance is None:
_Keyword_Spotting_Service._instance = _Keyword_Spotting_Service()
_Keyword_Spotting_Service.model = keras.models.load_model(MODEL_PATH)
return _Keyword_Spotting_Service._instance
if __name__ == "__main__":
kss = Keyword_Spotting_service()
keyword1 = kss.predict("test/black.wav")
keyword2 = kss.predict("test/blue.wav")
keyword3 = kss.predict("test/green.wav")
keyword4 = kss.predict("test/red.wav")
keyword5 = kss.predict("test/white.wav")
keyword6 = kss.predict("test/yellow.wav")
keyword7 = kss.predict("test/other.wav")
print(f"Predicted Keywords: {keyword1}")
print(f"Predicted Keywords: {keyword2}")
print(f"Predicted Keywords: {keyword3}")
print(f"Predicted Keywords: {keyword4}")
print(f"Predicted Keywords: {keyword5}")
print(f"Predicted Keywords: {keyword6}")
print(f"Predicted Keywords: {keyword7}")
\ No newline at end of file
API/model.h5 0 → 100644
View file @ b850fd89
File added
frontend/src/screen/activity/Blue.js
View file @ b850fd89
import { import { useNavigation } from "@react-navigation/native";
useNavigation
} from "@react-navigation/native";
import Orientation from 'react-native-orientation-locker'; import Orientation from 'react-native-orientation-locker';
import React, { import React, { useEffect, useState } from "react";
useEffect, import { Text,TouchableOpacity, StyleSheet, View, Dimensions, SafeAreaView, ImageBackground, Button, Image, StatusBar} from 'react-native'
useState
} from "react";
import {
Text,
TouchableOpacity,
StyleSheet,
View,
Dimensions,
SafeAreaView,
ImageBackground,
Button,
Image,
StatusBar
} from 'react-native'
import Permissions from 'react-native-permissions'; import Permissions from 'react-native-permissions';
import AudioRecord from 'react-native-audio-record'; import AudioRecord from 'react-native-audio-record';
import axios from "axios"; import axios from "axios";
import client from "../client/Client"; import client from "../client/Client";
import BackButton from "../../component/BackButton"
export default function Blue(color) { export default function Blue(color) {
const navigation = useNavigation(); const navigation = useNavigation();
...@@ -46,7 +31,6 @@ export default function Blue(color) { ...@@ -46,7 +31,6 @@ export default function Blue(color) {
return unsubscribe; return unsubscribe;
}, [navigation]); }, [navigation]);
// audioInit
function audioInit() { function audioInit() {
// console.log('audioInit'); // console.log('audioInit');
...@@ -61,10 +45,9 @@ export default function Blue(color) { ...@@ -61,10 +45,9 @@ export default function Blue(color) {
AudioRecord.init(colorAudio); AudioRecord.init(colorAudio);
} }
// checkPermission
async function checkPermission() { async function checkPermission() {
// console.log('checkPermission'); // console.log('checkPermission');
const p = await Permissions.check('microphone'); const p = await Permissions.check('microphone');
// console.log(p); // console.log(p);
// console.log('permission check', p); // console.log('permission check', p);
...@@ -79,13 +62,11 @@ export default function Blue(color) { ...@@ -79,13 +62,11 @@ export default function Blue(color) {
} }
// requestPermission
async function requestPermission() { async function requestPermission() {
const p = await Permissions.request('microphone'); const p = await Permissions.request('microphone');
// console.log('permission request', p); // console.log('permission request', p);
} }
// audioStart
async function audioStart() { async function audioStart() {
// console.log('audioStart'); // console.log('audioStart');
...@@ -96,7 +77,6 @@ export default function Blue(color) { ...@@ -96,7 +77,6 @@ export default function Blue(color) {
}, 2000); }, 2000);
} }
// audioStop
async function audioStop() { async function audioStop() {
// console.log('audioStop'); // console.log('audioStop');
...@@ -104,12 +84,66 @@ export default function Blue(color) { ...@@ -104,12 +84,66 @@ export default function Blue(color) {
let audioFile = await AudioRecord.stop(); let audioFile = await AudioRecord.stop();
console.log('userFile', audioFile); console.log('userFile', audioFile);
//call sendAudio funtion // sendAudio(audioFile);
sendAudio(audioFile); getAudio(audioFile);
} }
// send audio to server function function getAudio(audioFile) {
const formData = new FormData();
var obj = {
uri: `file://${audioFile}`,
type: 'audio/wav',
name: 'color.wav',
colorNmae: backColor
}
// formData.append(
// 'file',
// {
// uri: `file://${audioFile}`,
// type: 'audio/wav',
// name: 'color.wav',
// }
// );
// formData.append(
// 'name', backColor
// );
var data = JSON.stringify(obj)
console.log(data);
axios
.post("http://192.168.8.102:5000/predict", data, {
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
})
.then((response) => {
// setNames(response.data);
console.log('Success part : ',response.data);
})
.catch(function (error) {
if (error.response) {
console.log('Error data : ', error.response.data);
console.log('Error status : ', error.response.status);
console.log('Error headers : ', error.response.headers);
} else if (error.request) {
console.log('Error request : ', error.request);
} else {
console.log('Error message : ', error.message);
}
console.log('Error else part : ', error.config);
});
}
async function sendAudio(audioFile) { async function sendAudio(audioFile) {
// append form data and upload to api // append form data and upload to api
...@@ -193,16 +227,34 @@ export default function Blue(color) { ...@@ -193,16 +227,34 @@ export default function Blue(color) {
</View> </View>
<View style={{ flexDirection: "row", marginLeft: 480 }}> <View style={{ marginTop: 10 }}>
<Button style={styles.button} title="Recode" onPress={() => { audioStart(); }} /> <BackButton path="Color" />
<Button title="Return" color="#1DCE92" /> </View>
<Button title="Exit" color="#841584" />
<View style={{ flexDirection: "row", marginLeft: 420, marginTop: -80 }}>
<View style={{}}>
<TouchableOpacity onPress={() => { audioStart(); }}
style={{ width: "30%", height: "40%", borderRadius: 50 }}>
<Image source={require('../../assets/game/mic2.png')} resizeMode='contain' style={{ flex: 1, marginLeft: -190 }} />
</TouchableOpacity>
</View>
<View style={{ marginLeft: -100 }}>
<TouchableOpacity onPress={() => { }}
style={{ width: "60%", height: "40%", borderRadius: 50 }}>
<Image source={require('../../assets/game/next.png')} resizeMode='contain' style={{ flex: .9, marginLeft: -90 }} />
</TouchableOpacity>
</View>
</View> </View>
</ImageBackground> </ImageBackground>
</View> </View>
</SafeAreaView> </SafeAreaView>
) )
} }
const styles = StyleSheet.create({ const styles = StyleSheet.create({
...@@ -244,7 +296,8 @@ const styles = StyleSheet.create({ ...@@ -244,7 +296,8 @@ const styles = StyleSheet.create({
button: { button: {
padding: 10, padding: 10,
marginRight: 50, marginRight: 50,
color: "#000000" color: "#000000",
marginLeft: 50,
} }
}) })
\ No newline at end of file
frontend/src/screen/auth/Login.js
View file @ b850fd89
...@@ -93,13 +93,13 @@ const Login = () => { ...@@ -93,13 +93,13 @@ const Login = () => {
</View> </View>
<View style={styles.form_input}> <View style={styles.form_input}>
<TouchableOpacity onPress={submitForm} style={styles.btn}> <TouchableOpacity onPress={()=> { navigation.navigate("Home")}} style={styles.btn}>
<Text style={styles.btn_text}> <Text style={styles.btn_text}>
Sign In Sign In
</Text> </Text>
</TouchableOpacity> </TouchableOpacity>
</View> </View>
{/* onPress={submitForm} */}
<View style={styles.text_if}> <View style={styles.text_if}>
<TouchableOpacity onPress={()=> { navigation.navigate("Register")}}> <TouchableOpacity onPress={()=> { navigation.navigate("Register")}}>
<Text style={styles.btn_text2}> <Text style={styles.btn_text2}>
......
frontend/src/screen/auth/Register.js
View file @ b850fd89
...@@ -86,6 +86,8 @@ const Register = () => { ...@@ -86,6 +86,8 @@ const Register = () => {
} }
} }
// import client from "../client/Client";
return ( return (
<SafeAreaView> <SafeAreaView>
<ScrollView> <ScrollView>
......
frontend/src/screen/client/Client.js
View file @ b850fd89
import axios from 'axios'; import axios from 'axios';
export default axios.create({ baseURL: 'http://192.168.8.101:5000/',timeout: 15000, }); export default axios.create({ baseURL: 'http://192.168.8.102:5000/',timeout: 15000, });
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment