Skip to content

2
240
Commit 7cfb27ff authored by Malsha Rathnasiri's avatar Malsha Rathnasiri
Browse files
Options

Merge branch 'IT18094664' of http://gitlab.sliit.lk/2022-240/240 into IT18094664

parents 3ffcde55 af144043
Show whitespace changes
Inline Side-by-side
Showing with 179 additions and 256 deletions
backend/backend/cms/model/predict.py
View file @ 7cfb27ff
# import pickle import pickle
# from keras.models import load_model from keras.models import load_model
import numpy as np
# import numpy as np import IPython.display as ipd
import random
# import IPython.display as ipd from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
# import random def predict(samples):
#load the trained model
model=load_model(r'./best_model_final.hdf5')
# from sklearn.model_selection import train_test_split #load labels
# from sklearn.preprocessing import LabelEncoder f1 = open('all_label.txt', 'rb')
all_label = pickle.load(f1)
print('loaded labels')
#preprocess recorded audio
le = LabelEncoder()
y = le.fit_transform(all_label)
classes = list(le.classes_)
# def predict(samples): def predictSamples(audio):
# model=load_model(r'./best_model_final.hdf5') prob=model.predict(audio.reshape(1,8000,1))
index=np.argmax(prob[0])
return classes[index]
# f1 = open('all_label.txt', 'rb') ipd.Audio(samples, rate=8000)
# all_label = pickle.load(f1)
# print('loaded labels')
# # f2 = open('all_waves_file.txt', 'rb') #run the prediction
# # all_wave = pickle.load(f2) result = predictSamples(samples)
# # print('loaded waves')
# le = LabelEncoder() print("Text:",result)
# y = le.fit_transform(all_label)
# classes = list(le.classes_)
# # train_data_file = open("train_data_file.txt", 'rb') return result
# # [x_tr, x_val, y_tr, y_val] = np.load(train_data_file, allow_pickle=True) \ No newline at end of file
# # train_data_file.close()
# def predictSamples(audio):
# prob=model.predict(audio.reshape(1,8000,1))
# index=np.argmax(prob[0])
# return classes[index]
# # index=random.randint(0,len(x_val)-1)
# # samples=x_val[index].ravel()
# print(samples)
# # print("Audio:",classes[np.argmax(y_val[index])])
# ipd.Audio(samples, rate=8000)
# result = predictSamples(samples)
# print("Text:",result)
# return result
\ No newline at end of file
backend/backend/cms/model/train.py
View file @ 7cfb27ff
# import pickle import pickle
# from matplotlib import pyplot from matplotlib import pyplot
# import os import os
# import librosa import librosa
# import IPython.display as ipd import IPython.display as ipd
# import matplotlib.pyplot as plt import matplotlib.pyplot as plt
# import numpy as np import numpy as np
# from scipy.io import wavfile from scipy.io import wavfile
# import warnings import warnings
# from sklearn.preprocessing import LabelEncoder from sklearn.preprocessing import LabelEncoder
# from keras.utils import np_utils from keras.utils import np_utils
# from sklearn.model_selection import train_test_split
# from keras.layers import Dense, Dropout, Flatten, Conv1D, Input, MaxPooling1D
# from keras.models import Model
# from keras.callbacks import EarlyStopping, ModelCheckpoint
# from keras import backend as K
# K.clear_session()
# warnings.filterwarnings("ignore")
# # os.listdir('../../../data/')
# classes = ['down', 'go', 'left', 'no', 'off',
# 'on', 'right', 'stop', 'up', 'yes']
from sklearn.model_selection import train_test_split
# def train(): from keras.layers import Dense, Dropout, Flatten, Conv1D, Input, MaxPooling1D
from keras.models import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras import backend as K
K.clear_session()
# print('1')
# train_audio_path = r'./backend/data/train/train/audio/'
# samples, sample_rate = librosa.load(
# train_audio_path+'yes/0a7c2a8d_nohash_0.wav', sr=16000)
# # fig = plt.figure(figsize=(14, 8))
# # ax1 = fig.add_subplot(211)
# # ax1.set_title('Raw wave of ' + r'../input/train/audio/yes/0a7c2a8d_nohash_0.wav')
# # ax1.set_xlabel('time')
# # ax1.set_ylabel('Amplitude')
# # ax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)
# ipd.Audio(samples, rate=sample_rate) warnings.filterwarnings("ignore")
# print(sample_rate) # os.listdir('../../../data/')
# samples = librosa.resample(samples, sample_rate, 8000) # list of words to detect
# ipd.Audio(samples, rate=8000) classes = ['down', 'go', 'left', 'no', 'off',
'on', 'right', 'stop', 'up', 'yes']
# labels = os.listdir(train_audio_path)
# # find count of each label and plot bar graph def train():
# no_of_recordings = []
# for label in labels:
# waves = [f for f in os.listdir(
# train_audio_path + '/' + label) if f.endswith('.wav')]
# no_of_recordings.append(len(waves))
# # plot #data preprocessing
# # plt.figure(figsize=(30,5)) train_audio_path = r'./backend/data/train/train/audio/'
# index = np.arange(len(labels)) samples, sample_rate = librosa.load(
# # plt.bar(index, no_of_recordings) train_audio_path+'yes/0a7c2a8d_nohash_0.wav', sr=16000)
# # plt.xlabel('Commands', fontsize=12)
# # plt.ylabel('No of recordings', fontsize=12)
# # plt.xticks(index, labels, fontsize=15, rotation=60)
# # plt.title('No. of recordings for each command')
# # plt.show()
# print('2') # fig = plt.figure(figsize=(14, 8))
# ax1 = fig.add_subplot(211)
# ax1.set_title('Raw wave of ' + r'../input/train/audio/yes/0a7c2a8d_nohash_0.wav')
# ax1.set_xlabel('time')
# ax1.set_ylabel('Amplitude')
# ax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)
# labels = ["yes", "no", "up", "down", "left", ipd.Audio(samples, rate=sample_rate)
# "right", "on", "off", "stop", "go"]
# # labels_file = open('./labels_file.bin', 'wb+') print(sample_rate)
# # pickle.dump(obj=labels, file=labels_file)
# # labels_file.close()
# # # file = open('./labels_file.bin', 'rb') samples = librosa.resample(samples, sample_rate, 8000)
# # # dict = pickle.load(file) ipd.Audio(samples, rate=8000)
# # # print('loaded')
# # # print(dict)
# # # print('fdnasf')
# duration_of_recordings = [] labels = os.listdir(train_audio_path)
# for label in labels:
# print('2.1', label)
# waves = [f for f in os.listdir(
# train_audio_path + '/' + label) if f.endswith('.wav')]
# for wav in waves:
# sample_rate, samples = wavfile.read(
# train_audio_path + '/' + label + '/' + wav)
# duration_of_recordings.append(float(len(samples)/sample_rate))
# plt.hist(np.array(duration_of_recordings)) no_of_recordings = []
for label in labels:
waves = [f for f in os.listdir(
train_audio_path + '/' + label) if f.endswith('.wav')]
no_of_recordings.append(len(waves))
# train_audio_path = r'./backend/data/train/train/audio/' index = np.arange(len(labels))
# all_wave = [] labels = ["yes", "no", "up", "down", "left",
# all_label = [] "right", "on", "off", "stop", "go"]
# f1 = open('all_label.txt', 'rb') duration_of_recordings = []
# all_label = pickle.load(f1) for label in labels:
print('2.1', label)
waves = [f for f in os.listdir(
train_audio_path + '/' + label) if f.endswith('.wav')]
for wav in waves:
sample_rate, samples = wavfile.read(
train_audio_path + '/' + label + '/' + wav)
duration_of_recordings.append(float(len(samples)/sample_rate))
# f2 = open('all_waves_file.txt', 'rb') plt.hist(np.array(duration_of_recordings))
# all_wave = pickle.load(f2)
# if(all_wave and all_label): train_audio_path = r'./backend/data/train/train/audio/'
# print('loaded labels and waves')
# else:
# print('Creating labels and waves files')
# for label in labels:
# print(label)
# waves = [f for f in os.listdir(
# train_audio_path + '/' + label) if f.endswith('.wav')]
# for wav in waves:
# samples, sample_rate = librosa.load(
# train_audio_path + '/' + label + '/' + wav, sr=16000)
# samples = librosa.resample(samples, sample_rate, 8000)
# if(len(samples) == 8000):
# all_wave.append(samples)
# all_label.append(label)
# # print('3') all_wave = []
all_label = []
# all_labels_file = open('all_label.txt', 'wb+') #If labels are already generated load them
# pickle.dump(file=all_labels_file, obj=all_label) f1 = open('all_label.txt', 'rb')
# all_labels_file.close() all_label = pickle.load(f1)
f2 = open('all_waves_file.txt', 'rb')
all_wave = pickle.load(f2)
if(all_wave and all_label):
print('loaded labels and waves')
else:
print('Creating labels and waves files')
for label in labels:
print(label)
waves = [f for f in os.listdir(
train_audio_path + '/' + label) if f.endswith('.wav')]
for wav in waves:
samples, sample_rate = librosa.load(
train_audio_path + '/' + label + '/' + wav, sr=16000)
samples = librosa.resample(samples, sample_rate, 8000)
if(len(samples) == 8000):
all_wave.append(samples)
all_label.append(label)
# all_waves_file = open('all_waves_file.txt', 'wb+') all_labels_file = open('all_label.txt', 'wb+')
# pickle.dump(file=all_waves_file, obj=all_wave) pickle.dump(file=all_labels_file, obj=all_label)
# all_waves_file.close() all_labels_file.close()
# print('Done: creating labels and waves files') all_waves_file = open('all_waves_file.txt', 'wb+')
pickle.dump(file=all_waves_file, obj=all_wave)
all_waves_file.close()
print('Done: creating labels and waves files')
# le = LabelEncoder() le = LabelEncoder()
# y = le.fit_transform(all_label) y = le.fit_transform(all_label)
# classes = list(le.classes_) classes = list(le.classes_)
# print('4') y = np_utils.to_categorical(y, num_classes=len(labels))
# y = np_utils.to_categorical(y, num_classes=len(labels)) all_wave = np.array(all_wave).reshape(-1, 8000, 1)
# all_wave = np.array(all_wave).reshape(-1, 8000, 1) x_tr, x_val, y_tr, y_val = train_test_split(np.array(all_wave), np.array(
y), stratify=y, test_size=0.2, random_state=777, shuffle=True)
# x_tr, x_val, y_tr, y_val = train_test_split(np.array(all_wave), np.array( train_data_file = open('train_data_file.txt', 'wb+')
# y), stratify=y, test_size=0.2, random_state=777, shuffle=True) np.save(file=train_data_file, arr=np.array([x_tr, x_val, y_tr, y_val]))
train_data_file.close()
# train_data_file = open('train_data_file.txt', 'wb+') inputs = Input(shape=(8000, 1))
# np.save(file=train_data_file, arr=np.array([x_tr, x_val, y_tr, y_val]))
# train_data_file.close()
# inputs = Input(shape=(8000, 1)) # First Conv1D layer
conv = Conv1D(8, 13, padding='valid', activation='relu', strides=1)(inputs)
conv = MaxPooling1D(3)(conv)
conv = Dropout(0.3)(conv)
# # First Conv1D layer # Second Conv1D layer
# conv = Conv1D(8, 13, padding='valid', activation='relu', strides=1)(inputs) conv = Conv1D(16, 11, padding='valid', activation='relu', strides=1)(conv)
# conv = MaxPooling1D(3)(conv) conv = MaxPooling1D(3)(conv)
# conv = Dropout(0.3)(conv) conv = Dropout(0.3)(conv)
# # Second Conv1D layer # Third Conv1D layer
# conv = Conv1D(16, 11, padding='valid', activation='relu', strides=1)(conv) conv = Conv1D(32, 9, padding='valid', activation='relu', strides=1)(conv)
# conv = MaxPooling1D(3)(conv) conv = MaxPooling1D(3)(conv)
# conv = Dropout(0.3)(conv) conv = Dropout(0.3)(conv)
# # Third Conv1D layer # Fourth Conv1D layer
# conv = Conv1D(32, 9, padding='valid', activation='relu', strides=1)(conv) conv = Conv1D(64, 7, padding='valid', activation='relu', strides=1)(conv)
# conv = MaxPooling1D(3)(conv) conv = MaxPooling1D(3)(conv)
# conv = Dropout(0.3)(conv) conv = Dropout(0.3)(conv)
# # Fourth Conv1D layer # Flatten layer
# conv = Conv1D(64, 7, padding='valid', activation='relu', strides=1)(conv) conv = Flatten()(conv)
# conv = MaxPooling1D(3)(conv)
# conv = Dropout(0.3)(conv)
# # Flatten layer # Dense Layer 1
# conv = Flatten()(conv) conv = Dense(256, activation='relu')(conv)
conv = Dropout(0.3)(conv)
# # Dense Layer 1 # Dense Layer 2
# conv = Dense(256, activation='relu')(conv) conv = Dense(128, activation='relu')(conv)
# conv = Dropout(0.3)(conv) conv = Dropout(0.3)(conv)
# # Dense Layer 2 outputs = Dense(len(labels), activation='softmax')(conv)
# conv = Dense(128, activation='relu')(conv)
# conv = Dropout(0.3)(conv)
# outputs = Dense(len(labels), activation='softmax')(conv) # initializing the model
model = Model(inputs, outputs)
model.summary()
# model = Model(inputs, outputs) model.compile(loss='categorical_crossentropy',
# model.summary() optimizer='adam', metrics=['accuracy'])
# model.compile(loss='categorical_crossentropy', es = EarlyStopping(monitor='val_loss', mode='min',
# optimizer='adam', metrics=['accuracy']) verbose=1, patience=10, min_delta=0.0001)
mc = ModelCheckpoint('best_model.hdf5', monitor='val_accuracy',
verbose=1, save_best_only=True, mode='max')
# es = EarlyStopping(monitor='val_loss', mode='min', #training the model
# verbose=1, patience=10, min_delta=0.0001) history = model.fit(x_tr, y_tr, epochs=100, callbacks=[
# mc = ModelCheckpoint('best_model.hdf5', monitor='val_accuracy', es, mc], batch_size=32, validation_data=(x_val, y_val))
# verbose=1, save_best_only=True, mode='max')
# history = model.fit(x_tr, y_tr, epochs=100, callbacks=[ # code to plot the graphs
# es, mc], batch_size=32, validation_data=(x_val, y_val)) # pyplot.plot(history.history['loss'], label='train')
# pyplot.plot(history.history['val_loss'], label='test')
# pyplot.legend()
# # pyplot.plot(history.history['loss'], label='train') # pyplot.show()
# # pyplot.plot(history.history['val_loss'], label='test')
# # pyplot.legend()
# # pyplot.show() return history
# return history
backend/backend/cms/views.py
View file @ 7cfb27ff
...@@ -16,26 +16,23 @@ import librosa ...@@ -16,26 +16,23 @@ import librosa
from django.db.models import Q from django.db.models import Q
from django.core.files.storage import FileSystemStorage from django.core.files.storage import FileSystemStorage
from .models import Chat, Conversation, MlModel from .models import Chat, Conversation, MlModel
from rest_framework_simplejwt.views import TokenObtainPairView from rest_framework_simplejwt.views import TokenObtainPairView
# from .model.train import train from .model.train import train
from .model.predict import predict
# from .model.predict import predict
from pydub import AudioSegment from pydub import AudioSegment
import numpy as np import numpy as np
#Custom Viewset to get auth token
class ObtainTokenPairWithUserView(TokenObtainPairView): class ObtainTokenPairWithUserView(TokenObtainPairView):
permission_classes = (permissions.AllowAny,) permission_classes = (permissions.AllowAny,)
serializer_class = MyTokenObtainPairSerializer serializer_class = MyTokenObtainPairSerializer
class UserViewSet(viewsets.ModelViewSet): class UserViewSet(viewsets.ModelViewSet):
""" """
API endpoint that allows users to be viewed or edited. API endpoint that allows users to be viewed or edited.
...@@ -70,9 +67,9 @@ class MlModelViewSet(viewsets.ViewSet): ...@@ -70,9 +67,9 @@ class MlModelViewSet(viewsets.ViewSet):
permission_classes = [permissions.AllowAny] permission_classes = [permissions.AllowAny]
parser_classes = [MultiPartParser] parser_classes = [MultiPartParser]
#Custom api to add sample chats
@action(detail=False) @action(detail=False)
def addChats(*args, **kwargs): def addChats(*args, **kwargs):
admin = User.objects.get(username='admin') admin = User.objects.get(username='admin')
user2 = User.objects.get(username='user2') user2 = User.objects.get(username='user2')
...@@ -106,6 +103,7 @@ class MlModelViewSet(viewsets.ViewSet): ...@@ -106,6 +103,7 @@ class MlModelViewSet(viewsets.ViewSet):
conversation=convo, from_user_id=chat['from'], to_user_id=chat['to'], messsage=chat['message']) conversation=convo, from_user_id=chat['from'], to_user_id=chat['to'], messsage=chat['message'])
object.save() object.save()
#Custom api to train the model
@action(detail=False) @action(detail=False)
def train(*args, **kwargs): def train(*args, **kwargs):
print('Function ran') print('Function ran')
...@@ -113,6 +111,7 @@ class MlModelViewSet(viewsets.ViewSet): ...@@ -113,6 +111,7 @@ class MlModelViewSet(viewsets.ViewSet):
print(results) print(results)
return Response({'success': True}) return Response({'success': True})
#Custom api to convert audio to text
@action(detail=False, methods=["POST"]) @action(detail=False, methods=["POST"])
def detect(self, request, *args, **kwargs): def detect(self, request, *args, **kwargs):
print('Function ran') print('Function ran')
...@@ -135,7 +134,7 @@ class MlModelViewSet(viewsets.ViewSet): ...@@ -135,7 +134,7 @@ class MlModelViewSet(viewsets.ViewSet):
print('---------------------------------------------------------') print('---------------------------------------------------------')
if(samples.shape[0] > 8000): if(samples.shape[0] > 8000):
print('grateer -------------------------------------') print('grater -------------------------------------')
samples = samples[-8000:] samples = samples[-8000:]
print(samples.shape) print(samples.shape)
else: else:
...@@ -144,57 +143,17 @@ class MlModelViewSet(viewsets.ViewSet): ...@@ -144,57 +143,17 @@ class MlModelViewSet(viewsets.ViewSet):
samples = np.concatenate((samples, new_arr)) samples = np.concatenate((samples, new_arr))
print(samples.shape) print(samples.shape)
# audio_file = request.data results = predict(samples)
# # Using File storage to save file for future converting
# fs = FileSystemStorage()
# file_name = fs.save(audio_file.name, audio_file)
# audio_file_url = fs.url(file_name)
# # Preparing paths for convertion
# upstream = os.path.dirname(os.path.dirname(os.path.dirname(
# os.path.abspath(__file__))))
# path = os.path.join(upstream, 'media', audio_file.name)
# mp3_filename = '.'.join([audio_file.name.split('.')[0], 'mp3'])
# new_path = os.path.join(
# upstream, 'media', mp3_filename)
# # Converting to mp3 here
# wma_version = AudioSegment.from_file(path, "wav")
# wma_version.export(new_path, format="mp3")
# user_id = self.request.user.id
# # I was trying to create a Track instance, the mp3 get's saved but it is not being saved using location specified in models.
# track = Track.objects.create(user_id=user_id)
# track.file.name = mp3_filename
# track.save()
results = {} # predict(samples)
print(results) print(results)
return Response({'success': True, 'result': results}) return Response({'success': True, 'result': results})
class ChatViewSet(viewsets.ModelViewSet): class ChatViewSet(viewsets.ModelViewSet):
queryset = Chat.objects.all().order_by('-timestamp') queryset = Chat.objects.all().order_by('-timestamp')
serializer_class = ChatSerializer serializer_class = ChatSerializer
permission_classes = [permissions.IsAuthenticated] #change to permissions.isAutheniticated permission_classes = [permissions.IsAuthenticated]
@action(methods='POST', detail=True)
def getChats(self, request, *args, **kwargs):
#conversation_id hardcoded as we dont have many conversations at the moment
chats = Chat.objects \
.filter(conversation_id=1) \
.filter(
(Q(from_user__user_id=request.user.id)
| Q(to_user__user_id=request.user.id))
).order_by('-timestamp').values()
return Response({chats})
#ovveride defualt list action to get chats of specific user conversation
def list(self, request, pk=None): def list(self, request, pk=None):
if pk == None: if pk == None:
chats = Chat.objects \ chats = Chat.objects \
.filter(conversation_id=1) \ .filter(conversation_id=1) \
...@@ -210,7 +169,6 @@ class ChatViewSet(viewsets.ModelViewSet): ...@@ -210,7 +169,6 @@ class ChatViewSet(viewsets.ModelViewSet):
serializer = self.get_serializer(page, many=True) serializer = self.get_serializer(page, many=True)
result_set = serializer.data result_set = serializer.data
return Response(result_set) return Response(result_set)
def get_result_set(self, chats): def get_result_set(self, chats):
...@@ -218,7 +176,6 @@ class ChatViewSet(viewsets.ModelViewSet): ...@@ -218,7 +176,6 @@ class ChatViewSet(viewsets.ModelViewSet):
return result_set return result_set
class ConversationViewSet(viewsets.ModelViewSet): class ConversationViewSet(viewsets.ModelViewSet):
queryset = Conversation.objects.all().order_by('id') queryset = Conversation.objects.all().order_by('id')
serializer_class = ConversationSerializer serializer_class = ConversationSerializer
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment