Skip to content

2
2020-101
Commit 5382ffb9 authored by I.K Seneviratne's avatar I.K Seneviratne
Browse files
Options

Merge branch 'IT17100908' into 'QA_RELEASE' 

It17100908

See merge request !45
parents 27052e66 08f6d91b
Hide whitespace changes
Inline Side-by-side
Showing with 222 additions and 163 deletions
LectureSummarizingApp/api.py
View file @ 5382ffb9
from rest_framework.views import APIView from rest_framework.views import APIView
from rest_framework.response import Response from rest_framework.response import Response
from FirstApp.logic.id_generator import generate_new_id
from LectureSummarizingApp.models import LectureAudio, LectureAudioNoiseRemoved, LectureSpeechToText, \ from LectureSummarizingApp.models import LectureAudio, LectureAudioNoiseRemoved, LectureSpeechToText, \
LectureAudioSummary, LectureNotices LectureAudioSummary, LectureNotices
from LectureSummarizingApp.serializer import LectureAudioSerializer, LectureAudioNoiseRemovedSerializer, \ from LectureSummarizingApp.serializer import LectureAudioSerializer, LectureAudioNoiseRemovedSerializer, \
LectureSpeechToTextSerializer, LectureAudioSummarySerializer, LectureNoticesSerializer LectureSpeechToTextSerializer, LectureAudioSummarySerializer, LectureNoticesSerializer
from . import speech_to_text as stt from . import speech_to_text as stt
from . import noiseRemove as nr
import datetime
# APIs used in Lecture Summarizing Component # APIs used in Lecture Summarizing Component
from .noise import noise_removal
class LectureAudioAPI(APIView): class LectureAudioAPI(APIView):
def get(self, request): def get(self, request):
...@@ -21,9 +28,36 @@ class LectureAudioAPI(APIView): ...@@ -21,9 +28,36 @@ class LectureAudioAPI(APIView):
class audioNoiseRemovedList(APIView): class audioNoiseRemovedList(APIView):
def get(self, request): def get(self, request):
lecture_audio_noise_removed = LectureAudioNoiseRemoved.objects.all() # lecture_audio_noise_removed = LectureAudioNoiseRemoved.objects.all()
serializer = LectureAudioNoiseRemovedSerializer(lecture_audio_noise_removed, many=True) # serializer = LectureAudioNoiseRemovedSerializer(lecture_audio_noise_removed, many=True)
return Response(serializer.data) audio_noise_removed_list = LectureAudioNoiseRemoved.objects.order_by('lecture_audio_noise_removed_id').last()
audio_name = request.query_params.get("audio_name")
id = int(request.query_params.get("id"))
current_date = datetime.datetime.now().date()
fake_duration = datetime.timedelta(minutes=2, seconds=10, milliseconds=00)
# generate new id for audio noise removed
new_audio_noise_removed_id = generate_new_id(audio_noise_removed_list.lecture_audio_noise_removed_id)
# nr.noise_removalll(video_name)
noise_removal(audio_name)
LectureAudioNoiseRemoved(
lecture_audio_noise_removed_id=new_audio_noise_removed_id,
lecture_audio_id_id=id,
lecturer_date=current_date,
lecture_audio_name=audio_name,
lecture_audio_length=fake_duration
).save()
return Response({
"response": Response.status_code
})
def post(self, request): def post(self, request):
LectureAudioNoiseRemoved( LectureAudioNoiseRemoved(
...@@ -43,11 +77,14 @@ class audioToTextList(APIView): ...@@ -43,11 +77,14 @@ class audioToTextList(APIView):
serializer = LectureSpeechToTextSerializer(lecture_speech_to_text_id, many=True) serializer = LectureSpeechToTextSerializer(lecture_speech_to_text_id, many=True)
# return Response(serializer.data) # return Response(serializer.data)
video_name = request.query_params.get("video_name") # video_name = request.query_params.get("video_name")
#
print('video name: ', video_name) # print('video name: ', video_name)
#
# # nr.noise_removalll(video_name)
# noise_removal(video_name)
stt.speech_to_text(video_name) # stt.speech_to_text(video_name)
return Response({ return Response({
"response": "successful" "response": "successful"
......
LectureSummarizingApp/noise.py
View file @ 5382ffb9
import librosa import librosa
from pysndfx import AudioEffectsChain from pysndfx import AudioEffectsChain
import python_speech_features import python_speech_features
import os
def noise_removal(video_name):
def read_file(file_name): # sample_file = file_name
sample_file = file_name # sample_directory = 'lectures/'
sample_directory = 'lectures/' # sample_path = sample_directory + sample_file
sample_path = sample_directory + sample_file
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LECTURE_VIDEO_DIR = os.path.join(BASE_DIR, "LectureSummarizingApp\\lectures\\{}".format(video_name))
print('lecture audio directory: ', LECTURE_VIDEO_DIR)
# DESTINATION_DIR = os.path.join(BASE_DIR, "LectureSummarizingApp\\noise_removed_lectures")
DESTINATION_DIR = os.path.dirname(os.path.join(BASE_DIR, "LectureSummarizingApp\\noise_removed_lectures\\sample.txt"))
print('destination directory: ', DESTINATION_DIR)
# generating audio time series and a sampling rate (int) # generating audio time series and a sampling rate (int)
a, sr = librosa.load(sample_path) a, sr = librosa.load(path=LECTURE_VIDEO_DIR)
print('a: ', a)
print('sr: ', sr)
# speech_boosted = mffc_highshelf(a, sr)
output_file(destination=DESTINATION_DIR, filename=video_name, a=a, sr=sr)
return a, sr return a, sr
...@@ -24,6 +41,7 @@ def mffc_highshelf(a, sr): ...@@ -24,6 +41,7 @@ def mffc_highshelf(a, sr):
mfcc = python_speech_features.base.logfbank(a) mfcc = python_speech_features.base.logfbank(a)
mfcc = python_speech_features.base.lifter(mfcc) mfcc = python_speech_features.base.lifter(mfcc)
sum_of_squares = [] sum_of_squares = []
index = -1 index = -1
for r in mfcc: for r in mfcc:
...@@ -40,73 +58,49 @@ def mffc_highshelf(a, sr): ...@@ -40,73 +58,49 @@ def mffc_highshelf(a, sr):
speech_booster = AudioEffectsChain().highshelf(frequency=min_hz*(-1)*1.2, gain=-12.0, slope=0.6).limiter(gain=8.0) speech_booster = AudioEffectsChain().highshelf(frequency=min_hz*(-1)*1.2, gain=-12.0, slope=0.6).limiter(gain=8.0)
a_speach_boosted = speech_booster(a) a_speach_boosted = speech_booster(a)
# a_speach_boosted = speech_booster.
return (a_speach_boosted) return (a_speach_boosted)
def mfcc_lowshelf(a, sr):
mfcc = python_speech_features.base.mfcc(a)
mfcc = python_speech_features.base.logfbank(a)
mfcc = python_speech_features.base.lifter(mfcc)
sum_of_squares = []
index = -1
for x in mfcc:
sum_of_squares.append(0)
index = index + 1
for n in x:
sum_of_squares[index] = sum_of_squares[index] + n**2
strongest_frame = sum_of_squares.index(max(sum_of_squares))
hz = python_speech_features.base.mel2hz(mfcc[strongest_frame])
max_hz = max(hz)
min_hz = min(hz)
speech_booster = AudioEffectsChain().lowshelf(frequency=min_hz*(-1), gain=12.0, slope=0.5)
a_speach_boosted = speech_booster(a)
return (a_speach_boosted)
def trim_silence(y):
a_trimmed, index = librosa.effects.trim(y, top_db=20, frame_length=2, hop_length=500)
trimmed_length = librosa.get_duration(y) - librosa.get_duration(a_trimmed)
return a_trimmed, trimmed_length # def trim_silence(y):
# a_trimmed, index = librosa.effects.trim(y, top_db=20, frame_length=2, hop_length=500)
# trimmed_length = librosa.get_duration(y) - librosa.get_duration(a_trimmed)
#
# return a_trimmed, trimmed_length
def enhance(y): # def enhance(y):
apply_audio_effects = AudioEffectsChain().lowshelf(gain=10.0, frequency=260, slope=0.1).reverb(reverberance=25, hf_damping=5, room_scale=5, stereo_depth=50, pre_delay=20, wet_gain=0, wet_only=False)#.normalize() # apply_audio_effects = AudioEffectsChain().lowshelf(gain=10.0, frequency=260, slope=0.1).reverb(reverberance=25, hf_damping=5, room_scale=5, stereo_depth=50, pre_delay=20, wet_gain=0, wet_only=False)#.normalize()
a_enhanced = apply_audio_effects(y) # a_enhanced = apply_audio_effects(y)
#
return a_enhanced # return a_enhanced
def output_file(destination ,filename, a, sr, ext=""): def output_file(destination ,filename, a, sr, ext=""):
destination = destination + filename[:-4] + ext + '.wav' destination = destination + filename[:-4] + ext + '.wav'
librosa.output.write_wav(destination, a, sr) librosa.output.write_wav(destination, a, sr)
lectures = ['Lecture01.wav'] # lectures = ['Lecture01.wav', 'Lecture02.wav']
for s in lectures: # for s in lectures:
filename = s # filename = s
a, sr = read_file(filename) # a, sr = noise_removal(filename)
#
#
# a_reduced_centroid_s = reduce_noise_centroid_s(a, sr) # # a_reduced_centroid_s = reduce_noise_centroid_s(a, sr)
a_reduced_mfcc_lowshelf = mfcc_lowshelf(a, sr) # # a_reduced_mfcc_lowshelf = mfcc_lowshelf(a, sr)
a_reduced_mfcc_highshelf = mffc_highshelf(a, sr) # a_reduced_mfcc_highshelf = mffc_highshelf(a, sr)
#
#
# trimming silences # # trimming silences
# a_reduced_centroid_s, time_trimmed = trim_silence(a_reduced_centroid_s) # # a_reduced_centroid_s, time_trimmed = trim_silence(a_reduced_centroid_s)
a_reduced_mfcc_up, time_trimmed = trim_silence(mfcc_lowshelf) # # a_reduced_mfcc_up, time_trimmed = trim_silence(mfcc_lowshelf)
a_reduced_mfcc_down, time_trimmed = trim_silence(mffc_highshelf) # a_reduced_mfcc_down, time_trimmed = trim_silence(mffc_highshelf)
#
#
#
# output_file('lectures_trimmed_noise_reduced/' ,filename, y_reduced_centroid_s, sr, '_ctr_s') # # output_file('lectures_trimmed_noise_reduced/' ,filename, y_reduced_centroid_s, sr, '_ctr_s')
output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_up, sr, '_mfcc_up') # #output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_up, sr, '_mfcc_up')
# output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_down, sr, '_mfcc_down') # output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_down, sr, '_mfcc_down')
# output_file('lectures_trimmed_noise_reduced/' ,filename, a, sr, '_org') # # output_file('lectures_trimmed_noise_reduced/' ,filename, a, sr, '_org')
LectureSummarizingApp/noiseRemove.py
View file @ 5382ffb9
...@@ -4,54 +4,63 @@ from scipy.io.wavfile import read ...@@ -4,54 +4,63 @@ from scipy.io.wavfile import read
from scipy.io.wavfile import write from scipy.io.wavfile import write
from scipy import signal from scipy import signal
import matplotlib.pyplot as mplt import matplotlib.pyplot as mplt
import os
#get_ipython().magic('matplotlib inline') #get_ipython().magic('matplotlib inline')
(Frequency, array) = read('lectures/Lecture01.wav')
len(array) def noise_removalll(video_name):
mplt.plot(array) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
mplt.title('Original Signal Spectrum') LECTURE_VIDEO_DIR = os.path.join(BASE_DIR, "LectureSummarizingApp\\lectures\\{}".format(video_name))
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
fourierTransformation = sip.fft(array) # (Frequency, array) = read('lectures/Lecture01.wav')
# (Frequency, array) = read('lectures/{}'.format(video_name))
(Frequency, array) = read(LECTURE_VIDEO_DIR)
scale = sip.linspace(0, Frequency, len(array)) len(array)
mplt.stem(scale[0:5000], nump.abs(fourierTransformation[0:5000]), 'r') mplt.plot(array)
mplt.title('Signal spectrum after FFT') mplt.title('Original Signal Spectrum')
mplt.xlabel('Frequency(Hz)') mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude') mplt.ylabel('Amplitude')
fourierTransformation = sip.fft(array)
guassianNoise = nump.random.rand(len(fourierTransformation)) scale = sip.linspace(0, Frequency, len(array))
mplt.stem(scale[0:5000], nump.abs(fourierTransformation[0:5000]), 'r')
mplt.title('Signal spectrum after FFT')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
NewSound = guassianNoise + array
write("New-Sound-Added-With-Guassian-Noise.wav", Frequency, NewSound) guassianNoise = nump.random.rand(len(fourierTransformation))
u,v = signal.butter(5, 1000/(Frequency/2), btype='highpass')
filteredSignal = signal.lfilter(u,v,NewSound) NewSound = guassianNoise + array
# plotting the signal. write("New-Sound-Added-With-Guassian-Noise.wav", Frequency, NewSound)
mplt.plot(filteredSignal)
mplt.title('Highpass Filter')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
# ButterWorth low-filter u,v = signal.butter(5, 1000/(Frequency/2), btype='highpass')
x,y = signal.butter(5, 380/(Frequency/2), btype='lowpass')
# Applying the filter to the signal filteredSignal = signal.lfilter(u,v,NewSound)
newFilteredSignal = signal.lfilter(x,y,filteredSignal)
# plotting the signal. # plotting the signal.
mplt.plot(newFilteredSignal) mplt.plot(filteredSignal)
mplt.title('Lowpass Filter') mplt.title('Highpass Filter')
mplt.xlabel('Frequency(Hz)') mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude') mplt.ylabel('Amplitude')
write("removed.wav", Frequency, nump.int16(newFilteredSignal/nump.max(nump.abs(newFilteredSignal)) * 32767)) # ButterWorth low-filter
\ No newline at end of file x,y = signal.butter(5, 380/(Frequency/2), btype='lowpass')
# Applying the filter to the signal
newFilteredSignal = signal.lfilter(x,y,filteredSignal)
# plotting the signal.
mplt.plot(newFilteredSignal)
mplt.title('Lowpass Filter')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
write("removed.wav", Frequency, nump.int16(newFilteredSignal/nump.max(nump.abs(newFilteredSignal)) * 32767))
\ No newline at end of file
LectureSummarizingApp/templates/LectureSummarizingApp/RecordLecture.html
View file @ 5382ffb9
{% extends 'FirstApp/template.html' %} <!--{% extends 'FirstApp/template.html' %}-->
<!DOCTYPE html> <!--<!DOCTYPE html>-->
<html lang="en"> <!--<html lang="en">-->
<head> <!--<head>-->
<meta charset="UTF-8"> <!-- <meta charset="UTF-8">-->
<title>Lecture Recording</title> <!-- <title>Lecture Recording</title>-->
</head> <!--</head>-->
<body> <!--<body>-->
% block javascript %} <!--% block javascript %}-->
{% load static %} <!--{% load static %}-->
<!-- Bootstrap core JavaScript--> <!--&lt;!&ndash; Bootstrap core JavaScript&ndash;&gt;-->
<script src="{% static 'FirstApp/vendor/jquery/jquery.min.js' %}"></script> <!--<script src="{% static 'FirstApp/vendor/jquery/jquery.min.js' %}"></script>-->
<script src="{% static 'FirstApp/vendor/bootstrap/js/bootstrap.bundle.min.js' %}"></script> <!--<script src="{% static 'FirstApp/vendor/bootstrap/js/bootstrap.bundle.min.js' %}"></script>-->
<!-- Page level plugins --> <!--&lt;!&ndash; Page level plugins &ndash;&gt;-->
<script src="{% static 'FirstApp/vendor/datatables/jquery.dataTables.min.js' %}"></script> <!--<script src="{% static 'FirstApp/vendor/datatables/jquery.dataTables.min.js' %}"></script>-->
<script src="{% static 'FirstApp/vendor/datatables/dataTables.bootstrap4.min.js' %}"></script> <!--<script src="{% static 'FirstApp/vendor/datatables/dataTables.bootstrap4.min.js' %}"></script>-->
<!-- Page level custom scripts --> <!--&lt;!&ndash; Page level custom scripts &ndash;&gt;-->
<script src="{% static 'FirstApp/js/demo/datatables-demo.js' %}"></script> <!--<script src="{% static 'FirstApp/js/demo/datatables-demo.js' %}"></script>-->
<!-- Core plugin JavaScript--> <!--&lt;!&ndash; Core plugin JavaScript&ndash;&gt;-->
<script src="{% static 'FirstApp/vendor/jquery-easing/jquery.easing.min.js' %}"></script> <!--<script src="{% static 'FirstApp/vendor/jquery-easing/jquery.easing.min.js' %}"></script>-->
<!-- Load TensorFlow.js --> <!--&lt;!&ndash; Load TensorFlow.js &ndash;&gt;-->
<script src="https://unpkg.com/@tensorflow/tfjs"></script> <!--<script src="https://unpkg.com/@tensorflow/tfjs"></script>-->
<!-- Load Posenet --> <!--&lt;!&ndash; Load Posenet &ndash;&gt;-->
<script src="https://unpkg.com/@tensorflow-models/posenet"> <!--<script src="https://unpkg.com/@tensorflow-models/posenet">-->
</script> <!--</script>-->
{% endblock %} <!--{% endblock %}-->
<div id="wrapper"> <!--<div id="wrapper">-->
<div id="content-wrapper" class="d-flex flex-column"> <!-- <div id="content-wrapper" class="d-flex flex-column">-->
<div id="content"> <!-- <div id="content">-->
{% block 'container-fluid' %} <!-- {% block 'container-fluid' %}-->
<div class="container-fluid"> <!-- <div class="container-fluid">-->
{% load static %} <!-- {% load static %}-->
<div class="d-sm-flex align-items-center justify-content-between mb-4"> <!-- <div class="d-sm-flex align-items-center justify-content-between mb-4">-->
<h1 class="h3 mb-0 text-gray-800">Lecture Record</h1> <!-- <h1 class="h3 mb-0 text-gray-800">Lecture Record</h1>-->
</div> <!-- </div>-->
<div> <!-- <div>-->
<button TYPE="button" class="btn btn-success audio_process">Start Recording</button> <!-- <button TYPE="button" class="btn btn-success audio_process">Start Recording</button>-->
</div> <!-- </div>-->
</div> <!-- </div>-->
</div> <!-- </div>-->
</div> <!-- </div>-->
</div> <!--</div>-->
</body> <!--</body>-->
</html> <!--</html>-->
\ No newline at end of file \ No newline at end of file
LectureSummarizingApp/templates/LectureSummarizingApp/summarization.html
View file @ 5382ffb9
...@@ -48,18 +48,37 @@ ...@@ -48,18 +48,37 @@
}); });
<!-- background noise--> <!-- background noise-->
$('.audio_process').click(function() { $('.audio_process').click(function(e) {
alert('Processing');
let id = e.target.parentNode.parentNode.getAttribute('id');
let audio_name = e.target.parentNode.parentNode.getAttribute('data-audio-name');
audio_name = audio_name + ".wav";
alert('id: ' + id);
alert('audio name: ' + audio_name);
//call the fetch API //call the fetch API
fetch('http://127.0.0.1:8000/summary/lecture-audio-to-text/?video_name=Lecture01.wav') <!-- fetch('http://127.0.0.1:8000/summary/lecture-audio-to-text/?audio_name=')-->
<!-- .then((res) => res.json())-->
<!-- .then((out) => alert(out.response))-->
<!-- .catch((err) => alert('error: ' + err))-->
fetch('http://127.0.0.1:8000/summary/lecture-audio-noise-removed/?audio_name=' + audio_name + '&id=' + id)
.then((res) => res.json()) .then((res) => res.json())
.then((out) => alert(out.response)) .then((out) => handleNoiseRemoved(out.response))
.catch((err) => alert('error: ' + err)) .catch((err) => alert('error: ' + err))
}); });
//this function will handle the success respopnse for noise removed
function handleNoiseRemoved(response) {
if (response === 200) {
document.location.reload();
}
}
<!-- To summary--> <!-- To summary-->
$('.to_summary').click(function() { $('.to_summary').click(function() {
alert('Processing'); alert('Processing');
...@@ -121,7 +140,7 @@ ...@@ -121,7 +140,7 @@
<tbody> <tbody>
{% for lec_audio in lec_audio_data %} {% for lec_audio in lec_audio_data %}
<tr class="recordings not_clicked" id="{{ lec_audio.lecture_audio_id }}"> <tr class="recordings not_clicked" id="{{ lec_audio.id }}" data-audio-name="{{ lec_audio.lecture_audio_name }}">
<!-- <td>--> <!-- <td>-->
<!-- <div class="radio">--> <!-- <div class="radio">-->
<!-- <label><input type="radio"--> <!-- <label><input type="radio"-->
......
LectureSummarizingApp/urls.py
View file @ 5382ffb9
...@@ -8,7 +8,7 @@ router = routers.DefaultRouter() ...@@ -8,7 +8,7 @@ router = routers.DefaultRouter()
urlpatterns = [ urlpatterns = [
path('lecture', views.summarization), path('lecture', views.summarization),
path('record', views.lectureRecord), # path('record', views.lectureRecord),
# API to retrieve lecture summarizing details # API to retrieve lecture summarizing details
......
LectureSummarizingApp/views.py
View file @ 5382ffb9
...@@ -13,7 +13,7 @@ def lectureRecord(request): ...@@ -13,7 +13,7 @@ def lectureRecord(request):
print('lecture record data: ', lecture_audio_ser.data) print('lecture record data: ', lecture_audio_ser.data)
return render(request, "LectureSummarizationApp/RecordLecture.html") return render(request, "LectureSummarizingApp/RecordLecture.html")
# Views used in Lecture Summarization # Views used in Lecture Summarization
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment