Skip to content

2
2020-101
Commit 5382ffb9 authored by I.K Seneviratne's avatar I.K Seneviratne
Browse files
Options

Merge branch 'IT17100908' into 'QA_RELEASE' 

It17100908

See merge request !45
parents 27052e66 08f6d91b
Show whitespace changes
Inline Side-by-side
Showing with 222 additions and 163 deletions
LectureSummarizingApp/api.py
View file @ 5382ffb9
from rest_framework.views import APIView
from rest_framework.response import Response
from FirstApp.logic.id_generator import generate_new_id
from LectureSummarizingApp.models import LectureAudio, LectureAudioNoiseRemoved, LectureSpeechToText, \
LectureAudioSummary, LectureNotices
from LectureSummarizingApp.serializer import LectureAudioSerializer, LectureAudioNoiseRemovedSerializer, \
LectureSpeechToTextSerializer, LectureAudioSummarySerializer, LectureNoticesSerializer
from . import speech_to_text as stt
from . import noiseRemove as nr
import datetime
# APIs used in Lecture Summarizing Component
from .noise import noise_removal
class LectureAudioAPI(APIView):
def get(self, request):
......@@ -21,9 +28,36 @@ class LectureAudioAPI(APIView):
class audioNoiseRemovedList(APIView):
def get(self, request):
lecture_audio_noise_removed = LectureAudioNoiseRemoved.objects.all()
serializer = LectureAudioNoiseRemovedSerializer(lecture_audio_noise_removed, many=True)
return Response(serializer.data)
# lecture_audio_noise_removed = LectureAudioNoiseRemoved.objects.all()
# serializer = LectureAudioNoiseRemovedSerializer(lecture_audio_noise_removed, many=True)
audio_noise_removed_list = LectureAudioNoiseRemoved.objects.order_by('lecture_audio_noise_removed_id').last()
audio_name = request.query_params.get("audio_name")
id = int(request.query_params.get("id"))
current_date = datetime.datetime.now().date()
fake_duration = datetime.timedelta(minutes=2, seconds=10, milliseconds=00)
# generate new id for audio noise removed
new_audio_noise_removed_id = generate_new_id(audio_noise_removed_list.lecture_audio_noise_removed_id)
# nr.noise_removalll(video_name)
noise_removal(audio_name)
LectureAudioNoiseRemoved(
lecture_audio_noise_removed_id=new_audio_noise_removed_id,
lecture_audio_id_id=id,
lecturer_date=current_date,
lecture_audio_name=audio_name,
lecture_audio_length=fake_duration
).save()
return Response({
"response": Response.status_code
})
def post(self, request):
LectureAudioNoiseRemoved(
......@@ -43,11 +77,14 @@ class audioToTextList(APIView):
serializer = LectureSpeechToTextSerializer(lecture_speech_to_text_id, many=True)
# return Response(serializer.data)
video_name = request.query_params.get("video_name")
print('video name: ', video_name)
# video_name = request.query_params.get("video_name")
#
# print('video name: ', video_name)
#
# # nr.noise_removalll(video_name)
# noise_removal(video_name)
stt.speech_to_text(video_name)
# stt.speech_to_text(video_name)
return Response({
"response": "successful"
......
LectureSummarizingApp/noise.py
View file @ 5382ffb9
import librosa
from pysndfx import AudioEffectsChain
import python_speech_features
import os
def noise_removal(video_name):
def read_file(file_name):
sample_file = file_name
sample_directory = 'lectures/'
sample_path = sample_directory + sample_file
# sample_file = file_name
# sample_directory = 'lectures/'
# sample_path = sample_directory + sample_file
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LECTURE_VIDEO_DIR = os.path.join(BASE_DIR, "LectureSummarizingApp\\lectures\\{}".format(video_name))
print('lecture audio directory: ', LECTURE_VIDEO_DIR)
# DESTINATION_DIR = os.path.join(BASE_DIR, "LectureSummarizingApp\\noise_removed_lectures")
DESTINATION_DIR = os.path.dirname(os.path.join(BASE_DIR, "LectureSummarizingApp\\noise_removed_lectures\\sample.txt"))
print('destination directory: ', DESTINATION_DIR)
# generating audio time series and a sampling rate (int)
a, sr = librosa.load(sample_path)
a, sr = librosa.load(path=LECTURE_VIDEO_DIR)
print('a: ', a)
print('sr: ', sr)
# speech_boosted = mffc_highshelf(a, sr)
output_file(destination=DESTINATION_DIR, filename=video_name, a=a, sr=sr)
return a, sr
......@@ -24,6 +41,7 @@ def mffc_highshelf(a, sr):
mfcc = python_speech_features.base.logfbank(a)
mfcc = python_speech_features.base.lifter(mfcc)
sum_of_squares = []
index = -1
for r in mfcc:
......@@ -40,73 +58,49 @@ def mffc_highshelf(a, sr):
speech_booster = AudioEffectsChain().highshelf(frequency=min_hz*(-1)*1.2, gain=-12.0, slope=0.6).limiter(gain=8.0)
a_speach_boosted = speech_booster(a)
# a_speach_boosted = speech_booster.
return (a_speach_boosted)
def mfcc_lowshelf(a, sr):
mfcc = python_speech_features.base.mfcc(a)
mfcc = python_speech_features.base.logfbank(a)
mfcc = python_speech_features.base.lifter(mfcc)
sum_of_squares = []
index = -1
for x in mfcc:
sum_of_squares.append(0)
index = index + 1
for n in x:
sum_of_squares[index] = sum_of_squares[index] + n**2
strongest_frame = sum_of_squares.index(max(sum_of_squares))
hz = python_speech_features.base.mel2hz(mfcc[strongest_frame])
max_hz = max(hz)
min_hz = min(hz)
speech_booster = AudioEffectsChain().lowshelf(frequency=min_hz*(-1), gain=12.0, slope=0.5)
a_speach_boosted = speech_booster(a)
return (a_speach_boosted)
def trim_silence(y):
a_trimmed, index = librosa.effects.trim(y, top_db=20, frame_length=2, hop_length=500)
trimmed_length = librosa.get_duration(y) - librosa.get_duration(a_trimmed)
return a_trimmed, trimmed_length
# def trim_silence(y):
# a_trimmed, index = librosa.effects.trim(y, top_db=20, frame_length=2, hop_length=500)
# trimmed_length = librosa.get_duration(y) - librosa.get_duration(a_trimmed)
#
# return a_trimmed, trimmed_length
def enhance(y):
apply_audio_effects = AudioEffectsChain().lowshelf(gain=10.0, frequency=260, slope=0.1).reverb(reverberance=25, hf_damping=5, room_scale=5, stereo_depth=50, pre_delay=20, wet_gain=0, wet_only=False)#.normalize()
a_enhanced = apply_audio_effects(y)
return a_enhanced
# def enhance(y):
# apply_audio_effects = AudioEffectsChain().lowshelf(gain=10.0, frequency=260, slope=0.1).reverb(reverberance=25, hf_damping=5, room_scale=5, stereo_depth=50, pre_delay=20, wet_gain=0, wet_only=False)#.normalize()
# a_enhanced = apply_audio_effects(y)
#
# return a_enhanced
def output_file(destination ,filename, a, sr, ext=""):
destination = destination + filename[:-4] + ext + '.wav'
librosa.output.write_wav(destination, a, sr)
lectures = ['Lecture01.wav']
for s in lectures:
filename = s
a, sr = read_file(filename)
# a_reduced_centroid_s = reduce_noise_centroid_s(a, sr)
a_reduced_mfcc_lowshelf = mfcc_lowshelf(a, sr)
a_reduced_mfcc_highshelf = mffc_highshelf(a, sr)
# trimming silences
# a_reduced_centroid_s, time_trimmed = trim_silence(a_reduced_centroid_s)
a_reduced_mfcc_up, time_trimmed = trim_silence(mfcc_lowshelf)
a_reduced_mfcc_down, time_trimmed = trim_silence(mffc_highshelf)
# output_file('lectures_trimmed_noise_reduced/' ,filename, y_reduced_centroid_s, sr, '_ctr_s')
output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_up, sr, '_mfcc_up')
# output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_down, sr, '_mfcc_down')
# output_file('lectures_trimmed_noise_reduced/' ,filename, a, sr, '_org')
# lectures = ['Lecture01.wav', 'Lecture02.wav']
# for s in lectures:
# filename = s
# a, sr = noise_removal(filename)
#
#
# # a_reduced_centroid_s = reduce_noise_centroid_s(a, sr)
# # a_reduced_mfcc_lowshelf = mfcc_lowshelf(a, sr)
# a_reduced_mfcc_highshelf = mffc_highshelf(a, sr)
#
#
# # trimming silences
# # a_reduced_centroid_s, time_trimmed = trim_silence(a_reduced_centroid_s)
# # a_reduced_mfcc_up, time_trimmed = trim_silence(mfcc_lowshelf)
# a_reduced_mfcc_down, time_trimmed = trim_silence(mffc_highshelf)
#
#
#
# # output_file('lectures_trimmed_noise_reduced/' ,filename, y_reduced_centroid_s, sr, '_ctr_s')
# #output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_up, sr, '_mfcc_up')
# output_file('lectures_trimmed_noise_reduced/' ,filename, a_reduced_mfcc_down, sr, '_mfcc_down')
# # output_file('lectures_trimmed_noise_reduced/' ,filename, a, sr, '_org')
LectureSummarizingApp/noiseRemove.py
View file @ 5382ffb9
......@@ -4,54 +4,63 @@ from scipy.io.wavfile import read
from scipy.io.wavfile import write
from scipy import signal
import matplotlib.pyplot as mplt
import os
#get_ipython().magic('matplotlib inline')
(Frequency, array) = read('lectures/Lecture01.wav')
len(array)
def noise_removalll(video_name):
mplt.plot(array)
mplt.title('Original Signal Spectrum')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LECTURE_VIDEO_DIR = os.path.join(BASE_DIR, "LectureSummarizingApp\\lectures\\{}".format(video_name))
fourierTransformation = sip.fft(array)
# (Frequency, array) = read('lectures/Lecture01.wav')
# (Frequency, array) = read('lectures/{}'.format(video_name))
(Frequency, array) = read(LECTURE_VIDEO_DIR)
scale = sip.linspace(0, Frequency, len(array))
len(array)
mplt.stem(scale[0:5000], nump.abs(fourierTransformation[0:5000]), 'r')
mplt.title('Signal spectrum after FFT')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
mplt.plot(array)
mplt.title('Original Signal Spectrum')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
fourierTransformation = sip.fft(array)
guassianNoise = nump.random.rand(len(fourierTransformation))
scale = sip.linspace(0, Frequency, len(array))
mplt.stem(scale[0:5000], nump.abs(fourierTransformation[0:5000]), 'r')
mplt.title('Signal spectrum after FFT')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
NewSound = guassianNoise + array
write("New-Sound-Added-With-Guassian-Noise.wav", Frequency, NewSound)
guassianNoise = nump.random.rand(len(fourierTransformation))
u,v = signal.butter(5, 1000/(Frequency/2), btype='highpass')
filteredSignal = signal.lfilter(u,v,NewSound)
NewSound = guassianNoise + array
# plotting the signal.
mplt.plot(filteredSignal)
mplt.title('Highpass Filter')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
write("New-Sound-Added-With-Guassian-Noise.wav", Frequency, NewSound)
# ButterWorth low-filter
x,y = signal.butter(5, 380/(Frequency/2), btype='lowpass')
u,v = signal.butter(5, 1000/(Frequency/2), btype='highpass')
# Applying the filter to the signal
newFilteredSignal = signal.lfilter(x,y,filteredSignal)
filteredSignal = signal.lfilter(u,v,NewSound)
# plotting the signal.
mplt.plot(newFilteredSignal)
mplt.title('Lowpass Filter')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
# plotting the signal.
mplt.plot(filteredSignal)
mplt.title('Highpass Filter')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
write("removed.wav", Frequency, nump.int16(newFilteredSignal/nump.max(nump.abs(newFilteredSignal)) * 32767))
\ No newline at end of file
# ButterWorth low-filter
x,y = signal.butter(5, 380/(Frequency/2), btype='lowpass')
# Applying the filter to the signal
newFilteredSignal = signal.lfilter(x,y,filteredSignal)
# plotting the signal.
mplt.plot(newFilteredSignal)
mplt.title('Lowpass Filter')
mplt.xlabel('Frequency(Hz)')
mplt.ylabel('Amplitude')
write("removed.wav", Frequency, nump.int16(newFilteredSignal/nump.max(nump.abs(newFilteredSignal)) * 32767))
\ No newline at end of file
LectureSummarizingApp/templates/LectureSummarizingApp/RecordLecture.html
View file @ 5382ffb9
{% extends 'FirstApp/template.html' %}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Lecture Recording</title>
</head>
<body>
% block javascript %}
{% load static %}
<!-- Bootstrap core JavaScript-->
<script src="{% static 'FirstApp/vendor/jquery/jquery.min.js' %}"></script>
<script src="{% static 'FirstApp/vendor/bootstrap/js/bootstrap.bundle.min.js' %}"></script>
<!-- Page level plugins -->
<script src="{% static 'FirstApp/vendor/datatables/jquery.dataTables.min.js' %}"></script>
<script src="{% static 'FirstApp/vendor/datatables/dataTables.bootstrap4.min.js' %}"></script>
<!-- Page level custom scripts -->
<script src="{% static 'FirstApp/js/demo/datatables-demo.js' %}"></script>
<!-- Core plugin JavaScript-->
<script src="{% static 'FirstApp/vendor/jquery-easing/jquery.easing.min.js' %}"></script>
<!-- Load TensorFlow.js -->
<script src="https://unpkg.com/@tensorflow/tfjs"></script>
<!-- Load Posenet -->
<script src="https://unpkg.com/@tensorflow-models/posenet">
</script>
{% endblock %}
<div id="wrapper">
<div id="content-wrapper" class="d-flex flex-column">
<div id="content">
{% block 'container-fluid' %}
<div class="container-fluid">
{% load static %}
<div class="d-sm-flex align-items-center justify-content-between mb-4">
<h1 class="h3 mb-0 text-gray-800">Lecture Record</h1>
</div>
<div>
<button TYPE="button" class="btn btn-success audio_process">Start Recording</button>
</div>
</div>
</div>
</div>
</div>
</body>
</html>
\ No newline at end of file
<!--{% extends 'FirstApp/template.html' %}-->
<!--<!DOCTYPE html>-->
<!--<html lang="en">-->
<!--<head>-->
<!-- <meta charset="UTF-8">-->
<!-- <title>Lecture Recording</title>-->
<!--</head>-->
<!--<body>-->
<!--% block javascript %}-->
<!--{% load static %}-->
<!--&lt;!&ndash; Bootstrap core JavaScript&ndash;&gt;-->
<!--<script src="{% static 'FirstApp/vendor/jquery/jquery.min.js' %}"></script>-->
<!--<script src="{% static 'FirstApp/vendor/bootstrap/js/bootstrap.bundle.min.js' %}"></script>-->
<!--&lt;!&ndash; Page level plugins &ndash;&gt;-->
<!--<script src="{% static 'FirstApp/vendor/datatables/jquery.dataTables.min.js' %}"></script>-->
<!--<script src="{% static 'FirstApp/vendor/datatables/dataTables.bootstrap4.min.js' %}"></script>-->
<!--&lt;!&ndash; Page level custom scripts &ndash;&gt;-->
<!--<script src="{% static 'FirstApp/js/demo/datatables-demo.js' %}"></script>-->
<!--&lt;!&ndash; Core plugin JavaScript&ndash;&gt;-->
<!--<script src="{% static 'FirstApp/vendor/jquery-easing/jquery.easing.min.js' %}"></script>-->
<!--&lt;!&ndash; Load TensorFlow.js &ndash;&gt;-->
<!--<script src="https://unpkg.com/@tensorflow/tfjs"></script>-->
<!--&lt;!&ndash; Load Posenet &ndash;&gt;-->
<!--<script src="https://unpkg.com/@tensorflow-models/posenet">-->
<!--</script>-->
<!--{% endblock %}-->
<!--<div id="wrapper">-->
<!-- <div id="content-wrapper" class="d-flex flex-column">-->
<!-- <div id="content">-->
<!-- {% block 'container-fluid' %}-->
<!-- <div class="container-fluid">-->
<!-- {% load static %}-->
<!-- <div class="d-sm-flex align-items-center justify-content-between mb-4">-->
<!-- <h1 class="h3 mb-0 text-gray-800">Lecture Record</h1>-->
<!-- </div>-->
<!-- <div>-->
<!-- <button TYPE="button" class="btn btn-success audio_process">Start Recording</button>-->
<!-- </div>-->
<!-- </div>-->
<!-- </div>-->
<!-- </div>-->
<!--</div>-->
<!--</body>-->
<!--</html>-->
\ No newline at end of file
LectureSummarizingApp/templates/LectureSummarizingApp/summarization.html
View file @ 5382ffb9
......@@ -48,18 +48,37 @@
});
<!-- background noise-->
$('.audio_process').click(function() {
alert('Processing');
$('.audio_process').click(function(e) {
let id = e.target.parentNode.parentNode.getAttribute('id');
let audio_name = e.target.parentNode.parentNode.getAttribute('data-audio-name');
audio_name = audio_name + ".wav";
alert('id: ' + id);
alert('audio name: ' + audio_name);
//call the fetch API
fetch('http://127.0.0.1:8000/summary/lecture-audio-to-text/?video_name=Lecture01.wav')
<!-- fetch('http://127.0.0.1:8000/summary/lecture-audio-to-text/?audio_name=')-->
<!-- .then((res) => res.json())-->
<!-- .then((out) => alert(out.response))-->
<!-- .catch((err) => alert('error: ' + err))-->
fetch('http://127.0.0.1:8000/summary/lecture-audio-noise-removed/?audio_name=' + audio_name + '&id=' + id)
.then((res) => res.json())
.then((out) => alert(out.response))
.then((out) => handleNoiseRemoved(out.response))
.catch((err) => alert('error: ' + err))
});
//this function will handle the success respopnse for noise removed
function handleNoiseRemoved(response) {
if (response === 200) {
document.location.reload();
}
}
<!-- To summary-->
$('.to_summary').click(function() {
alert('Processing');
......@@ -121,7 +140,7 @@
<tbody>
{% for lec_audio in lec_audio_data %}
<tr class="recordings not_clicked" id="{{ lec_audio.lecture_audio_id }}">
<tr class="recordings not_clicked" id="{{ lec_audio.id }}" data-audio-name="{{ lec_audio.lecture_audio_name }}">
<!-- <td>-->
<!-- <div class="radio">-->
<!-- <label><input type="radio"-->
......
LectureSummarizingApp/urls.py
View file @ 5382ffb9
......@@ -8,7 +8,7 @@ router = routers.DefaultRouter()
urlpatterns = [
path('lecture', views.summarization),
path('record', views.lectureRecord),
# path('record', views.lectureRecord),
# API to retrieve lecture summarizing details
......
LectureSummarizingApp/views.py
View file @ 5382ffb9
......@@ -13,7 +13,7 @@ def lectureRecord(request):
print('lecture record data: ', lecture_audio_ser.data)
return render(request, "LectureSummarizationApp/RecordLecture.html")
return render(request, "LectureSummarizingApp/RecordLecture.html")
# Views used in Lecture Summarization
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment