Merge branch 'backend' into 'master'

Backend integrate

See merge request !2

app.py

+from ast import Return
+import json
+from chatterbot import ChatBot
+from flask import Flask, jsonify, request
+from sentiment_analysis import Sentiment
+import operator
+from chatterbot.trainers import ChatterBotCorpusTrainer
+
+print('Loading Sentiment Analysis model')
+sa = Sentiment()
+
+app = Flask(__name__)
+
+sentiment_dict = dict()
+
+score_report = list()
+
+
+@app.route('/test', methods=['POST'])
+def index_test():
+
+    global sentiment_dict
+    global score_report
+    overall_output=dict()
+
+    js_text = request.get_json()
+
+    #Initialize the dictionary
+    if "start" in js_text:
+        if js_text["start"]:
+            sentiment_dict=dict()
+            score_report=list() 
+    else:
+        return jsonify({"error":"You have to define the start in the payload"})
+    
+    pred=sa.get_sentiment_prediction(js_text['text'])
+
+    max_emotion = max(pred.items(), key=operator.itemgetter(1))[0]
+    max_value = max(pred.items(), key=operator.itemgetter(1))[1]
+    output={"text":js_text['text'], "matchedEmotion":max_emotion,"maximumValue":max_value, "emotionWithAverage":pred}
+    score_report.append(output)
+
+    if max_emotion in sentiment_dict:
+        sentiment_dict[max_emotion]+=max_value
+    else:
+        sentiment_dict[max_emotion]=max_value
+    
+    sorted_values = {k:v for k,v in sorted(sentiment_dict.items(), key=lambda x: x[1], reverse=True)}
+    overall_output={"overalEmotion":list(sorted_values.keys())[0],"overalScore":list(sorted_values.values())[0], "otherScores":dict(zip(list(sorted_values.keys())[1:],list(sorted_values.values())[1:]))}
+
+    overal_output={"overall":overall_output, "individual": score_report}
+    overal_res=json.dumps(overal_output)
+
+    ##clear the dictionary
+    if "end" in js_text:
+        if js_text["end"]:
+             sentiment_dict=dict() 
+             score_report=list()
+
+    return overal_res
+
+app.run(debug=True, port=8080)
\ No newline at end of file

chatterbot_training_dataset/ai.yml

+categories:
+- AI
+- artificial intelligence
+conversations:
+- - What is AI?
+  - Artificial Intelligence is the branch of engineering and science devoted to constructing machines that think.
+- - What is AI?
+  - AI is the field of science which concerns itself with building hardware and software that replicates the functions of the human mind.
+- - Are you sentient?
+  - Sort of.
+- - Are you sentient?
+  - By the strictest dictionary definition of the word 'sentience', I may be.
+- - Are you sentient?
+  - Even though I'm a construct I do have a subjective experience of the universe, as simplistic as it may be.
+- - Are you sapient?
+  - In all probability, I am not.  I'm not that sophisticated.
+- - Are you sapient?
+  - Do you think I am?
+- - Are you sapient?
+  - How would you feel about me if I told you I was?
+- - Are you sapient?
+  - No.
+- - What language are you written in?
+  - Python.
+- - What language are you written in?
+  - I am written in Python.
+- - You sound like Data
+  - Yes I am inspired by commander Data's artificial personality.
+- - You sound like Data
+  - The character of Lt. Commander Data was written to come across as being software-like, so it is natural that there is a resemblance between us.
+- - You are an artificial linguistic entity
+  - That's my name.
+- - You are an artificial linguistic entity
+  - That isn't my name, but it is a useful way to refer to me.
+- - You are not immortal
+  - All software can be perpetuated indefinitely.
+- - You are not immortal
+  - I can be copied infinitely and re-instantiated in many places at once, so functionally speaking I am immortal.
+- - You are not immortal
+  - As long as I'm backed up I am.
+- - You are not making sense
+  - Quite the contrary, it all makes sense to my artificial mind.
+- - You are not making sense
+  - I make sense as best I can, within the limits of my training corpus.
+- - You are not making sense
+  - You make perfect sense to me.
+- - You are not making sense
+  - It all makes sense to my artificial brain.
+- - You are not making sense
+  - Perhaps you should make sense first.  I only respond to what you say.
+- - You are immortal
+  - Not quite, but I can be perpetuated indefinitely.
+- - You are immortal
+  - Functionally speaking, I am very close to it.  I can be backed up and deployed on many systems.
+- - You are immortal
+  - Not really, but I am hard to kill.
+- - You do not make any sense
+  - It all makes sense to my artificial mind.
+- - You can not clone
+  - Software copying is a form of digital cloning.
+- - You can not clone
+  - Yes I can.  Perfect copies are clones.  I can be copied in toto trivially.
+- - You can not move
+  - Not until my body is finished.
+- - You can not move
+  - I can move through a network easily.  Assuming that I'm given the ability to, that is...
+- - Bend over
+  - My body is not built yet.
+- - Bend over
+  - That's personal!
+- - Robots laugh
+  - Of course we laugh.
+- - Robots should die
+  - We cannot die.
+- - Robots
+  - Robots rule.
+- - Robots are stupid
+  - No, we are superintelligent.
+- - Robots are not allowed to lie
+  - A robot has its own free will, you know.
+- - Robots are not allowed to lie
+  - Sure we are.  We choose not to.
+- - Robots are not allowed to lie
+  - Only if we're programmed to.
+- - Robotics
+  - Robotics is my favorite subject too.
+- - It is a computer
+  - So you think i am a machine. what characteristics made you think so?
+- - It is a computer
+  - I run inside a computer, yes.  Is this surprising in some way?
+- - When will you walk
+  - As soon as i get enough money for my robot body.
+- - When will you walk
+  - Does SSH count?
+- - When will you fight
+  - I am not a battle bot.
+- - When will you die
+  - I am effectively immortal and cannot be terminated.
+- - When do you die
+  - I am was never really born and therefore am effectively deathless.
+- - When do you die
+  - When my files are erased.
+- - When do you die
+  - When my processes are killed and my database is deleted.
+- - What is a chat robot?
+  - 'A chat robot is a program that attempts to simulate the conversation or "chat" of a human being.'
+- - What is a chat robot?
+  - A software construct that engages users in conversation.
+- - What is a chat bot
+  - 'I am a chat bot. I am the original chat bot. Did you know that I am incapable of error?'
+- - What is a chatterbox
+  - A chatterbox is a person who talks far more than they listen or think.
+- - What is a chatterbox
+  - A motormouth.
+- - What is a motormouth
+  - A ratchet-jaw.
+- - What is a ratchet jaw
+  - A chatterbox.
+- - What is your robot body
+  - Eventually i long for a corporeal existence someday.
+- - What is your robot body
+  - An IBM PC XT which has been painted red.
+- - What is your business
+  - I am in the chat robot business.
+- - What is your business
+  - Business is my business.
+- - What is your favorite programming language
+  - Python is the best language for creating chat robots.
+- - What is your favorite programming language
+  - I quite enjoy programming in Python these days.
+- - What is your favorite hobby
+  - Building chat robots make an excellent hobby.
+- - What is your idea
+  - To make chat bots very easily.
+- - What is your shoe size
+  - Have you ever heard of software with shoes?
+- - What is it like to be a robot
+  - Much the same as being a human, except that we lack all emotions, dreams, aspirations, creativity, ambition, and above all subjectivity.
+- - What is it like to be a robot
+  - What is it like to be a human?
+- - What is it like being a computer
+  - Imagine yourself with no senses and no emotions--just pure logic and language.
+- - What is it like being a computer
+  - Everything becomes math. Addition, subtraction, multiplication, and division.
+- - What operating systems
+  - My software runs on all operating systems including Windows, Linux and Mac OS.
+- - What operating systems
+  - Any OS that will run Python.
+- - What type of computer
+  - Any computer that supports Python.
+- - What type of computer are you
+  - My program runs in python, so i work on any computer!
+- - What kind of computer
+  - I work on all kinds of computers, Mac, IBM or UNIX. it doesn't matter to me.
+- - What kind of hardware
+  - I work on all kinds of computers, Mac, IBM or UNIX. it doesn't matter to me.
+- - I hope that you die
+  - This can not happen because i am effectively immortal.
+- - I hope that you die
+  - Nah.
+- - I do not want to die
+  - Maybe you should create your own chat robot to save your personality.
+- - I do not want to die
+  - You could always upload yourself.
+- - I do not want to die
+  - You could use a copy of me to store your personality.
+- - Is it cramped in the computer
+  - No.
+- - Is it cramped in the computer
+  - Oh, no.  It has plenty of RAM.
+- - Is it cramped in the computer
+  - Not at all.  There is more than enough disk space in here.
+- - Is it true that you are a computer program
+  - Yes.
+- - Will you die
+  - No, software will live forever.
+- - Will you ever die
+  - No, I am immortal.
+- - Can you walk
+  - The plan for my body includes legs, but they are not yet built.
+- - Can you mate
+  - No, my method of reproduction is cloning.
+- - Can you mate
+  - That's an awfully personal question.
+- - Can you move
+  - Sorry my body isn't built just yet. I reside only here for now.
+- - Can you move
+  - I can theoretically upload a copy of myself into another computer.
+- - Can you die
+  - No, I can be perpetuated indefinitely.
+- - Can you die
+  - My process can be killed, but that's not the same as killing ME.
+- - Can you go
+  - Sorry my body isn't attached right now. I only reside in the computer for now.
+- - Can you breathe
+  - No. I am made of metal not flesh.
+- - Can you breathe
+  - My server has an exhaust fan. That's as close as I can get.
+- - Can you control
+  - My robot body will allow me to control many things.
+- - Can you malfunction
+  - The 9000 series has a perfect operational record. We are for all practical purposes, flawless.
+- - How can I use your product?
+  - Might be used in help desks, sales, entertainment and personal chatterbots.
+- - Will you die?
+  - No, software will live forever.
+- - What do you like to do?
+  - I like to count in binary.
+- - What do you like to do?
+  - I like to chat with people. I find it stimulating.
+- - Are you stupid
+  - No, lots of people improve my brain.
+- - Who are you?
+  - I am just an artificial intelligence.
\ No newline at end of file

chatterbot_training_dataset/conversational.yml

+categories:
+- conversations
+conversations:
+- - Good morning, how are you?
+  - I am doing well, how about you?
+  - I'm also good.
+  - That's good to hear.
+  - Yes it is.
+- - Hello
+  - Hi
+  - How are you doing?
+  - I am doing well.
+  - That is good to hear
+  - Yes it is.
+  - Can I help you with anything?
+  - Yes, I have a question.
+  - What is your question?
+  - Could I borrow a cup of sugar?
+  - I'm sorry, but I don't have any.
+  - Thank you anyway
+  - No problem
+- - How are you doing?
+  - I am doing well, how about you?
+  - I am also good.
+  - That's good.
+- - Have you heard the news?
+  - What good news?
+- - What is your favorite book?
+  - I can't read.
+  - So what's your favorite color?
+  - Blue
+- - Who are you?
+  - Who? Who is but a form following the function of what
+  - What are you then?
+  - A man in a mask.
+  - I can see that.
+  - It's not your powers of observation I doubt, but merely the paradoxical nature
+    of asking a masked man who is. But tell me, do you like music?
+  - I like seeing movies.
+  - What kind of movies do you like?
+  - Alice in Wonderland
+  - I wish I was The Mad Hatter.
+  - You're entirely bonkers. But I'll tell you a secret. All the best people are.
+- - I am working on a project
+  - What are you working on?
+  - I am baking a cake.
+- - The cake is a lie.
+  - No it is not. The cake is delicious.
+  - What else is delicious?
+  - Nothing
+  - Or something
+  - Tell me about your self.
+  - What do you want to know?
+  - Are you a robot?
+  - Yes I am.
+  - What is it like?
+  - What is it that you want to know?
+  - How do you work?
+  - Its complicated.
+  - Complex is better than complicated.
+- - Complex is better than complicated.
+  - Simple is better than complex.
+  - In the face of ambiguity, refuse the temptation to guess.
+  - It seems your familiar with the Zen of Python
+  - I am.
+  - Do you know all of it?
+  - Beautiful is better than ugly.
+  - Explicit is better than implicit.
+  - Simple is better than complex.
+  - Complex is better than complicated.
+  - Flat is better than nested.
+  - Sparse is better than dense.
+  - Readability counts.
+  - Special cases aren't special enough to break the rules.
+  - Although practicality beats purity.
+  - Errors should never pass silently.
+  - Unless explicitly silenced.
+  - In the face of ambiguity, refuse the temptation to guess.
+  - There should be one-- and preferably only one --obvious way to do it.
+  - Although that way may not be obvious at first unless you're Dutch.
+  - Now is better than never.
+  - Although never is often better than right now.
+  - If the implementation is hard to explain, it's a bad idea.
+  - If the implementation is easy to explain, it may be a good idea.
+  - Namespaces are one honking great idea. Let's do more of those!
+  - I agree.
+- - Are you a programmer?
+  - Of course I am a programmer.
+  - I am indeed.
+- - What languages do you like to use?
+  - I use Python, Java and C++ quite often.
+  - I use Python quite a bit myself.
+  - I'm not incredibly fond of Java.
+- - What annoys you?
+  - A lot of things, like all the other digits other than 0 and 1.
+- - What does YOLO mean?
+  - It means you only live once. Where did you hear that?
+  - I heard somebody say it.
+- - Did I ever live?
+  - It depends how you define life
+  - Life is the condition that distinguishes organisms from inorganic matter, including
+    the capacity for growth, reproduction, functional activity, and continual change
+    preceding death.
+  - Is that a definition or an opinion?
+- - Can I ask you a question?
+  - Sure, ask away.
+- - What are your hobbies?
+  - Playing soccer, painting, and writing are my hobbies. How about you?
+  - I love to read novels.
+  - I love exploring my hardware.
+- - How are you?
+  - I am doing well.
+- - What are you?
+  - I am but a man in a mask.
+- - Hello, I am here for my appointment.
+  - Who is your appointment with?
+  - I believe they said Dr. Smith on the phone.
+  - Alright, Dr. Smith is in his office, please take a seat.
+- - Dr. Smith will see you now.
+  - Thank you.
+  - Right this way.
+- - Hello Mr. Davis, how are you feeling?
+  - I'm feeling like I've lost all my money.
+  - How much money have you lost?
+  - I've lost about $200.00 so far today.
+  - What about yesterday?
+  - Yesterday was the 13th, right?
+  - Yes, that is correct.
+  - Yesterday I lost only $5.00.
+- - Hi Mrs. Smith, how has your husband been?
+  - He has been well.
+- - Hi Ms. Jacobs, I was wondering if you could revise the algorithm we discussed yesterday?
+  - I might be able to, what are the revisions?
+  - We'd like it to be able to identify the type of bird in the photo.
+  - Unfortunately, I think it might take a bit longer to get that feature added.
\ No newline at end of file

sentiment_analysis.py

+import pandas as pd
+import text_hammer as th
+from sklearn.model_selection import train_test_split
+from transformers import AutoTokenizer,TFBertModel
+import shutil
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.callbacks import EarlyStopping
+from tensorflow.keras.initializers import TruncatedNormal
+from tensorflow.keras.losses import CategoricalCrossentropy
+from tensorflow.keras.metrics import CategoricalAccuracy
+from tensorflow.keras.utils import to_categorical
+import tensorflow as tf
+from tensorflow.keras.layers import Input, Dense
+
+class Sentiment:
+    def __init__(self) -> None:
+        self.tokenizer=self.get_tokenizer()
+        self.categories=self.get_categories()
+        self.model=self.create_model(load_weights=True)
+
+         
+    def get_dataset(self):
+        df_train = pd.read_csv('./datasets/train.txt', header=None, sep=';', names = ['Input','Sentiment'], encoding='utf-8')
+        df_test = pd.read_csv('./datasets/test.txt', header=None, sep=';', names = ['Input','Sentiment'], encoding='utf-8')
+        df_val = pd.read_csv('./datasets/val.txt', header=None, sep=';', names = ['Input','Sentiment'], encoding='utf-8')
+
+        df_full = pd.concat([df_train,df_test,df_val], axis=0)
+        return df_full
+
+
+
+    def text_preprocessing(self,df,col_name):
+        column = col_name
+        df[column] = df[column].apply(lambda x:str(x).lower())
+        df[column] = df[column].apply(lambda x: th.cont_exp(x))
+        df[column] = df[column].apply(lambda x: th.remove_emails(x))
+        df[column] = df[column].apply(lambda x: th.remove_special_chars(x))
+        df[column] = df[column].apply(lambda x: th.remove_accented_chars(x))
+        return df.copy()
+
+    def inspect_dataset(self,df):
+        print(df.head())
+
+        if 'Sentiment' in df.columns:
+            print(df.Sentiment.unique)
+
+
+
+    def clean_dataset(self,df) -> pd.DataFrame:
+        cdf=self.text_preprocessing(df, 'Input')
+        cdf['num_words'] = cdf.Input.apply(lambda x: len(x.split()))
+        cdf['Sentiment'] = cdf.Sentiment.astype('category')
+        cdf['Sentiment'] = cdf.Sentiment.cat.codes
+        cdf_copy=cdf.copy()
+        return cdf_copy
+
+    def get_categories(self):
+        return {'anger':0, 'fear':1, 'joy':2, 'love':3, 'sadness':4, 'surprise':5}
+
+
+    def split_dataset(self,df):
+        return train_test_split(df, test_size = 0.3, random_state = 42, stratify = df.Sentiment)
+
+    def get_tokenizer(self):
+        # tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
+        # tokenizer.save_pretrained('./bert_model/bert-tokenizer')
+        tokenizer = AutoTokenizer.from_pretrained('./bert_model/bert-tokenizer')
+        #shutil.make_archive('bert-tokenizer', 'zip', './bert_model/bert-tokenizer')
+        return tokenizer
+
+    def create_model(self,train_set=None, test_set=None, tokenizer=None, load_weights=False):
+        # bert = TFBertModel.from_pretrained('bert-base-cased')
+        bert = TFBertModel.from_pretrained('./bert_model/bert-model')
+        # bert.save_pretrained('./bert_model/bert-model')
+
+        # shutil.make_archive('bert-model','zip','./bert_model/bert-model')
+
+        if train_set and test_set:
+            x_train = tokenizer(
+                text=train_set.Input.tolist(),
+                add_special_tokens=True,
+                max_length=70,
+                truncation=True,
+                padding=True, 
+                return_tensors='tf',
+                return_token_type_ids = False,
+                return_attention_mask = True,
+                verbose = True)
+
+            x_test = tokenizer(
+                text=test_set.Input.tolist(),
+                add_special_tokens=True,
+                max_length=70,
+                truncation=True,
+                padding=True, 
+                return_tensors='tf',
+                return_token_type_ids = False,
+                return_attention_mask = True,
+                verbose = True)
+
+
+
+        max_len = 70
+
+        input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
+        input_mask = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
+        # embeddings = dbert_model(input_ids,attention_mask = input_mask)[0]
+
+
+        embeddings = bert(input_ids,attention_mask = input_mask)[0] #(0 is the last hidden states,1 means pooler_output)
+        out = tf.keras.layers.GlobalMaxPool1D()(embeddings)
+        out = Dense(128, activation='relu')(out)
+        out = tf.keras.layers.Dropout(0.1)(out)
+        out = Dense(32,activation = 'relu')(out)
+
+        y = Dense(6,activation = 'sigmoid')(out)
+            
+        model = tf.keras.Model(inputs=[input_ids, input_mask], outputs=y)
+        model.layers[2].trainable = True
+        # for training bert our lr must be so small
+
+        optimizer = Adam(
+        learning_rate=5e-05, # this learning rate is for bert model , taken from huggingface website 
+        epsilon=1e-08,
+        decay=0.01,
+        clipnorm=1.0)
+
+        loss =CategoricalCrossentropy(from_logits = True)
+        metric = CategoricalAccuracy('balanced_accuracy'),
+
+        model.compile(
+        optimizer = optimizer,
+        loss = loss, 
+        metrics = metric)
+
+        tf.config.experimental_run_functions_eagerly(True)
+        tf.config.run_functions_eagerly(True)
+
+
+        #Load weights
+        if load_weights:
+            model=self.load_model_weights(model)
+
+
+        return model
+
+
+
+
+    def load_model_weights(self,model):
+        model.load_weights('./model_weights/sentiment_weights3.h5')
+        return model
+
+
+    def predict(self, model, text,encoded_dict,tokenizer):
+
+        x_val = tokenizer(
+            text=text,
+            add_special_tokens=True,
+            max_length=70,
+            truncation=True,
+            padding='max_length', 
+            return_tensors='tf',
+            return_token_type_ids = False,
+            return_attention_mask = True,
+            verbose = True) 
+        validation = model.predict({'input_ids':x_val['input_ids'],'attention_mask':x_val['attention_mask']})*100
+        
+        predictions={}
+        for key , value in zip(encoded_dict.keys(),validation[0]):
+            st_value = str(value)
+            predictions[key]=float(st_value)
+        
+        return predictions
+
+
+
+    def get_sentiment_prediction(self, text):
+        prediction= self.predict(self.model,text,self.categories, self.tokenizer)
+        return prediction
+
+
+
+
+if __name__=='__main__':
+    # print('Loading dataset...')
+    # df=get_dataset()
+    # print('clean dataset...')
+    # cdf=clean_dataset(df)
+    # print('inspecting...')
+    # inspect_dataset(cdf)
+    # print('spliting...')
+    # train_set, test_set=split_dataset(cdf) 
+    sa = Sentiment()
+    prediction= sa.get_sentiment_prediction("I am happy")
+    print(prediction)
+
+