bert usage

Indexing/bert.py

+import nltk
+import question_generator as q_gen
+
+# nltk.download('words')
+
+from bertopic import BERTopic
+from nltk.corpus import words
+
+model = BERTopic(verbose=True)
+
+
+def get_topics(file):
+    topics_outputs = []
+
+    docs = []
+    with open(file) as file:
+        for line in file:
+            docs.append(line.rstrip())
+
+    topics, probabilities = model.fit_transform(docs)
+    #
+    print(model.get_topic_freq())
+    #
+    print('done')
+    #
+    # model.get_topic_freq().head(11)
+
+    print(model.get_topics())
+
+    for i in model.get_topic(0):
+        if i[0] in words.words():
+            pass
+        else:
+            print(i[0])
+            topics_outputs.append(i[0])
+
+    return topics_outputs
+
+
+def get_topics_new(text):
+    topics = q_gen.get_keywords(text, q_gen.summarizer(text))
+    # print(topics)
+    return topics