feat(useCaseDiagramModelDetection) create end point for save text associated...

feat(useCaseDiagramModelDetection) create end point for save text associated with the use cases and actors

backend/app.py

@@ -21,6 +21,8 @@ OUTPUTS_GENERATED_CLASS_FILES_PATH = os.path.join('outputs', 'generated_class_fi
 OUTPUTS_FOLDER = os.path.join(APP_ROOT, 'outputs')
 UML_GENERATOR_UPLOAD_FOLDER = os.path.join(APP_ROOT, 'uploads')
 SUBMISSION_PATH = os.path.join(APP_ROOT, 'submissions/use_case')
+USE_CASE_SAVED_MODEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/use_case/model')
+USE_CASE_SAVED_LABEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/use_case/label')
 app = Flask(__name__, static_folder='outputs')
 CORS(app)
 

backend/models/actor.py

@@ -9,8 +9,8 @@ class Actor(db.Model):
     y_min = db.Column(db.String(50), nullable=False)
     x_max = db.Column(db.String(50), nullable=False)
     y_max = db.Column(db.String(50), nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
 
     def __repr__(self) -> str:
         return 'Actor>>> {self.content}'
\ No newline at end of file

backend/models/use_case.py

@@ -9,8 +9,8 @@ class UseCase(db.Model):
     y_min = db.Column(db.String(50), nullable=False)
     x_max = db.Column(db.String(50), nullable=False)
     y_max = db.Column(db.String(50), nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
 
     def __repr__(self) -> str:
         return 'UseCase>>> {self.code}'

backend/requirements.txt

 attrs==21.4.0

backend/routes/submission_routes.py

@@ -2,8 +2,11 @@ import json
 
 from flask import request, jsonify, Blueprint
 from flask_jwt_extended import jwt_required, get_jwt_identity
+
 from constants.http_status_codes_constant import HTTP_400_BAD_REQUEST, HTTP_200_OK
 from services.submission_service import save_submission
+from services.use_case_model_detection_service import model_object_detection
+
 submission = Blueprint('submissions', __name__, url_prefix='/api/v1/submissions')
 
 
@@ -21,10 +24,11 @@ def upload_submission():
     if submission_type is None or image is None or assignment_id is None:
         return jsonify({'err': 'invalid request '}), HTTP_400_BAD_REQUEST
     elif submission_type == 'use case':
-        save_submission(assignment_id, image, submission_type, comment, user_id)
-        return HTTP_200_OK
+        use_case_obj = save_submission(assignment_id, image, submission_type, comment, user_id)
+        model_object_detection(image.filename, use_case_obj.id)
+        return jsonify({'filename': image.filename}), HTTP_200_OK
     elif submission_type == 'class':
-        save_submission(assignment_id, image, submission_type, comment, user_id)
-        return HTTP_200_OK
+        class_obj = save_submission(assignment_id, image, submission_type, comment, user_id)
+        return jsonify({'id': str(class_obj.id)}), HTTP_200_OK
     else:
         return jsonify({'err': 'invalid request '}), HTTP_400_BAD_REQUEST

backend/services/submission_service.py

 import os
 import app
-from datetime import datetime
 
 from config.database import db
 from models.class_diagram_answer import ClassAnswer
@@ -8,8 +7,6 @@ from models.use_case_answer import UseCaseAnswer
 
 
 def save_submission(assignment_id, image, submission_type, comment, user_id):
-    # date_now = str(datetime.now())
-    # file_name = image.filename + '-' + date_now+'.jpg'
 
     if submission_type == 'use case':
         image.save(os.path.join(app.SUBMISSION_PATH, image.filename))
@@ -17,9 +14,11 @@ def save_submission(assignment_id, image, submission_type, comment, user_id):
                                      comments=comment)
         db.session.add(use_case_obj)
         db.session.commit()
+        return use_case_obj
     else:
         image.save(os.path.join(app.SUBMISSION_PATH, image.filename))
         class_obj = ClassAnswer(user=user_id, assignment=assignment_id, file_name=image.filename,
                                 comments=comment)
         db.session.add(class_obj)
         db.session.commit()
+        return class_obj

backend/services/use_case_model_detection_service.py

+import operator
+import re
+
+import cv2
+import numpy as np
+import pytesseract as pytesseract
+from PIL import Image
+
+from object_detection.utils import label_map_util
+
+import app
+import tensorflow as tf
+
+from config.database import db
+from models.actor import Actor
+from models.use_case import UseCase
+
+# pytesseract.pytesseract.tesseract_cmd = 'C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe'
+
+
+def model_object_detection(filename, use_case_id):
+    detect_fn = tf.saved_model.load(app.USE_CASE_SAVED_MODEL_PATH)
+    category_index = label_map_util.create_category_index_from_labelmap(
+        app.USE_CASE_SAVED_LABEL_PATH + "/label_map.pbtxt",
+        use_display_name=True)
+    image_np = np.array(Image.open(app.SUBMISSION_PATH + '/' + filename))
+    input_tensor = tf.convert_to_tensor(image_np)
+
+    input_tensor = input_tensor[tf.newaxis, ...]
+
+    detections = detect_fn(input_tensor)
+
+    num_detections = int(detections.pop('num_detections'))
+    detections = {key: value[0, :num_detections].numpy()
+                  for key, value in detections.items()}
+    detections['num_detections'] = num_detections
+
+    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
+
+    accurate_indexes = [k for k, v in enumerate(detections['detection_scores']) if (v > 0.4)]
+
+    class_id = operator.itemgetter(*accurate_indexes)(detections['detection_classes'])
+    boxes = detections['detection_boxes']
+    text_extraction(filename, class_id, boxes, accurate_indexes, category_index, use_case_id)
+
+
+def text_extraction(filename, class_id, boxes, accurate_indexes, category_index, use_case_id):
+    image = cv2.imread(app.SUBMISSION_PATH + '/' + filename)
+    for i in range(0, len(accurate_indexes)):
+        if category_index[class_id[i]]['name'] != "relationship":
+
+            height, width, c = image.shape
+
+            ymin = boxes[i][0] * height
+            xmin = boxes[i][1] * width
+            ymax = boxes[i][2] * height
+            xmax = boxes[i][3] * width
+
+            crop_img = image[int(ymin):int(ymax), int(xmin):int(xmax)]
+
+            gray_img = cv2.cvtColor(crop_img, cv2.COLOR_BGR2GRAY)
+
+            thresh, bw_img = cv2.threshold(gray_img, 160, 255, cv2.THRESH_TOZERO)
+
+            resize_img = cv2.resize(bw_img, None, fx=1, fy=1)
+
+            ocr_result = pytesseract.image_to_string(resize_img, config='1 eng --oem 1 --psm 13')
+            result = ocr_result.strip().replace("\\", "")
+            text = re.sub("=|,", "", result)
+
+            if category_index[class_id[i]]['name'] == 'actor':
+                actor_obj = Actor(use_case_answer=use_case_id, text=text, x_min=xmin, y_min=ymin, x_max=xmax,
+                                  y_max=ymax)
+                db.session.add(actor_obj)
+                db.session.commit()
+            else:
+                use_case_obj = UseCase(use_case_answer=use_case_id, text=text, x_min=xmin, y_min=ymin, x_max=xmax,
+                                       y_max=ymax)
+                db.session.add(use_case_obj)
+                db.session.commit()

backend/tensorflow_models/use_case/label/label_map.pbtxt

+item {
+    id: 1
+    name: 'actor'
+}
+
+item {
+    id: 2
+    name: 'relationship'
+}
+
+item {
+    id: 3
+    name: 'use case'
+}
\ No newline at end of file