Merge branch 'master' into 'IT19076362'

Master

See merge request !62

backend/app.py

@@ -10,6 +10,7 @@ from routes.auth_routes import auth
 from routes.module_routes import module
 from routes.assignment_routes import assignment
 from routes.diagram_routes import diagram
+from routes.submission_routes import submission
 
 APP_ROOT = os.path.dirname(os.path.abspath(__file__))
 
@@ -19,13 +20,21 @@ OUTPUTS_GENERATED_CLASS_DIAGRAMS_PATH = os.path.join('outputs', 'generated_class
 OUTPUTS_GENERATED_CLASS_FILES_PATH = os.path.join('outputs', 'generated_class_files')
 OUTPUTS_FOLDER = os.path.join(APP_ROOT, 'outputs')
 UML_GENERATOR_UPLOAD_FOLDER = os.path.join(APP_ROOT, 'uploads')
-
+SUBMISSION_PATH = os.path.join(APP_ROOT, 'submissions/use_case')
+SUBMISSION_PATH_CLASS = os.path.join(APP_ROOT, 'submissions/class')
+
+USE_CASE_SAVED_MODEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/use_case/model')
+USE_CASE_SAVED_LABEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/use_case/label')
+CLASS_SAVED_MODEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/class/model')
+CLASS_SAVED_LABEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/class/label/label_map.pbtxt')
+CLASS_COMP_SAVED_MODEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/class_component/model')
+CLASS_COMP_SAVED_LABEL_PATH = os.path.join(APP_ROOT, 'tensorflow_models/class_component/label/label_map.pbtxt')
 app = Flask(__name__, static_folder='outputs')
 CORS(app)
 
 app.config.from_mapping(SECRET_KEY=os.environ.get('SECRET_KEY'),
-                        SQLALCHEMY_DATABASE_URI=os.environ.get('SQLALCHEMY_DATABASE_URI'),
-                        SQLALCHEMY_TRACK_MODIFICATIONS=False, JWT_SECRET_KEY=os.environ.get('JWT_SECRET_KEY'))
+                        SQLALCHEMY_DATABASE_URI='sqlite:///database.db',
+                        SQLALCHEMY_TRACK_MODIFICATIONS=False, JWT_SECRET_KEY='JWT_SECRET_KEY')
 app.config['UML_GENERATOR_UPLOAD_FOLDER'] = UML_GENERATOR_UPLOAD_FOLDER
 
 db.app = app
@@ -37,6 +46,7 @@ app.register_blueprint(auth)
 app.register_blueprint(module)
 app.register_blueprint(assignment)
 app.register_blueprint(diagram)
+app.register_blueprint(submission)
 
 
 @app.before_first_request

backend/models/actor.py

-from config.database import db
-from models.use_case_answer import UseCaseAnswer
-
-
-class Actor(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    use_case_answer = db.Column(UseCaseAnswer, nullable=False)
-    text = db.Column(db.String(500), nullable=False)
-    x_min = db.Column(db.String(50), nullable=False)
-    y_min = db.Column(db.String(50), nullable=False)
-    x_max = db.Column(db.String(50), nullable=False)
-    y_max = db.Column(db.String(50), nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
\ No newline at end of file

backend/models/use_case.py → backend/models/actor_and_use_case.py

 from config.database import db
-from models.use_case_answer import UseCaseAnswer
 
 
-class UseCase(db.Model):
+class ActorANDUseCase(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    use_case_answer = db.Column(UseCaseAnswer, nullable=False)
+    use_case_answer = db.Column(db.Integer, nullable=False)
+    type = db.Column(db.String(50), nullable=False)
     text = db.Column(db.String(500), nullable=False)
     x_min = db.Column(db.String(50), nullable=False)
     y_min = db.Column(db.String(50), nullable=False)
     x_max = db.Column(db.String(50), nullable=False)
     y_max = db.Column(db.String(50), nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
+
+    def __repr__(self) -> str:
+        return 'ActorANDUseCase>>> {self.content}'

backend/models/actor_generalization_relationship.py

 from config.database import db
-from models.actor import Actor
-from models.use_case_answer import UseCaseAnswer
 
 
 class ActorGeneralizationRelationship(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    use_case_answer = db.Column(UseCaseAnswer, nullable=False)
-    connected_component_01 = db.Column(Actor, nullable=False)
-    connected_component_02 = db.Column(Actor, nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    use_case_answer = db.Column(db.Integer, nullable=False)
+    connected_component_01 = db.Column(db.Integer, nullable=False)
+    connected_component_02 = db.Column(db.Integer, nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
+
+    def __repr__(self) -> str:
+        return 'ActorGeneralizationRelationship>>> {self.content}'

backend/models/assignment.py

-from datetime import datetime
-from config.database import db
-from models.user_model import User
-
-
-class Assignment(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    user = db.Column(User, nullable=False)
-    name = db.Column(db.String(250), nullable=False)
-    description = db.Column(db.String(800), nullable=False)
-    created_at = db.Column(db.DateTime, default=datetime.now())
-    start_at = db.Column(db.DateTime)
-    end_at = db.Column(db.DateTime)
-    type = db.Column(db.String(50), nullable=False)

backend/models/class_diagram_answer.py

+from datetime import datetime
+from config.database import db
+
+
+class ClassAnswer(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    user = db.Column(db.Integer, nullable=False)
+    assignment = db.Column(db.Integer, nullable=False)
+    comments = db.Column(db.String(800), nullable=False)
+    submitted_at = db.Column(db.DateTime, default=datetime.now())
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
+    file_name = db.Column(db.String(50), nullable=False)
+
+    def __repr__(self) -> str:
+        return 'ClassAnswer>>> {self.content}'

backend/models/extend_relationship.py

 from config.database import db
-from models.use_case import UseCase
-from models.use_case_answer import UseCaseAnswer
 
 
 class ExtendRelationship(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    use_case_answer = db.Column(UseCaseAnswer, nullable=False)
-    connected_component_01 = db.Column(UseCase, nullable=False)
-    connected_component_02 = db.Column(UseCase, nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    use_case_answer = db.Column(db.Integer, nullable=False)
+    connected_component_01 = db.Column(db.Integer, nullable=False)
+    connected_component_02 = db.Column(db.Integer, nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
 
+    def __repr__(self) -> str:
+        return 'ExtendRelationship>>> {self.content}'

backend/models/include_relationship.py

 from config.database import db
-from models.use_case import UseCase
-from models.use_case_answer import UseCaseAnswer
 
 
 class IncludeRelationship(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    use_case_answer = db.Column(UseCaseAnswer, nullable=False)
-    connected_component_01 = db.Column(UseCase, nullable=False)
-    connected_component_02 = db.Column(UseCase, nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    use_case_answer = db.Column(db.Integer, nullable=False)
+    connected_component_01 = db.Column(db.Integer, nullable=False)
+    connected_component_02 = db.Column(db.Integer, nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
+
+    def __repr__(self) -> str:
+        return 'IncludeRelationship>>> {self.content}'
\ No newline at end of file

backend/models/use_case_answer.py

 from datetime import datetime
 from config.database import db
-from models.assignment import Assignment
-from models.user_model import User
 
 
 class UseCaseAnswer(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    user = db.Column(User, nullable=False)
-    assignment = db.Column(Assignment, nullable=False)
+    user = db.Column(db.Integer, nullable=False)
+    assignment = db.Column(db.Integer, nullable=False)
     comments = db.Column(db.String(800), nullable=False)
     submitted_at = db.Column(db.DateTime, default=datetime.now())
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
+    file_name = db.Column(db.String(50), nullable=False)
+
+    def __repr__(self) -> str:
+        return 'UseCaseAnswer>>> {self.code}'
\ No newline at end of file

backend/models/use_case_association_relationship.py

 from config.database import db
-from models.actor import Actor
-from models.use_case import UseCase
-from models.use_case_answer import UseCaseAnswer
 
 
 class UseCaseAssociationRelationship(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    use_case_answer = db.Column(UseCaseAnswer, nullable=False)
-    connected_component_01 = db.Column(Actor, nullable=False)
-    connected_component_02 = db.Column(UseCase, nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    use_case_answer = db.Column(db.Integer, nullable=False)
+    connected_component_01 = db.Column(db.Integer, nullable=False)
+    connected_component_02 = db.Column(db.Integer, nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
+
+    def __repr__(self) -> str:
+        return 'UseCaseAssociationRelationship>>> {self.code}'

backend/models/use_case_generalization_relationship.py

 from config.database import db
-from models.use_case import UseCase
-from models.use_case_answer import UseCaseAnswer
 
 
 class UseCaseGeneralizationRelationship(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    use_case_answer = db.Column(UseCaseAnswer, nullable=False)
-    connected_component_01 = db.Column(UseCase, nullable=False)
-    connected_component_02 = db.Column(UseCase, nullable=False)
-    plagiarism_count = db.Column(db.String(50), nullable=False)
-    correctness_count = db.Column(db.String(50), nullable=False)
+    use_case_answer = db.Column(db.Integer, nullable=False)
+    connected_component_01 = db.Column(db.Integer, nullable=False)
+    connected_component_02 = db.Column(db.Integer, nullable=False)
+    plagiarism_count = db.Column(db.String(50))
+    correctness_count = db.Column(db.String(50))
+
+    def __repr__(self) -> str:
+        return 'UseCaseGeneralizationRelationship>>> {self.code}'
\ No newline at end of file

backend/requirements.txt

 attrs==21.4.0

backend/routes/submission_routes.py

+import json
+
+from flask import request, jsonify, Blueprint
+from flask_jwt_extended import jwt_required, get_jwt_identity
+
+from constants.http_status_codes_constant import HTTP_400_BAD_REQUEST, HTTP_200_OK
+from services.submission_service import save_submission
+from services.use_case_model_detection_service import model_object_detection
+
+submission = Blueprint('submissions', __name__, url_prefix='/api/v1/submissions')
+
+
+@submission.post('/upload')
+@jwt_required()
+def upload_submission():
+    user_id = get_jwt_identity()
+
+    image = request.files['file']
+    json_data = json.loads(request.form['data'])
+    submission_type = json_data['type']
+    assignment_id = json_data['id']
+    comment = json_data['comment']
+
+    if submission_type is None or image is None or assignment_id is None:
+        return jsonify({'err': 'invalid request '}), HTTP_400_BAD_REQUEST
+    elif submission_type == 'use case':
+        use_case_obj = save_submission(assignment_id, image, submission_type, comment, user_id)
+        model_object_detection(image.filename, use_case_obj.id)
+        return jsonify({'message': 'upload successful '}), HTTP_200_OK
+    elif submission_type == 'class':
+        class_obj = save_submission(assignment_id, image, submission_type, comment, user_id)
+        # model_object_detection(image.filename, class_obj.id)
+        return jsonify({'id': str(class_obj.id)}), HTTP_200_OK
+    else:
+        return jsonify({'err': 'invalid request '}), HTTP_400_BAD_REQUEST

backend/services/class_model_detection_service.py

+import operator
+import re
+
+import cv2
+import numpy as np
+import pytesseract as pytesseract
+from PIL import Image
+
+from object_detection.utils import label_map_util
+
+import app
+import tensorflow as tf
+
+from config.database import db
+
+
+def class_object_detection(filename, class_comp_id, model_path, label_path, image_path):
+    detect_fn = tf.saved_model.load(model_path)
+    category_index = label_map_util.create_category_index_from_labelmap(label_path, use_display_name=True)
+    image_np = np.array(Image.open(image_path))
+    input_tensor = tf.convert_to_tensor(image_np)
+
+    input_tensor = input_tensor[tf.newaxis, ...]
+
+    detections = detect_fn(input_tensor)
+
+    num_detections = int(detections.pop('num_detections'))
+    detections = {key: value[0, :num_detections].numpy()
+                  for key, value in detections.items()}
+    detections['num_detections'] = num_detections
+
+    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
+
+    accurate_indexes = [k for k, v in enumerate(detections['detection_scores']) if (v > 0.6)]
+    num_entities = len(accurate_indexes)
+
+    class_id = operator.itemgetter(*accurate_indexes)(detections['detection_classes'])
+    boxes = detections['detection_boxes']
+    return boxes, num_entities, accurate_indexes, num_entities, category_index, class_id
+
+
+def component_separation(filename, class_comp_id):
+    boxes, num_entities, accurate_indexes, num_entities, category_index, class_id = class_object_detection(filename,
+                                                                                                           class_comp_id,
+                                                                                                           app.CLASS_SAVED_MODEL_PATH,
+                                                                                                           app.CLASS_SAVED_LABEL_PATH,
+                                                                                                           app.SUBMISSION_PATH_CLASS + '/' + filename)
+    # Convert the class id in their name
+    if num_entities > 0:
+        for i in range(0, len(accurate_indexes)):
+
+            if category_index[class_id]['name'] == 'class':
+                class_image = crop_and_image_resolution(filename, boxes)
+                cv2.imwrite(app.SUBMISSION_PATH_CLASS, "class" + str(i))
+                # boxes,num_entities,accurate_indexes,num_entities,category_index,class_id = class_object_detection(class_image, class_comp_id, app.CLASS_SAVED_MODEL_PATH, app.CLASS_SAVED_LABEL_PATH,app.SUBMISSION_PATH_CLASS+'/'+"class"+str(i))
+
+            elif category_index[class_id]['name'] == 'interface':
+                image = crop_and_image_resolution(filename, boxes)
+
+
+def crop_and_image_resolution(filename, boxes):
+    image = cv2.imread(app.SUBMISSION_PATH + '/' + filename)
+    height, width, c = image.shape
+    # crop box format: xmin, ymin, xmax, ymax
+    ymin = boxes[0] * height
+    xmin = boxes[1] * width
+    ymax = boxes[2] * height
+    xmax = boxes[3] * width
+
+    cropped_image = image[int(ymin):int(ymax), int(xmin):int(xmax)]
+    image = cv2.imwrite('image.jpg', cropped_image)
+    # convert values to int
+    black = (0, 0, 0)
+    white = (255, 255, 255)
+    threshold = (160, 160, 160)
+
+    # Open input image in grayscale mode and get its pixels.
+    img = Image.open("image.jpg").convert("LA")
+    pixels = img.getdata()
+
+    new_pixels = []
+
+    # Compare each pixel
+    for pixel in pixels:
+        if pixel < threshold:
+            new_pixels.append(black)
+        else:
+            new_pixels.append(white)
+
+            # Create and save new image.
+    new_img = Image.new("RGB", img.size)
+    new_img.putdata(new_pixels)
+
+    return image

backend/services/extend_include_relationship_detection_service.py

+import math
+
+import cv2
+import keras_ocr
+import numpy as np
+
+import app
+from config.database import db
+from models.actor_and_use_case import ActorANDUseCase
+from decimal import Decimal
+
+from models.extend_relationship import ExtendRelationship
+from models.include_relationship import IncludeRelationship
+
+
+def detect_extend_include_relationship(filename, boxes, accurate_indexes, use_case_id, category_index, class_id):
+    image = cv2.imread(app.SUBMISSION_PATH + '/' + filename)
+    height, width, c = image.shape
+    use_case_objects = ActorANDUseCase.query.filter_by(use_case_answer=use_case_id, type='use case').all()
+
+    for i in range(0, len(accurate_indexes)):
+
+        if category_index[class_id[i]]['name'] == 'relationship':
+            ymin = boxes[i][0] * height
+            xmin = boxes[i][1] * width
+            ymax = boxes[i][2] * height
+            xmax = boxes[i][3] * width
+
+            crop_img = image[int(ymin):int(ymax), int(xmin):int(xmax)]
+
+            r_type = text_detection(crop_img)
+            line_recovery_image = line_recovery(crop_img)
+
+            gray_image = cv2.cvtColor(line_recovery_image, cv2.COLOR_BGR2GRAY)
+            _, thresh_image = cv2.threshold(gray_image, 100, 255, cv2.THRESH_BINARY_INV)
+            arrow_image = get_filter_arrow_image(thresh_image)
+
+            if arrow_image is not None:
+                arrow_info_image, point1, point2 = get_arrow_info(arrow_image)
+
+                point1_x = int(xmin) + point1[0]
+                point1_y = int(ymin) + point1[1]
+                point2_x = int(xmin) + point2[0]
+                point2_y = int(ymin) + point2[1]
+
+                line_point1 = (point1_x, point1_y)
+                line_point2 = (point2_x, point2_y)
+
+                u1_object = find_closest_components_length(line_point1, use_case_objects)
+
+                u2_object = find_closest_components_length(line_point2, use_case_objects)
+
+                if (r_type == 'include'):
+                    include_obj = IncludeRelationship(use_case_answer=use_case_id,
+                                                      connected_component_01=u1_object.id,
+                                                      connected_component_02=u2_object.id)
+                    db.session.add(include_obj)
+                    db.session.commit()
+
+                if (r_type == "extend"):
+                    extend_obj = ExtendRelationship(use_case_answer=use_case_id,
+                                                    connected_component_01=u1_object.id,
+                                                    connected_component_02=u2_object.id)
+                    db.session.add(extend_obj)
+                    db.session.commit()
+
+
+def line_recovery(img):
+    kernel1 = np.ones((3, 5), np.uint8)
+    kernel2 = np.ones((9, 9), np.uint8)
+
+    imgGray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    imgBW = cv2.threshold(imgGray, 230, 255, cv2.THRESH_BINARY_INV)[1]
+
+    img1 = cv2.erode(imgBW, kernel1, iterations=1)
+    img2 = cv2.dilate(img1, kernel2, iterations=3)
+    img3 = cv2.bitwise_and(imgBW, img2)
+    img3 = cv2.bitwise_not(img3)
+    img4 = cv2.bitwise_and(imgBW, imgBW, mask=img3)
+    imgLines = cv2.HoughLinesP(img4, 1, np.pi / 180, 20, minLineLength=0, maxLineGap=5)
+
+    for i in range(len(imgLines)):
+        for x1, y1, x2, y2 in imgLines[i]:
+            cv2.line(img, (x1, y1), (x2, y2), (0, 0, 0), 2)
+
+    return img
+
+
+def text_detection(c_img):
+    pipeline = keras_ocr.pipeline.Pipeline()
+
+    prediction_groups = pipeline.recognize([c_img])
+
+    for prg in prediction_groups[0]:
+        print(prg[0])
+        if prg[0].find('clu') >= 0:
+            return "include"
+        else:
+            if prg[0].find('ten') >= 0:
+                return "extend"
+
+
+def get_filter_arrow_image(threslold_image):
+    blank_image = np.zeros_like(threslold_image)
+
+    kernel_dilate = cv2.getStructuringElement(cv2.MORPH_RECT, (2, 2))
+    threslold_image = cv2.dilate(threslold_image, kernel_dilate, iterations=1)
+
+    contours, hierarchy = cv2.findContours(threslold_image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+    if hierarchy is not None:
+
+        threshold_distnace = 100
+
+        for cnt in contours:
+            hull = cv2.convexHull(cnt, returnPoints=False)
+            defects = cv2.convexityDefects(cnt, hull)
+            if defects is not None:
+                for i in range(defects.shape[0]):
+                    start_index, end_index, farthest_index, distance = defects[i, 0]
+
+                    if distance > threshold_distnace:
+                        cv2.drawContours(blank_image, [cnt], -1, 225, -1)
+
+        return blank_image
+    else:
+        return None
+
+
+def get_length(p1, p2):
+    line_length = ((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) ** 0.5
+    return line_length
+
+
+def find_max_length(contours):
+    max_lenth = 0
+
+    for cnt in contours:
+        p1, p2 = get_max_distace_point(cnt)
+        line_length = ((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) ** 0.5
+
+        if line_length > max_lenth:
+            max_lenth = line_length
+
+    return max_lenth
+
+
+def get_max_distace_point(cnt):
+    max_distance = 0
+    max_points = None
+    for [[x1, y1]] in cnt:
+        for [[x2, y2]] in cnt:
+            distance = get_length((x1, y1), (x2, y2))
+
+            if distance > max_distance:
+                max_distance = distance
+                max_points = [(x1, y1), (x2, y2)]
+
+    return max_points
+
+
+def angle_beween_points(a, b):
+    arrow_slope = (a[0] - b[0]) / (a[1] - b[1])
+    arrow_angle = math.degrees(math.atan(arrow_slope))
+    return arrow_angle
+
+
+def get_arrow_info(arrow_image):
+    arrow_info_image = cv2.cvtColor(arrow_image.copy(), cv2.COLOR_GRAY2BGR)
+    contours, hierarchy = cv2.findContours(arrow_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    if hierarchy is not None:
+
+        max_lenth = find_max_length(contours)
+
+        for cnt in contours:
+
+            blank_image = np.zeros_like(arrow_image)
+            cv2.drawContours(blank_image, [cnt], -1, 255, -1)
+
+            point1, point2 = get_max_distace_point(cnt)
+
+            lenght = get_length(point1, point2)
+
+            if lenght == max_lenth:
+                cv2.circle(arrow_info_image, point1, 2, (255, 0, 0), 3)
+                cv2.circle(arrow_info_image, point2, 2, (255, 0, 0), 3)
+
+                cv2.putText(arrow_info_image, "point 1 : %s" % (str(point1)), point2, cv2.FONT_HERSHEY_PLAIN, 0.8,
+                            (0, 0, 255), 1)
+                cv2.putText(arrow_info_image, "point 2 : %s" % (str(point2)), (point2[0], point2[1] + 20),
+                            cv2.FONT_HERSHEY_PLAIN, 0.8, (0, 0, 255), 1)
+
+                return arrow_info_image, point1, point2
+    else:
+        return None, None
+
+
+def find_closest_components_length(point, use_case_objects):
+    u_object = 0
+    min_length = 1000000000000
+    for obj in use_case_objects:
+
+        ymin = Decimal(obj.y_min)
+        xmin = Decimal(obj.x_min)
+        ymax = Decimal(obj.y_max)
+        xmax = Decimal(obj.x_max)
+
+        usecase_x = xmin + (xmax - xmin) / 2
+        usecase_y = ymin + (ymax - ymin) / 2
+
+        usecase_point = (int(usecase_x), int(usecase_y))
+
+        l_length = ((point[0] - usecase_point[0]) ** 2 + (point[1] - usecase_point[1]) ** 2) ** 0.5
+
+        if min_length > l_length:
+            min_length = l_length
+            u_object = obj
+
+    return u_object

backend/services/generalization_relationship_detection_service.py

+import math
+
+import cv2
+import numpy as np
+
+import app
+from config.database import db
+from decimal import Decimal
+from models.actor_and_use_case import ActorANDUseCase
+from models.actor_generalization_relationship import ActorGeneralizationRelationship
+from models.use_case_association_relationship import UseCaseAssociationRelationship
+from models.use_case_generalization_relationship import UseCaseGeneralizationRelationship
+
+
+def detect_generalization_relationship(filename, boxes, accurate_indexes, use_case_id):
+    image = cv2.imread(app.SUBMISSION_PATH + '/' + filename)
+    img1 = hide_detected_components(image, boxes, accurate_indexes)
+    img2 = remove_rectangle(img1)
+    img3 = recover_broke_line(img2)
+    gray_image = cv2.cvtColor(img3, cv2.COLOR_BGR2GRAY)
+    _, thresh_image = cv2.threshold(gray_image, 100, 255, cv2.THRESH_BINARY_INV)
+    arrow_image = get_filter_arrow_image(thresh_image)
+
+    if arrow_image is not None:
+        all_objects = ActorANDUseCase.query.filter_by(use_case_answer=use_case_id).all()
+        result = get_arrow_info(arrow_image, all_objects, use_case_id)
+
+
+def hide_detected_components(image, boxes, accurate_indexes):
+    height, width, c = image.shape
+    for i in range(0, len(accurate_indexes)):
+        ymin = boxes[i][0] * height
+        xmin = boxes[i][1] * width
+        ymax = boxes[i][2] * height
+        xmax = boxes[i][3] * width
+
+        cv2.rectangle(image, (int(xmin), int(ymin)), (int(xmax), int(ymax)), (255, 255, 255), -1)
+    return image
+
+
+def remove_rectangle(image):
+    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    _, thrash = cv2.threshold(gray_image, 240, 255, cv2.THRESH_BINARY)
+    contours, _ = cv2.findContours(thrash, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
+
+    for contour in contours:
+        shape = cv2.approxPolyDP(contour, 0.05 * cv2.arcLength(contour, True), True)
+
+        if len(shape) == 4:
+            cv2.drawContours(image, [shape], 0, (255, 255, 255), 3)
+
+    return image
+
+
+def recover_broke_line(image):
+    kernel1 = np.ones((3, 5), np.uint8)
+    kernel2 = np.ones((9, 9), np.uint8)
+
+    imgGray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    imgBW = cv2.threshold(imgGray, 230, 255, cv2.THRESH_BINARY_INV)[1]
+
+    img1 = cv2.erode(imgBW, kernel1, iterations=1)
+    img2 = cv2.dilate(img1, kernel2, iterations=3)
+    img3 = cv2.bitwise_and(imgBW, img2)
+    img3 = cv2.bitwise_not(img3)
+    img4 = cv2.bitwise_and(imgBW, imgBW, mask=img3)
+    imgLines = cv2.HoughLinesP(img4, 1, np.pi / 180, 40, minLineLength=0, maxLineGap=10)
+
+    for i in range(len(imgLines)):
+        for x1, y1, x2, y2 in imgLines[i]:
+            cv2.line(image, (x1, y1), (x2, y2), (0, 0, 0), 2)
+
+    return image
+
+
+def get_filter_arrow_image(threslold_image):
+    blank_image = np.zeros_like(threslold_image)
+
+    kernel_dilate = cv2.getStructuringElement(cv2.MORPH_RECT, (2, 2))
+    threslold_image = cv2.dilate(threslold_image, kernel_dilate, iterations=1)
+
+    contours, hierarchy = cv2.findContours(threslold_image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+    if hierarchy is not None:
+
+        threshold_distnace = 100
+
+        for cnt in contours:
+            hull = cv2.convexHull(cnt, returnPoints=False)
+            defects = cv2.convexityDefects(cnt, hull)
+            if defects is not None:
+                for i in range(defects.shape[0]):
+                    start_index, end_index, farthest_index, distance = defects[i, 0]
+
+                    if distance > threshold_distnace:
+                        cv2.drawContours(blank_image, [cnt], -1, 225, -1)
+
+        return blank_image
+    else:
+        return None
+
+
+def get_length(p1, p2):
+    line_length = ((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) ** 0.5
+    return line_length
+
+
+def get_max_distace_point(cnt):
+    max_distance = 0
+    max_points = None
+    for [[x1, y1]] in cnt:
+        for [[x2, y2]] in cnt:
+            distance = get_length((x1, y1), (x2, y2))
+
+            if distance > max_distance:
+                max_distance = distance
+                max_points = [(x1, y1), (x2, y2)]
+
+    return max_points
+
+
+def angle_beween_points(a, b):
+    arrow_slope = (a[0] - b[0]) / (a[1] - b[1])
+    arrow_angle = math.degrees(math.atan(arrow_slope))
+    return arrow_angle
+
+
+def get_arrow_info(arrow_image, all_objects, use_case_id):
+    contours, hierarchy = cv2.findContours(arrow_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    if hierarchy is not None:
+
+        for cnt in contours:
+            blank_image = np.zeros_like(arrow_image)
+            cv2.drawContours(blank_image, [cnt], -1, 255, -1)
+
+            point1, point2 = get_max_distace_point(cnt)
+
+            component_1 = find_closest_components_length(point1, all_objects)
+            component_2 = find_closest_components_length(point2, all_objects)
+
+            if component_1.type == 'use case' and component_2.type == 'use case':
+                use_case_generalization_obj = UseCaseGeneralizationRelationship(use_case_answer=use_case_id,
+                                                                                connected_component_01=component_1.id,
+                                                                                connected_component_02=component_2.id)
+                db.session.add(use_case_generalization_obj)
+                db.session.commit()
+
+            elif component_1.type == "actor" and component_2.type == "actor":
+                actor_generalization_obj = ActorGeneralizationRelationship(use_case_answer=use_case_id,
+                                                                           connected_component_01=component_1.id,
+                                                                           connected_component_02=component_2.id)
+                db.session.add(actor_generalization_obj)
+                db.session.commit()
+
+            else:
+                association_obj = UseCaseAssociationRelationship(use_case_answer=use_case_id,
+                                                                 connected_component_01=component_1.id,
+                                                                 connected_component_02=component_2.id)
+                db.session.add(association_obj)
+                db.session.commit()
+
+
+    else:
+        return None
+
+
+def find_closest_components_length(point, all_objects):
+    component = None
+    min_length = 1000000000
+    for obj in all_objects:
+
+        ymin = Decimal(obj.y_min)
+        xmin = Decimal(obj.x_min)
+        ymax = Decimal(obj.y_max)
+        xmax = Decimal(obj.x_max)
+
+        usecase_x = xmin + (xmax - xmin) / 2
+        usecase_y = ymin + (ymax - ymin) / 2
+
+        usecase_point = (int(usecase_x), int(usecase_y))
+
+        l_length = ((point[0] - usecase_point[0]) ** 2 + (point[1] - usecase_point[1]) ** 2) ** 0.5
+
+        if min_length > l_length:
+            min_length = l_length
+            component = obj
+
+    return component

backend/services/submission_service.py

+import os
+import app
+
+from config.database import db
+from models.class_diagram_answer import ClassAnswer
+from models.use_case_answer import UseCaseAnswer
+
+
+def save_submission(assignment_id, image, submission_type, comment, user_id):
+
+    if submission_type == 'use case':
+        image.save(os.path.join(app.SUBMISSION_PATH, image.filename))
+        use_case_obj = UseCaseAnswer(user=user_id, assignment=assignment_id, file_name=image.filename,
+                                     comments=comment)
+        db.session.add(use_case_obj)
+        db.session.commit()
+        return use_case_obj
+    else:
+        image.save(os.path.join(app.SUBMISSION_PATH_CLASS, image.filename))
+        class_obj = ClassAnswer(user=user_id, assignment=assignment_id, file_name=image.filename,
+                                comments=comment)
+        db.session.add(class_obj)
+        db.session.commit()
+        return class_obj

backend/services/use_case_model_detection_service.py

+import operator
+import re
+
+import cv2
+import numpy as np
+import pytesseract as pytesseract
+from PIL import Image
+
+from object_detection.utils import label_map_util
+
+import app
+import tensorflow as tf
+
+from config.database import db
+from models.actor_and_use_case import ActorANDUseCase
+
+# pytesseract.pytesseract.tesseract_cmd = 'C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe'
+from services.extend_include_relationship_detection_service import detect_extend_include_relationship
+from services.generalization_relationship_detection_service import detect_generalization_relationship
+
+
+def model_object_detection(filename, use_case_id):
+    detect_fn = tf.saved_model.load(app.USE_CASE_SAVED_MODEL_PATH)
+    category_index = label_map_util.create_category_index_from_labelmap(
+        app.USE_CASE_SAVED_LABEL_PATH + "/label_map.pbtxt",
+        use_display_name=True)
+    image_np = np.array(Image.open(app.SUBMISSION_PATH + '/' + filename))
+    input_tensor = tf.convert_to_tensor(image_np)
+
+    input_tensor = input_tensor[tf.newaxis, ...]
+
+    detections = detect_fn(input_tensor)
+
+    num_detections = int(detections.pop('num_detections'))
+    detections = {key: value[0, :num_detections].numpy()
+                  for key, value in detections.items()}
+    detections['num_detections'] = num_detections
+
+    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
+
+    accurate_indexes = [k for k, v in enumerate(detections['detection_scores']) if (v > 0.4)]
+
+    class_id = operator.itemgetter(*accurate_indexes)(detections['detection_classes'])
+    boxes = detections['detection_boxes']
+    text_extraction(filename, class_id, boxes, accurate_indexes, category_index, use_case_id)
+    detect_generalization_relationship(filename, boxes, accurate_indexes, use_case_id)
+    detect_extend_include_relationship(filename, boxes, accurate_indexes, use_case_id, category_index, class_id)
+
+
+def text_extraction(filename, class_id, boxes, accurate_indexes, category_index, use_case_id):
+    image = cv2.imread(app.SUBMISSION_PATH + '/' + filename)
+    for i in range(0, len(accurate_indexes)):
+        if category_index[class_id[i]]['name'] != "relationship":
+
+            height, width, c = image.shape
+
+            ymin = boxes[i][0] * height
+            xmin = boxes[i][1] * width
+            ymax = boxes[i][2] * height
+            xmax = boxes[i][3] * width
+
+            crop_img = image[int(ymin):int(ymax), int(xmin):int(xmax)]
+
+            gray_img = cv2.cvtColor(crop_img, cv2.COLOR_BGR2GRAY)
+
+            thresh, bw_img = cv2.threshold(gray_img, 160, 255, cv2.THRESH_TOZERO)
+
+            resize_img = cv2.resize(bw_img, None, fx=1, fy=1)
+
+            ocr_result = pytesseract.image_to_string(resize_img, config='1 eng --oem 1 --psm 13')
+            result = ocr_result.strip().replace("\\", "")
+            text = re.sub("=|,", "", result)
+
+            if category_index[class_id[i]]['name'] == 'actor':
+                actor_obj = ActorANDUseCase(use_case_answer=use_case_id, type='actor', text=text, x_min=xmin,
+                                            y_min=ymin, x_max=xmax,
+                                            y_max=ymax)
+                db.session.add(actor_obj)
+                db.session.commit()
+            else:
+                use_case_obj = ActorANDUseCase(use_case_answer=use_case_id, type='use case', text=text, x_min=xmin,
+                                               y_min=ymin, x_max=xmax,
+                                               y_max=ymax)
+                db.session.add(use_case_obj)
+                db.session.commit()

backend/tensorflow_models/class/label/label_map.pbtxt

+item {
+    id: 1
+    name: 'class'
+}
+
+item {
+    id: 2
+    name: 'relationship'
+}
\ No newline at end of file

backend/tensorflow_models/class_component/label/label_map.pbtxt

+item {
+    id: 1
+    name: 'class_attributes'
+}
+
+item {
+    id: 2
+    name: 'class_methods'
+}
+
+item {
+    id: 3
+    name: 'class_name'
+}
\ No newline at end of file

backend/tensorflow_models/use_case/label/label_map.pbtxt

+item {
+    id: 1
+    name: 'actor'
+}
+
+item {
+    id: 2
+    name: 'relationship'
+}
+
+item {
+    id: 3
+    name: 'use case'
+}
\ No newline at end of file