Merge branch 'IT20097660-Sashini' into 'master'

Backend Completion | SVM Completion

See merge request !9

deployment/backend/.gitignore

+/app/models
+/app/src
+/app/configs
\ No newline at end of file

deployment/backend/Dockerfile

+
+# Define function directory
+ARG FUNCTION_DIR="/function"
+
+FROM python:3.9-buster as build-image
+
+# Install aws-lambda-cpp build dependencies
+RUN apt-get update && \
+  apt-get install -y \
+  g++ \
+  make \
+  cmake \
+  unzip \
+  libcurl4-openssl-dev
+
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+
+# Create function directory
+RUN mkdir -p ${FUNCTION_DIR}
+
+# Install pip dependencies
+COPY cpu-requirements.txt .
+RUN pip install -r cpu-requirements.txt --index-url https://download.pytorch.org/whl/cpu --target ${FUNCTION_DIR}
+COPY requirements.txt .
+RUN pip install -r requirements.txt --target ${FUNCTION_DIR}
+# Install the runtime interface client
+RUN pip install \
+        --target ${FUNCTION_DIR} \
+        awslambdaric
+
+# Multi-stage build: grab a fresh copy of the base image
+FROM python:3.9-buster
+
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+
+# Copy in the build image dependencies
+COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
+
+# Download nltk data
+RUN python3 -m nltk.downloader --dir /usr/share/nltk_data wordnet punkt stopwords averaged_perceptron_tagger tagsets
+
+# copy function code
+COPY app/models ${FUNCTION_DIR}/models
+COPY app/configs ${FUNCTION_DIR}/configs
+COPY app/src ${FUNCTION_DIR}/src
+COPY app/handlers ${FUNCTION_DIR}/handlers
+COPY app/app.py ${FUNCTION_DIR}/app.py
+
+# Setting the entry point
+ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
+CMD [ "app.handler" ]
+

deployment/backend/app/app.py

+from typing import Dict
+from handlers import evaluate, analyze
+import traceback
+
+
+def handler(event: Dict, context: Dict):
+    task = event["task"]
+    payload = event["payload"]
+    try:
+        if task == "evaluation":
+            body = evaluate(payload)
+            return {"status": 200, "body": body}
+        elif task == "analysis":
+            body = analyze(payload)
+            return {"status": 200, "body": body}
+        else:
+            body = "Invocation error"
+            return {"status": 400, "body": body}
+    except Exception as e:
+        body = str(e)
+        print("Error:", body)
+        print("Input:", event)
+        traceback.print_exc()
+        return {"status": 500, "body": body}

deployment/backend/app/handlers/__init__.py

+from .evaluation import evaluate
+from .analysis import analyze

deployment/backend/app/handlers/analysis.py

+from typing import Dict
+from src import TestBench
+
+tb_kwargs = {
+    "svm": {"cf_generator_config": "configs/models/wf-cf-generator.yaml"},
+    "knn": {"cf_generator_config": "configs/models/wf-cf-generator.yaml"},
+    "rf": {
+        "threshold_classifier": 0.49339999999983775,
+        "max_iter": 50,
+        "time_maximum": 120,
+    },
+    "lr": {
+        "threshold_classifier": 0.49179999999978463,
+        "max_iter": 50,
+        "time_maximum": 120,
+    },
+}
+
+
+def analyze(payload: Dict):
+    model_name = payload["model_name"]
+    configurations = payload["configurations"]
+    prompt = payload["prompt"]
+    variations = payload["variations"]
+    tb = TestBench(
+        model_path=f"models/analysis-models/{model_name}.pkl",
+        vectorizer_path="models/analysis-models/tfidf.pkl",
+        analyzer_name=model_name,
+        **tb_kwargs[model_name],
+    )
+    reports = tb(configurations, prompt, variations)
+    reports = "\n\n".join(reports)
+    return reports

deployment/backend/app/handlers/evaluation.py

+from src.models import AnalysisModels as Models
+from typing import Dict
+
+
+def evaluate(payload: Dict) -> Dict:
+    texts = payload["texts"]
+    model_name = payload["model_name"]
+    models = Models("configs/models/analysis-models.yaml", "models/analysis-models/")
+    model = getattr(models, model_name)
+    scores, preds = model(texts)
+
+    return {"scores": scores, "predictions": preds}

deployment/backend/build-and-push.bash

+#!/bin/bash
+
+
+if [ -e Dockerfile ]; then
+    # remove old repititions if they exist
+    if [ -e app/models ]; then
+        rm -r app/models
+    fi
+    if [ -e app/configs ]; then
+        rm -r app/configs
+    fi
+    if [ -e app/src ]; then
+        rm -r app/src
+    fi
+
+    # copy repetitions
+    cp -r ../../models app/models
+    cp -r ../../configs app/configs
+    cp -r ../../src app/src
+
+    # log into docker
+    aws ecr get-login-password --region ap-south-1 | sudo docker login --username AWS --password-stdin 065257926712.dkr.ecr.ap-south-1.amazonaws.com
+
+    # build and push
+    sudo docker build -t 065257926712.dkr.ecr.ap-south-1.amazonaws.com/xai:latest .
+    sudo docker push 065257926712.dkr.ecr.ap-south-1.amazonaws.com/xai:latest
+else
+    echo "Please change the working directory to the directory containing the Dockerfile"
+    exit 1
+fi
\ No newline at end of file

deployment/backend/cpu-requirements.txt

+torch==2.0.1
\ No newline at end of file

deployment/backend/requirements.txt

+scikit-learn==1.2.2
+nltk==3.8.1
+ipykernel==6.24.0
+ipywidgets==7.6.5
+pyyaml==6.0
+pandas==2.0.3
+beautifulsoup4==4.12.2
+wget==3.2
+numpy==1.23.5
+shap==0.41.0
+matplotlib==3.5.1
+seaborn==0.11.2
+ordered-set==4.1.0
+boto3==1.27.0
+transformers==4.31.0
+sagemaker==2.173.0
+sentencepiece==0.1.99
\ No newline at end of file