Backend added

parent 6550c505
from django.contrib import admin
# Register your models here.
from django.apps import AppConfig
class CgpappConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'CgpApp'
title,category
Assistant Director (Social Mobilization),Administrative
Technical Officer,Engineering
Development Officer,Operations
Data Processing Assistant,Operations
Community Decelopment Assistant,Operations
Management Assistant,Administrative
Technical Assistant,Engineering
Social Development Assistant,Administrative
Office Aid,Other
Record Keeper,Other
Chairman,Administrative
Administrative Appeals Tribunal,Administrative
Assistant Director (Legal),Administrative
Vocational Instructor,Academic
Medical Consultant,Healthcare
Medical Officer,Healthcare
Dental Surgeon,Healthcare
Para Medical,Healthcare
Matron,Healthcare
Nursing Officer,Healthcare
Nursing Sister,Healthcare
Pharmacist,Healthcare
Therapist,Healthcare
Dental Technician,Healthcare
Orthoptist,Healthcare
Public Health Inspector,Healthcare
Midwife,Healthcare
Management Assistant,Administrative
Driver,Other
Attendant,Healthcare
Registrar,Administrative
Teacher,Academic
class cgphelper:
skip_words = ["qualification", "qualifications", "experience", "proficiency", "knowledge", "skills", "interview", "example", "example : ", "general conditions : ",
"n.b.", "note", "age", "salary", "salary", "educational qualifications", "other qualifications", "method of application", "clarity in communication.",
"punishment", "paste", "certification", "paste", "receipt", "\"", "n.b", "w", "fldgi", "fPoh", "time management.", "presentation methods.", "admission to the examination",
"appointing authority", "penalty for false information",
]
department_identifier = {"public service commision", "registrar",
"judicial service commision", "national dangerous drugs control board", "parliament of sri lanka"}
skip_substring = []
from spellchecker import SpellChecker
class Spell:
def spellcheck_(entry):
print(entry)
listofwords = entry.split()
spell = SpellChecker()
# find those words that may be misspelled
misspelled = spell.unknown(listofwords)
for word in misspelled:
# Get the one `most likely` answer
print(spell.correction(word))
# Get a list of `likely` options
print(spell.candidates(word))
listofwords = spell.candidates(word)
newlist = []
for item in listofwords:
newlist.append(item)
return newlist
# Generated by Django 3.2.8 on 2021-10-14 11:30
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Commodity',
fields=[
('ITEM', models.AutoField(primary_key=True, serialize=False)),
('GAZETTE', models.CharField(default='N/A', max_length=4)),
('NAME', models.CharField(max_length=200)),
('CURRENTPRICE', models.CharField(default='0', max_length=50)),
('NEWPRICE', models.CharField(default='0', max_length=50)),
],
),
migrations.CreateModel(
name='G_POST_V',
fields=[
('GID', models.AutoField(primary_key=True, serialize=False)),
('GazetteNo', models.IntegerField()),
('PostTitle', models.CharField(max_length=200)),
('District', models.CharField(max_length=50)),
('Description', models.CharField(max_length=2000)),
],
),
migrations.CreateModel(
name='Gazette',
fields=[
('GazetteNo', models.IntegerField(primary_key=True, serialize=False)),
('GazetteDate', models.CharField(max_length=100)),
('PriceAvl', models.CharField(default='0', max_length=100)),
],
),
migrations.CreateModel(
name='Gazette_mst',
fields=[
('GID', models.AutoField(primary_key=True, serialize=False)),
('GType', models.CharField(max_length=10)),
('GTitle', models.CharField(max_length=100)),
('GDesc', models.CharField(max_length=200)),
],
),
migrations.CreateModel(
name='Users',
fields=[
('UserID', models.AutoField(primary_key=True, serialize=False)),
('UserEmpID', models.CharField(default='00000', max_length=5)),
('UserDepartment', models.CharField(max_length=20)),
('UserPassword', models.CharField(max_length=20)),
],
),
migrations.CreateModel(
name='G_GEN',
fields=[
('rowId', models.AutoField(primary_key=True, serialize=False)),
('GazetteNo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='CgpApp.gazette')),
],
),
]
from django.db import models
# Create your models here.
from django.db import models
from django.db.models.fields import AutoField
# Create your models here.
class Users(models.Model):
UserID = models.AutoField(primary_key=True)
UserEmpID = models.CharField(max_length=5, default='00000')
UserDepartment = models.CharField(max_length=20)
UserPassword = models.CharField(max_length=20)
class Gazette(models.Model):
GazetteNo = models.IntegerField(primary_key=True)
GazetteDate = models.CharField(max_length=100)
PriceAvl = models.CharField(
max_length=100, default="0")
class Gazette_mst(models.Model):
GID = models.AutoField(primary_key=True)
GType = models.CharField(max_length=10)
GTitle = models.CharField(max_length=100)
GDesc = models.CharField(max_length=200)
class G_GEN(models.Model):
rowId = models.AutoField(primary_key=True)
GazetteNo = models.ForeignKey(Gazette, on_delete=models.CASCADE)
class G_POST_V(models.Model):
GID = models.AutoField(primary_key=True)
GazetteNo = models.IntegerField()
PostTitle = models.CharField(max_length=200)
District = models.CharField(max_length=50)
Description = models.CharField(max_length=2000)
class Commodity(models.Model):
ITEM = models.AutoField(primary_key=True)
GAZETTE = models.CharField(max_length=4, default="N/A")
NAME = models.CharField(max_length=200)
CURRENTPRICE = models.CharField(max_length=50, default="0")
NEWPRICE = models.CharField(max_length=50, default="0")
class Vac(models.Model):
id = models.AutoField(primary_key=True)
department = models.CharField(max_length=20000)
title = models.CharField(max_length=20000)
description = models.CharField(max_length=20000)
summary = models.CharField(max_length=20000)
date = models.CharField(max_length=20000)
url = models.CharField(max_length=20000)
cat = models.CharField(max_length=100)
class Exm(models.Model):
id = models.AutoField(primary_key=True)
department = models.CharField(max_length=20000)
title = models.CharField(max_length=20000)
description = models.CharField(max_length=20000)
summary = models.CharField(max_length=20000)
date = models.CharField(max_length=20000)
url = models.CharField(max_length=20000)
cat = models.CharField(max_length=100)
class ComCompare(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=100)
DateofMonth = models.CharField(max_length=100)
price = models.CharField(max_length=10)
Item,Importers Maximum,Maximum Retail Price,Maximum Retail Price
"",Wholesale Price,(Unpacketed),(Packeted)
White Sugar,Rs. 116/- per kg.,Rs. 122/- per kg.,Rs. 125/- per kg.
Brown or Red Sugar,,Rs. 125/- per kg.,Rs. 128/- per kg.
"",Variety,Maximum Retail
"",,Price
Keeri Samba,,Rs. 125/- per kg.
White/Red Samba - Steamed/Boiled (excluding Suduru Samba),,Rs. 103/- per kg.
White/Red Nadu - Steamed/Boiled (excluding Mottaikarupan and Attakari),,Rs. 98/- per kg.
White/Red Raw Rice,,Rs. 95/- per kg.
"I fldgi ( ^I& fPoh - YS% ,xld m%cd;dka;s%l iudcjd ckrcfha w;s fYI .ei m;%h - 2021'09'041A PART I : SEC. (I) - GAZETTE EXTRAORDINARY OF THE DEMOCRATIC SOCIALIST REPUBLIC OF SRI LANKA - 04.09.2021"
matchers = ['department', 'ministry']
matching = [s for s in departmentList if any(
xs in s for xs in matchers)]
# to english
sent = " ".join(w for w in nltk.wordpunct_tokenize(sent)
if w.lower() in words or not w.isalpha())
from django.db.models import fields
from rest_framework import serializers
from CgpApp.models import Gazette, Users, Gazette_mst, G_POST_V, Commodity, Vac, ComCompare
class UserSerializer(serializers.ModelSerializer):
class Meta:
model = Users
fields = ('UserID', 'UserEmpID', 'UserDepartment', 'UserPassword')
class GazetteSerializer(serializers.ModelSerializer):
class Meta:
model = Gazette
fields = ('GazetteNo', 'GazetteDate', 'PriceAvl')
class GazetteMasterSerializer(serializers.ModelSerializer):
class Meta:
model = Gazette_mst
fields = ('GID', 'GTitle', 'GType', 'GDesc')
class PostVSerializer(serializers.ModelSerializer):
class Meta:
model = G_POST_V
fields = ('GID', 'GazetteNo', 'PostTitle', 'District', 'Description')
class CommoditySerializer(serializers.ModelSerializer):
class Meta:
model = Commodity
fields = ('ITEM', 'GAZETTE', 'NAME', 'CURRENTPRICE', 'NEWPRICE')
class VacSerializer(serializers.ModelSerializer):
class Meta:
model = Vac
fields = ('id', 'department', 'title',
'description', 'summary', 'date', 'url', 'cat')
class ComSerializer(serializers.ModelSerializer):
class Meta:
model = ComCompare
fields = ('id', 'name', 'DateofMonth', 'price')
from json.decoder import JSONDecoder
import matplotlib.pyplot as plt
from nltk import word_tokenize
from typing import Any
from django.http import response
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt
from rest_framework.parsers import JSONParser
from django.http.response import JsonResponse
from CgpApp.models import Gazette, Users, Gazette_mst, Commodity
from CgpApp.serializers import GazetteMasterSerializer, GazetteSerializer, UserSerializer, CommoditySerializer
from CgpApp.cgphelper import cgphelper
from CgpApp.topicclustering import TopicCluster
import numpy
from clint.textui import progress
import datetime
import calendar
import os.path
import glob
import re
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize, sent_tokenize
import enchant
import spacy
import dotenv
nltk.download('punkt')
dotenv.load_dotenv()
class Summarizer_:
def summarizer(text):
sentenceText = text.replace(" ", "")
lll = ''
if not sentenceText == "":
# Tokenizing the text
stopWords = set(stopwords.words("english"))
words = word_tokenize(text)
# Creating a frequency table to keep the
# score of each word
freqTable = dict()
for word in words:
word = word.lower()
if word in stopWords:
continue
if word in freqTable:
freqTable[word] += 1
else:
freqTable[word] = 1
# Creating a dictionary to keep the score
# of each sentence
sentences = sent_tokenize(text)
sentenceValue = dict()
for sentence in sentences:
for word, freq in freqTable.items():
if word in sentence.lower():
if sentence in sentenceValue:
sentenceValue[sentence] += freq
else:
sentenceValue[sentence] = freq
sumValues = 0
for sentence in sentenceValue:
sumValues += sentenceValue[sentence]
# Average value of a sentence from the original text
average = int(sumValues / len(sentenceValue))
# Storing sentences into our summary.
summary = ''
for sentence in sentences:
if (sentence in sentenceValue) and (sentenceValue[sentence] > (1.2 * average)):
summary += " " + sentence
def clean_text(x):
x = " ".join(x.split())
x = " ".join((" ".join(x.split("[**"))).split("**]"))
x = re.sub(r"\([^()]*\)", "", x)
key_value_strip = (x.split(":"))
string = " ".join([sub_unit for sub_unit in key_value_strip if len(
sub_unit) > 10]) # Remove strings less than 10 Characters
# Remove serialization (eg 1. 1)
x = re.sub(r"(\d+)+(\.|\))", "", string)
x = re.sub(r"(\*|\?|=)+", "", x) # Removing ( *, ? and = )
# Removing dupicate words
x = re.sub(r"\b(\w+)( \1\b)+", r"\1", x)
x = x.replace("FOLLOW UP", "FOLLOWUP")
x = x.replace("FOLLOW-UP", "FOLLOWUP")
# Remove firstname
x = re.sub(
r"(\b)(f|F)(irst)(\b)?[\d\-\d]*(\s)*(\b)?(n|N)(ame)[\d\-\d]*(\s)*[\d\-\d]*(\b)", "", x)
x = re.sub(
r"(\b)(l|L)(ast)(\b)?[\d\-\d]*(\s)*(\b)?(n|N)(ame)[\d\-\d]*(\s)*[\d\-\d]*(\b)", "", x)
x = re.sub(r"(\b)(d|D)\.?(r|R)\.?(\b)",
"", x) # Remove abreviation
# Remove punctuations
x = re.sub(r"([^A-Za-z0-9\s](\s)){2,}", "", x)
return(x.replace(" ", " "))
summaryCleaned = clean_text(summary)
# Process the text (to find Out dependencies and hirerachy of the text)
spacy.prefer_gpu()
nlp = spacy.load("en_core_web_sm")
doc = nlp(summaryCleaned)
lll = doc
# print("look here:::::::::::::::::", doc)
print("Input Description:")
print(text)
# print(lll)
else:
print("")
return "Summarized result: \n" + str(lll)
def commodityPriceHandler():
pdfFiles = []
for file in glob.glob("CgpApp/scrape_output/price/*.pdf"):
t = file.split("\\")
pdfFiles.append(t[1])
for iFile in pdfFiles:
iFile = iFile.replace(".pdf", "")
csvFile = "CgpApp/scrape_output/price/"+iFile+".csv"
file = "CgpApp/scrape_output/price/"+iFile+".pdf"
df = read_pdf(file, pages="all") # address of pdf file
# print(tabulate(df))
tabula.convert_into(file, csvFile)
gnumber = os.path.basename(file)[0:4]
gazetteRecord = Gazette.objects.get(GazetteNo=gnumber)
# update gazette table
gazetteRecord.PriceAvl = "11"
gazetteRecord.save()
df = pd.read_csv(csvFile,
sep=",", encoding="cp1252")
c = 0
if "Item" in df:
priceData = pd.read_csv(
csvFile, sep=",", encoding='cp1252', usecols=[0, 3], header=None)
c = 1
elif "Variety" in df:
priceData = pd.read_csv(
csvFile, sep=",", encoding='cp1252', usecols=[0, 2], header=None)
c = 2
if c == 1:
for index, row in priceData.iterrows():
if 'Rs.' in str(row):
print(row[0], row[3])
Commodity.objects.create(
NAME=row[0], NEWPRICE=row[3], GAZETTE=gnumber)
elif c == 2:
for index, row in priceData.iterrows():
if 'Rs.' in str(row):
print(row[0], row[2])
Commodity.objects.create(
NAME=row[0], NEWPRICE=row[2], GAZETTE=gnumber)
from django.test import TestCase
# Create your tests here.
This diff is collapsed.
from django.conf.urls import url
from CgpApp import views
urlpatterns = [
url(r'^gazette/$', views.gazetteApi),
url(r'^gazette/([0-9]+)$', views.gazetteApi),
url(r'^gazettemst/$', views.gazetteMSTApi),
url(r'^postvacancy/([a-zA-Z]+)$', views.vacancyAPI),
url(r'^userhandler/([a-zA-Z]+[0-9]+)$', views.userhandlerAPI),
url(r'^getalldata/$', views.getAllData),
url(r'^getalldata/$', views.getAllData),
url(r'^examinations/$', views.examinationAPI),
url(r'^gazetteDownloadDetailsApi/$', views.gazetteDownloadDetailsApi),
url(r'^commodity/$', views.commodityAPI),
url(r'^vacancy/([a-zA-Z]+)$', views.vacancy),
url(r'^examination/([a-zA-Z]+)$', views.examination),
url(r'^testapi/$', views.testapi),
url(r'^spellingChecker/([a-zA-Z]+)$', views.spellingChecker),
url(r'^comcompare/$', views.comcompare),
]
from django.shortcuts import render
# Create your views here.
This diff is collapsed.
"""
ASGI config for cgp1 project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cgp1.settings')
application = get_asgi_application()
"""
Django settings for cgp1 project.
Generated by 'django-admin startproject' using Django 3.2.5.
For more information on this file, see
https://docs.djangoproject.com/en/3.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.2/ref/settings/
"""
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-nx*xc56u1xj)393=*v!b8p66ba!xjk=hzv^h6k6avgs8k3p0uf'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
CORS_ALLOWED_ORIGINS = [
"https://example.com",
"https://sub.example.com",
"http://localhost:8080",
"http://127.0.0.1:9000"
]
CORS_ORIGIN_ALLOW_ALL = True
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'corsheaders',
'CgpApp.apps.CgpappConfig',
'rest_framework'
]
MIDDLEWARE = [
'corsheaders.middleware.CorsMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'cgp1.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'cgp1.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
# Password validation
# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.2/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
STATIC_URL = '/static/'
# Default primary key field type
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
"""cgp1 URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.conf.urls import url, include
urlpatterns = [
path('admin/', admin.site.urls),
url(r'^', include('CgpApp.urls'))
]
"""
WSGI config for cgp1 project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cgp1.settings')
application = get_wsgi_application()
File added
[]
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
[{"state": "unavailable"}]
\ No newline at end of file
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
[{"state": "unavailable"}]
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment