Skip to content

2
2022-226
Commit 05e607a1 authored by Wickramasinghe R.J.P's avatar Wickramasinghe R.J.P
Browse files
Options

changes added for ontology generation

parent c7ac0681
Hide whitespace changes
Inline Side-by-side
Showing with 40434 additions and 0 deletions
Ontology_Based_Information_System/Backend With Ontology/.idea/.gitignore 0 → 100644
View file @ 05e607a1
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
Ontology_Based_Information_System/Backend With Ontology/.idea/CanisCare_Backend.iml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="FacetManager">
<facet type="django" name="Django">
<configuration>
<option name="rootFolder" value="$MODULE_DIR$" />
<option name="settingsModule" value="djangoProject/settings.py" />
<option name="manageScript" value="$MODULE_DIR$/manage.py" />
<option name="environment" value="&lt;map/&gt;" />
<option name="doNotUseTestRunner" value="false" />
<option name="trackFilePattern" value="migrations" />
</configuration>
</facet>
</component>
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Django" />
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/../djangoProject\templates" />
</list>
</option>
</component>
</module>
\ No newline at end of file
Ontology_Based_Information_System/Backend With Ontology/.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ 05e607a1
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N806" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
Ontology_Based_Information_System/Backend With Ontology/.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ 05e607a1
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
Ontology_Based_Information_System/Backend With Ontology/.idea/misc.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (djangoProject)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
Ontology_Based_Information_System/Backend With Ontology/.idea/modules.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/CanisCare_Backend.iml" filepath="$PROJECT_DIR$/.idea/CanisCare_Backend.iml" />
</modules>
</component>
</project>
\ No newline at end of file
Ontology_Based_Information_System/Backend With Ontology/.idea/vcs.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
Ontology_Based_Information_System/Backend With Ontology/Procfile 0 → 100644
View file @ 05e607a1
web gunicorn djangoProject.wsgi:application --log-file -
\ No newline at end of file
Ontology_Based_Information_System/Backend With Ontology/api/admin.py 0 → 100644
View file @ 05e607a1
from django.contrib import admin
from .models import Note
admin.site.register(Note)
Ontology_Based_Information_System/Backend With Ontology/api/apps.py 0 → 100644
View file @ 05e607a1
from django.apps import AppConfig
class ApiConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'api'
Ontology_Based_Information_System/Backend With Ontology/api/migrations/0001_initial.py 0 → 100644
View file @ 05e607a1
# Generated by Django 4.1 on 2022-08-25 06:14
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Note',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('body', models.TextField()),
('updated', models.DateTimeField(auto_now=True)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'ordering': ['-updated'],
},
),
]
Ontology_Based_Information_System/Backend With Ontology/api/models.py 0 → 100644
View file @ 05e607a1
from django.db import models
class Note(models.Model):
body = models.TextField()
updated = models.DateTimeField(auto_now=True)
created = models.DateTimeField(auto_now_add=True)
def __str__(self):
return self.body[0:50]
class Meta:
ordering = ['-updated']
Ontology_Based_Information_System/Backend With Ontology/api/serializers.py 0 → 100644
View file @ 05e607a1
from rest_framework.serializers import ModelSerializer
from .models import Note
class NoteSerializer(ModelSerializer):
class Meta:
model = Note
fields = '__all__'
Ontology_Based_Information_System/Backend With Ontology/api/tests.py 0 → 100644
View file @ 05e607a1
from django.test import TestCase
# Create your tests here.
Ontology_Based_Information_System/Backend With Ontology/api/urls.py 0 → 100644
View file @ 05e607a1
from django.urls import path
from . import views
urlpatterns = [
path('', views.getRoutes),
path('diseasecategory/<str:pk>/', views.get_categorized_diseases),
path('diseases/', views.get_diseases),
path('disease/<str:pk>/', views.get_disease_description),
path('disease/<str:pk>/<str:pk2>/', views.get_disease_sub),
path('disease/<str:pk>/<str:pk2>/<str:pk3>/', views.get_disease_sub_info),
path('diseaselink/<str:pk>/', views.get_link),
]
Ontology_Based_Information_System/Backend With Ontology/api/views.py 0 → 100644
View file @ 05e607a1
from django.shortcuts import render
from rest_framework.decorators import api_view
from rest_framework.response import Response
from owlready2 import *
from rdflib import *
onto = get_ontology("dogDisease.owl").load(reload_if_newer=True)
graph = default_world.as_rdflib_graph()
@api_view(['GET'])
def getRoutes(request):
routes = [
{
'EndPoint': '/diseases/',
'method': 'GET',
'body': None,
'description': 'Returns a List of Diseases'
},
{
'EndPoint': '/diseasecategory/diseasename',
'method': 'GET',
'body': None,
'description': 'Returns Categorized Diseases '
},
{
'EndPoint': '/disease/diseasename/',
'method': 'GET',
'body': None,
'description': 'Get Description for the Disease'
},
{
'EndPoint': '/disease/diseasename/infotype/',
'method': 'GET',
'body': None,
'description': 'Get List of Individuals Relevant to Sub Topic'
},
{
'EndPoint': '/disease/diseasename/infotype/individual',
'method': 'GET',
'body': None,
'description': 'Get Relevant Information to Each Individual'
},
{
'EndPoint': '/diseaselink/diseasename',
'method': 'GET',
'body': None,
'description': 'Get SeeAlso Link for Disease'
}
]
return Response(routes)
@api_view(['GET'])
def get_disease_description(request, pk):
disease_description = list(graph.query(
"""
SELECT ?description
WHERE { ?disease <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#NamedDisease>;
FILTER(?disease = <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl#""" + pk + """>).
?disease <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDescription> ?description.
}"""
))
new_disease_description = [str(x[0]) for x in disease_description]
new_disease_description = ' '.join(new_disease_description)
new_disease_description = new_disease_description.split('.')
new_disease_description = new_disease_description[0:-1]
return Response(new_disease_description)
@api_view(['GET'])
def get_disease_sub_info(request, pk, pk2, pk3):
new_pk2 = pk2[:-1]
new_pk3 = new_pk2
if new_pk3 == "Cause":
new_pk3 = "Casue"
sub_individual_list_info = list(graph.query(
"""
SELECT ?individualinfo
WHERE { ?disease <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#NamedDisease>;
FILTER((?disease = <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl#""" + pk + """>) && (?treat = <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl#""" + pk3 + """>)).
?disease <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#has""" + new_pk2 + """> ?treat.
?treat <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#has""" + new_pk3 + """Description> ?individualinfo.
}"""))
new_sub_individual_list_info = [str(x[0]) for x in sub_individual_list_info]
new_sub_individual_list_info = ' '.join(new_sub_individual_list_info)
new_sub_individual_list_info = new_sub_individual_list_info.split('.')
new_sub_individual_list_info = new_sub_individual_list_info[0:-1]
return Response(new_sub_individual_list_info)
@api_view(['GET'])
def get_disease_sub(request, pk, pk2):
new_pk2 = pk2[:-1]
sub_individual_list = list(graph.query(
"""
SELECT ?subindividual
WHERE { ?disease <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#NamedDisease>;
FILTER(?disease = <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl#""" + pk + """>).
?disease <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#has""" + new_pk2 + """> ?subindividual.
}"""))
new_sub_individual_list = [str(x[0]) for x in sub_individual_list]
fin_sub_list = []
for y in new_sub_individual_list:
itm = y[y.find('#'):][1:]
fin_sub_list.append(itm)
return Response(new_sub_individual_list)
@api_view(['GET'])
def get_diseases(request):
disease_list = list(graph.query("""
SELECT ?disease
WHERE {
?disease <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#NamedDisease> .
}"""))
new_disease_list = [str(x[0]) for x in disease_list]
fin_disease_list = []
for y in new_disease_list:
des = y[y.find('#'):][1:]
fin_disease_list.append(des)
return Response(fin_disease_list)
@api_view(['GET'])
def get_categorized_diseases(request, pk):
categorized_diseases = list(graph.query("""
SELECT ?s
WHERE { ?s <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection> <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl#""" + pk + """> }
"""))
new_categorized_diseases = [str(x[0]) for x in categorized_diseases]
fin_des_list = []
for y in new_categorized_diseases:
des = y[y.find('#'):][1:]
fin_des_list.append(des)
return Response(fin_des_list)
@api_view(['GET'])
def get_link(request, pk):
seeAlsoLink = list(graph.query("""
SELECT ?link
WHERE { <http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl#""" + pk + """> <http://www.w3.org/2000/01/rdf-schema#seeAlso> ?link }
"""))
new_link_list = [str(x[0]) for x in seeAlsoLink]
fin_link_list = []
for link in new_link_list:
fin_link_list.append(link)
return Response(fin_link_list)
Ontology_Based_Information_System/Backend With Ontology/djangoProject/asgi.py 0 → 100644
View file @ 05e607a1
"""
ASGI config for djangoProject project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djangoProject.settings')
application = get_asgi_application()
Ontology_Based_Information_System/Backend With Ontology/djangoProject/settings.py 0 → 100644
View file @ 05e607a1
"""
Django settings for djangoProject project.
Generated by 'django-admin startproject' using Django 4.1.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.1/ref/settings/
"""
import os
from pathlib import Path
import django_heroku
import dj_database_url
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-3ts!3sf#jntwaa-9r(^t%7ty=nc03p_eil3yha%#hinc30b&ww'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = ['*']
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'api.apps.ApiConfig',
'rest_framework',
'corsheaders'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
"corsheaders.middleware.CorsMiddleware",
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'djangoProject.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [BASE_DIR / 'templates']
,
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'djangoProject.wsgi.application'
# Database
# https://docs.djangoproject.com/en/4.1/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
# Password validation
# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.1/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.1/howto/static-files/
STATIC_URL = 'static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
STATICFILES_DIRS = (os.path.join(BASE_DIR, 'static'),)
django_heroku.settings(locals())
# Default primary key field type
# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
CORS_ALLOW_ALL_ORIGINS = True
Ontology_Based_Information_System/Backend With Ontology/djangoProject/urls.py 0 → 100644
View file @ 05e607a1
from django.contrib import admin
from django.urls import path,include
urlpatterns = [
path('admin/', admin.site.urls),
path('', include('api.urls'))
]
Ontology_Based_Information_System/Backend With Ontology/djangoProject/wsgi.py 0 → 100644
View file @ 05e607a1
"""
WSGI config for djangoProject project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djangoProject.settings')
application = get_wsgi_application()
Ontology_Based_Information_System/Backend With Ontology/dogDisease.owl 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Backend With Ontology/manage.py 0 → 100644
View file @ 05e607a1
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
from owlready2 import *
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djangoProject.settings')
# onto = get_ontology("C:\\Users\\LENOVO\\Documents\\Research\\dogSkinDisease.owl").load()
#
# default_world.set_backend(filename="disease.sqlite3")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()
Ontology_Based_Information_System/Backend With Ontology/templates/base.html 0 → 100644
View file @ 05e607a1
<!DOCTYPE html>
<html lang="en">
{#<link rel="stylesheet" href="{{ url_for('static', filename='style.css')}}">#}
<head>
<meta charset="UTF-8">
<title>Chatbot</title>
</head>
<body>
<div class="container">
<div class="chatbox">
<div class="chatbox__support">
<div class="chatbox__header">
<div class="chatbox__image--header">
<img src="https://img.icons8.com/color/48/000000/circled-user-female-skin-type-5--v1.png" alt="image">
</div>
<div class="chatbox__content--header">
<h4 class="chatbox__heading--header">Chat support</h4>
<p class="chatbox__description--header">Hi. My name is Sam. How can I help you?</p>
</div>
</div>
<div class="chatbox__messages">
<div></div>
</div>
<div class="chatbox__footer">
<input type="text" placeholder="Write a message...">
<button class="chatbox__send--footer send__button">Send</button>
</div>
</div>
<div class="chatbox__button">
{# <button><img src="{{ url_for('static', filename='images/chatbox-icon.svg') }}" /></button>#}
</div>
</div>
</div>
</body>
</html>
\ No newline at end of file
Ontology_Based_Information_System/Ontology Generator/New Scripts/.idea/.gitignore 0 → 100644
View file @ 05e607a1
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
Ontology_Based_Information_System/Ontology Generator/New Scripts/.idea/buildOntology.iml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
Ontology_Based_Information_System/Ontology Generator/New Scripts/.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ 05e607a1
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N806" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
Ontology_Based_Information_System/Ontology Generator/New Scripts/.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ 05e607a1
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
Ontology_Based_Information_System/Ontology Generator/New Scripts/.idea/misc.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (buildOntology)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
Ontology_Based_Information_System/Ontology Generator/New Scripts/.idea/modules.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/buildOntology.iml" filepath="$PROJECT_DIR$/.idea/buildOntology.iml" />
</modules>
</component>
</project>
\ No newline at end of file
Ontology_Based_Information_System/Ontology Generator/New Scripts/catalog-v001.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<catalog prefer="public" xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog">
<group id="Folder Repository, directory=, recursive=true, Auto-Update=true, version=2" prefer="public" xml:base="">
<uri id="Automatically generated entry, Timestamp=1665432340207" name="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl" uri="dogDisease.owl"/>
<uri id="Automatically generated entry, Timestamp=1665432340207" name="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease" uri="dogSkinDisease.owl"/>
</group>
</catalog>
Ontology_Based_Information_System/Ontology Generator/New Scripts/csv_convertor.py 0 → 100644
View file @ 05e607a1
import re
import pandas as pd
if __name__ == '__main__':
df = pd.read_csv('petmd.csv')
link_df = pd.read_csv('link.csv')
df_col = df.columns
link_df_col = link_df.columns
data_dict = {
'Disease': [],
'DiseaseDescription': [],
'DiseaseCauseDescription': [],
'DiseaseDiagnoseDescription': [],
'DiseasePreventionDescription': [],
'DiseaseSymptomDescription': [],
'DiseaseTreatmentDescription': [],
'DiseaseInfection': [],
'DiseaseLink': []
}
disease = ''
diseases = []
for index, row in df.iterrows():
new_disease = row[df_col[0]]
if disease != new_disease:
disease = new_disease
diseases.append(disease)
for disease in diseases:
grouped_rows = df[df[df_col[0]] == disease]
link_rows = link_df[link_df[link_df_col[1]] == 'Disease Link']
disease_link_row = link_rows[link_rows[link_df_col[0]] == disease]
if disease_link_row.empty:
data_dict['DiseaseLink'].append('https://www.petmd.com')
else:
for row in disease_link_row.iterrows():
description = row[1][df_col[2]]
data_dict['DiseaseLink'].append(str(description).split(' ')[1][:-1])
break
regex = re.compile('[^a-zA-Z]')
txt = str(disease).strip().replace(" ", "").replace("'", "").replace("inDogs", "")
head, sep, tail = txt.partition(":")
txt2 = head
head, sep, tail = txt2.partition("(")
txt3 = head
head, sep, tail = txt3.partition("/")
txt4 = regex.sub('', str(head))
data_dict['Disease'].append(txt4)
temp = 0
for row in grouped_rows.iterrows():
regexp = re.compile(r'What|what')
topic = row[1][df_col[1]]
description = row[1][df_col[2]]
if regexp.search(str(topic)):
data_dict['DiseaseDescription'].append(str(description))
temp = 1
break
if temp == 0:
data_dict['DiseaseDescription'].append('')
temp = 0
for row in grouped_rows.iterrows():
regexp = re.compile(r'Cause|cause')
topic = row[1][df_col[1]]
description = row[1][df_col[2]]
if regexp.search(str(topic)):
data_dict['DiseaseCauseDescription'].append(str(description))
temp = 1
break
if temp == 0:
data_dict['DiseaseCauseDescription'].append('')
temp = 0
for row in grouped_rows.iterrows():
regexp = re.compile(r'Diagnose|diagnose')
topic = row[1][df_col[1]]
description = row[1][df_col[2]]
if regexp.search(str(topic)):
data_dict['DiseaseDiagnoseDescription'].append(str(description))
temp = 1
break
if temp == 0:
data_dict['DiseaseDiagnoseDescription'].append('')
temp = 0
for row in grouped_rows.iterrows():
regexp = re.compile(r'Prevention|prevention')
topic = row[1][df_col[1]]
description = row[1][df_col[2]]
if regexp.search(str(topic)):
data_dict['DiseasePreventionDescription'].append(str(description))
temp = 1
break
if temp == 0:
data_dict['DiseasePreventionDescription'].append('')
temp = 0
for row in grouped_rows.iterrows():
regexp = re.compile(r'Symptoms|symptoms')
topic = row[1][df_col[1]]
description = row[1][df_col[2]]
if regexp.search(str(topic)):
data_dict['DiseaseSymptomDescription'].append(str(description))
temp = 1
break
if temp == 0:
data_dict['DiseaseSymptomDescription'].append('')
temp = 0
for row in grouped_rows.iterrows():
regexp = re.compile(r'Treatment|treatment')
topic = row[1][df_col[1]]
description = row[1][df_col[2]]
if regexp.search(str(topic)):
data_dict['DiseaseTreatmentDescription'].append(str(description))
temp = 1
break
if temp == 0:
data_dict['DiseaseTreatmentDescription'].append('')
temp = 0
for row in grouped_rows.iterrows():
description = row[1][df_col[2]]
regexp = re.compile(r'Allerg|allerg')
if regexp.search(str(description)):
data_dict['DiseaseInfection'].append('Allergic')
temp = 1
break
regexp = re.compile(r'Bacter|bacter')
if regexp.search(str(description)):
data_dict['DiseaseInfection'].append('Bacterial')
temp = 1
break
regexp = re.compile(r'Flea|flea')
if regexp.search(str(description)):
data_dict['DiseaseInfection'].append('Fleas')
temp = 1
break
regexp = re.compile(r'Fung|fung')
if regexp.search(str(description)):
data_dict['DiseaseInfection'].append('Fungal')
temp = 1
break
regexp = re.compile(r'Virus|virus')
if regexp.search(str(description)):
data_dict['DiseaseInfection'].append('Viral')
temp = 1
break
if temp == 0:
data_dict['DiseaseInfection'].append('')
pd.DataFrame(data_dict).drop_duplicates().to_csv('dogDisease.csv', index=False)
pd.read_csv('test.csv').to_csv('dogDisease.csv', mode='a', index=False, header=False)
Ontology_Based_Information_System/Ontology Generator/New Scripts/dogDisease.csv 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Ontology Generator/New Scripts/dogDisease.owl 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Ontology Generator/New Scripts/dogDisease.properties 0 → 100644
View file @ 05e607a1
#Mon Sep 12 20:54:26 IST 2022
jdbc.url=
jdbc.driver=
jdbc.user=
jdbc.password=
Ontology_Based_Information_System/Ontology Generator/New Scripts/dogSkinDisease.owl 0 → 100644
View file @ 05e607a1
<?xml version="1.0"?>
<rdf:RDF xmlns="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#"
xml:base="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease"
xmlns:owl="http://www.w3.org/2002/07/owl#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:xml="http://www.w3.org/XML/1998/namespace"
xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xmlns:disease="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#">
<owl:Ontology rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease">
<rdfs:comment rdf:datatype="http://www.w3.org/2001/XMLSchema#string">Ontology for Dog Skin Disease Domain.</rdfs:comment>
<rdfs:comment rdf:datatype="http://www.w3.org/2001/XMLSchema#string">This has been developed with the purpose of providing sufficient information to the users relevant to the most common skin diseases of dogs.</rdfs:comment>
</owl:Ontology>
<!--
///////////////////////////////////////////////////////////////////////////////////////
//
// Object Properties
//
///////////////////////////////////////////////////////////////////////////////////////
-->
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasArea -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasArea">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasCause -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasCause">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo"/>
<owl:inverseOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isCauseOf"/>
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseCause"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDiagnose -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDiagnose">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo"/>
<owl:inverseOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isDiagnoseOf"/>
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseDiagnose"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo"/>
<owl:inverseOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfectionOf"/>
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo">
<owl:inverseOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasPrevention -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasPrevention">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo"/>
<owl:inverseOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isPreventionOf"/>
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseasePrevention"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasSymptom -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasSymptom">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo"/>
<owl:inverseOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isSymptomOf"/>
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseSymptom"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasTreatment -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasTreatment">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfo"/>
<owl:inverseOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isTreatmentOf"/>
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseTreatment"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isCauseOf -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isCauseOf">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isDiagnoseOf -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isDiagnoseOf">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfectionOf -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfectionOf">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isPreventionOf -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isPreventionOf">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isSymptomOf -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isSymptomOf">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
</owl:ObjectProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isTreatmentOf -->
<owl:ObjectProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isTreatmentOf">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#isInfoOf"/>
</owl:ObjectProperty>
<!--
///////////////////////////////////////////////////////////////////////////////////////
//
// Data properties
//
///////////////////////////////////////////////////////////////////////////////////////
-->
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasAllergicType -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasAllergicType">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfectionType"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasAreaDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasAreaDescription">
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasBehaviouralSymptomDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasBehaviouralSymptomDescription">
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasCasueDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasCasueDescription">
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseCause"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDescription">
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDiagnoseDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDiagnoseDescription">
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasFungalType -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasFungalType">
<rdfs:subPropertyOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfectionType"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfectionType -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfectionType">
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasPreventionDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasPreventionDescription">
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasScientificName -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasScientificName">
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#FunctionalProperty"/>
<rdfs:domain rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasSymptomDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasSymptomDescription">
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasTreatmentDescription -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasTreatmentDescription">
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasType -->
<owl:DatatypeProperty rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasType">
<rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#string"/>
</owl:DatatypeProperty>
<!--
///////////////////////////////////////////////////////////////////////////////////////
//
// Classes
//
///////////////////////////////////////////////////////////////////////////////////////
-->
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Allergic -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Allergic">
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#AllergicDisease -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#AllergicDisease">
<owl:equivalentClass>
<owl:Class>
<owl:intersectionOf rdf:parseType="Collection">
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Allergic"/>
</owl:Restriction>
</owl:intersectionOf>
</owl:Class>
</owl:equivalentClass>
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Bacterial -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Bacterial">
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#BacterialDisease -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#BacterialDisease">
<owl:equivalentClass>
<owl:Class>
<owl:intersectionOf rdf:parseType="Collection">
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Bacterial"/>
</owl:Restriction>
</owl:intersectionOf>
</owl:Class>
</owl:equivalentClass>
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease">
<rdfs:subClassOf>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasArea"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseArea"/>
</owl:Restriction>
</rdfs:subClassOf>
<rdfs:subClassOf>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasDiagnose"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseDiagnose"/>
</owl:Restriction>
</rdfs:subClassOf>
<rdfs:subClassOf>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
</owl:Restriction>
</rdfs:subClassOf>
<rdfs:subClassOf>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasPrevention"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseasePrevention"/>
</owl:Restriction>
</rdfs:subClassOf>
<rdfs:subClassOf>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasSymptom"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseSymptom"/>
</owl:Restriction>
</rdfs:subClassOf>
<rdfs:subClassOf>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasTreatment"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseTreatment"/>
</owl:Restriction>
</rdfs:subClassOf>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseArea -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseArea"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseCause -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseCause"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseDiagnose -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseDiagnose"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseasePrevention -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseasePrevention"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseSymptom -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseSymptom"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseTreatment -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseTreatment"/>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Fleas -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Fleas">
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#FleasDisease -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#FleasDisease">
<owl:equivalentClass>
<owl:Class>
<owl:intersectionOf rdf:parseType="Collection">
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Fleas"/>
</owl:Restriction>
</owl:intersectionOf>
</owl:Class>
</owl:equivalentClass>
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Fungal -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Fungal">
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#FungalDisease -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#FungalDisease">
<owl:equivalentClass>
<owl:Class>
<owl:intersectionOf rdf:parseType="Collection">
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Fungal"/>
</owl:Restriction>
</owl:intersectionOf>
</owl:Class>
</owl:equivalentClass>
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#NamedDisease -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#NamedDisease">
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Viral -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Viral">
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
</owl:Class>
<!-- http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#ViralDisease -->
<owl:Class rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#ViralDisease">
<owl:equivalentClass>
<owl:Class>
<owl:intersectionOf rdf:parseType="Collection">
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<owl:Restriction>
<owl:onProperty rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#hasInfection"/>
<owl:someValuesFrom rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Viral"/>
</owl:Restriction>
</owl:intersectionOf>
</owl:Class>
</owl:equivalentClass>
<rdfs:subClassOf rdf:resource="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
</owl:Class>
<!--
///////////////////////////////////////////////////////////////////////////////////////
//
// General axioms
//
///////////////////////////////////////////////////////////////////////////////////////
-->
<rdf:Description>
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AllDisjointClasses"/>
<owl:members rdf:parseType="Collection">
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#Disease"/>
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseArea"/>
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseCause"/>
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseDiagnose"/>
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseInfection"/>
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseasePrevention"/>
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseSymptom"/>
<rdf:Description rdf:about="http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogSkinDisease#DiseaseTreatment"/>
</owl:members>
</rdf:Description>
</rdf:RDF>
<!-- Generated by the OWL API (version 4.5.9.2019-02-01T07:24:44Z) https://github.com/owlcs/owlapi -->
Ontology_Based_Information_System/Ontology Generator/New Scripts/link.csv 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Ontology Generator/New Scripts/main.py 0 → 100644
View file @ 05e607a1
# This is a sample Python script.
# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
def print_hi(name):
# Use a breakpoint in the code line below to debug your script.
print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint.
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
print_hi('PyCharm')
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
Ontology_Based_Information_System/Ontology Generator/New Scripts/owl_generator.py 0 → 100644
View file @ 05e607a1
from owlready2 import *
import csv, types
if __name__ == '__main__':
onto = get_ontology("dogSkinDisease.owl").load()
onto_individuals = get_ontology("http://www.semanticweb.org/dogdisease/ontologies/2022/4/dogDisease.owl")
onto_individuals.imported_ontologies.append(onto)
f = open("dogDisease.csv", encoding='utf-8')
reader = csv.reader(f)
next(reader)
with onto_individuals:
for row in reader:
Disease, DiseaseDescription, DiseaseCauseDescription, DiseaseDiagnoseDescription, DiseasePreventionDescription, DiseaseSymptomDescription, DiseaseTreatmentDescription, DiseaseInfection, DiseaseLink = row
individual = onto.NamedDisease(str(Disease))
if DiseaseDescription:
individual.hasDescription.append(DiseaseDescription)
if DiseaseLink:
individual.seeAlso.append(DiseaseLink)
if DiseaseCauseDescription:
Sub_Class_name_cause = Disease + "Cause"
ClassCause = types.new_class(Sub_Class_name_cause, (onto.DiseaseCause,))
individualCause = ClassCause()
individualCause.hasCasueDescription.append(DiseaseCauseDescription)
individual.hasCause.append(individualCause)
if DiseaseDiagnoseDescription:
Sub_Class_name_diagnose = Disease + "Diagnose"
ClassDiagnose = types.new_class(Sub_Class_name_diagnose, (onto.DiseaseDiagnose,))
individualDiagnose = ClassDiagnose()
individualDiagnose.hasDiagnoseDescription.append(DiseaseDiagnoseDescription)
individual.hasDiagnose.append(individualDiagnose)
if DiseasePreventionDescription:
Sub_Class_name_prevention = Disease + "Prevention"
ClassPrevention = types.new_class(Sub_Class_name_prevention, (onto.DiseasePrevention,))
individualPrevention = ClassPrevention()
individualPrevention.hasPreventionDescription.append(DiseasePreventionDescription)
individual.hasPrevention.append(individualPrevention)
if DiseaseSymptomDescription:
Sub_Class_name_symptom = Disease + "Symptom"
ClassSymptom = types.new_class(Sub_Class_name_symptom, (onto.DiseaseSymptom,))
individualSymptom = ClassSymptom()
individualSymptom.hasSymptomDescription.append(DiseaseSymptomDescription)
individual.hasSymptom.append(individualSymptom)
if DiseaseTreatmentDescription:
Sub_Class_name_treatment = Disease + "Treatment"
ClassTreatment = types.new_class(Sub_Class_name_treatment, (onto.DiseaseTreatment,))
individualTreatment = ClassTreatment()
individualTreatment.hasTreatmentDescription.append(DiseaseTreatmentDescription)
individual.hasTreatment.append(individualTreatment)
if DiseaseInfection:
if DiseaseInfection == 'Allergic':
ClassAllergic = onto.Allergic
individualInfection = ClassAllergic('allergic')
individual.hasInfection.append(individualInfection)
elif DiseaseInfection == 'Bacterial':
ClassBacterial = onto.Bacterial
individualInfection = ClassBacterial('bacterial')
individual.hasInfection.append(individualInfection)
elif DiseaseInfection == 'Fleas':
ClassFleas = onto.Fleas
individualInfection = ClassFleas('fleas')
individual.hasInfection.append(individualInfection)
elif DiseaseInfection == 'Fungal':
ClassFungal = onto.Fungal
individualInfection = ClassFungal('fungal')
individual.hasInfection.append(individualInfection)
elif DiseaseInfection == 'Viral':
ClassViral = onto.Viral
individualInfection = ClassViral('viral')
individual.hasInfection.append(individualInfection)
onto_individuals.save("dogDisease.owl")
Ontology_Based_Information_System/Ontology Generator/New Scripts/petmd.csv 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Ontology Generator/New Scripts/test.csv 0 → 100644
View file @ 05e607a1
Disease,DiseaseDescription,DiseaseCauseDescription,DiseaseDiagnoseDescription,DiseasePreventionDescription,DiseaseSymptomDescription,DiseaseTreatmentDescription,DiseaseInfection,DiseaseLink
Dandruff,"If you’ve noticed white flakes in your dog’s fur, you might be wondering if they have dandruff or whether dogs even get dandruff. Yes, they can. Dandruff, or seborrheic dermatitis, is common in dogs and humans alike. Dandruff is not typically a sign of a serious condition, but you can talk with a veterinarian to find out what may be causing it. Make an appointment sooner rather than later if you see symptoms like extreme itchiness or a change in weight or behavior. Not all dandruff in dogs looks like white flakes. It can be either dry or oily, or it may not even be true dandruff. The underlying skin may or may not be red or patchy from hair loss. Here are the most common types: Seborrhea sicca (dry seborrhea): This dry dandruff may appear as white flakes with crusty skin. Seborrhea oleosa (oily seborrhea): Your dog’s skin may have an oily feel and give off an odor. Walking dandruff: If the dandruff seems like it’s moving, this is called Cheyletiella and is actually a type of mite.","Primary seborrhea is a congenital, genetic disease that typically starts at a young age and gets worse as your dog gets older. West Highland White Terriers, Basset Hounds, American Cocker Spaniels, and English Springer Spaniels are most commonly affected.","A skin scraping to test for mites and lice.An impression cytology (collection) of skin and ear debris to test for a yeast or bacterial infection that looks like seborrhea, such as Malassezia yeast.A blood chemistry panel to screen for diabetes or Cushing’s disease (your vet will need further tests to confirm the diagnosis before starting treatment).A blood test for thyroid hormone levels to determine whether your dog has hypothyroidism.A biopsy to look for autoimmune disease or cancer.","Keep your dog properly fed to prevent dry skin.When bathing your dog, use dog-formulated shampoo to prevent dry skin.Groom your dog regularly; some problems are caused by matted hair that provides breeding grounds for a variety of skin diseases. Regular grooming also helps keep you aware of any initial problems.Keep your dog flea and parasite free.Check your dog regularly for foxtails, burrs, and other sharp objects they may pick up when outside.You should also look for seeds, burrs and sharp plants or objects they may have picked up on a walk.suggest bathing your dog more frequently to prevent dandruff. using a specially formulated dog dandruff shampoo and making sure you dry your pet thoroughly after their bath.omega-3 fatty acids and vitamin E that promote a healthy coat and immune system",Itching that ranges from mild to severe.,"Needs frequent baths with anti-seborrheic shampoos, typically every 2 or 3 days to start with. These shampoos typically contain coal tar and salicylic acid.Frequent bathing is continued for 2-3 weeks or longer, until the skin improves. The goal of bathing is to remove excess keratin. Depending on how your dog responds to treatment, bathing frequency may decrease to every 1 to 2 weeks, or it may stay at every 2 to 3 days.clean your dog’s ears with a medicated ear cleaner every 2 to 3 days. If there is an infection in the ears, your veterinarian will prescribe an ear medication as well. dog may also be started on prednisone to decrease inflammation and debris buildup. Regular rechecks with your veterinarian, typically every one to three weeks, are important to monitor how your dog is responding to treatment.",Allergic,https://www.petmd.com/dog/conditions/skin/c_dg_canine_seborrhea
Dandruff,,"Skin allergies to fleas, food, and the environment.Hypothyroidism, caused by an underactive thyroid gland.Cushing’s disease, caused by an overactive adrenal gland.Diabetes mellitus.Mites and lice.Autoimmune diseases like pemphigus foliaceus, sebaceous adenitis, and lupus erythematosus.A type of cancer called cutaneous epitheliotropic lymphoma.Vitamin deficiencies like zinc-responsive dermatosis and vitamin A-responsive dermatosis.Dogs with lots of skin folds, like Basset Hounds, usually experience more affected skin in those folds.",,,"Very dry, dull coat.Dandruff.Greasy, oily skin that smells bad.Crusted, plaque-like (rough and scaly) skin lesions.Large amount of earwax and ear debris.",,,
Ringworm,,,,,"Itchiness, scratching, or excessive grooming",,,
Ringworm,,,,,"Circular areas of hair loss, often with a red and crusty edge.Broken hair and a poor hair coat.Dry, scaly skin or areas of excessive dandruff. Inflamed areas of skin.Darkened patches of skin. Inflamed nail beds or darkened or dry nails.Dry, brittle, or misshapen nails. circular itchy rash that typically appears on the skin.",,,
YeastInfection,"Yeast are spore-producing fungi that are always present on a dog’s skin, usually in low numbers, as part of the normal flora. A yeast infection happens when there’s an excessive amount of yeast in a certain area. Yeast infections in dogs are quite common and can occur anywhere on the skin, including the ears. Generally, yeast infections are caused by another issue. Anything that diminishes the normal defenses in the skin can make yeast infections more likely. Itchy, irritated skin with a distinct odor can be an indication of a yeast infection, a common skin concern for dogs. A dog will typically develop a yeast infection on the skin or in the ears. Regardless of location, a yeast infection can cause extreme discomfort and can be an indication of a more serious issue.",Yeast infections in dogs are usually secondary problems. This means that there is some other issue that is weakening the skin’s defense mechanisms to allow the yeast to grow in higher numbers than normal.It is very common to see yeast infections in a dog’s ears or on their skin if they have food allergies or environmental allergies. Other underlying issues that may cause yeast infections in dogs include hormonal problems or other diseases that suppress the immune system.,"Yeast infections in a dog’s ears generally cause redness, a brown discharge, head shaking or rubbing, odor, and itching.","Prevention of yeast infections in dogs must include addressing the underlying cause to reduce the likelihood that the infection will reoccur.Routine bathing with an antifungal shampoo may be beneficial. However, for shampoo treatment to be effective, the lather must sit on a dog’s skin for a minimum of 10 minutes before rinsing.Dogs with skin folds may need to have maintenance treatment to keep these areas from becoming too moist, as yeast thrive in moist, dark places such as skin folds and ears.Dogs that have had allergy testing and are determined to be allergic to yeast can be desensitized by including yeast as an ingredient in immunotherapy (allergy vaccine). If you suspect that your dog has a yeast infection, consult your regular veterinarian for a diagnosis and treatment plan that is appropriate for your pet.","Yeast infections on a dog’s mouth or face can cause extreme itching or face rubbing.Dogs with yeast infections on the paws usually lick their paws more than normal. Yeast infections in a dog’s ears can be very itchy, causing dogs to scratch their ears or rub their head excessively.","Your veterinarian may perform cytology (taking a swab of the discharge and staining it to look at it under the microscope) to diagnose a yeast infection in a dog’s ears.Prescription treatment may include antifungal drops or ointment, an ear cleaner, and in severe or difficult-to-treat cases, an oral antifungal medication.",Fungal,https://www.petmd.com
YeastInfection,,,,,"The affected skin may be red, irritated, itchy, greasy, or flaky, and there may be hair loss.If the infection is chronic, the skin may thicken and become darker in color. A dog with yeast infections on their paws can have red, irritated, and itchy paws.The underside of the paws, between the pads, is affected most often, but yeast can occur anywhere on the paws. Sometimes a brown discharge can be seen in the nail beds.Dog ear yeast infections are quite common, and the ears often smell sweet or musty.Usually, you will see redness, which may extend onto the flap of the ear, and the discharge is generally brown. The ear may appear to be greasy, and the hair may be matted.",,,
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/.idea/.gitignore 0 → 100644
View file @ 05e607a1
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ 05e607a1
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/.idea/misc.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (pythonProject) (2)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/.idea/modules.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/pythonProject.iml" filepath="$PROJECT_DIR$/.idea/pythonProject.iml" />
</modules>
</component>
</project>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/.idea/pythonProject.iml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/book_crawler/book_crawler/items.py 0 → 100644
View file @ 05e607a1
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class BookCrawlerItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/book_crawler/book_crawler/middlewares.py 0 → 100644
View file @ 05e607a1
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class BookCrawlerSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class BookCrawlerDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/book_crawler/book_crawler/pipelines.py 0 → 100644
View file @ 05e607a1
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class BookCrawlerPipeline:
def process_item(self, item, spider):
return item
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/book_crawler/book_crawler/settings.py 0 → 100644
View file @ 05e607a1
# Scrapy settings for book_crawler project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'book_crawler'
SPIDER_MODULES = ['book_crawler.spiders']
NEWSPIDER_MODULE = 'book_crawler.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'book_crawler (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'book_crawler.middlewares.BookCrawlerSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'book_crawler.middlewares.BookCrawlerDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'book_crawler.pipelines.BookCrawlerPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/book_crawler/book_crawler/spiders/__init__.py 0 → 100644
View file @ 05e607a1
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/book_crawler/book_crawler/spiders/books.py 0 → 100644
View file @ 05e607a1
from time import sleep
from scrapy import Spider
from selenium import webdriver
from scrapy.selector import Selector
from scrapy.http import Request
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
class BooksSpider(Spider):
name = 'books'
allowed_domains = ['books.toscrape.com']
def start_requests(self):
self.driver = webdriver.Chrome('C:/Users/Asus/Documents/chromedriver')
self.driver.get('http://books.toscrape.com')
sel = Selector(text=self.driver.page_source)
books = sel.xpath('//h3/a/@href').extract()
for book in books:
url = 'http://books.toscrape.com/' + book
yield Request(url, callback=self.parse_book)
while True:
try:
next_page = self.driver.find_element(by=By.XPATH, value='//a[text()="next"]')
sleep(3)
self.logger.info('Sleeping for 3 seconds')
next_page.click()
sel = Selector(text=self.driver.page_source)
books = sel.xpath('//h3/a/@href').extract()
for book in books:
url = 'http://books.toscrape.com/catalogue/' + book
yield Request(url, callback=self.parse_book)
except NoSuchElementException:
self.logger.info('No more pages to load.')
self.driver.quit()
break
def parse_book(self, response):
pass
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/book_crawler/scrapy.cfg 0 → 100644
View file @ 05e607a1
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = book_crawler.settings
[deploy]
#url = http://localhost:6800/
project = book_crawler
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/items.csv 0 → 100644
View file @ 05e607a1
Text,Author,Tags
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,Albert Einstein,"change,deep-thoughts,thinking,world"
"“It is our choices, Harry, that show what we truly are, far more than our abilities.”",J.K. Rowling,"abilities,choices"
“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”,Albert Einstein,"inspirational,life,live,miracle,miracles"
"“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”",Jane Austen,"aliteracy,books,classic,humor"
"“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”",Marilyn Monroe,"be-yourself,inspirational"
“Try not to become a man of success. Rather become a man of value.”,Albert Einstein,"adulthood,success,value"
“It is better to be hated for what you are than to be loved for what you are not.”,André Gide,"life,love"
"“I have not failed. I've just found 10,000 ways that won't work.”",Thomas A. Edison,"edison,failure,inspirational,paraphrased"
“A woman is like a tea bag; you never know how strong it is until it's in hot water.”,Eleanor Roosevelt,misattributed-eleanor-roosevelt
"“A day without sunshine is like, you know, night.”",Steve Martin,"humor,obvious,simile"
"“This life is what you make it. No matter what, you're going to mess up sometimes, it's a universal truth. But the good part is you get to decide how you're going to mess it up. Girls will be your friends - they'll act like it anyway. But just remember, some come, some go. The ones that stay with you through everything - they're your true best friends. Don't let go of them. Also remember, sisters make the best friends in the world. As for lovers, well, they'll come and go too. And baby, I hate to say it, most of them - actually pretty much all of them are going to break your heart, but you can't give up because if you give up, you'll never find your soulmate. You'll never find that half who makes you whole and that goes for everything. Just because you fail once, doesn't mean you're gonna fail at everything. Keep trying, hold on, and always, always, always believe in yourself, because if you don't, then who will, sweetie? So keep your head high, keep your chin up, and most importantly, keep smiling, because life's a beautiful thing and there's so much to smile about.”",Marilyn Monroe,"friends,heartbreak,inspirational,life,love,sisters"
"“It takes a great deal of bravery to stand up to our enemies, but just as much to stand up to our friends.”",J.K. Rowling,"courage,friends"
"“If you can't explain it to a six year old, you don't understand it yourself.”",Albert Einstein,"simplicity,understand"
"“You may not be her first, her last, or her only. She loved before she may love again. But if she loves you now, what else matters? She's not perfect—you aren't either, and the two of you may never be perfect together but if she can make you laugh, cause you to think twice, and admit to being human and making mistakes, hold onto her and give her the most you can. She may not be thinking about you every second of the day, but she will give you a part of her that she knows you can break—her heart. So don't hurt her, don't change her, don't analyze and don't expect more than she can give. Smile when she makes you happy, let her know when she makes you mad, and miss her when she's not there.”",Bob Marley,love
"“I like nonsense, it wakes up the brain cells. Fantasy is a necessary ingredient in living.”",Dr. Seuss,fantasy
"“I may not have gone where I intended to go, but I think I have ended up where I needed to be.”",Douglas Adams,"life,navigation"
"“The opposite of love is not hate, it's indifference. The opposite of art is not ugliness, it's indifference. The opposite of faith is not heresy, it's indifference. And the opposite of life is not death, it's indifference.”",Elie Wiesel,"activism,apathy,hate,indifference,inspirational,love,opposite,philosophy"
"“It is not a lack of love, but a lack of friendship that makes unhappy marriages.”",Friedrich Nietzsche,"friendship,lack-of-friendship,lack-of-love,love,marriage,unhappy-marriage"
"“Good friends, good books, and a sleepy conscience: this is the ideal life.”",Mark Twain,"books,contentment,friends,friendship,life"
“Life is what happens to us while we are making other plans.”,Allen Saunders,"fate,life,misattributed-john-lennon,planning,plans"
"“I love you without knowing how, or when, or from where. I love you simply, without problems or pride: I love you in this way because I do not know any other way of loving but this, in which there is no I or you, so intimate that your hand upon my chest is my hand, so intimate that when I fall asleep your eyes close.”",Pablo Neruda,"love,poetry"
“For every minute you are angry you lose sixty seconds of happiness.”,Ralph Waldo Emerson,happiness
"“If you judge people, you have no time to love them.”",Mother Teresa,attributed-no-source
“Anyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.”,Garrison Keillor,"humor,religion"
“Beauty is in the eye of the beholder and it may be necessary from time to time to give a stupid or misinformed beholder a black eye.”,Jim Henson,humor
"“Today you are You, that is truer than true. There is no one alive who is Youer than You.”",Dr. Seuss,"comedy,life,yourself"
"“If you want your children to be intelligent, read them fairy tales. If you want them to be more intelligent, read them more fairy tales.”",Albert Einstein,"children,fairy-tales"
"“It is impossible to live without failing at something, unless you live so cautiously that you might as well not have lived at all - in which case, you fail by default.”",J.K. Rowling,
“Logic will get you from A to Z; imagination will get you everywhere.”,Albert Einstein,imagination
"“One good thing about music, when it hits you, you feel no pain.”",Bob Marley,music
"“The more that you read, the more things you will know. The more that you learn, the more places you'll go.”",Dr. Seuss,"learning,reading,seuss"
"“Of course it is happening inside your head, Harry, but why on earth should that mean that it is not real?”",J.K. Rowling,dumbledore
"“The truth is, everyone is going to hurt you. You just got to find the ones worth suffering for.”",Bob Marley,friendship
“Not all of us can do great things. But we can do small things with great love.”,Mother Teresa,"misattributed-to-mother-teresa,paraphrased"
"“To the well-organized mind, death is but the next great adventure.”",J.K. Rowling,"death,inspirational"
“All you need is love. But a little chocolate now and then doesn't hurt.”,Charles M. Schulz,"chocolate,food,humor"
“We read to know we're not alone.”,William Nicholson,"misattributed-to-c-s-lewis,reading"
“Any fool can know. The point is to understand.”,Albert Einstein,"knowledge,learning,understanding,wisdom"
“I have always imagined that Paradise will be a kind of library.”,Jorge Luis Borges,"books,library"
“It is never too late to be what you might have been.”,George Eliot,inspirational
"“A reader lives a thousand lives before he dies, said Jojen. The man who never reads lives only one.”",George R.R. Martin,"read,readers,reading,reading-books"
“You can never get a cup of tea large enough or a book long enough to suit me.”,C.S. Lewis,"books,inspirational,reading,tea"
“You believe lies so you eventually learn to trust no one but yourself.”,Marilyn Monroe,
"“If you can make a woman laugh, you can make her do anything.”",Marilyn Monroe,"girls,love"
"“Life is like riding a bicycle. To keep your balance, you must keep moving.”",Albert Einstein,"life,simile"
“The real lover is the man who can thrill you by kissing your forehead or smiling into your eyes or just staring into space.”,Marilyn Monroe,love
"“A wise girl kisses but doesn't love, listens but doesn't believe, and leaves before she is left.”",Marilyn Monroe,attributed-no-source
“Only in the darkness can you see the stars.”,Martin Luther King Jr.,"hope,inspirational"
"“It matters not what someone is born, but what they grow to be.”",J.K. Rowling,dumbledore
"“Love does not begin and end the way we seem to think it does. Love is a battle, love is a war; love is a growing up.”",James Baldwin,love
"“There is nothing I would not do for those who are really my friends. I have no notion of loving people by halves, it is not my nature.”",Jane Austen,"friendship,love"
“Do one thing every day that scares you.”,Eleanor Roosevelt,"attributed,fear,inspiration"
"“I am good, but not an angel. I do sin, but I am not the devil. I am just a small girl in a big world trying to find someone to love.”",Marilyn Monroe,attributed-no-source
"“If I were not a physicist, I would probably be a musician. I often think in music. I live my daydreams in music. I see my life in terms of music.”",Albert Einstein,music
"“If you only read the books that everyone else is reading, you can only think what everyone else is thinking.”",Haruki Murakami,"books,thought"
“The difference between genius and stupidity is: genius has its limits.”,Alexandre Dumas fils,misattributed-to-einstein
"“He's like a drug for you, Bella.”",Stephenie Meyer,"drug,romance,simile"
“There is no friend as loyal as a book.”,Ernest Hemingway,"books,friends,novelist-quotes"
"“When one door of happiness closes, another opens; but often we look so long at the closed door that we do not see the one which has been opened for us.”",Helen Keller,inspirational
“Life isn't about finding yourself. Life is about creating yourself.”,George Bernard Shaw,"inspirational,life,yourself"
"“That's the problem with drinking, I thought, as I poured myself a drink. If something bad happens you drink in an attempt to forget; if something good happens you drink in order to celebrate; and if nothing happens you drink to make something happen.”",Charles Bukowski,alcohol
“You don’t forget the face of the person who was your last hope.”,Suzanne Collins,the-hunger-games
"“Remember, we're madly in love, so it's all right to kiss me anytime you feel like it.”",Suzanne Collins,humor
"“To love at all is to be vulnerable. Love anything and your heart will be wrung and possibly broken. If you want to make sure of keeping it intact you must give it to no one, not even an animal. Wrap it carefully round with hobbies and little luxuries; avoid all entanglements. Lock it up safe in the casket or coffin of your selfishness. But in that casket, safe, dark, motionless, airless, it will change. It will not be broken; it will become unbreakable, impenetrable, irredeemable. To love is to be vulnerable.”",C.S. Lewis,love
“Not all those who wander are lost.”,J.R.R. Tolkien,"bilbo,journey,lost,quest,travel,wander"
"“Do not pity the dead, Harry. Pity the living, and, above all those who live without love.”",J.K. Rowling,live-death-love
“There is nothing to writing. All you do is sit down at a typewriter and bleed.”,Ernest Hemingway,"good,writing"
“Finish each day and be done with it. You have done what you could. Some blunders and absurdities no doubt crept in; forget them as soon as you can. Tomorrow is a new day. You shall begin it serenely and with too high a spirit to be encumbered with your old nonsense.”,Ralph Waldo Emerson,"life,regrets"
“I have never let my schooling interfere with my education.”,Mark Twain,education
“I have heard there are troubles of more than one kind. Some come from ahead and some come from behind. But I've bought a big bat. I'm all ready you see. Now my troubles are going to have troubles with me!”,Dr. Seuss,troubles
“If I had a flower for every time I thought of you...I could walk through my garden forever.”,Alfred Tennyson,"friendship,love"
“Some people never go crazy. What truly horrible lives they must lead.”,Charles Bukowski,humor
"“The trouble with having an open mind, of course, is that people will insist on coming along and trying to put things in it.”",Terry Pratchett,"humor,open-mind,thinking"
"“Think left and think right and think low and think high. Oh, the thinks you can think up if only you try!”",Dr. Seuss,"humor,philosophy"
"“What really knocks me out is a book that, when you're all done reading it, you wish the author that wrote it was a terrific friend of yours and you could call him up on the phone whenever you felt like it. That doesn't happen much, though.”",J.D. Salinger,"authors,books,literature,reading,writing"
“The reason I talk to myself is because I’m the only one whose answers I accept.”,George Carlin,"humor,insanity,lies,lying,self-indulgence,truth"
"“You may say I'm a dreamer, but I'm not the only one. I hope someday you'll join us. And the world will live as one.”",John Lennon,"beatles,connection,dreamers,dreaming,dreams,hope,inspirational,peace"
“I am free of all prejudice. I hate everyone equally. ”,W.C. Fields,"humor,sinister"
“The question isn't who is going to let me; it's who is going to stop me.”,Ayn Rand,
“′Classic′ - a book which people praise and don't read.”,Mark Twain,"books,classic,reading"
“Anyone who has never made a mistake has never tried anything new.”,Albert Einstein,mistakes
"“A lady's imagination is very rapid; it jumps from admiration to love, from love to matrimony in a moment.”",Jane Austen,"humor,love,romantic,women"
"“Remember, if the time should come when you have to make a choice between what is right and what is easy, remember what happened to a boy who was good, and kind, and brave, because he strayed across the path of Lord Voldemort. Remember Cedric Diggory.”",J.K. Rowling,integrity
"“I declare after all there is no enjoyment like reading! How much sooner one tires of any thing than of a book! -- When I have a house of my own, I shall be miserable if I have not an excellent library.”",Jane Austen,"books,library,reading"
"“There are few people whom I really love, and still fewer of whom I think well. The more I see of the world, the more am I dissatisfied with it; and every day confirms my belief of the inconsistency of all human characters, and of the little dependence that can be placed on the appearance of merit or sense.”",Jane Austen,"elizabeth-bennet,jane-austen"
“Some day you will be old enough to start reading fairy tales again.”,C.S. Lewis,"age,fairytales,growing-up"
“We are not necessarily doubting that God will do the best for us; we are wondering how painful the best will turn out to be.”,C.S. Lewis,god
“The fear of death follows from the fear of life. A man who lives fully is prepared to die at any time.”,Mark Twain,"death,life"
“A lie can travel half way around the world while the truth is putting on its shoes.”,Mark Twain,"misattributed-mark-twain,truth"
"“I believe in Christianity as I believe that the sun has risen: not only because I see it, but because by it I see everything else.”",C.S. Lewis,"christianity,faith,religion,sun"
"“The truth."" Dumbledore sighed. ""It is a beautiful and terrible thing, and should therefore be treated with great caution.”",J.K. Rowling,truth
"“I'm the one that's got to die when it's time for me to die, so let me live my life the way I want to.”",Jimi Hendrix,"death,life"
“To die will be an awfully big adventure.”,J.M. Barrie,"adventure,love"
“It takes courage to grow up and become who you really are.”,E.E. Cummings,courage
“But better to get hurt by the truth than comforted with a lie.”,Khaled Hosseini,life
“You never really understand a person until you consider things from his point of view... Until you climb inside of his skin and walk around in it.”,Harper Lee,better-life-empathy
"“You have to write the book that wants to be written. And if the book will be too difficult for grown-ups, then you write it for children.”",Madeleine L'Engle,"books,children,difficult,grown-ups,write,writers,writing"
“Never tell the truth to people who are not worthy of it.”,Mark Twain,truth
"“A person's a person, no matter how small.”",Dr. Seuss,inspirational
"“... a mind needs books as a sword needs a whetstone, if it is to keep its edge.”",George R.R. Martin,"books,mind"
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/items.json 0 → 100644
View file @ 05e607a1
[
{"Text": "\u201cThe world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.\u201d", "Author": "Albert Einstein", "Tags": "change,deep-thoughts,thinking,world"},
{"Text": "\u201cIt is our choices, Harry, that show what we truly are, far more than our abilities.\u201d", "Author": "J.K. Rowling", "Tags": "abilities,choices"},
{"Text": "\u201cThere are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.\u201d", "Author": "Albert Einstein", "Tags": "inspirational,life,live,miracle,miracles"},
{"Text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d", "Author": "Jane Austen", "Tags": "aliteracy,books,classic,humor"},
{"Text": "\u201cImperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.\u201d", "Author": "Marilyn Monroe", "Tags": "be-yourself,inspirational"},
{"Text": "\u201cTry not to become a man of success. Rather become a man of value.\u201d", "Author": "Albert Einstein", "Tags": "adulthood,success,value"},
{"Text": "\u201cIt is better to be hated for what you are than to be loved for what you are not.\u201d", "Author": "Andr\u00e9 Gide", "Tags": "life,love"},
{"Text": "\u201cI have not failed. I've just found 10,000 ways that won't work.\u201d", "Author": "Thomas A. Edison", "Tags": "edison,failure,inspirational,paraphrased"},
{"Text": "\u201cA woman is like a tea bag; you never know how strong it is until it's in hot water.\u201d", "Author": "Eleanor Roosevelt", "Tags": "misattributed-eleanor-roosevelt"},
{"Text": "\u201cA day without sunshine is like, you know, night.\u201d", "Author": "Steve Martin", "Tags": "humor,obvious,simile"},
{"Text": "\u201cThis life is what you make it. No matter what, you're going to mess up sometimes, it's a universal truth. But the good part is you get to decide how you're going to mess it up. Girls will be your friends - they'll act like it anyway. But just remember, some come, some go. The ones that stay with you through everything - they're your true best friends. Don't let go of them. Also remember, sisters make the best friends in the world. As for lovers, well, they'll come and go too. And baby, I hate to say it, most of them - actually pretty much all of them are going to break your heart, but you can't give up because if you give up, you'll never find your soulmate. You'll never find that half who makes you whole and that goes for everything. Just because you fail once, doesn't mean you're gonna fail at everything. Keep trying, hold on, and always, always, always believe in yourself, because if you don't, then who will, sweetie? So keep your head high, keep your chin up, and most importantly, keep smiling, because life's a beautiful thing and there's so much to smile about.\u201d", "Author": "Marilyn Monroe", "Tags": "friends,heartbreak,inspirational,life,love,sisters"},
{"Text": "\u201cIt takes a great deal of bravery to stand up to our enemies, but just as much to stand up to our friends.\u201d", "Author": "J.K. Rowling", "Tags": "courage,friends"},
{"Text": "\u201cIf you can't explain it to a six year old, you don't understand it yourself.\u201d", "Author": "Albert Einstein", "Tags": "simplicity,understand"},
{"Text": "\u201cYou may not be her first, her last, or her only. She loved before she may love again. But if she loves you now, what else matters? She's not perfect\u2014you aren't either, and the two of you may never be perfect together but if she can make you laugh, cause you to think twice, and admit to being human and making mistakes, hold onto her and give her the most you can. She may not be thinking about you every second of the day, but she will give you a part of her that she knows you can break\u2014her heart. So don't hurt her, don't change her, don't analyze and don't expect more than she can give. Smile when she makes you happy, let her know when she makes you mad, and miss her when she's not there.\u201d", "Author": "Bob Marley", "Tags": "love"},
{"Text": "\u201cI like nonsense, it wakes up the brain cells. Fantasy is a necessary ingredient in living.\u201d", "Author": "Dr. Seuss", "Tags": "fantasy"},
{"Text": "\u201cI may not have gone where I intended to go, but I think I have ended up where I needed to be.\u201d", "Author": "Douglas Adams", "Tags": "life,navigation"},
{"Text": "\u201cThe opposite of love is not hate, it's indifference. The opposite of art is not ugliness, it's indifference. The opposite of faith is not heresy, it's indifference. And the opposite of life is not death, it's indifference.\u201d", "Author": "Elie Wiesel", "Tags": "activism,apathy,hate,indifference,inspirational,love,opposite,philosophy"},
{"Text": "\u201cIt is not a lack of love, but a lack of friendship that makes unhappy marriages.\u201d", "Author": "Friedrich Nietzsche", "Tags": "friendship,lack-of-friendship,lack-of-love,love,marriage,unhappy-marriage"},
{"Text": "\u201cGood friends, good books, and a sleepy conscience: this is the ideal life.\u201d", "Author": "Mark Twain", "Tags": "books,contentment,friends,friendship,life"},
{"Text": "\u201cLife is what happens to us while we are making other plans.\u201d", "Author": "Allen Saunders", "Tags": "fate,life,misattributed-john-lennon,planning,plans"},
{"Text": "\u201cI love you without knowing how, or when, or from where. I love you simply, without problems or pride: I love you in this way because I do not know any other way of loving but this, in which there is no I or you, so intimate that your hand upon my chest is my hand, so intimate that when I fall asleep your eyes close.\u201d", "Author": "Pablo Neruda", "Tags": "love,poetry"},
{"Text": "\u201cFor every minute you are angry you lose sixty seconds of happiness.\u201d", "Author": "Ralph Waldo Emerson", "Tags": "happiness"},
{"Text": "\u201cIf you judge people, you have no time to love them.\u201d", "Author": "Mother Teresa", "Tags": "attributed-no-source"},
{"Text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d", "Author": "Garrison Keillor", "Tags": "humor,religion"},
{"Text": "\u201cBeauty is in the eye of the beholder and it may be necessary from time to time to give a stupid or misinformed beholder a black eye.\u201d", "Author": "Jim Henson", "Tags": "humor"},
{"Text": "\u201cToday you are You, that is truer than true. There is no one alive who is Youer than You.\u201d", "Author": "Dr. Seuss", "Tags": "comedy,life,yourself"},
{"Text": "\u201cIf you want your children to be intelligent, read them fairy tales. If you want them to be more intelligent, read them more fairy tales.\u201d", "Author": "Albert Einstein", "Tags": "children,fairy-tales"},
{"Text": "\u201cIt is impossible to live without failing at something, unless you live so cautiously that you might as well not have lived at all - in which case, you fail by default.\u201d", "Author": "J.K. Rowling", "Tags": ""},
{"Text": "\u201cLogic will get you from A to Z; imagination will get you everywhere.\u201d", "Author": "Albert Einstein", "Tags": "imagination"},
{"Text": "\u201cOne good thing about music, when it hits you, you feel no pain.\u201d", "Author": "Bob Marley", "Tags": "music"},
{"Text": "\u201cThe more that you read, the more things you will know. The more that you learn, the more places you'll go.\u201d", "Author": "Dr. Seuss", "Tags": "learning,reading,seuss"},
{"Text": "\u201cOf course it is happening inside your head, Harry, but why on earth should that mean that it is not real?\u201d", "Author": "J.K. Rowling", "Tags": "dumbledore"},
{"Text": "\u201cThe truth is, everyone is going to hurt you. You just got to find the ones worth suffering for.\u201d", "Author": "Bob Marley", "Tags": "friendship"},
{"Text": "\u201cNot all of us can do great things. But we can do small things with great love.\u201d", "Author": "Mother Teresa", "Tags": "misattributed-to-mother-teresa,paraphrased"},
{"Text": "\u201cTo the well-organized mind, death is but the next great adventure.\u201d", "Author": "J.K. Rowling", "Tags": "death,inspirational"},
{"Text": "\u201cAll you need is love. But a little chocolate now and then doesn't hurt.\u201d", "Author": "Charles M. Schulz", "Tags": "chocolate,food,humor"},
{"Text": "\u201cWe read to know we're not alone.\u201d", "Author": "William Nicholson", "Tags": "misattributed-to-c-s-lewis,reading"},
{"Text": "\u201cAny fool can know. The point is to understand.\u201d", "Author": "Albert Einstein", "Tags": "knowledge,learning,understanding,wisdom"},
{"Text": "\u201cI have always imagined that Paradise will be a kind of library.\u201d", "Author": "Jorge Luis Borges", "Tags": "books,library"},
{"Text": "\u201cIt is never too late to be what you might have been.\u201d", "Author": "George Eliot", "Tags": "inspirational"},
{"Text": "\u201cA reader lives a thousand lives before he dies, said Jojen. The man who never reads lives only one.\u201d", "Author": "George R.R. Martin", "Tags": "read,readers,reading,reading-books"},
{"Text": "\u201cYou can never get a cup of tea large enough or a book long enough to suit me.\u201d", "Author": "C.S. Lewis", "Tags": "books,inspirational,reading,tea"},
{"Text": "\u201cYou believe lies so you eventually learn to trust no one but yourself.\u201d", "Author": "Marilyn Monroe", "Tags": ""},
{"Text": "\u201cIf you can make a woman laugh, you can make her do anything.\u201d", "Author": "Marilyn Monroe", "Tags": "girls,love"},
{"Text": "\u201cLife is like riding a bicycle. To keep your balance, you must keep moving.\u201d", "Author": "Albert Einstein", "Tags": "life,simile"},
{"Text": "\u201cThe real lover is the man who can thrill you by kissing your forehead or smiling into your eyes or just staring into space.\u201d", "Author": "Marilyn Monroe", "Tags": "love"},
{"Text": "\u201cA wise girl kisses but doesn't love, listens but doesn't believe, and leaves before she is left.\u201d", "Author": "Marilyn Monroe", "Tags": "attributed-no-source"},
{"Text": "\u201cOnly in the darkness can you see the stars.\u201d", "Author": "Martin Luther King Jr.", "Tags": "hope,inspirational"},
{"Text": "\u201cIt matters not what someone is born, but what they grow to be.\u201d", "Author": "J.K. Rowling", "Tags": "dumbledore"},
{"Text": "\u201cLove does not begin and end the way we seem to think it does. Love is a battle, love is a war; love is a growing up.\u201d", "Author": "James Baldwin", "Tags": "love"},
{"Text": "\u201cThere is nothing I would not do for those who are really my friends. I have no notion of loving people by halves, it is not my nature.\u201d", "Author": "Jane Austen", "Tags": "friendship,love"},
{"Text": "\u201cDo one thing every day that scares you.\u201d", "Author": "Eleanor Roosevelt", "Tags": "attributed,fear,inspiration"},
{"Text": "\u201cI am good, but not an angel. I do sin, but I am not the devil. I am just a small girl in a big world trying to find someone to love.\u201d", "Author": "Marilyn Monroe", "Tags": "attributed-no-source"},
{"Text": "\u201cIf I were not a physicist, I would probably be a musician. I often think in music. I live my daydreams in music. I see my life in terms of music.\u201d", "Author": "Albert Einstein", "Tags": "music"},
{"Text": "\u201cIf you only read the books that everyone else is reading, you can only think what everyone else is thinking.\u201d", "Author": "Haruki Murakami", "Tags": "books,thought"},
{"Text": "\u201cThe difference between genius and stupidity is: genius has its limits.\u201d", "Author": "Alexandre Dumas fils", "Tags": "misattributed-to-einstein"},
{"Text": "\u201cHe's like a drug for you, Bella.\u201d", "Author": "Stephenie Meyer", "Tags": "drug,romance,simile"},
{"Text": "\u201cThere is no friend as loyal as a book.\u201d", "Author": "Ernest Hemingway", "Tags": "books,friends,novelist-quotes"},
{"Text": "\u201cWhen one door of happiness closes, another opens; but often we look so long at the closed door that we do not see the one which has been opened for us.\u201d", "Author": "Helen Keller", "Tags": "inspirational"},
{"Text": "\u201cLife isn't about finding yourself. Life is about creating yourself.\u201d", "Author": "George Bernard Shaw", "Tags": "inspirational,life,yourself"},
{"Text": "\u201cThat's the problem with drinking, I thought, as I poured myself a drink. If something bad happens you drink in an attempt to forget; if something good happens you drink in order to celebrate; and if nothing happens you drink to make something happen.\u201d", "Author": "Charles Bukowski", "Tags": "alcohol"},
{"Text": "\u201cYou don\u2019t forget the face of the person who was your last hope.\u201d", "Author": "Suzanne Collins", "Tags": "the-hunger-games"},
{"Text": "\u201cRemember, we're madly in love, so it's all right to kiss me anytime you feel like it.\u201d", "Author": "Suzanne Collins", "Tags": "humor"},
{"Text": "\u201cTo love at all is to be vulnerable. Love anything and your heart will be wrung and possibly broken. If you want to make sure of keeping it intact you must give it to no one, not even an animal. Wrap it carefully round with hobbies and little luxuries; avoid all entanglements. Lock it up safe in the casket or coffin of your selfishness. But in that casket, safe, dark, motionless, airless, it will change. It will not be broken; it will become unbreakable, impenetrable, irredeemable. To love is to be vulnerable.\u201d", "Author": "C.S. Lewis", "Tags": "love"},
{"Text": "\u201cNot all those who wander are lost.\u201d", "Author": "J.R.R. Tolkien", "Tags": "bilbo,journey,lost,quest,travel,wander"},
{"Text": "\u201cDo not pity the dead, Harry. Pity the living, and, above all those who live without love.\u201d", "Author": "J.K. Rowling", "Tags": "live-death-love"},
{"Text": "\u201cThere is nothing to writing. All you do is sit down at a typewriter and bleed.\u201d", "Author": "Ernest Hemingway", "Tags": "good,writing"},
{"Text": "\u201cFinish each day and be done with it. You have done what you could. Some blunders and absurdities no doubt crept in; forget them as soon as you can. Tomorrow is a new day. You shall begin it serenely and with too high a spirit to be encumbered with your old nonsense.\u201d", "Author": "Ralph Waldo Emerson", "Tags": "life,regrets"},
{"Text": "\u201cI have never let my schooling interfere with my education.\u201d", "Author": "Mark Twain", "Tags": "education"},
{"Text": "\u201cI have heard there are troubles of more than one kind. Some come from ahead and some come from behind. But I've bought a big bat. I'm all ready you see. Now my troubles are going to have troubles with me!\u201d", "Author": "Dr. Seuss", "Tags": "troubles"},
{"Text": "\u201cIf I had a flower for every time I thought of you...I could walk through my garden forever.\u201d", "Author": "Alfred Tennyson", "Tags": "friendship,love"},
{"Text": "\u201cSome people never go crazy. What truly horrible lives they must lead.\u201d", "Author": "Charles Bukowski", "Tags": "humor"},
{"Text": "\u201cThe trouble with having an open mind, of course, is that people will insist on coming along and trying to put things in it.\u201d", "Author": "Terry Pratchett", "Tags": "humor,open-mind,thinking"},
{"Text": "\u201cThink left and think right and think low and think high. Oh, the thinks you can think up if only you try!\u201d", "Author": "Dr. Seuss", "Tags": "humor,philosophy"},
{"Text": "\u201cWhat really knocks me out is a book that, when you're all done reading it, you wish the author that wrote it was a terrific friend of yours and you could call him up on the phone whenever you felt like it. That doesn't happen much, though.\u201d", "Author": "J.D. Salinger", "Tags": "authors,books,literature,reading,writing"},
{"Text": "\u201cThe reason I talk to myself is because I\u2019m the only one whose answers I accept.\u201d", "Author": "George Carlin", "Tags": "humor,insanity,lies,lying,self-indulgence,truth"},
{"Text": "\u201cYou may say I'm a dreamer, but I'm not the only one. I hope someday you'll join us. And the world will live as one.\u201d", "Author": "John Lennon", "Tags": "beatles,connection,dreamers,dreaming,dreams,hope,inspirational,peace"},
{"Text": "\u201cI am free of all prejudice. I hate everyone equally. \u201d", "Author": "W.C. Fields", "Tags": "humor,sinister"},
{"Text": "\u201cThe question isn't who is going to let me; it's who is going to stop me.\u201d", "Author": "Ayn Rand", "Tags": ""},
{"Text": "\u201c\u2032Classic\u2032 - a book which people praise and don't read.\u201d", "Author": "Mark Twain", "Tags": "books,classic,reading"},
{"Text": "\u201cAnyone who has never made a mistake has never tried anything new.\u201d", "Author": "Albert Einstein", "Tags": "mistakes"},
{"Text": "\u201cA lady's imagination is very rapid; it jumps from admiration to love, from love to matrimony in a moment.\u201d", "Author": "Jane Austen", "Tags": "humor,love,romantic,women"},
{"Text": "\u201cRemember, if the time should come when you have to make a choice between what is right and what is easy, remember what happened to a boy who was good, and kind, and brave, because he strayed across the path of Lord Voldemort. Remember Cedric Diggory.\u201d", "Author": "J.K. Rowling", "Tags": "integrity"},
{"Text": "\u201cI declare after all there is no enjoyment like reading! How much sooner one tires of any thing than of a book! -- When I have a house of my own, I shall be miserable if I have not an excellent library.\u201d", "Author": "Jane Austen", "Tags": "books,library,reading"},
{"Text": "\u201cThere are few people whom I really love, and still fewer of whom I think well. The more I see of the world, the more am I dissatisfied with it; and every day confirms my belief of the inconsistency of all human characters, and of the little dependence that can be placed on the appearance of merit or sense.\u201d", "Author": "Jane Austen", "Tags": "elizabeth-bennet,jane-austen"},
{"Text": "\u201cSome day you will be old enough to start reading fairy tales again.\u201d", "Author": "C.S. Lewis", "Tags": "age,fairytales,growing-up"},
{"Text": "\u201cWe are not necessarily doubting that God will do the best for us; we are wondering how painful the best will turn out to be.\u201d", "Author": "C.S. Lewis", "Tags": "god"},
{"Text": "\u201cThe fear of death follows from the fear of life. A man who lives fully is prepared to die at any time.\u201d", "Author": "Mark Twain", "Tags": "death,life"},
{"Text": "\u201cA lie can travel half way around the world while the truth is putting on its shoes.\u201d", "Author": "Mark Twain", "Tags": "misattributed-mark-twain,truth"},
{"Text": "\u201cI believe in Christianity as I believe that the sun has risen: not only because I see it, but because by it I see everything else.\u201d", "Author": "C.S. Lewis", "Tags": "christianity,faith,religion,sun"},
{"Text": "\u201cThe truth.\" Dumbledore sighed. \"It is a beautiful and terrible thing, and should therefore be treated with great caution.\u201d", "Author": "J.K. Rowling", "Tags": "truth"},
{"Text": "\u201cI'm the one that's got to die when it's time for me to die, so let me live my life the way I want to.\u201d", "Author": "Jimi Hendrix", "Tags": "death,life"},
{"Text": "\u201cTo die will be an awfully big adventure.\u201d", "Author": "J.M. Barrie", "Tags": "adventure,love"},
{"Text": "\u201cIt takes courage to grow up and become who you really are.\u201d", "Author": "E.E. Cummings", "Tags": "courage"},
{"Text": "\u201cBut better to get hurt by the truth than comforted with a lie.\u201d", "Author": "Khaled Hosseini", "Tags": "life"},
{"Text": "\u201cYou never really understand a person until you consider things from his point of view... Until you climb inside of his skin and walk around in it.\u201d", "Author": "Harper Lee", "Tags": "better-life-empathy"},
{"Text": "\u201cYou have to write the book that wants to be written. And if the book will be too difficult for grown-ups, then you write it for children.\u201d", "Author": "Madeleine L'Engle", "Tags": "books,children,difficult,grown-ups,write,writers,writing"},
{"Text": "\u201cNever tell the truth to people who are not worthy of it.\u201d", "Author": "Mark Twain", "Tags": "truth"},
{"Text": "\u201cA person's a person, no matter how small.\u201d", "Author": "Dr. Seuss", "Tags": "inspirational"},
{"Text": "\u201c... a mind needs books as a sword needs a whetstone, if it is to keep its edge.\u201d", "Author": "George R.R. Martin", "Tags": "books,mind"}
]
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/items.py 0 → 100644
View file @ 05e607a1
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class FirstscrapyItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/items.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="utf-8"?>
<items>
<item><Text>“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</Text><Author>Albert Einstein</Author><Tags>change,deep-thoughts,thinking,world</Tags></item>
<item><Text>“It is our choices, Harry, that show what we truly are, far more than our abilities.”</Text><Author>J.K. Rowling</Author><Tags>abilities,choices</Tags></item>
<item><Text>“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</Text><Author>Albert Einstein</Author><Tags>inspirational,life,live,miracle,miracles</Tags></item>
<item><Text>“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</Text><Author>Jane Austen</Author><Tags>aliteracy,books,classic,humor</Tags></item>
<item><Text>“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”</Text><Author>Marilyn Monroe</Author><Tags>be-yourself,inspirational</Tags></item>
<item><Text>“Try not to become a man of success. Rather become a man of value.”</Text><Author>Albert Einstein</Author><Tags>adulthood,success,value</Tags></item>
<item><Text>“It is better to be hated for what you are than to be loved for what you are not.”</Text><Author>André Gide</Author><Tags>life,love</Tags></item>
<item><Text>“I have not failed. I've just found 10,000 ways that won't work.”</Text><Author>Thomas A. Edison</Author><Tags>edison,failure,inspirational,paraphrased</Tags></item>
<item><Text>“A woman is like a tea bag; you never know how strong it is until it's in hot water.”</Text><Author>Eleanor Roosevelt</Author><Tags>misattributed-eleanor-roosevelt</Tags></item>
<item><Text>“A day without sunshine is like, you know, night.”</Text><Author>Steve Martin</Author><Tags>humor,obvious,simile</Tags></item>
<item><Text>“This life is what you make it. No matter what, you're going to mess up sometimes, it's a universal truth. But the good part is you get to decide how you're going to mess it up. Girls will be your friends - they'll act like it anyway. But just remember, some come, some go. The ones that stay with you through everything - they're your true best friends. Don't let go of them. Also remember, sisters make the best friends in the world. As for lovers, well, they'll come and go too. And baby, I hate to say it, most of them - actually pretty much all of them are going to break your heart, but you can't give up because if you give up, you'll never find your soulmate. You'll never find that half who makes you whole and that goes for everything. Just because you fail once, doesn't mean you're gonna fail at everything. Keep trying, hold on, and always, always, always believe in yourself, because if you don't, then who will, sweetie? So keep your head high, keep your chin up, and most importantly, keep smiling, because life's a beautiful thing and there's so much to smile about.”</Text><Author>Marilyn Monroe</Author><Tags>friends,heartbreak,inspirational,life,love,sisters</Tags></item>
<item><Text>“It takes a great deal of bravery to stand up to our enemies, but just as much to stand up to our friends.”</Text><Author>J.K. Rowling</Author><Tags>courage,friends</Tags></item>
<item><Text>“If you can't explain it to a six year old, you don't understand it yourself.”</Text><Author>Albert Einstein</Author><Tags>simplicity,understand</Tags></item>
<item><Text>“You may not be her first, her last, or her only. She loved before she may love again. But if she loves you now, what else matters? She's not perfect—you aren't either, and the two of you may never be perfect together but if she can make you laugh, cause you to think twice, and admit to being human and making mistakes, hold onto her and give her the most you can. She may not be thinking about you every second of the day, but she will give you a part of her that she knows you can break—her heart. So don't hurt her, don't change her, don't analyze and don't expect more than she can give. Smile when she makes you happy, let her know when she makes you mad, and miss her when she's not there.”</Text><Author>Bob Marley</Author><Tags>love</Tags></item>
<item><Text>“I like nonsense, it wakes up the brain cells. Fantasy is a necessary ingredient in living.”</Text><Author>Dr. Seuss</Author><Tags>fantasy</Tags></item>
<item><Text>“I may not have gone where I intended to go, but I think I have ended up where I needed to be.”</Text><Author>Douglas Adams</Author><Tags>life,navigation</Tags></item>
<item><Text>“The opposite of love is not hate, it's indifference. The opposite of art is not ugliness, it's indifference. The opposite of faith is not heresy, it's indifference. And the opposite of life is not death, it's indifference.”</Text><Author>Elie Wiesel</Author><Tags>activism,apathy,hate,indifference,inspirational,love,opposite,philosophy</Tags></item>
<item><Text>“It is not a lack of love, but a lack of friendship that makes unhappy marriages.”</Text><Author>Friedrich Nietzsche</Author><Tags>friendship,lack-of-friendship,lack-of-love,love,marriage,unhappy-marriage</Tags></item>
<item><Text>“Good friends, good books, and a sleepy conscience: this is the ideal life.”</Text><Author>Mark Twain</Author><Tags>books,contentment,friends,friendship,life</Tags></item>
<item><Text>“Life is what happens to us while we are making other plans.”</Text><Author>Allen Saunders</Author><Tags>fate,life,misattributed-john-lennon,planning,plans</Tags></item>
<item><Text>“I love you without knowing how, or when, or from where. I love you simply, without problems or pride: I love you in this way because I do not know any other way of loving but this, in which there is no I or you, so intimate that your hand upon my chest is my hand, so intimate that when I fall asleep your eyes close.”</Text><Author>Pablo Neruda</Author><Tags>love,poetry</Tags></item>
<item><Text>“For every minute you are angry you lose sixty seconds of happiness.”</Text><Author>Ralph Waldo Emerson</Author><Tags>happiness</Tags></item>
<item><Text>“If you judge people, you have no time to love them.”</Text><Author>Mother Teresa</Author><Tags>attributed-no-source</Tags></item>
<item><Text>“Anyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.”</Text><Author>Garrison Keillor</Author><Tags>humor,religion</Tags></item>
<item><Text>“Beauty is in the eye of the beholder and it may be necessary from time to time to give a stupid or misinformed beholder a black eye.”</Text><Author>Jim Henson</Author><Tags>humor</Tags></item>
<item><Text>“Today you are You, that is truer than true. There is no one alive who is Youer than You.”</Text><Author>Dr. Seuss</Author><Tags>comedy,life,yourself</Tags></item>
<item><Text>“If you want your children to be intelligent, read them fairy tales. If you want them to be more intelligent, read them more fairy tales.”</Text><Author>Albert Einstein</Author><Tags>children,fairy-tales</Tags></item>
<item><Text>“It is impossible to live without failing at something, unless you live so cautiously that you might as well not have lived at all - in which case, you fail by default.”</Text><Author>J.K. Rowling</Author><Tags></Tags></item>
<item><Text>“Logic will get you from A to Z; imagination will get you everywhere.”</Text><Author>Albert Einstein</Author><Tags>imagination</Tags></item>
<item><Text>“One good thing about music, when it hits you, you feel no pain.”</Text><Author>Bob Marley</Author><Tags>music</Tags></item>
<item><Text>“The more that you read, the more things you will know. The more that you learn, the more places you'll go.”</Text><Author>Dr. Seuss</Author><Tags>learning,reading,seuss</Tags></item>
<item><Text>“Of course it is happening inside your head, Harry, but why on earth should that mean that it is not real?”</Text><Author>J.K. Rowling</Author><Tags>dumbledore</Tags></item>
<item><Text>“The truth is, everyone is going to hurt you. You just got to find the ones worth suffering for.”</Text><Author>Bob Marley</Author><Tags>friendship</Tags></item>
<item><Text>“Not all of us can do great things. But we can do small things with great love.”</Text><Author>Mother Teresa</Author><Tags>misattributed-to-mother-teresa,paraphrased</Tags></item>
<item><Text>“To the well-organized mind, death is but the next great adventure.”</Text><Author>J.K. Rowling</Author><Tags>death,inspirational</Tags></item>
<item><Text>“All you need is love. But a little chocolate now and then doesn't hurt.”</Text><Author>Charles M. Schulz</Author><Tags>chocolate,food,humor</Tags></item>
<item><Text>“We read to know we're not alone.”</Text><Author>William Nicholson</Author><Tags>misattributed-to-c-s-lewis,reading</Tags></item>
<item><Text>“Any fool can know. The point is to understand.”</Text><Author>Albert Einstein</Author><Tags>knowledge,learning,understanding,wisdom</Tags></item>
<item><Text>“I have always imagined that Paradise will be a kind of library.”</Text><Author>Jorge Luis Borges</Author><Tags>books,library</Tags></item>
<item><Text>“It is never too late to be what you might have been.”</Text><Author>George Eliot</Author><Tags>inspirational</Tags></item>
<item><Text>“A reader lives a thousand lives before he dies, said Jojen. The man who never reads lives only one.”</Text><Author>George R.R. Martin</Author><Tags>read,readers,reading,reading-books</Tags></item>
<item><Text>“You can never get a cup of tea large enough or a book long enough to suit me.”</Text><Author>C.S. Lewis</Author><Tags>books,inspirational,reading,tea</Tags></item>
<item><Text>“You believe lies so you eventually learn to trust no one but yourself.”</Text><Author>Marilyn Monroe</Author><Tags></Tags></item>
<item><Text>“If you can make a woman laugh, you can make her do anything.”</Text><Author>Marilyn Monroe</Author><Tags>girls,love</Tags></item>
<item><Text>“Life is like riding a bicycle. To keep your balance, you must keep moving.”</Text><Author>Albert Einstein</Author><Tags>life,simile</Tags></item>
<item><Text>“The real lover is the man who can thrill you by kissing your forehead or smiling into your eyes or just staring into space.”</Text><Author>Marilyn Monroe</Author><Tags>love</Tags></item>
<item><Text>“A wise girl kisses but doesn't love, listens but doesn't believe, and leaves before she is left.”</Text><Author>Marilyn Monroe</Author><Tags>attributed-no-source</Tags></item>
<item><Text>“Only in the darkness can you see the stars.”</Text><Author>Martin Luther King Jr.</Author><Tags>hope,inspirational</Tags></item>
<item><Text>“It matters not what someone is born, but what they grow to be.”</Text><Author>J.K. Rowling</Author><Tags>dumbledore</Tags></item>
<item><Text>“Love does not begin and end the way we seem to think it does. Love is a battle, love is a war; love is a growing up.”</Text><Author>James Baldwin</Author><Tags>love</Tags></item>
<item><Text>“There is nothing I would not do for those who are really my friends. I have no notion of loving people by halves, it is not my nature.”</Text><Author>Jane Austen</Author><Tags>friendship,love</Tags></item>
<item><Text>“Do one thing every day that scares you.”</Text><Author>Eleanor Roosevelt</Author><Tags>attributed,fear,inspiration</Tags></item>
<item><Text>“I am good, but not an angel. I do sin, but I am not the devil. I am just a small girl in a big world trying to find someone to love.”</Text><Author>Marilyn Monroe</Author><Tags>attributed-no-source</Tags></item>
<item><Text>“If I were not a physicist, I would probably be a musician. I often think in music. I live my daydreams in music. I see my life in terms of music.”</Text><Author>Albert Einstein</Author><Tags>music</Tags></item>
<item><Text>“If you only read the books that everyone else is reading, you can only think what everyone else is thinking.”</Text><Author>Haruki Murakami</Author><Tags>books,thought</Tags></item>
<item><Text>“The difference between genius and stupidity is: genius has its limits.”</Text><Author>Alexandre Dumas fils</Author><Tags>misattributed-to-einstein</Tags></item>
<item><Text>“He's like a drug for you, Bella.”</Text><Author>Stephenie Meyer</Author><Tags>drug,romance,simile</Tags></item>
<item><Text>“There is no friend as loyal as a book.”</Text><Author>Ernest Hemingway</Author><Tags>books,friends,novelist-quotes</Tags></item>
<item><Text>“When one door of happiness closes, another opens; but often we look so long at the closed door that we do not see the one which has been opened for us.”</Text><Author>Helen Keller</Author><Tags>inspirational</Tags></item>
<item><Text>“Life isn't about finding yourself. Life is about creating yourself.”</Text><Author>George Bernard Shaw</Author><Tags>inspirational,life,yourself</Tags></item>
<item><Text>“That's the problem with drinking, I thought, as I poured myself a drink. If something bad happens you drink in an attempt to forget; if something good happens you drink in order to celebrate; and if nothing happens you drink to make something happen.”</Text><Author>Charles Bukowski</Author><Tags>alcohol</Tags></item>
<item><Text>“You don’t forget the face of the person who was your last hope.”</Text><Author>Suzanne Collins</Author><Tags>the-hunger-games</Tags></item>
<item><Text>“Remember, we're madly in love, so it's all right to kiss me anytime you feel like it.”</Text><Author>Suzanne Collins</Author><Tags>humor</Tags></item>
<item><Text>“To love at all is to be vulnerable. Love anything and your heart will be wrung and possibly broken. If you want to make sure of keeping it intact you must give it to no one, not even an animal. Wrap it carefully round with hobbies and little luxuries; avoid all entanglements. Lock it up safe in the casket or coffin of your selfishness. But in that casket, safe, dark, motionless, airless, it will change. It will not be broken; it will become unbreakable, impenetrable, irredeemable. To love is to be vulnerable.”</Text><Author>C.S. Lewis</Author><Tags>love</Tags></item>
<item><Text>“Not all those who wander are lost.”</Text><Author>J.R.R. Tolkien</Author><Tags>bilbo,journey,lost,quest,travel,wander</Tags></item>
<item><Text>“Do not pity the dead, Harry. Pity the living, and, above all those who live without love.”</Text><Author>J.K. Rowling</Author><Tags>live-death-love</Tags></item>
<item><Text>“There is nothing to writing. All you do is sit down at a typewriter and bleed.”</Text><Author>Ernest Hemingway</Author><Tags>good,writing</Tags></item>
<item><Text>“Finish each day and be done with it. You have done what you could. Some blunders and absurdities no doubt crept in; forget them as soon as you can. Tomorrow is a new day. You shall begin it serenely and with too high a spirit to be encumbered with your old nonsense.”</Text><Author>Ralph Waldo Emerson</Author><Tags>life,regrets</Tags></item>
<item><Text>“I have never let my schooling interfere with my education.”</Text><Author>Mark Twain</Author><Tags>education</Tags></item>
<item><Text>“I have heard there are troubles of more than one kind. Some come from ahead and some come from behind. But I've bought a big bat. I'm all ready you see. Now my troubles are going to have troubles with me!”</Text><Author>Dr. Seuss</Author><Tags>troubles</Tags></item>
<item><Text>“If I had a flower for every time I thought of you...I could walk through my garden forever.”</Text><Author>Alfred Tennyson</Author><Tags>friendship,love</Tags></item>
<item><Text>“Some people never go crazy. What truly horrible lives they must lead.”</Text><Author>Charles Bukowski</Author><Tags>humor</Tags></item>
<item><Text>“The trouble with having an open mind, of course, is that people will insist on coming along and trying to put things in it.”</Text><Author>Terry Pratchett</Author><Tags>humor,open-mind,thinking</Tags></item>
<item><Text>“Think left and think right and think low and think high. Oh, the thinks you can think up if only you try!”</Text><Author>Dr. Seuss</Author><Tags>humor,philosophy</Tags></item>
<item><Text>“What really knocks me out is a book that, when you're all done reading it, you wish the author that wrote it was a terrific friend of yours and you could call him up on the phone whenever you felt like it. That doesn't happen much, though.”</Text><Author>J.D. Salinger</Author><Tags>authors,books,literature,reading,writing</Tags></item>
<item><Text>“The reason I talk to myself is because I’m the only one whose answers I accept.”</Text><Author>George Carlin</Author><Tags>humor,insanity,lies,lying,self-indulgence,truth</Tags></item>
<item><Text>“You may say I'm a dreamer, but I'm not the only one. I hope someday you'll join us. And the world will live as one.”</Text><Author>John Lennon</Author><Tags>beatles,connection,dreamers,dreaming,dreams,hope,inspirational,peace</Tags></item>
<item><Text>“I am free of all prejudice. I hate everyone equally. ”</Text><Author>W.C. Fields</Author><Tags>humor,sinister</Tags></item>
<item><Text>“The question isn't who is going to let me; it's who is going to stop me.”</Text><Author>Ayn Rand</Author><Tags></Tags></item>
<item><Text>“′Classic′ - a book which people praise and don't read.”</Text><Author>Mark Twain</Author><Tags>books,classic,reading</Tags></item>
<item><Text>“Anyone who has never made a mistake has never tried anything new.”</Text><Author>Albert Einstein</Author><Tags>mistakes</Tags></item>
<item><Text>“A lady's imagination is very rapid; it jumps from admiration to love, from love to matrimony in a moment.”</Text><Author>Jane Austen</Author><Tags>humor,love,romantic,women</Tags></item>
<item><Text>“Remember, if the time should come when you have to make a choice between what is right and what is easy, remember what happened to a boy who was good, and kind, and brave, because he strayed across the path of Lord Voldemort. Remember Cedric Diggory.”</Text><Author>J.K. Rowling</Author><Tags>integrity</Tags></item>
<item><Text>“I declare after all there is no enjoyment like reading! How much sooner one tires of any thing than of a book! -- When I have a house of my own, I shall be miserable if I have not an excellent library.”</Text><Author>Jane Austen</Author><Tags>books,library,reading</Tags></item>
<item><Text>“There are few people whom I really love, and still fewer of whom I think well. The more I see of the world, the more am I dissatisfied with it; and every day confirms my belief of the inconsistency of all human characters, and of the little dependence that can be placed on the appearance of merit or sense.”</Text><Author>Jane Austen</Author><Tags>elizabeth-bennet,jane-austen</Tags></item>
<item><Text>“Some day you will be old enough to start reading fairy tales again.”</Text><Author>C.S. Lewis</Author><Tags>age,fairytales,growing-up</Tags></item>
<item><Text>“We are not necessarily doubting that God will do the best for us; we are wondering how painful the best will turn out to be.”</Text><Author>C.S. Lewis</Author><Tags>god</Tags></item>
<item><Text>“The fear of death follows from the fear of life. A man who lives fully is prepared to die at any time.”</Text><Author>Mark Twain</Author><Tags>death,life</Tags></item>
<item><Text>“A lie can travel half way around the world while the truth is putting on its shoes.”</Text><Author>Mark Twain</Author><Tags>misattributed-mark-twain,truth</Tags></item>
<item><Text>“I believe in Christianity as I believe that the sun has risen: not only because I see it, but because by it I see everything else.”</Text><Author>C.S. Lewis</Author><Tags>christianity,faith,religion,sun</Tags></item>
<item><Text>“The truth." Dumbledore sighed. "It is a beautiful and terrible thing, and should therefore be treated with great caution.”</Text><Author>J.K. Rowling</Author><Tags>truth</Tags></item>
<item><Text>“I'm the one that's got to die when it's time for me to die, so let me live my life the way I want to.”</Text><Author>Jimi Hendrix</Author><Tags>death,life</Tags></item>
<item><Text>“To die will be an awfully big adventure.”</Text><Author>J.M. Barrie</Author><Tags>adventure,love</Tags></item>
<item><Text>“It takes courage to grow up and become who you really are.”</Text><Author>E.E. Cummings</Author><Tags>courage</Tags></item>
<item><Text>“But better to get hurt by the truth than comforted with a lie.”</Text><Author>Khaled Hosseini</Author><Tags>life</Tags></item>
<item><Text>“You never really understand a person until you consider things from his point of view... Until you climb inside of his skin and walk around in it.”</Text><Author>Harper Lee</Author><Tags>better-life-empathy</Tags></item>
<item><Text>“You have to write the book that wants to be written. And if the book will be too difficult for grown-ups, then you write it for children.”</Text><Author>Madeleine L'Engle</Author><Tags>books,children,difficult,grown-ups,write,writers,writing</Tags></item>
<item><Text>“Never tell the truth to people who are not worthy of it.”</Text><Author>Mark Twain</Author><Tags>truth</Tags></item>
<item><Text>“A person's a person, no matter how small.”</Text><Author>Dr. Seuss</Author><Tags>inspirational</Tags></item>
<item><Text>“... a mind needs books as a sword needs a whetstone, if it is to keep its edge.”</Text><Author>George R.R. Martin</Author><Tags>books,mind</Tags></item>
</items>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/middlewares.py 0 → 100644
View file @ 05e607a1
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class FirstscrapySpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class FirstscrapyDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/pipelines.py 0 → 100644
View file @ 05e607a1
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class FirstscrapyPipeline:
def process_item(self, item, spider):
return item
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/settings.py 0 → 100644
View file @ 05e607a1
# Scrapy settings for firstScrapy project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'firstScrapy'
SPIDER_MODULES = ['firstScrapy.spiders']
NEWSPIDER_MODULE = 'firstScrapy.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'firstScrapy (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'firstScrapy.middlewares.FirstscrapySpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'firstScrapy.middlewares.FirstscrapyDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'firstScrapy.pipelines.FirstscrapyPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/spiders/__init__.py 0 → 100644
View file @ 05e607a1
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/spiders/quotes2.py 0 → 100644
View file @ 05e607a1
import scrapy
class Quotes2Spider(scrapy.Spider):
name = 'quotes2'
allowed_domains = ['quotes.toscrape.com']
start_urls = ['http://quotes.toscrape.com/']
def parse(self, response):
# h1_tag = response.xpath('//h1/a/text()').extract_first()
# tag = response.xpath('//*[@class="tag-item"]/a/text()').extract()
# yield {'H1 Tag': h1_tag, 'Tags': tag}
quotes = response.xpath('//*[@class="quote"]')
for quote in quotes:
text = quote.xpath('.//*[@class="text"]/text()').extract_first()
author = quote.xpath('.//*[@itemprop="author"]/text()').extract_first()
tags = quote.xpath('.//*[@itemprop="keywords"]/@content').extract_first()
yield {'Text': text,
'Author': author,
'Tags': tags}
next_page_url = response.xpath('//*[@class="next"]/a/@href').extract_first()
absolute_next_page_url = response.urljoin(next_page_url)
yield scrapy.Request(absolute_next_page_url)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/firstScrapy/spiders/spider1.py 0 → 100644
View file @ 05e607a1
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
def start_requests(self):
urls = ['http://quotes.toscrape.com/page/1/'
'http://quotes.toscrape.com/page/2/']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response, **kwargs):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
with open(filename, 'wb') as f:
f.write(response.body)
self.log('Saved file %s' % filename)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstScrapy/scrapy.cfg 0 → 100644
View file @ 05e607a1
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = firstScrapy.settings
[deploy]
#url = http://localhost:6800/
project = firstScrapy
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/firstvet/items.py 0 → 100644
View file @ 05e607a1
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class FirstvetItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/firstvet/middlewares.py 0 → 100644
View file @ 05e607a1
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class FirstvetSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class FirstvetDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/firstvet/pipelines.py 0 → 100644
View file @ 05e607a1
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class FirstvetPipeline:
def process_item(self, item, spider):
return item
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/firstvet/settings.py 0 → 100644
View file @ 05e607a1
# Scrapy settings for firstvet project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'firstvet'
SPIDER_MODULES = ['firstvet.spiders']
NEWSPIDER_MODULE = 'firstvet.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'firstvet (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'firstvet.middlewares.FirstvetSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'firstvet.middlewares.FirstvetDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'firstvet.pipelines.FirstvetPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/firstvet/spiders/__init__.py 0 → 100644
View file @ 05e607a1
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/firstvet/spiders/firstvetspider.py 0 → 100644
View file @ 05e607a1
import scrapy
class FirstvetspiderSpider(scrapy.Spider):
name = 'firstvetspider'
allowed_domains = ['firstvet.com']
start_urls = ['https://firstvet.com/us/articles/ringworm-in-dogs/']
def parse(self, response):
info_ringworm = response.xpath('//p/text()').extract()
yield {'Info': info_ringworm}
print(info_ringworm)
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/info.json 0 → 100644
View file @ 05e607a1
[
{"Info": ["Does your pet have ringworm? What can you do to help? Keep reading to learn about the different types and treatments of ringworm in cats and dogs.", "Book a video consultation with an experienced veterinarian within minutes.", "Ringworm is not a worm! Actually, it\u2019s an infection caused by a type of fungus. These are also called dermatophytes. A \u201cringworm\u201d infection is also known as \u201cdermatophytosis\u201d, and it can infect many animals including dogs, cats, and people. Several species of fungi can cause infection in the superficial layers of skin, and also hair and nails.", "The most common dermatophytes causing infection in dogs and cats:", "Ringworm tends to affect the young, old, and immunocompromised. A healthy adult animal may come into contact with these organisms without becoming infected by them. Most pets become infected through contact with other animals. It is not uncommon to see dermatophytosis in puppies and kittens, rescue and shelter pets or overcrowding situations, as well as hunting dogs or animals in warm environments. Animals that are under stress, malnourished, or harboring an underlying disease may be more likely to become infected. Interestingly, cats with FIV or Feline Leukemia are ", "more susceptible to dermatophytosis.", "Dermatophytosis is a zoonotic infection, meaning humans can become infected by contact with infected animals. The name \u2018ringworm\u2019 comes from its red, round appearance surrounded by a scaly ring (on human skin).", "A combination of moisture on the skin, fungal spores, and microtrauma to the superficial layers of the skin can cause a lesion. The severity of lesions is correlated with immune response. There are no \u201cmore virulent\u201d or \u201cless virulent\u201d strains, the infection is dictated by the host\u2019s immune system. Many things can cause micro-abrasions to the skin, such as grooming and bathing, fleas, and mites.", "Direct contact is the main mode of transmission of ringworm among dogs and in between dogs and other animals. Transmission often happens when a dog is in contact with the infected animal or any contaminated object like a carpet, food bowl, or bedding. Infected animals spread fungal spores into the environment when they shed off infected hair. Fungal spores can stay viable for up to 18 months.", "However, contact alone is not enough to cause an infection in dogs and humans. Host factors like immunity, age, health condition, nutrition, and grooming behavior can influence and affect the risk of infection even with direct contact with the fungal spores. Also, infected animals that have recovered can develop some degree of resistance against dermatophytes that protect them against reinfection for a short time.", "Dermatophytosis in dogs typically causes hair loss and itchiness. Lesions are often seen in bald patches, and the skin can become scaly and produce dandruff. The skin can also become darker and occasionally red from inflammation. Secondary bacterial infection may occur on the affected parts of the skin and pustular nodules may start to develop.", "The commonly affected parts of the dog\u2019s body are the feet, face, ears, and tail, as these are the ones that come in contact with various objects in the environment or other animals. The nails and nailbeds can also become infected, which can result in ", ", ", ", or brittle nails in dogs. Redness and darkly pigmented skin are often seen in dogs with ringworm infection on their nails and nailbeds.", "Dogs can become carriers of dermatophytes, and not show any signs even if they are carrying the fungal organism. Asymptomatic carriers can still transfer the infection to other animals and humans through contact.", "Diagnosis by your veterinarian may be immediate or take some time for testing through a laboratory. Tests for ringworm include:", "Treatment requires persistent and appropriate medication, time, monitoring, and patience. Or doing absolutely nothing at all!", "Since ringworm is primarily transmitted through direct contact, isolation of the infected animal and daily cleaning of the house and objects that might have come in contact with an infected animal can help prevent transmission and control the spread of ringworm. Supplements that help improve your dog\u2019s immune system and general health can help prevent infection even if there\u2019s contact with infectious spores.", "Recently, a vaccine is being studied that can offer protection against some species of dermatophytes that cause ringworm in dogs. Having your dog vaccinated can help offer protection but does not eliminate the risk entirely, since other fungal species can cause skin infection in dogs.", "Click ", " to schedule a video consult to speak to one of our vets. You can also download the FirstVet app from the Apple App Store and Google Play Stores.", "Crystals in your dog\u2019s urine (crystalluria) are formed when there is an excessive amount (oversaturation) of various min...", "\nRead full article\n", "An ectopic ureter is a congenital condition, which means that the anatomical defect is already present at birth. Affecte...", "\nRead full article\n", "In many ways, we share a lot of similarities with our canine buddies in terms of anatomy and physiology. Canines also su...", "\nRead full article\n", "Book a video consultation with an experienced veterinarian within minutes.", "Video call a licensed vet to get expert advice. Open 24 hours a day, 365 days a year."]}
]
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/firstvet/scrapy.cfg 0 → 100644
View file @ 05e607a1
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = firstvet.settings
[deploy]
#url = http://localhost:6800/
project = firstvet
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/main.py 0 → 100644
View file @ 05e607a1
# This is a sample Python script.
# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
def print_hi(name):
# Use a breakpoint in the code line below to debug your script.
print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint.
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
print_hi('PyCharm')
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/.idea/.gitignore 0 → 100644
View file @ 05e607a1
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ 05e607a1
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="17">
<item index="0" class="java.lang.String" itemvalue="queuelib" />
<item index="1" class="java.lang.String" itemvalue="python-dateutil" />
<item index="2" class="java.lang.String" itemvalue="zope.interface" />
<item index="3" class="java.lang.String" itemvalue="incremental" />
<item index="4" class="java.lang.String" itemvalue="numpy" />
<item index="5" class="java.lang.String" itemvalue="pyasn1" />
<item index="6" class="java.lang.String" itemvalue="pycparser" />
<item index="7" class="java.lang.String" itemvalue="constantly" />
<item index="8" class="java.lang.String" itemvalue="jmespath" />
<item index="9" class="java.lang.String" itemvalue="pandas" />
<item index="10" class="java.lang.String" itemvalue="pyasn1-modules" />
<item index="11" class="java.lang.String" itemvalue="certifi" />
<item index="12" class="java.lang.String" itemvalue="typing_extensions" />
<item index="13" class="java.lang.String" itemvalue="PyDispatcher" />
<item index="14" class="java.lang.String" itemvalue="lxml" />
<item index="15" class="java.lang.String" itemvalue="urllib3" />
<item index="16" class="java.lang.String" itemvalue="Django" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N806" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ 05e607a1
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/.idea/misc.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/.idea/modules.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/petmd.iml" filepath="$PROJECT_DIR$/.idea/petmd.iml" />
</modules>
</component>
</project>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/.idea/petmd.iml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/link.csv 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd.csv 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd.json 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd.xml 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd/items.py 0 → 100644
View file @ 05e607a1
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class PetmdItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd/middlewares.py 0 → 100644
View file @ 05e607a1
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class PetmdSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class PetmdDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd/pipelines.py 0 → 100644
View file @ 05e607a1
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class PetmdPipeline:
def process_item(self, item, spider):
return item
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd/settings.py 0 → 100644
View file @ 05e607a1
# Scrapy settings for petmd project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'petmd'
SPIDER_MODULES = ['petmd.spiders']
NEWSPIDER_MODULE = 'petmd.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'cosmos/0.9_(robot@xyleme.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'petmd.middlewares.PetmdSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'petmd.middlewares.PetmdDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'petmd.pipelines.PetmdPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd/spiders/__init__.py 0 → 100644
View file @ 05e607a1
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/petmd/spiders/dogdiseases.py 0 → 100644
View file @ 05e607a1
from scrapy import Spider
from scrapy.http import Request
class DogdiseasesSpider(Spider):
name = 'dogdiseases'
allowed_domains = ['petmd.com']
start_urls = ['https://www.petmd.com/dog/conditions']
def __init__(self, disease=None):
self.disease = disease
def parse(self, response):
if self.disease:
disease_url = response.xpath('//*[contains(@href,"' + self.disease + '")]/@href').extract_first()
yield Request(response.urljoin(disease_url), callback=self.parse_disease)
else:
self.logger.info('Scraping all articles.')
diseases = response.xpath('//*[@class="kib-grid__item kib-grid__item--span-4@min-xs kib-grid__item--span-4@md kib-grid__item--span-4@min-lg"]/a/@href').extract()
for disease in diseases:
yield Request(response.urljoin(disease), callback=self.parse_disease)
def parse_disease(self, response):
disease_name = response.xpath('//title/text()').extract_first()
disease_name = disease_name.split(' | ')
disease_name = disease_name[0]
root = response.xpath('//*[@class="article_article_body__JKAEF"]')
topics = response.xpath('//h2/text()').extract()
for index, topic in enumerate(topics):
topic_heading = topic
path = './/p[count(preceding-sibling::h2)=' + str(index) + ']/text()'
topic_description = root.xpath(path).extract()
yield {
'disease_name': disease_name,
'topic_heading': topic_heading,
'topic_description': topic_description
}
yield {
'disease_name': disease_name,
'topic_heading': 'Disease Link',
'topic_description': response
}
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/scrapy.cfg 0 → 100644
View file @ 05e607a1
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = petmd.settings
[deploy]
#url = http://localhost:6800/
project = petmd
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petmd/test.json 0 → 100644
View file @ 05e607a1
This source diff could not be displayed because it is too large. You can view the blob instead.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/items.csv 0 → 100644
View file @ 05e607a1
Name,Info
Dog Skin Problems ,The sound of a dog constantly scratching or licking can be as irritating as nails on a chalkboard. But don’t blame your pooch for these bad habits -- a skin condition is probably the culprit. Possible causes range from parasites to allergies to underlying illness. WebMD has compiled images of some of the most common canine skin problems.
Allergic Dermatitis ,"Dogs can have allergic reactions to grooming products, food, and environmental irritants, such as pollen or insect bites. A dog with allergies may scratch relentlessly, and a peek at the skin often reveals an ugly rash. Corticosteroids or other, newer medicines can help with itchy rashes. But the most effective treatment is to identify and avoid exposure to the allergens."
Yeast Infection ,"If your dog can't seem to stop scratching an ear or licking and chewing their toes, ask your veterinarian to check for a yeast infection. Symptoms include irritated, itchy, or discolored skin. The infection usually strikes the paws or ears, where yeast have a cozy space to grow. Yeast infections are easy to diagnose and often respond well to a topical cream. In some cases, your veterinarian may prescribe oral drugs, medicated sprays, or medicated baths."
Folliculitis ,"Superficial bacterial folliculitis is an infection that causes sores, bumps, and scabs on the skin. These skin abnormalities are easier to see in shorthaired dogs. In longhaired dogs, the most obvious symptoms may be a dull coat and shedding with scaly skin underneath. Folliculitis often occurs in conjunction with other skin problems, such as mange, allergies, or injury. Treatment may include oral antibiotics and antibacterial ointments or shampoos."
Impetigo ,"Another type of bacterial infection, impetigo is most common in puppies. It causes pus-filled blisters that may break and crust over. The blisters usually develop on the hairless portion of the abdomen. Impetigo is rarely serious and can be treated with a topical solution. In a small number of cases, the infection may spread or persist."
Seborrhea ,"Seborrhea causes a dog's skin to become greasy and develop scales (dandruff). In some cases, it's a genetic disease that begins when a dog is young and lasts a lifetime. But most dogs with seborrhea develop the scaling as a complication of another medical problem, such as allergies or hormonal abnormalities. In these cases, it is vital to treat the underlying cause so symptoms do not recur. The seborrhea itself typically can be treated with certain medicated shampoos."
Ringworm ,"Despite its name, ringworm is not caused by a worm, but by a fungus. The term ""ring"" comes from the circular patches that can form anywhere, but are often found on a dog's head, paws, ears, and forelegs. Inflammation, scaly patches, and hair loss often surround the lesions. Puppies less than a year old are the most susceptible, and the infection can spread quickly between dogs in a kennel or to pet owners at home. Various anti-fungal treatments are available."
Shedding and Hair Loss (Alopecia) ,"Anyone who shares their home with dogs knows that they shed. How much shedding is normal depends on breed, time of year, and environment. But sometimes stress, poor nutrition, or illness can cause a dog to lose more hair than usual. If abnormal or excessive shedding persists for more than a week, or you notice patches of missing fur, check with your veterinarian."
Mange (Mites) ,Mange is a skin disorder caused by tiny parasites called mites.
Fleas ,"Fleas are the bane of any pet owner. You may not see the tiny insects themselves, but flea droppings or eggs are usually visible in a dog's coat. Other symptoms include excessive licking or scratching, scabs, and hot spots. Severe flea infestations can cause blood loss and anemia, and even expose your dog to other parasites, such as tapeworms. Treatment may include a topical and/or oral flea killer and a thorough cleaning of the pet's home and yard."
Ticks ,"Ticks, like fleas, are external parasites that feed on the blood of their hosts. You can spot a tick feeding on your dog with the naked eye. To properly remove a tick, grasp the tick with tweezers close to the dog’s skin, and gently pull it straight out. Twisting or pulling too hard may cause the head to remain lodged in your dog’s skin, which can lead to infection. Place the tick in a jar with some alcohol for a couple of days. If your pet gets ill, your vet may need it to analyze what's wrong. In addition to causing blood loss and anemia, ticks can transmit Lyme disease and other potentially serious bacterial infections. If you live in an area where ticks are common, talk to your veterinarian about tick control products."
Color or Texture Changes ,Changes in a dog's skin color or coat texture can be a warning sign of several common metabolic or hormone problems. They can also result from an infection or other skin disorder. Usually a simple blood test can identify the underlying cause. Be sure to ask your veterinarian about any significant changes to your dog’s coat.
" Dry, Flaky Skin ","Dry, flaky skin can be a red flag for a number of problems. It's a common symptom of allergies, mange, and other skin diseases. But most often, dry or flaky skin is nothing serious. Make sure you are feeding Fido high quality food. Like people, some dogs simply get dry skin in the winter. If this seems to cause your pet discomfort, consult your veterinarian. Ask whether a fatty acid supplement or a humidifier might help."
Acral Lick Granuloma ,"Also called acral lick dermatitis, this is a frustrating skin condition caused by compulsive, relentless licking of a single area -- most often on the front of the lower leg. The area is unable to heal, and the resulting pain and itching can lead the dog to keep licking the same spot. Treatment includes discouraging the dog from licking, either by using a bad-tasting topical solution or an Elizabethan collar. Also ask your dog's vet about other treatment options."
Skin Tumors ,"If you notice a lump on your dog's skin, point it out to your vet as soon as possible. Dogs can develop cancerous tumors in their skin. The only way to confirm a diagnosis of cancer is to biopsy the tumor. If the lump is small enough, your veterinarian may recommend removing it entirely. This can yield a diagnosis and treatment with a single procedure. For tumors that have not spread, this may be the only treatment needed."
Hot Spots ,"Hot spots, also called acute moist dermatitis, are small areas that appear red, irritated, and inflamed. They are most commonly found on a dog's head, hips, or chest, and often feel hot to the touch. Hot spots can result from a wide range of conditions, including infections, allergies, insect bites, or excessive licking and chewing. Treatment consists of cleansing the hot spot and addressing the underlying condition."
Immune Disorders ,"In rare cases, skin lesions or infections that won’t heal can indicate an immune disorder in your dog. One of the best known is lupus, a disease that affects dogs and people. Lupus is an autoimmune disorder, meaning the body’s immune system attacks its own cells. Symptoms include skin abnormalities and kidney problems. It can be fatal if untreated."
Anal Sac Disease ,"As if dog poop weren't smelly enough, dogs release a foul-smelling substance when they do their business. The substance comes from small anal sacs, which can become impacted if they don't empty properly. The hallmark of impacted anal sacs is a dog scooting their bottom along the ground. Other symptoms include biting or licking the anal area. A vet can manually express full anal sacs, but in severe cases, the sacs may be surgically removed."
When to See the Vet ,"Although most skin problems are not emergencies, it is important to get an accurate diagnosis so the condition can be treated. See your veterinarian if your dog is scratching or licking excessively, or if you notice any changes in your pet's coat or skin, including scaling, redness, discoloration, or bald patches. Once the cause is identified, most skin problems respond well to treatment."
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/items.json 0 → 100644
View file @ 05e607a1
[
{"Name": " Dog Skin Problems ", "Info": "The sound of a dog constantly scratching or licking can be as irritating as nails on a chalkboard. But don\u2019t blame your pooch for these bad habits -- a skin condition is probably the culprit. Possible causes range from parasites to allergies to underlying illness. WebMD has compiled images of some of the most common canine skin problems."},
{"Name": " Allergic Dermatitis ", "Info": "Dogs can have allergic reactions to grooming products, food, and environmental irritants, such as pollen or insect bites. A dog with allergies may scratch relentlessly, and a peek at the skin often reveals an ugly rash. Corticosteroids or other, newer medicines can help with itchy rashes.\u00a0But the most effective treatment is to identify and avoid exposure to the allergens."},
{"Name": " Yeast Infection ", "Info": "If your dog can't seem to stop scratching an ear or licking and chewing their toes, ask your veterinarian to check for a yeast infection. Symptoms include irritated, itchy, or discolored skin. The infection usually strikes the paws or ears, where yeast have a cozy space to grow. Yeast infections are easy to diagnose and often respond well to a topical cream. In some cases, your veterinarian may prescribe oral drugs, medicated sprays,\u00a0or medicated baths."},
{"Name": " Folliculitis ", "Info": "Superficial bacterial folliculitis is an infection that causes sores, bumps, and scabs on the skin. These skin abnormalities are easier to see in shorthaired dogs. In longhaired dogs, the most obvious symptoms may be a dull coat and shedding with scaly skin underneath. Folliculitis often occurs in conjunction with other skin problems, such as mange, allergies, or injury. Treatment may include oral antibiotics and antibacterial ointments or shampoos."},
{"Name": " Impetigo ", "Info": "Another type of bacterial infection, impetigo is most common in puppies. It causes pus-filled blisters that may break and crust over. The blisters usually develop on the hairless portion of the abdomen. Impetigo is rarely serious and can be treated with a topical solution. In a small number of cases, the infection may spread or persist."},
{"Name": " Seborrhea ", "Info": "Seborrhea causes a dog's skin to become greasy and develop scales (dandruff). In some cases, it's a genetic disease that begins when a dog is young and lasts a lifetime. But most dogs with seborrhea develop the scaling as a complication of another medical problem, such as allergies or hormonal abnormalities. In these cases, it is vital to treat the underlying cause so symptoms do not recur. The seborrhea itself typically can be treated with certain medicated shampoos."},
{"Name": " Ringworm ", "Info": "Despite its name, ringworm is not caused by a worm, but by a fungus. The term \"ring\" comes from the circular patches that can form anywhere, but are often found on a dog's head, paws, ears, and forelegs. Inflammation, scaly patches, and hair loss often surround the lesions. Puppies less than a year old are the most susceptible, and the infection can spread quickly between dogs in a kennel or to pet owners at home. Various anti-fungal treatments are available."},
{"Name": " Shedding and Hair Loss (Alopecia) ", "Info": "Anyone who shares their home with dogs knows that they shed. How much shedding is normal depends on breed, time of year, and environment. But sometimes stress, poor nutrition, or illness can cause a dog to lose more hair than usual. If abnormal or excessive shedding persists for more than a week, or you notice patches of missing fur, check with your veterinarian."},
{"Name": " Mange (Mites) ", "Info": "Mange is a skin disorder caused by tiny parasites called mites. "},
{"Name": " Fleas ", "Info": "Fleas are the bane of any pet owner. You may not see the tiny insects themselves, but flea droppings or eggs are usually visible in a dog's coat. Other symptoms include excessive licking or scratching, scabs, and hot spots. Severe flea infestations can cause blood loss and anemia, and even expose your dog to other parasites, such as tapeworms. Treatment may include a topical and/or oral flea killer and a thorough cleaning of the pet's home and yard."},
{"Name": " Ticks ", "Info": "Ticks, like fleas, are external parasites that feed on the blood of their hosts. You can spot a tick feeding on your dog with the naked eye. To properly remove a tick, grasp the tick with tweezers close to the dog\u2019s skin, and gently pull it straight out. Twisting or pulling too hard may cause the head to remain lodged in your dog\u2019s skin, which can lead to infection. Place the tick in a jar with some alcohol for a couple of days. If your pet gets ill, your vet may need it to analyze what's wrong.\u00a0In addition to causing blood loss and anemia, ticks can transmit Lyme disease and other potentially serious bacterial infections. If you live in an area where ticks are common, talk to your veterinarian about tick control products."},
{"Name": " Color or Texture Changes ", "Info": "Changes in a dog's skin color or coat texture can be a warning sign of several common metabolic or hormone problems. They can also result from an infection or other skin disorder. Usually a simple blood test can identify the underlying cause.\u00a0Be sure to ask your veterinarian about any significant changes to your dog\u2019s coat."},
{"Name": " Dry, Flaky Skin ", "Info": "Dry, flaky skin can be a red flag for a number of problems. It's a common symptom of allergies, mange, and other skin diseases. But most often, dry or flaky skin is nothing serious. Make sure you are feeding Fido high quality food. Like people, some dogs simply get dry skin in the winter. If this seems to cause your pet discomfort, consult your veterinarian. Ask whether a fatty acid supplement\u00a0or a humidifier might help."},
{"Name": " Acral Lick Granuloma ", "Info": "Also called acral lick dermatitis, this is a frustrating skin condition caused by compulsive, relentless licking of a single area -- most often on the front of the lower leg. The area is unable to heal, and the resulting pain and itching can lead the dog to keep licking the same spot. Treatment includes discouraging the dog from licking, either by using a bad-tasting topical solution or an Elizabethan collar. Also ask your dog's vet about other treatment options."},
{"Name": " Skin Tumors ", "Info": "If you notice a lump on your dog's skin, point it out to your vet as soon as possible. Dogs can develop cancerous tumors in their skin. The only way to confirm a diagnosis of cancer is to biopsy the tumor. If the lump is small enough, your veterinarian may recommend removing it entirely. This can yield a diagnosis and treatment with a single procedure. For tumors that have not spread, this may be the only treatment needed."},
{"Name": " Hot Spots ", "Info": "Hot spots, also called acute moist dermatitis, are small areas that appear red, irritated, and inflamed. They are most commonly found on a dog's head, hips, or chest, and often feel hot to the touch. Hot spots can result from a wide range of conditions, including infections, allergies, insect bites, or excessive licking and chewing. Treatment consists of cleansing the hot spot and addressing the underlying condition."},
{"Name": " Immune Disorders ", "Info": "In rare cases, skin lesions or infections that won\u2019t heal can indicate an immune disorder in your dog. One of the best known is lupus, a disease that affects dogs and people. Lupus is an autoimmune disorder, meaning the body\u2019s immune system attacks its own cells. Symptoms include skin abnormalities and kidney problems. It can be fatal if untreated."},
{"Name": " Anal Sac Disease ", "Info": "As if dog poop weren't smelly enough, dogs release a foul-smelling substance when they do their business. The substance comes from small anal sacs, which can become impacted if they don't empty properly. The hallmark of impacted anal sacs is a dog scooting their bottom along the ground. Other symptoms include biting or licking the anal area. A vet can manually express full anal sacs, but in severe cases, the sacs may be surgically removed."},
{"Name": " When to See the Vet ", "Info": "Although most skin problems are not emergencies, it is important to get an accurate diagnosis so the condition can be treated. See your veterinarian if your dog is scratching or licking excessively, or if you notice any changes in your pet's coat or skin, including scaling, redness, discoloration, or bald patches. Once the cause is identified, most skin problems respond well to treatment."}
]
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/items.xml 0 → 100644
View file @ 05e607a1
<?xml version="1.0" encoding="utf-8"?>
<items>
<item><Name> Dog Skin Problems </Name><Info>The sound of a dog constantly scratching or licking can be as irritating as nails on a chalkboard. But don’t blame your pooch for these bad habits -- a skin condition is probably the culprit. Possible causes range from parasites to allergies to underlying illness. WebMD has compiled images of some of the most common canine skin problems.</Info></item>
<item><Name> Allergic Dermatitis </Name><Info>Dogs can have allergic reactions to grooming products, food, and environmental irritants, such as pollen or insect bites. A dog with allergies may scratch relentlessly, and a peek at the skin often reveals an ugly rash. Corticosteroids or other, newer medicines can help with itchy rashes. But the most effective treatment is to identify and avoid exposure to the allergens.</Info></item>
<item><Name> Yeast Infection </Name><Info>If your dog can't seem to stop scratching an ear or licking and chewing their toes, ask your veterinarian to check for a yeast infection. Symptoms include irritated, itchy, or discolored skin. The infection usually strikes the paws or ears, where yeast have a cozy space to grow. Yeast infections are easy to diagnose and often respond well to a topical cream. In some cases, your veterinarian may prescribe oral drugs, medicated sprays, or medicated baths.</Info></item>
<item><Name> Folliculitis </Name><Info>Superficial bacterial folliculitis is an infection that causes sores, bumps, and scabs on the skin. These skin abnormalities are easier to see in shorthaired dogs. In longhaired dogs, the most obvious symptoms may be a dull coat and shedding with scaly skin underneath. Folliculitis often occurs in conjunction with other skin problems, such as mange, allergies, or injury. Treatment may include oral antibiotics and antibacterial ointments or shampoos.</Info></item>
<item><Name> Impetigo </Name><Info>Another type of bacterial infection, impetigo is most common in puppies. It causes pus-filled blisters that may break and crust over. The blisters usually develop on the hairless portion of the abdomen. Impetigo is rarely serious and can be treated with a topical solution. In a small number of cases, the infection may spread or persist.</Info></item>
<item><Name> Seborrhea </Name><Info>Seborrhea causes a dog's skin to become greasy and develop scales (dandruff). In some cases, it's a genetic disease that begins when a dog is young and lasts a lifetime. But most dogs with seborrhea develop the scaling as a complication of another medical problem, such as allergies or hormonal abnormalities. In these cases, it is vital to treat the underlying cause so symptoms do not recur. The seborrhea itself typically can be treated with certain medicated shampoos.</Info></item>
<item><Name> Ringworm </Name><Info>Despite its name, ringworm is not caused by a worm, but by a fungus. The term "ring" comes from the circular patches that can form anywhere, but are often found on a dog's head, paws, ears, and forelegs. Inflammation, scaly patches, and hair loss often surround the lesions. Puppies less than a year old are the most susceptible, and the infection can spread quickly between dogs in a kennel or to pet owners at home. Various anti-fungal treatments are available.</Info></item>
<item><Name> Shedding and Hair Loss (Alopecia) </Name><Info>Anyone who shares their home with dogs knows that they shed. How much shedding is normal depends on breed, time of year, and environment. But sometimes stress, poor nutrition, or illness can cause a dog to lose more hair than usual. If abnormal or excessive shedding persists for more than a week, or you notice patches of missing fur, check with your veterinarian.</Info></item>
<item><Name> Mange (Mites) </Name><Info>Mange is a skin disorder caused by tiny parasites called mites. </Info></item>
<item><Name> Fleas </Name><Info>Fleas are the bane of any pet owner. You may not see the tiny insects themselves, but flea droppings or eggs are usually visible in a dog's coat. Other symptoms include excessive licking or scratching, scabs, and hot spots. Severe flea infestations can cause blood loss and anemia, and even expose your dog to other parasites, such as tapeworms. Treatment may include a topical and/or oral flea killer and a thorough cleaning of the pet's home and yard.</Info></item>
<item><Name> Ticks </Name><Info>Ticks, like fleas, are external parasites that feed on the blood of their hosts. You can spot a tick feeding on your dog with the naked eye. To properly remove a tick, grasp the tick with tweezers close to the dog’s skin, and gently pull it straight out. Twisting or pulling too hard may cause the head to remain lodged in your dog’s skin, which can lead to infection. Place the tick in a jar with some alcohol for a couple of days. If your pet gets ill, your vet may need it to analyze what's wrong. In addition to causing blood loss and anemia, ticks can transmit Lyme disease and other potentially serious bacterial infections. If you live in an area where ticks are common, talk to your veterinarian about tick control products.</Info></item>
<item><Name> Color or Texture Changes </Name><Info>Changes in a dog's skin color or coat texture can be a warning sign of several common metabolic or hormone problems. They can also result from an infection or other skin disorder. Usually a simple blood test can identify the underlying cause. Be sure to ask your veterinarian about any significant changes to your dog’s coat.</Info></item>
<item><Name> Dry, Flaky Skin </Name><Info>Dry, flaky skin can be a red flag for a number of problems. It's a common symptom of allergies, mange, and other skin diseases. But most often, dry or flaky skin is nothing serious. Make sure you are feeding Fido high quality food. Like people, some dogs simply get dry skin in the winter. If this seems to cause your pet discomfort, consult your veterinarian. Ask whether a fatty acid supplement or a humidifier might help.</Info></item>
<item><Name> Acral Lick Granuloma </Name><Info>Also called acral lick dermatitis, this is a frustrating skin condition caused by compulsive, relentless licking of a single area -- most often on the front of the lower leg. The area is unable to heal, and the resulting pain and itching can lead the dog to keep licking the same spot. Treatment includes discouraging the dog from licking, either by using a bad-tasting topical solution or an Elizabethan collar. Also ask your dog's vet about other treatment options.</Info></item>
<item><Name> Skin Tumors </Name><Info>If you notice a lump on your dog's skin, point it out to your vet as soon as possible. Dogs can develop cancerous tumors in their skin. The only way to confirm a diagnosis of cancer is to biopsy the tumor. If the lump is small enough, your veterinarian may recommend removing it entirely. This can yield a diagnosis and treatment with a single procedure. For tumors that have not spread, this may be the only treatment needed.</Info></item>
<item><Name> Hot Spots </Name><Info>Hot spots, also called acute moist dermatitis, are small areas that appear red, irritated, and inflamed. They are most commonly found on a dog's head, hips, or chest, and often feel hot to the touch. Hot spots can result from a wide range of conditions, including infections, allergies, insect bites, or excessive licking and chewing. Treatment consists of cleansing the hot spot and addressing the underlying condition.</Info></item>
<item><Name> Immune Disorders </Name><Info>In rare cases, skin lesions or infections that won’t heal can indicate an immune disorder in your dog. One of the best known is lupus, a disease that affects dogs and people. Lupus is an autoimmune disorder, meaning the body’s immune system attacks its own cells. Symptoms include skin abnormalities and kidney problems. It can be fatal if untreated.</Info></item>
<item><Name> Anal Sac Disease </Name><Info>As if dog poop weren't smelly enough, dogs release a foul-smelling substance when they do their business. The substance comes from small anal sacs, which can become impacted if they don't empty properly. The hallmark of impacted anal sacs is a dog scooting their bottom along the ground. Other symptoms include biting or licking the anal area. A vet can manually express full anal sacs, but in severe cases, the sacs may be surgically removed.</Info></item>
<item><Name> When to See the Vet </Name><Info>Although most skin problems are not emergencies, it is important to get an accurate diagnosis so the condition can be treated. See your veterinarian if your dog is scratching or licking excessively, or if you notice any changes in your pet's coat or skin, including scaling, redness, discoloration, or bald patches. Once the cause is identified, most skin problems respond well to treatment.</Info></item>
</items>
\ No newline at end of file
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/petswebmd/items.py 0 → 100644
View file @ 05e607a1
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class PetswebmdItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/petswebmd/middlewares.py 0 → 100644
View file @ 05e607a1
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class PetswebmdSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class PetswebmdDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/petswebmd/pipelines.py 0 → 100644
View file @ 05e607a1
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class PetswebmdPipeline:
def process_item(self, item, spider):
return item
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/petswebmd/settings.py 0 → 100644
View file @ 05e607a1
# Scrapy settings for petswebmd project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'petswebmd'
SPIDER_MODULES = ['petswebmd.spiders']
NEWSPIDER_MODULE = 'petswebmd.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'petswebmd (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'petswebmd.middlewares.PetswebmdSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'petswebmd.middlewares.PetswebmdDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'petswebmd.pipelines.PetswebmdPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/petswebmd/spiders/__init__.py 0 → 100644
View file @ 05e607a1
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/petswebmd/spiders/petswebmd_dog.py 0 → 100644
View file @ 05e607a1
import scrapy
class PetswebmdDogSpider(scrapy.Spider):
name = 'petswebmd_dog'
allowed_domains = ['pets.webmd.com']
start_urls = ['http://pets.webmd.com/dogs/ss/slideshow-skin-problems-in-dogs/']
def parse(self, response):
disease_info = response.xpath('//*[@class="slide"]')
for disease in disease_info:
topic = disease.xpath('.//*[@class="slide-header inf_sliderheader"]/h2/text()').extract_first()
information = disease.xpath('.//*[@class="caption inf_caption"]/p/text()').extract_first()
yield {'Name': topic,
'Info': information}
print('\n')
print(topic)
print(information)
print('\n')
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/petswebmd/scrapy.cfg 0 → 100644
View file @ 05e607a1
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = petswebmd.settings
[deploy]
#url = http://localhost:6800/
project = petswebmd
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/testing/scrapy.cfg 0 → 100644
View file @ 05e607a1
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = testing.settings
[deploy]
#url = http://localhost:6800/
project = testing
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/testing/testing/items.py 0 → 100644
View file @ 05e607a1
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class TestingItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/testing/testing/middlewares.py 0 → 100644
View file @ 05e607a1
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class TestingSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class TestingDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/testing/testing/pipelines.py 0 → 100644
View file @ 05e607a1
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class TestingPipeline:
def process_item(self, item, spider):
return item
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/testing/testing/settings.py 0 → 100644
View file @ 05e607a1
# Scrapy settings for testing project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'testing'
SPIDER_MODULES = ['testing.spiders']
NEWSPIDER_MODULE = 'testing.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'testing (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'testing.middlewares.TestingSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'testing.middlewares.TestingDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'testing.pipelines.TestingPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/testing/testing/spiders/__init__.py 0 → 100644
View file @ 05e607a1
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
Ontology_Based_Information_System/Web Crawlers/Modified Crawlers/testing/testing/spiders/test.py 0 → 100644
View file @ 05e607a1
from scrapy import Spider
from scrapy.http import Request
class TestSpider(Spider):
name = 'test'
allowed_domains = ['eplanning.ie']
start_urls = ['http://eplanning.ie/']
def parse(self, response):
urls = response.xpath('//a/@href').extract()
for url in urls:
if '#' == url:
pass
else:
yield Request(url, callback=self.parse_application)
def parse_application(self, response):
pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment