Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
2
2021-155
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
2021-155
2021-155
Commits
70363aeb
Commit
70363aeb
authored
Aug 20, 2021
by
Ekanayake P.M.D.P IT18013610
☺
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'it18013610' into 'master'
It18013610 See merge request
!9
parents
7963cda7
17231558
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
322 additions
and
48 deletions
+322
-48
backend/controllers/automated-answer.js
backend/controllers/automated-answer.js
+63
-0
backend/controllers/questions.js
backend/controllers/questions.js
+9
-18
backend/python/auto-answer/Dev.py
backend/python/auto-answer/Dev.py
+29
-9
backend/python/auto-answer/github.py
backend/python/auto-answer/github.py
+85
-0
backend/python/auto-answer/scrapper.py
backend/python/auto-answer/scrapper.py
+24
-15
backend/routes.js
backend/routes.js
+6
-3
client/components/automated-answer-swiper/AutomatedAnswerSwiper.jsx
...ponents/automated-answer-swiper/AutomatedAnswerSwiper.jsx
+27
-0
client/components/automated-answer-swiper/AutomatedAnswerSwiper.module.css
.../automated-answer-swiper/AutomatedAnswerSwiper.module.css
+0
-0
client/components/pagination/Pagination.jsx
client/components/pagination/Pagination.jsx
+42
-0
client/components/pagination/Pagination.module.css
client/components/pagination/Pagination.module.css
+31
-0
client/components/youtube-videos/YoutubeVideoWrapper.js
client/components/youtube-videos/YoutubeVideoWrapper.js
+6
-3
No files found.
backend/controllers/automated-answer.js
0 → 100644
View file @
70363aeb
const
{
spawn
}
=
require
(
'
child_process
'
);
const
path
=
require
(
'
path
'
);
const
AutomatedAnswer
=
require
(
'
../models/automatedAnswer
'
);
const
Question
=
require
(
'
../models/question
'
);
/**
* @Method - POST
* @param {*} question_id - ID of the question to create automated answer for
* @returns Whether the automated answer request created or not
*/
exports
.
requestAutomatedAnswer
=
async
(
req
,
res
,
next
)
=>
{
const
{
question_id
}
=
req
.
body
;
try
{
const
question
=
await
Question
.
findById
(
question_id
);
if
(
!
question
)
{
return
res
.
status
(
404
)
.
json
({
message
:
'
No question found for this id, Please try again with another question
'
});
}
let
autoAnswer
;
autoAnswer
=
await
AutomatedAnswer
.
findOne
({
question
:
question_id
});
if
(
autoAnswer
)
{
return
res
.
status
(
303
)
.
json
({
message
:
'
Automated answer is already generated
'
,
automatedanswer
:
autoAnswer
});
}
res
.
status
(
201
).
json
({
message
:
'
Automated answer requested! Please wait for few seconds
'
});
autoAnswer
=
await
AutomatedAnswer
.
create
({
question
:
question
.
_id
,
loading
:
true
});
spawn
(
'
python
'
,
[
path
.
join
(
__dirname
,
'
..
'
,
'
python
'
,
'
auto-answer
'
,
'
scrapper.py
'
),
question
.
title
,
question
.
tags
,
autoAnswer
.
_id
]);
}
catch
(
err
)
{
next
(
err
);
}
};
/**
*
* @Method - GET
* @qid {*} - question id
* @returns the automated answer for the question(qid) if there is one
*/
exports
.
getAutomatedAnswer
=
async
(
req
,
res
,
next
)
=>
{
const
{
qid
}
=
req
.
params
;
console
.
log
(
'
answer is here
'
+
qid
);
try
{
const
answer
=
await
AutomatedAnswer
.
findOne
({
question
:
qid
});
if
(
!
answer
)
{
return
res
.
status
(
404
).
json
({
message
:
'
No automated answer found for this question
'
});
}
return
res
.
status
(
200
).
json
({
message
:
'
Automated answer found
'
,
automatedanswer
:
answer
});
}
catch
(
err
)
{
next
(
err
);
}
};
backend/controllers/questions.js
View file @
70363aeb
const
Question
=
require
(
'
../models/question
'
);
const
User
=
require
(
'
../models/user
'
);
const
AutomatedAnswer
=
require
(
'
../models/automatedAnswer
'
);
const
User
=
require
(
'
../models/user
'
);
const
{
body
,
validationResult
}
=
require
(
'
express-validator
'
);
const
{
spawn
}
=
require
(
'
child_process
'
);
const
path
=
require
(
'
path
'
);
...
...
@@ -35,12 +35,18 @@ exports.createQuestion = async (req, res, next) => {
});
res
.
status
(
201
).
json
(
question
);
const
autoAnswer
=
await
AutomatedAnswer
.
create
({
question
:
question
.
_id
});
spawn
(
'
python
'
,
[
const
pythonScript
=
spawn
(
'
python
'
,
[
path
.
join
(
__dirname
,
'
..
'
,
'
python
'
,
'
auto-answer
'
,
'
scrapper.py
'
),
title
,
tags
,
autoAnswer
.
_id
]);
pythonScript
.
stdout
.
on
(
'
data
'
,
(
data
)
=>
{
console
.
log
(
'
DATA FROM PYTHON IS HERE
'
+
data
);
});
pythonScript
.
stdout
.
on
(
'
end
'
,
function
()
{
console
.
log
(
'
PYTHON SCRIPT HAS BEEN ENDED
'
);
});
}
catch
(
error
)
{
next
(
error
);
}
...
...
@@ -96,6 +102,7 @@ exports.removeQuestion = async (req, res, next) => {
try
{
await
req
.
question
.
remove
();
res
.
json
({
message
:
'
Your question successfully deleted.
'
});
await
AutomatedAnswer
.
findOneAndRemove
({
question
:
req
.
question
.
_id
});
}
catch
(
error
)
{
next
(
error
);
}
...
...
@@ -113,22 +120,6 @@ exports.loadComment = async (req, res, next, id) => {
next
();
};
exports
.
getAutomatedAnswer
=
async
(
req
,
res
,
next
)
=>
{
console
.
log
(
'
HERE
'
);
try
{
const
{
q
}
=
req
.
params
;
const
answer
=
await
AutomatedAnswer
.
findOne
({
question
:
q
});
if
(
answer
)
{
res
.
json
(
answer
);
}
else
{
res
.
status
(
404
).
json
({
msg
:
'
No automated answer found
'
});
}
}
catch
(
error
)
{
next
(
error
);
}
};
exports
.
questionValidate
=
[
body
(
'
title
'
)
.
exists
()
...
...
backend/python/auto-answer/Dev.py
View file @
70363aeb
...
...
@@ -9,7 +9,11 @@ class DevTo:
self
.
title
=
title
self
.
tags
=
tags
def
getApiKey
(
self
):
def
get_api_key
(
self
):
"""
get random api key from api keys of rss2json.com
:return: string
"""
api_keys
=
[
"2rk1eg4sexdnp5umrwtwbtwd2insqvgzvejooqrn"
,
"yit6ytfcs3ziawdgasfd3bgkbf4tef1m2nzdxvnz"
,
...
...
@@ -23,7 +27,10 @@ class DevTo:
gresults
=
gsearch
.
search
(
*
search_args
)
return
gresults
[
"links"
]
def
getValidUrls
(
self
,
links
):
def
get_valid_urls
(
self
,
links
):
"""
filter out invalid urls
"""
validUrls
=
[]
for
i
in
links
:
if
"dev.to"
in
i
:
...
...
@@ -32,7 +39,10 @@ class DevTo:
validUrls
.
append
(
ur
)
return
validUrls
def
getValidSets
(
self
,
validUrls
):
def
get_valid_sets
(
self
,
validUrls
):
"""
extract valid usernames and tags from valid dev.to urls
"""
validSets
=
[]
for
url
in
validUrls
:
try
:
...
...
@@ -48,11 +58,14 @@ class DevTo:
continue
return
validSets
def
getBlogs
(
self
,
username
,
tag
):
def
get_blogs
(
self
,
username
,
tag
):
"""
get the contents of the dev.to article
"""
blog
=
{}
try
:
response
=
requests
.
get
(
f
"https://api.rss2json.com/v1/api.json?rss_url=https
%3
A
%2
F
%2
Fdev.to
%2
Ffeed
%2
F{username}&api_key={self.get
ApiK
ey()}"
f
"https://api.rss2json.com/v1/api.json?rss_url=https
%3
A
%2
F
%2
Fdev.to
%2
Ffeed
%2
F{username}&api_key={self.get
_api_k
ey()}"
)
if
response
.
status_code
==
200
:
res
=
response
.
json
()
...
...
@@ -63,13 +76,20 @@ class DevTo:
print
(
e
)
return
blog
def
getDevArticles
(
self
):
def
get_dev_articles
(
self
):
"""
Search google for dev.to articles
return a list of urls
filter out invalid urls
get content of the valid urls
return the content of valid dev.to articles
"""
links
=
self
.
google
(
f
"site:dev.to {self.title} after:2020-01-01"
)
validUrls
=
self
.
get
ValidU
rls
(
links
)
validSets
=
self
.
get
ValidS
ets
(
validUrls
)
validUrls
=
self
.
get
_valid_u
rls
(
links
)
validSets
=
self
.
get
_valid_s
ets
(
validUrls
)
blogs
=
[]
for
validset
in
validSets
:
blog
=
self
.
get
B
logs
(
validset
[
"username"
],
validset
[
"tag"
])
blog
=
self
.
get
_b
logs
(
validset
[
"username"
],
validset
[
"tag"
])
if
bool
(
blog
):
blogs
.
append
(
blog
)
return
{
"blogs"
:
blogs
,
"resources"
:
validUrls
}
backend/python/auto-answer/github.py
View file @
70363aeb
import
requests
from
search_engine_parser
import
GoogleSearch
import
re
class
Github
:
"""
A class to manage the Github API.
"""
def
__init__
(
self
):
"""
Initialize the Github API.
"""
def
get_github_resources
(
self
,
query
):
"""
this function will search github for a query and return a list of links if available,
if not available it will search google for the query and return a list of links.
"""
github
=
{}
github_repos
=
self
.
search_github_repos
(
query
)
github_links
=
self
.
search_github_repos_in_google
(
query
)
valid_github_links
=
self
.
get_valid_urls
(
github_links
)
github
[
"links"
]
=
valid_github_links
github
[
"repos"
]
=
self
.
get_first_ten_repos
(
github_repos
[
"items"
])
return
github
def
get_first_ten_repos
(
self
,
repos
):
"""
Get the first ten repos if more than 10 repos are found.
otherwise return all repos.
:param repos: The repos.
"""
if
len
(
repos
)
>
10
:
return
repos
[:
10
]
else
:
return
repos
def
get_user_repos
(
self
,
user
):
"""
Get the repos of a user.
:param user: The Github user.
:return: The repos of the user.
"""
url
=
"https://api.github.com/users/{}/repos"
.
format
(
user
)
response
=
requests
.
get
(
url
)
response
.
raise_for_status
()
return
response
.
json
()
def
search_github_repos
(
self
,
query
):
"""
Search for repos on Github.
:param query: The search query.
:return: The repos found.
"""
url
=
"https://api.github.com/search/repositories?q={}"
.
format
(
query
)
response
=
requests
.
get
(
url
)
response
.
raise_for_status
()
return
response
.
json
()
def
search_github_repos_in_google
(
self
,
query
):
"""
Search for repos on Github using google search enging.
:param query: The search query.
:return: The repos found.
"""
google_query
=
"site:github.com {}"
.
format
(
query
)
search_args
=
(
google_query
,
1
)
gsearch
=
GoogleSearch
()
gresults
=
gsearch
.
search
(
*
search_args
)
return
gresults
[
"links"
]
def
get_valid_urls
(
self
,
links
):
"""
filter out invalid urls
"""
validUrls
=
[]
for
i
in
links
:
if
"github.com"
in
i
:
uriTrimmed
=
re
.
match
(
r"^.*?\&sa="
,
i
[
29
:])
.
group
(
0
)
ur
=
uriTrimmed
.
replace
(
"&sa="
,
""
)
validUrls
.
append
(
ur
)
return
validUrls
\ No newline at end of file
backend/python/auto-answer/scrapper.py
View file @
70363aeb
...
...
@@ -2,11 +2,12 @@ from youtube import Youtube
from
Medium
import
Medium
from
Dev
import
DevTo
from
stof
import
STOF
from
Github
import
Github
import
sys
from
database
import
get_database
def
saveAnswer
(
ans_id
,
stackoverflow
,
videos
,
medium_r
,
dev_r
):
def
saveAnswer
(
ans_id
,
stackoverflow
,
videos
,
medium_r
,
dev_r
,
github_r
):
db
=
get_database
()
try
:
from
bson.objectid
import
ObjectId
...
...
@@ -16,12 +17,15 @@ def saveAnswer(ans_id, stackoverflow, videos, medium_r, dev_r):
{
"_id"
:
ObjectId
(
ans_id
)},
{
"$set"
:
{
"loading"
:
False
,
"youtube"
:
videos
,
"stackoverflow"
:
stackoverflow
,
"medium_articles"
:
medium_r
[
"blogs"
],
"dev_articles"
:
dev_r
[
"blogs"
],
"medium_resources"
:
medium_r
[
"resources"
],
"dev_resources"
:
dev_r
[
"resources"
],
"medium_articles"
:
medium_r
.
get
(
"blogs"
,
[]),
"dev_articles"
:
dev_r
.
get
(
"blogs"
,
[]),
"medium_resources"
:
medium_r
.
get
(
"resources"
,
[]),
"dev_resources"
:
dev_r
.
get
(
"resources"
,
[]),
"github_repos"
:
github_r
.
get
(
"repos"
,
[]),
"github_links"
:
github_r
.
get
(
"links"
,
[]),
}
},
)
...
...
@@ -33,29 +37,34 @@ def saveAnswer(ans_id, stackoverflow, videos, medium_r, dev_r):
"dev_articles"
:
dev_r
[
"blogs"
],
"medium_resources"
:
medium_r
[
"resources"
],
"dev_resources"
:
dev_r
[
"resources"
],
"github_repos"
:
github_r
[
"repos"
],
"github_links"
:
github_r
[
"links"
],
}
)
except
NameError
as
err
:
print
(
"ERRORRR"
)
print
(
err
)
if
__name__
==
"__main__"
:
# title = input("Enter question title: ")
title
=
sys
.
argv
[
1
]
# "python django or flask for web development"
tags
=
sys
.
argv
[
2
]
# ["react"]
AUTO_ANS_ID
=
sys
.
argv
[
3
]
# "60dc9a5f84692f001569d7ab"
title
=
sys
.
argv
[
1
]
# "what are the benefits of using java for mobile app development over flutter"
tags
=
sys
.
argv
[
2
]
# ["flutter","java"]
AUTO_ANS_ID
=
sys
.
argv
[
3
]
# "611feaff2c4db730e56d78e8"
stack
=
STOF
(
title
)
ans
=
stack
.
searchQuestion
()
print
(
ans
)
medium
=
Medium
(
title
,
tags
)
medium_articels
=
medium
.
getMediumArticles
()
devto
=
DevTo
(
title
,
tags
)
dev_articles
=
devto
.
getDevArticles
()
youtube
=
Youtube
(
title
,
tags
)
github
=
Github
()
ans
=
stack
.
searchQuestion
()
medium_articels
=
medium
.
getMediumArticles
()
dev_articles
=
devto
.
get_dev_articles
()
videos
=
youtube
.
find_videos
()
saveAnswer
(
AUTO_ANS_ID
,
ans
,
videos
,
medium_articels
,
dev_articles
)
github_resources
=
github
.
get_github_resources
(
title
)
saveAnswer
(
AUTO_ANS_ID
,
ans
,
videos
,
medium_articels
,
dev_articles
,
github_resources
)
print
(
"WORKED"
)
sys
.
stdout
.
flush
()
backend/routes.js
View file @
70363aeb
...
...
@@ -14,8 +14,7 @@ const {
listQuestions
,
listByTags
,
listByUser
,
removeQuestion
,
getAutomatedAnswer
removeQuestion
}
=
require
(
'
./controllers/questions
'
);
const
{
loadAnswers
,
...
...
@@ -31,6 +30,7 @@ const requireAuth = require('./middlewares/requireAuth');
const
questionAuth
=
require
(
'
./middlewares/questionAuth
'
);
const
commentAuth
=
require
(
'
./middlewares/commentAuth
'
);
const
answerAuth
=
require
(
'
./middlewares/answerAuth
'
);
const
{
requestAutomatedAnswer
,
getAutomatedAnswer
}
=
require
(
'
./controllers/automated-answer
'
);
const
router
=
require
(
'
express
'
).
Router
();
...
...
@@ -51,7 +51,6 @@ router.get('/question', listQuestions);
router
.
get
(
'
/questions/:tags
'
,
listByTags
);
router
.
get
(
'
/question/user/:username
'
,
listByUser
);
router
.
delete
(
'
/question/:question
'
,
[
requireAuth
,
questionAuth
],
removeQuestion
);
router
.
get
(
'
/automatedanswer/:q
'
,
getAutomatedAnswer
);
//tags
router
.
get
(
'
/tags/populertags
'
,
listPopulerTags
);
...
...
@@ -74,6 +73,10 @@ router.post('/comment/:question/:answer?', [requireAuth, validate], createCommen
router
.
delete
(
'
/comment/:question/:comment
'
,
[
requireAuth
,
commentAuth
],
removeComment
);
router
.
delete
(
'
/comment/:question/:answer/:comment
'
,
[
requireAuth
,
commentAuth
],
removeComment
);
//automate-answer
router
.
post
(
'
/automatedanswer
'
,
requireAuth
,
requestAutomatedAnswer
);
router
.
get
(
'
/automatedanswer/:qid
'
,
getAutomatedAnswer
);
module
.
exports
=
(
app
)
=>
{
app
.
use
(
'
/api
'
,
router
);
...
...
client/components/automated-answer-swiper/AutomatedAnswerSwiper.jsx
0 → 100644
View file @
70363aeb
import
React
,
{
useState
}
from
'
react
'
import
Pagination
from
'
../pagination/Pagination
'
const
AutomatedAnswerSwiper
=
({
children
})
=>
{
const
[
currentIndex
,
setCurrentIndex
]
=
useState
(
0
)
const
onBtnClick
=
(
index
)
=>
{
console
.
log
(
'
here
'
+
index
)
if
(
index
<
children
.
length
)
{
setCurrentIndex
(
index
)
}
}
return
(
<
div
className=
"automated"
>
<
Pagination
currentIndex=
{
currentIndex
}
length=
{
children
.
length
}
onBtnClick=
{
onBtnClick
}
onClickFirst=
{
()
=>
setCurrentIndex
(
0
)
}
onClickLast=
{
()
=>
setCurrentIndex
(
children
.
length
-
1
)
}
/>
{
children
[
currentIndex
]
}
</
div
>
)
}
export
default
AutomatedAnswerSwiper
client/components/automated-answer-swiper/AutomatedAnswerSwiper.module.css
0 → 100644
View file @
70363aeb
client/components/pagination/Pagination.jsx
0 → 100644
View file @
70363aeb
import
React
from
'
react
'
import
styles
from
'
./Pagination.module.css
'
const
Pagination
=
({
currentIndex
,
length
,
onBtnClick
,
onClickFirst
,
onClickLast
})
=>
{
return
(
<
div
className=
{
styles
.
pagination_wrapper
}
>
<
span
onClick=
{
onClickFirst
}
className=
{
styles
.
cursor
}
>
first
</
span
>
<
div
className=
{
styles
.
numbering
}
>
{
new
Array
(
length
).
fill
(
0
).
map
((
item
,
index
)
=>
{
return
(
<
span
className=
{
index
==
currentIndex
?
styles
.
active_number
:
styles
.
muted_number
}
key=
{
index
}
onClick=
{
()
=>
onBtnClick
(
index
)
}
>
{
index
+
1
}
</
span
>
)
})
}
{
/* <span className={styles.active_number}>{currentIndex + 1}</span>/
<span className={styles.muted_number}>{length}</span> */
}
</
div
>
<
span
onClick=
{
onClickLast
}
className=
{
styles
.
cursor
}
>
last
</
span
>
</
div
>
)
}
export
default
Pagination
client/components/pagination/Pagination.module.css
0 → 100644
View file @
70363aeb
.pagination_wrapper
{
display
:
flex
;
flex-direction
:
row
;
justify-content
:
space-between
;
}
.numbering
{
color
:
#999
;
*
{
margin
:
0
2px
;
cursor
:
pointer
;
}
}
.active_number
{
color
:
#fff
;
padding
:
2px
10px
;
background-color
:
#13ff625
c
;
border
:
1px
solid
#78ff78
;
}
.muted_number
{
color
:
#fff
;
padding
:
2px
10px
;
background-color
:
#5d5d5d5
c
;
border
:
1px
solid
#9c9c9c
;
}
.cursor
{
cursor
:
pointer
;
}
client/components/youtube-videos/YoutubeVideoWrapper.js
View file @
70363aeb
import
React
from
'
react
'
import
AutomatedAnswerSwiper
from
'
../automated-answer-swiper/AutomatedAnswerSwiper
'
import
YoutubeVideo
from
'
./youtube-video/YoutubeVideo
'
import
styles
from
'
./YoutubeVideoWrapper.module.css
'
const
YoutubeVideoWrapper
=
({
videos
})
=>
{
...
...
@@ -9,9 +10,11 @@ const YoutubeVideoWrapper = ({ videos }) => {
youtube
<
/h1
>
<
div
className
=
{
styles
.
wrapper
}
>
{
videos
.
map
((
video
,
index
)
=>
{
return
<
YoutubeVideo
video
=
{
video
}
key
=
{
index
}
/
>
})}
<
AutomatedAnswerSwiper
>
{
videos
.
map
((
video
,
index
)
=>
{
return
<
YoutubeVideo
video
=
{
video
}
key
=
{
index
}
/
>
})}
<
/AutomatedAnswerSwiper
>
<
/div
>
<
/
>
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment