Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
2
2021-155
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
2021-155
2021-155
Commits
83c0705f
Commit
83c0705f
authored
Aug 15, 2021
by
Kamal Thennakoon
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
add finding fav language feature
parent
37cb3b82
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
118 additions
and
0 deletions
+118
-0
backend/python/auto-answer/requirements.txt
backend/python/auto-answer/requirements.txt
+11
-0
backend/python/portfolio/stof-scrapper.py
backend/python/portfolio/stof-scrapper.py
+107
-0
No files found.
backend/python/auto-answer/requirements.txt
0 → 100644
View file @
83c0705f
bson==0.5.10
beautifulsoup4==4.9.3
dnspython==2.1.0
lxml==4.6.1
pymongo==3.11.4
regex==2020.7.14
requests==2.24.0
requests-html==0.10.0
scipy==1.5.4
search-engine-parser==0.6.2
youtube-search-python==1.4.6
backend/python/portfolio/stof-scrapper.py
0 → 100644
View file @
83c0705f
from
bs4
import
BeautifulSoup
from
lxml
import
etree
import
requests
import
re
# we need to get the stack url with the user id
url
=
'https://stackoverflow.com/users/351048/joshmaker'
# we need to get the favourite language list from gitHub
git_languages
=
[
'javascript'
,
'typescript'
,
'dart'
,
'java'
,
'php'
]
sourceHtml
=
requests
.
get
(
url
)
.
text
soup
=
BeautifulSoup
(
sourceHtml
,
'lxml'
)
user
=
soup
.
find
(
'h2'
,
class_
=
'fs-headline1'
)
top_tags
=
[]
user_reputation
=
0
user_score
=
0
if
user
is
None
:
print
(
'404'
)
else
:
print
(
user
.
text
)
# check user reputation
dom
=
etree
.
HTML
(
str
(
soup
))
user_rep_str
=
dom
.
xpath
(
'//*[@id="user-card"]/div[1]/div/div[2]/div[1]/div'
)[
0
]
.
text
user_rep
=
re
.
sub
(
'[^0-9]+'
,
''
,
user_rep_str
)
user_reputation
=
int
(
user_rep
)
print
(
user_reputation
)
if
user_reputation
>
15
:
url
+=
'?tab=tags&sort=votes'
print
(
url
)
# get all tags
source_user_stats
=
requests
.
get
(
url
)
.
text
soup
=
BeautifulSoup
(
source_user_stats
,
'lxml'
)
tags_dom
=
etree
.
HTML
(
str
(
soup
))
tags_xpath
=
tags_dom
.
xpath
(
'//*[@id="user-tab-tags"]/div[2]/table/tbody/tr[*]/td[*]/a'
)
for
tag
in
tags_xpath
:
print
(
tag
.
text
)
top_tags
.
append
(
tag
.
text
)
else
:
print
(
'user is a noob!'
)
def
find_fav_language
(
tag_list
,
language_list_git
):
if
not
tag_list
and
not
language_list_git
:
print
(
'both null'
)
elif
not
tag_list
:
print
(
'no tags found'
)
elif
not
language_list_git
:
print
(
'no lang from git'
)
else
:
for
lang
in
language_list_git
:
for
tag
in
tag_list
:
print
(
lang
,
'::'
,
tag
)
if
tag
==
lang
:
print
(
'we have got a match'
,
tag
)
break
else
:
continue
break
find_fav_language
(
top_tags
,
[
'js'
,
'dart'
,
'delphi'
,
'r'
,
'php'
])
def
calculate_user_score
():
if
user_reputation
<=
10
:
return
15
elif
10
<
user_reputation
<=
50
:
return
25
elif
50
<
user_reputation
<=
150
:
return
35
elif
150
<
user_reputation
<=
500
:
return
45
elif
500
<
user_reputation
<=
1000
:
return
55
elif
1000
<
user_reputation
<=
3000
:
return
65
elif
3000
<
user_reputation
<=
8000
:
return
75
elif
8000
<
user_reputation
<=
15000
:
return
85
elif
15000
<
user_reputation
<=
55000
:
return
90
elif
55000
<
user_reputation
<=
100000
:
return
95
else
:
return
100
print
(
'your score is:'
,
calculate_user_score
())
# html_data=soup.prettify()
# scrapped_html = open("stack.html", "w")
# scrapped_html.write(html_data)
# scrapped_html.close()
# cyberMsg = soup.find('p', class_='MuiTypography-colorTextSecondary').text
# print(cyberMsg)
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment