implement stackoverflow child process

backend/controllers/portfolio.js

@@ -2,6 +2,7 @@ const { spawn } = require('child_process');
 const path=require('path');
 
 exports.testPortfolio=async(req,res,next)=>{
+    let temp='';
     try{
         const {title,tag,id}=req.body;
         const pythonScript=spawn('python',[
@@ -10,9 +11,10 @@ exports.testPortfolio=async(req,res,next)=>{
             tag,
             id
         ]);
-        pythonScript.stdout.on('data',(data)=>{
+        pythonScript.stdout.on('data',(data)=>{   
             console.log('Python test output',data.toString());
         });
+   
         pythonScript.stdout.on('end',()=>{
             console.log('Python script has finshed executing')
         })

backend/python/portfolio/db.py

+DATABASE_URL_PROD = "mongodb+srv://admin2:admin12345@cluster0.u4vl4.mongodb.net/production?retryWrites=true&w=majority"
+DATABASE_URL_DEV = "mongodb+srv://admin:admin1234@cluster0.u4vl4.mongodb.net/test?retryWrites=true&w=majority"
+
+# Change Environment accordingly
+ENV = "DEV"
+
+
+def get_database():
+    
+    from pymongo import MongoClient
+
+    # Mongodb atlas url to connect python to mongodb using pymongo
+    CONNECTION_STRING = ENV == "DEV" and DATABASE_URL_DEV or DATABASE_URL_PROD
+
+    # Create connection to the MongoClient
+    client = MongoClient(CONNECTION_STRING)
+
+    print(client.list_database_names())
+
+    return ENV == "DEV" and client["test"] or client["production"]

backend/python/portfolio/scrapper.py

+from stof_scrapper import StofStat
+from db import get_database
 import sys
 
+def save_stof_stat():
+    db=get_database()
+    try:
+        portfolios=db["portfolios"]
+        temp_user={"name":"kamal","username":"crhunter","score":789}
+        result= portfolios.insert_one(temp_user)
+        print(result)
+    except NameError as err:
+        print(err)
+         
+    
+
+
 if __name__ == "__main__":
-    # title = input("Enter question title: ")
-    title = sys.argv[1]  # "python django or flask for web development"
-    tags = sys.argv[2]  # ["react"]
-    AUTO_ANS_ID = sys.argv[3]  # "60dc9a5f84692f001569d7ab"
+    
+    #title = sys.argv[1] 
+    #tags = sys.argv[2] 
+    url='https://stackoverflow.com/users/351048/joshmaker'
+    git_languages = ['javascript', 'django','typescript', 'dart', 'java', 'php','python']
+    stof=StofStat(url,git_languages)
+    stof_stat=stof.get_user_stats()
+
+    save_stof_stat()
 
-    print('title: ',title)
+    print(stof_stat)
     sys.stdout.flush()

backend/python/portfolio/stof_scrapper.py

+from bs4 import BeautifulSoup
+from lxml import etree
+import requests
+import re
+
+class StofStat:
+    def __init__(self,url,git_languages):
+        self.url=url
+        self.git_languages=git_languages
+    
+    def get_user_stats(self):
+
+        sourceHtml = requests.get(self.url).text
+        soup = BeautifulSoup(sourceHtml, 'lxml')
+        user = soup.find('div', class_='fs-headline2')
+
+        top_tags = []
+        user_score = 0
+        fav_language = "none"
+        user_reputation = 0
+
+        if user is None:
+            print('404')
+        else:
+            # print(user.text)
+            # check user reputation
+            dom = etree.HTML(str(soup))
+            user_rep_str = dom.xpath(
+                '//*[@id="main-content"]/div[1]/div[1]/div/div[2]/div/div[1]/div')[0].text
+            user_rep = re.sub('[^0-9]+', '', user_rep_str)
+            user_reputation = int(user_rep)
+            # print(user_reputation)
+
+            if user_reputation > 15:
+                self.url += '?tab=tags&sort=votes'
+                # print(self.url)
+                # get all tags
+                source_user_stats = requests.get(self.url).text
+
+                soup = BeautifulSoup(source_user_stats, 'lxml')
+                tags_dom = etree.HTML(str(soup))
+                tags_xpath = tags_dom.xpath(
+                    '//*[@id="user-tab-tags"]/div[2]/table/tbody/tr[*]/td[*]/a')
+                for tag in tags_xpath:
+                    # print(tag.text)
+                    top_tags.append(tag.text)
+
+            else:
+                pass
+                # print('noob user')
+
+
+        def find_fav_language(tag_list, language_list_git):
+            if not tag_list and not language_list_git:
+                # print('both null')
+                return '404'
+            elif not tag_list:
+                # print('no tags found')
+                return '404'
+            elif not language_list_git:
+                # print('no lang from git')
+                return '404'
+            else:
+                for lang in language_list_git:
+                    for tag in tag_list:
+                        # print(lang, '::', tag)
+                        if tag == lang:
+                            # print('we have got a match', tag)
+                            return tag
+                    else:
+                        continue
+                return '404'
+
+
+        fav_language=find_fav_language(top_tags,self.git_languages)
+
+
+        def calculate_user_score():
+            if user_reputation <= 10:
+                return 15
+            elif 10 < user_reputation <= 50:
+                return 25
+            elif 50 < user_reputation <= 150:
+                return 35
+            elif 150 < user_reputation <= 500:
+                return 45
+            elif 500 < user_reputation <= 1000:
+                return 55
+            elif 1000 < user_reputation <= 3000:
+                return 65
+            elif 3000 < user_reputation <= 8000:
+                return 75
+            elif 8000 < user_reputation <= 15000:
+                return 85
+            elif 15000 < user_reputation <= 55000:
+                return 90
+            elif 55000 < user_reputation <= 100000:
+                return 95
+            else:
+                return 100
+
+
+        # print('your score is:', calculate_user_score())
+        stof_score = calculate_user_score()
+        stof_stat = {"tags": top_tags, "stof_score": stof_score, "fav_language": fav_language}
+        return stof_stat
+