import csv
import requests
from bs4 import BeautifulSoup
from config import HEADER

# html tags need to be scrapped
TAGS = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'div', 'span', 'td', 'li', 'a']
# keywords csv file path
KEYWORDS_PATH = 'keywords.csv'
# init KEYWORDS global dictionary to store keywords and their respective weight
KEYWORDS = dict()


def test_with_bs4(url):
    response = requests.get(url, headers=HEADER)
    soup = BeautifulSoup(response.text, "html.parser")
    file = open("test.html", "w+")
    file.write(str(soup))
    file.close()


def load_keywords():
    # access global KEYWORDS
    global KEYWORDS
    KEYWORDS = dict()
    # read csv file
    with open(KEYWORDS_PATH) as csv_file:
        # init csv reader
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        for row in csv_reader:
            # header row
            if line_count == 0:
                print(f'Column names are {", ".join(row)}')
                line_count += 1
            # other rows
            else:
                KEYWORDS[row[0]] = int(row[1])
                line_count += 1
        print(f'Processed {line_count} lines.')


def scrapping_words(url):
    # init word dictionary
    words = dict()
    # download html page of the website
    response = requests.get(url, headers=HEADER)
    # parse with bs4
    soup = BeautifulSoup(response.text, "html.parser")
    # iterate through html tags
    for tag in TAGS:
        # find all inner texts for the tag
        rows = soup.find_all(tag)
        # iterate through all rows found related to the given tag
        for row in rows:
            # inner text to lower
            sentence = row.get_text().lower()
            # keep only alphabet
            sentence = ''.join(x for x in sentence if x.isalpha() or x == ' ')
            # split into words
            array = sentence.split(' ')
            # cleaning array
            modified_array = [e.strip() for e in array if len(e.strip()) > 0]
            # iterate through each word
            for word in modified_array:
                # if word not exists in dict add it
                if word not in words.keys():
                    words[word] = 0
                # increase count by 1
                words[word] += 1
    # return scrapped words from the given webpage
    return words


def calculate_score(words):
    # init total score to zero initially
    total_score = 0
    # iterate through scrapped words
    for word, frequency in words.items():
        # if scrapped word exists in keywords
        if word in KEYWORDS.keys():
            # multiply weight of the keyword by frequency and add it to total score
            total_score += (KEYWORDS[word] * words[word])
    return total_score


if __name__ == "__main__":
    load_keywords()
    scrapped_words = scrapping_words("https://cointelegraph.com/")
    score = calculate_score(scrapped_words)
    print(f"Score = {score}")
    # if score > 0:
    #     print("========> POSITIVE")
    # else:
    #     print("========> NEGATIVE")
