import requests
from googlesearch import search
from bs4 import BeautifulSoup
import re
import random
class WebScarper:
    def __init__(self):
        pass
    def get_url(self,query):
        results = []
        for result in search(query, num_results=3):
            results.append(result)
        if(results!=[]):
            return random.choice(results)
        else:
            return None
    def fetch_url(self, url):
        try: 
            headers = { 
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
                'Accept-Language': 'en-US,en;q=0.9',
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
            } 

            response = requests.get(url, headers=headers)
            if response.status_code != 200:
                raise Exception(f"Unable to fetch URL, status code: {response.status_code}")
            return response.text
        
        except Exception as e:
            print(f"Error: {e}")
            return None


    def get_text(self, data):
        soup = BeautifulSoup(data, 'html.parser')
        text = soup.get_text() 
        cleaned_text = re.sub(r'\s+', ' ', text).strip() 
        if(len(cleaned_text)>4000):
            return cleaned_text[:4000]
        else:
            return cleaned_text
        

    def scarpe(self,query):
        url = self.get_url(query)
        data = self.fetch_url(url)
        if(data==None or url==None):
            return None
        return self.get_text(data)