Spaces:
Sleeping
Sleeping
| ############################################################################################################################################################### | |
| # _____ _ ___ _ ___ | |
| # |_ _|| |_ ___ | _ )(_) __ _ / __| __ _ _ __ _ _ __ ___ _ _ | |
| # | | | ' \ / -_) | _ \| |/ _` | \__ \/ _|| '_|/ _` || '_ \/ -_)| '_| | |
| # |_| |_||_|\___| |___/|_|\__, | |___/\__||_| \__,_|| .__/\___||_| | |
| # |___/ |_| | |
| # | |
| ############################################################################################################################################################## | |
| # _ ______ _ _ _______ _ _ | |
| # _ | | (_____ \ | | (_) (_______) (_) (_) | |
| # _____ _ _ _| |_ | |__ ___ ____ _ _____) ) ___ __| | ____ _ ____ ___ _ _ _ _____ ___ _ ____ _ | |
| # (____ || | | |(_ _)| _ \ / _ \ / ___)(_) | __ / / _ \ / _ | / ___)| | / _ | / _ \ | ||_|| |(____ | /___)| || _ \ | | | |
| # / ___ || |_| | | |_ | | | || |_| || | _ | | \ \ | |_| |( (_| || | | |( (_| || |_| | | | | |/ ___ ||___ || || | | || | | |
| # \_____||____/ \__)|_| |_| \___/ |_| (_) |_| |_| \___/ \____||_| |_| \___ | \___/ |_| |_|\_____|(___/ |_||_| |_||_| | |
| # (_____| | |
| ############################################################################################################################################################### | |
| # | |
| # Last updated in: 8/15/2024 | |
| # | |
| ############################################################################################################################################################### | |
| # ------------------------------------------------------------------------------ | |
| # IMPORTS | |
| # ------------------------------------------------------------------------------ | |
| import gradio as gr | |
| from bs4 import BeautifulSoup as Soup | |
| from langchain_community.document_loaders import (AsyncHtmlLoader, | |
| NewsURLLoader, PubMedLoader, | |
| PlaywrightURLLoader, | |
| RecursiveUrlLoader, | |
| SeleniumURLLoader, | |
| UnstructuredURLLoader, | |
| WebBaseLoader) | |
| # ------------------------------------------------------------------------------ | |
| # THE BIG SCRAPER METHOD | |
| # ------------------------------------------------------------------------------ | |
| def extractDataFromUrls(urls: str, loader_type: str): | |
| """Extracts data from provided URLs using specified loader type. | |
| Args: | |
| urls (str): Comma-separated URLs to extract data from. | |
| loader_type (str): Type of loader to use for data extraction. | |
| Returns: | |
| tuple: A tuple containing the extracted data in JSON format and as a list of Document objects. | |
| Returns error messages if an exception occurs. | |
| """ | |
| try: | |
| urls = urls.split(',') | |
| data = [] | |
| # Instantiate the selected loader based on loader_type | |
| if loader_type == 'AsyncHtmlLoader': | |
| loader = AsyncHtmlLoader(urls) | |
| elif loader_type == 'UnstructuredURL': | |
| loader = UnstructuredURLLoader(urls=urls) | |
| elif loader_type == 'RecursiveURL': | |
| loader = RecursiveUrlLoader( | |
| url=urls[0], max_depth=2, extractor=lambda x: Soup(x, "html.parser").text | |
| ) | |
| elif loader_type == 'SeleniumURL': | |
| loader = SeleniumURLLoader(urls=urls) | |
| elif loader_type == 'SeleniumURLH': | |
| loader = SeleniumURLLoader(urls=urls, headless=False) | |
| elif loader_type == 'PlaywrightURL': | |
| loader = PlaywrightURLLoader(urls=urls) | |
| elif loader_type == 'PubMed': | |
| loader = PubMedLoader(urls[0]) | |
| elif loader_type == 'NewsURL': | |
| loader = NewsURLLoader(urls) | |
| elif loader_type == 'WebBaseLoader': | |
| loader = WebBaseLoader(urls) | |
| else: | |
| return "Not Implemented. Development in Progress", "Work In Progress" | |
| # Load data using the selected loader | |
| data = loader.load() | |
| # Convert data to JSON format | |
| jsonData = [] | |
| for item in data: | |
| jsonData.append(item.to_json()) | |
| return jsonData, data | |
| except Exception as err: | |
| return "An Error Occurred. Contact Developer" + str(err), "Error Occured. Boom" | |
| # ------------------------------------------------------------------------------ | |
| # GRADIO | |
| # ------------------------------------------------------------------------------ | |
| # Define choices for the dropdown menu | |
| choices = [ | |
| 'AsyncHtmlLoader', 'UnstructuredURL', 'RecursiveURL', 'PubMed', | |
| 'WebBaseLoader', 'Scrapy', 'PySpider', 'Beautiful Soup', | |
| 'SeleniumURL', 'SeleniumURLH', 'PlaywrightURL', 'NewsURL', | |
| ] | |
| # Create the Gradio interface | |
| demo = gr.Interface( | |
| fn=extractDataFromUrls, | |
| inputs=[ | |
| gr.Textbox(label="Enter your comma separated URLs here"), | |
| gr.Dropdown(choices=choices, label="Pick your Loader from here") | |
| ], | |
| outputs=["json", "textbox"], | |
| allow_flagging='never', | |
| theme="sudeepshouche/minimalist" | |
| ) | |
| # Launch the Gradio interface | |
| demo.launch() |