From 012b6163188c32709e9ff072c9248dbfbb0e1f06 Mon Sep 17 00:00:00 2001 From: Sushama Shroff Date: Wed, 6 Dec 2023 17:12:50 -0800 Subject: [PATCH 1/3] Add support to crawl website --- app/api/api_v1/endpoints/load.py | 18 +++++++++++++++++- app/helper/utility.py | 16 ++++++++++++++++ main.py | 1 + 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/app/api/api_v1/endpoints/load.py b/app/api/api_v1/endpoints/load.py index 4498b52..411bc8b 100644 --- a/app/api/api_v1/endpoints/load.py +++ b/app/api/api_v1/endpoints/load.py @@ -6,12 +6,28 @@ from starlette.responses import Response from app.helper.url_loader import URLLoader -from app.helper.utility import load_file, fetch_contents_from_api_endpoint +from app.helper.utility import web_crawl, load_file, fetch_contents_from_api_endpoint from app.schemas.loader import BaseLoader router = APIRouter() +@router.post("/crawl", status_code=200) +def crawl_website(document: BaseLoader) -> Any: + """ + This endpoint is used to load any type of document by crawling a website url + """ + try: + content, err = web_crawl(document) + if err != "": + return Response(status_code=400, content=str(err)) + except Exception as e: + return Response(status_code=400, content=str(e)) + if not content: + return Response(status_code=400, content="could not crawl provided url") + else: + return content + @router.post("/", status_code=200) def load_document(file: UploadFile = File(...)) -> Any: diff --git a/app/helper/utility.py b/app/helper/utility.py index f743099..91de14b 100644 --- a/app/helper/utility.py +++ b/app/helper/utility.py @@ -8,6 +8,8 @@ from app.helper.url_loader import URLLoader from app.schemas.loader import BaseLoader +from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader +from bs4 import BeautifulSoup as Soup def load_file(document: BaseLoader) -> (List[Document], str): """ @@ -63,3 +65,17 @@ def fetch_contents_from_api_endpoint(document: BaseLoader): # delete the temporary file temp_loader.del_file() return content + +def web_crawl(document: BaseLoader) -> (List[Document], str): + """ + This endpoint is used to crawl a website url provided + :param document: url of website to crawl + :return: List of documents + """ + #print(document) + loader = RecursiveUrlLoader(url=document.url, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text) + docs = loader.load() + + print(docs) + + return docs, "" diff --git a/main.py b/main.py index 7a9b805..bc11546 100644 --- a/main.py +++ b/main.py @@ -22,3 +22,4 @@ app.include_router(api_v1_router, prefix=settings.API_V1_STR) + From bd24940744a30ad2710597816badadfaf1fa0a67 Mon Sep 17 00:00:00 2001 From: Sushama Shroff Date: Thu, 14 Dec 2023 12:27:26 -0800 Subject: [PATCH 2/3] Additional data loaders --- app/api/api_v1/endpoints/load.py | 7 +++- app/helper/utility.py | 63 +++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/app/api/api_v1/endpoints/load.py b/app/api/api_v1/endpoints/load.py index 411bc8b..75ac41b 100644 --- a/app/api/api_v1/endpoints/load.py +++ b/app/api/api_v1/endpoints/load.py @@ -6,7 +6,7 @@ from starlette.responses import Response from app.helper.url_loader import URLLoader -from app.helper.utility import web_crawl, load_file, fetch_contents_from_api_endpoint +from app.helper.utility import web_crawl_RecursiveUrlLoader, web_crawl_AsyncHtmlLoader, web_crawl_AsyncChromiumLoader, web_crawl_SimpleWebPageLoader, load_file, fetch_contents_from_api_endpoint from app.schemas.loader import BaseLoader @@ -18,7 +18,10 @@ def crawl_website(document: BaseLoader) -> Any: This endpoint is used to load any type of document by crawling a website url """ try: - content, err = web_crawl(document) + #content, err = web_crawl_RecursiveUrlLoader(document) + #content, err = web_crawl_AsyncHtmlLoader(document) + content, err = web_crawl_AsyncChromiumLoader(document) + #content, err = web_crawl_SimpleWebPageLoader(document) if err != "": return Response(status_code=400, content=str(err)) except Exception as e: diff --git a/app/helper/utility.py b/app/helper/utility.py index 91de14b..3d6fd24 100644 --- a/app/helper/utility.py +++ b/app/helper/utility.py @@ -10,6 +10,13 @@ from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader from bs4 import BeautifulSoup as Soup +from fake_useragent import UserAgent + +from langchain.document_loaders import AsyncHtmlLoader + +from langchain.document_loaders import AsyncChromiumLoader + +from llama_index import download_loader def load_file(document: BaseLoader) -> (List[Document], str): """ @@ -66,16 +73,68 @@ def fetch_contents_from_api_endpoint(document: BaseLoader): temp_loader.del_file() return content -def web_crawl(document: BaseLoader) -> (List[Document], str): +def web_crawl_RecursiveUrlLoader(document: BaseLoader) -> (List[Document], str): """ This endpoint is used to crawl a website url provided :param document: url of website to crawl :return: List of documents """ - #print(document) + + #https://github.com/langchain-ai/langchain/issues/11540 -> 403 errors for website + header_template = {} + header_template["User-Agent"] = UserAgent().random + loader = RecursiveUrlLoader(url=document.url, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text) docs = loader.load() print(docs) return docs, "" + +def web_crawl_AsyncHtmlLoader(document: BaseLoader) -> (List[Document], str): + """ + This endpoint is used to crawl a website url provided + :param document: url of website to crawl + :return: List of documents + """ + urls = [] + urls.append(document.url) + loader = AsyncHtmlLoader(urls) + docs = loader.load() + + print(docs) + + return docs, "" + +def web_crawl_AsyncChromiumLoader(document: BaseLoader) -> (List[Document], str): + """ + This endpoint is used to crawl a website url provided + :param document: url of website to crawl + :return: List of documents + """ + urls = [] + urls.append(document.url) + loader = AsyncChromiumLoader(urls) + docs = loader.load() + + #print(docs) + #print(docs[0].page_content[0:]) + + return docs, "" + +def web_crawl_SimpleWebPageLoader(document: BaseLoader) -> (List[Document], str): + """ + This endpoint is used to crawl a website url provided + :param document: url of website to crawl + :return: List of documents + """ + urls = [] + urls.append(document.url) + + SimpleWebPageReader = download_loader("SimpleWebPageReader") + loader = SimpleWebPageReader() + docs = loader.load_data(urls) + + print(docs) + + return docs, "" From 56d868142ba64d9d8c10877dda38f910ba4e46d8 Mon Sep 17 00:00:00 2001 From: Sushama Shroff Date: Mon, 18 Dec 2023 14:16:32 -0800 Subject: [PATCH 3/3] Add support for medium crawling --- app/api/api_v1/endpoints/load.py | 6 +- app/helper/utility.py | 144 +++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/app/api/api_v1/endpoints/load.py b/app/api/api_v1/endpoints/load.py index 75ac41b..45d63e0 100644 --- a/app/api/api_v1/endpoints/load.py +++ b/app/api/api_v1/endpoints/load.py @@ -6,7 +6,7 @@ from starlette.responses import Response from app.helper.url_loader import URLLoader -from app.helper.utility import web_crawl_RecursiveUrlLoader, web_crawl_AsyncHtmlLoader, web_crawl_AsyncChromiumLoader, web_crawl_SimpleWebPageLoader, load_file, fetch_contents_from_api_endpoint +from app.helper.utility import web_crawl_MediumUrlLoader, web_crawl_MediumPubUrlLoader, web_crawl_RecursiveUrlLoader, web_crawl_AsyncHtmlLoader, web_crawl_AsyncChromiumLoader, web_crawl_SimpleWebPageLoader, load_file, fetch_contents_from_api_endpoint from app.schemas.loader import BaseLoader @@ -18,9 +18,11 @@ def crawl_website(document: BaseLoader) -> Any: This endpoint is used to load any type of document by crawling a website url """ try: + content, err = web_crawl_MediumUrlLoader(document) + #content, err = web_crawl_MediumPubUrlLoader(document) #wip #content, err = web_crawl_RecursiveUrlLoader(document) #content, err = web_crawl_AsyncHtmlLoader(document) - content, err = web_crawl_AsyncChromiumLoader(document) + #content, err = web_crawl_AsyncChromiumLoader(document) #content, err = web_crawl_SimpleWebPageLoader(document) if err != "": return Response(status_code=400, content=str(err)) diff --git a/app/helper/utility.py b/app/helper/utility.py index 3d6fd24..eff9b27 100644 --- a/app/helper/utility.py +++ b/app/helper/utility.py @@ -18,6 +18,12 @@ from llama_index import download_loader +import requests +from bs4 import BeautifulSoup +import pandas + +from urllib.parse import urljoin, urlparse + def load_file(document: BaseLoader) -> (List[Document], str): """ This endpoint is used to load any type of document from a local filesystem or remote location @@ -73,6 +79,144 @@ def fetch_contents_from_api_endpoint(document: BaseLoader): temp_loader.del_file() return content + +def web_crawl_MediumUrlLoader(document: BaseLoader) -> (List[Document], str): + """ + This endpoint is used to crawl a single medium article per url provided + :param document: url of medium article to crawl + :return: List of documents and metadata related to the medium article url input + """ + + # The below uses + # 1. BeautifulSoup to parse the metadata from the medium article url. + # (other frameworks like scrapy work as well for more advanced usecases) + # 2. RecursiveUrlLoader to parse the page_content for the medium article url. + # (can use vanilla Beautiful Soup or another Loader type as well) + # 3. How to combine metadata obtained from the 2 methods as required. + + response = requests.get(document.url) + + post = {} + response = requests.get(document.url) + article = BeautifulSoup(response.content, 'html.parser') + #title + if article.find('h1',attrs={'class':'a8db'}): + h1 = article.find('h1',attrs={'class':'a8db'}) + else: + h1 = article.find('h1') + article_title = h1.text + + post = { + 'title':article_title, + 'link':document.url + } + + print("Additional metadata :", post) + loader = RecursiveUrlLoader(url=document.url, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text) + docs = loader.load() + + # if we want to combine metadata obtained by Beautiful soup parsing + obtained from loader, that can be done here + # Additional metadata can be added to the metadata field of langchain document + # The merge it with the docs to return + # https://js.langchain.com/docs/modules/data_connection/document_loaders/how_to/creating_documents + + #print(docs) + + return docs, "" + + +def web_crawl_MediumPubUrlLoader(document: BaseLoader) -> (List[Document], str): + """ + This endpoint is used to crawl a medium publication url provided + :param document: url of medium publication to crawl + :return: List of documents and metadata related to each document in the publication + """ + + # todo - get all articles under a publication + # currently limited articles are returned, when input publication url as https://medium.com/cisco-fpie + # Medium Apis do not offer functionality to get all articles under a publication + # https://github.com/Medium/medium-api-docs#32-publications + # Alternative use inputs that can be considered - publication archives for medium archive url ? + + # This example demonstrates - How to get specific metadata about medium article (that is not provided by other default loaders), + # The medium article can be scraped to get that metadata + merge with metadata provided by the Langchain Loaders as required. + + response = requests.get(document.url) + content = BeautifulSoup(response.content, 'html.parser') + container = content.find('div', attrs={'class':'js-collectionStream'}) + a = container.findAll('a') + + links = [] + for link in a: + link = link['href'] + links.append(link) + + cleaned_links = [] + + # scrape link metadata only for required medium pub (Eg. medium.com/cisco-fpie if input url=https://medium.com/cisco-fpie) + url_pub_path = document.url.split('/')[2] + "/" + document.url.split('/')[3] + for link in links: + if url_pub_path in link: + cleaned_links.append(link) + cleaned_links = list(dict.fromkeys(cleaned_links)) + + print(cleaned_links) + + archive = [] + docs = [] #List[Document] + + for article_link in cleaned_links: + post = {} + response = requests.get(article_link) + article = BeautifulSoup(response.content, 'html.parser') + #title + if article.find('h1',attrs={'class':'a8db'}): + h1 = article.find('h1',attrs={'class':'a8db'}) + else: + h1 = article.find('h1') + article_title = h1.text + #subtitle + #if article.find('h2', attrs={'class':'3dfe'}): + # h2 = article.find('h2', attrs={'class':'3dfe'}) + #else: + # subh = article.findAll('h2') + # h2 = subh[1] + #article_subtitle=h2.text + #img + ''' + #fig = article.find('figure') + #print(fig) + ''' + #img = article.find('img',attrs={'class':'s'}) + #article_img = img['src'] + post = { + 'title':article_title, + #'subtitle':article_subtitle, + #'img': article_img, + 'link':article_link + } + archive.append(post) + + url_path_noquery = article_link.split('?')[0] + print(url_path_noquery) + + # use any appropriate/chosen loader/ vanilla beautiful soup to extract contents of each article wihin the provided medium publication + loader = RecursiveUrlLoader(url=article_link, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text) + docs_for_link = loader.load() + print(docs_for_link) + + # todo + # if we want to combine metadata obtained by Beautiful soup parsing + obtained from loader, that can be done here + # Additional metadata can be added to the metadata field of langchain document + # The merge it with the docs to return + # https://js.langchain.com/docs/modules/data_connection/document_loaders/how_to/creating_documents + docs.extend(docs_for_link) + + print("********************************************************************************************") + print(docs) + + return docs, "" + def web_crawl_RecursiveUrlLoader(document: BaseLoader) -> (List[Document], str): """ This endpoint is used to crawl a website url provided