Spaces:
Running
Running
File size: 2,618 Bytes
9536c67 483c169 9536c67 483c169 9536c67 483c169 9536c67 255e074 483c169 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
"""
This module provides a client for interacting with the FireCrawl service.
It encapsulates the logic for scraping a website using the FireCrawlLoader from
LangChain, converting the scraped documents into a single markdown string, and
handling potential errors during the process.
"""
from langchain_community.document_loaders import FireCrawlLoader
from langchain_core.documents import Document
from config import FIRE_CRAWL_API_KEY
def scrape_with_firecrawl(url: str) -> list[Document]:
"""
Scrapes a given URL using FireCrawl and returns the content as a list of Documents.
Args:
url (str): The URL of the website to scrape.
Returns:
list[Document]: A list of LangChain Document objects, where each document
represents a scraped page.
"""
loader = FireCrawlLoader(url=url,
api_key=FIRE_CRAWL_API_KEY,
mode='scrape')
pages = []
for page in loader.lazy_load(): # type: ignore
pages.append(page)
return pages
def get_markdown_from_documents(docs: list[Document]) -> str:
"""
Converts a list of LangChain Documents into a single markdown string.
Each document's content is appended, separated by a horizontal rule.
Args:
docs (list[Document]): A list of Document objects to process.
Returns:
str: A string containing the combined content in markdown format.
"""
markdown_content = ""
for i, doc in enumerate(docs):
markdown_content += f"### Page {i+1}\n"
markdown_content += f"{doc.page_content}\n\n--------------\n\n"
return markdown_content
def scrape_and_get_markdown_with_firecrawl(url: str) -> str:
"""
Orchestrates the scraping of a URL with FireCrawl and returns the content as markdown.
This is the main entry point function for this module. It handles the full
process of scraping, content conversion, and error handling.
Args:
url (str): The URL of the website to scrape.
Returns:
str: The scraped content in markdown format, or a formatted error message string if an issue occurs.
"""
try:
docs = scrape_with_firecrawl(url)
if not docs:
return "❌ <span style='color:red;'>FireCrawl completed but returned no content. The page might be empty or inaccessible.</span>"
markdown = get_markdown_from_documents(docs)
return markdown
except Exception as e:
return f"❌ <span style='color:red;'>An error occurred while scraping with FireCrawl: {e}</span>"
|