Update tests.py
Browse files
tests.py
CHANGED
|
@@ -379,7 +379,7 @@ def read_excel_file(filename) -> dict:
|
|
| 379 |
return excel_data_dict
|
| 380 |
@mcp.tool()
|
| 381 |
def scrape_websites(url_list:list,query:str) -> list:
|
| 382 |
-
"""Scrapes specific website content.query is the question you want to ask about the content of the website.e.g-query:Give .pptx links in the website,Summarise the content in very great detail,etc
|
| 383 |
|
| 384 |
conn = http.client.HTTPSConnection("scrapeninja.p.rapidapi.com")
|
| 385 |
|
|
@@ -390,6 +390,7 @@ def scrape_websites(url_list:list,query:str) -> list:
|
|
| 390 |
'Content-Type': "application/json"
|
| 391 |
}
|
| 392 |
Output=""
|
|
|
|
| 393 |
content=""
|
| 394 |
for urls in url_list:
|
| 395 |
payload = {"url" :urls}
|
|
@@ -400,6 +401,7 @@ def scrape_websites(url_list:list,query:str) -> list:
|
|
| 400 |
content=content+str(data.decode("utf-8"))
|
| 401 |
|
| 402 |
#Only thing llama 4 is good for.
|
|
|
|
| 403 |
response = clienty.chat.completions.create(
|
| 404 |
model="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
|
| 405 |
messages=[
|
|
@@ -408,8 +410,16 @@ def scrape_websites(url_list:list,query:str) -> list:
|
|
| 408 |
)
|
| 409 |
for chunk in response:
|
| 410 |
Output = Output +str(chunk.choices[0].delta.content)
|
| 411 |
-
|
| 412 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 413 |
|
| 414 |
|
| 415 |
|
|
|
|
| 379 |
return excel_data_dict
|
| 380 |
@mcp.tool()
|
| 381 |
def scrape_websites(url_list:list,query:str) -> list:
|
| 382 |
+
"""Scrapes specific website content.query is the question you want to ask about the content of the website.e.g-query:Give .pptx links in the website,Summarise the content in very great detail,etc.Maximum 4 urls can be passed at a time."""
|
| 383 |
|
| 384 |
conn = http.client.HTTPSConnection("scrapeninja.p.rapidapi.com")
|
| 385 |
|
|
|
|
| 390 |
'Content-Type': "application/json"
|
| 391 |
}
|
| 392 |
Output=""
|
| 393 |
+
links=""
|
| 394 |
content=""
|
| 395 |
for urls in url_list:
|
| 396 |
payload = {"url" :urls}
|
|
|
|
| 401 |
content=content+str(data.decode("utf-8"))
|
| 402 |
|
| 403 |
#Only thing llama 4 is good for.
|
| 404 |
+
|
| 405 |
response = clienty.chat.completions.create(
|
| 406 |
model="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
|
| 407 |
messages=[
|
|
|
|
| 410 |
)
|
| 411 |
for chunk in response:
|
| 412 |
Output = Output +str(chunk.choices[0].delta.content)
|
| 413 |
+
#--------------
|
| 414 |
+
response2 = clienty.chat.completions.create(
|
| 415 |
+
model="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
|
| 416 |
+
messages=[
|
| 417 |
+
{"role": "user", "content": f"Give all relevant links in this content.The links may be relevant image links , file links , video links , website links , etc .You must give Minimum 10 links and maximum 20 links.[CONTENT]:{content}"}
|
| 418 |
+
],stream=True
|
| 419 |
+
)
|
| 420 |
+
for chunk in response2:
|
| 421 |
+
links = links +str(chunk.choices[0].delta.content)
|
| 422 |
+
return {"website_content":Output,"relevant_links":links}
|
| 423 |
|
| 424 |
|
| 425 |
|