Update app.py

app.py

@@ -40,21 +40,21 @@ PLAYWRIGHT_STATE: Dict = {}
 REVOLVER = CredentialRevolver(os.getenv("PROXY_LIST", ""))
 
 SEARCH_ENGINES = {
-    "Google": ("https://www.google.com/search?q={query}&hl=en", '#rso, #search, #botstuff'),
-    "DuckDuckGo": ("https://duckduckgo.com/html/?q={query}", '#links'),
-    "Bing": ("https://www.bing.com/search?q={query}", '#b_results'),
-    "Brave": ("https://search.brave.com/search?q={query}", '#results'),
-    "Ecosia": ("https://www.ecosia.org/search?q={query}", 'main[role="main"]'),
-    "Yahoo": ("https://search.yahoo.com/search?p={query}", '#web'),
-    "Startpage": ("https://www.startpage.com/sp/search?q={query}", '#main'),
-    "Qwant": ("https://www.qwant.com/?q={query}", '[data-testid="web-results"]'),
-    "Swisscows": ("https://swisscows.com/web?query={query}", '.web-results'),
-    "You.com": ("https://you.com/search?q={query}", '#search-results'),
-    "SearXNG": ("https://searx.be/search?q={query}", '#results'),
-    "MetaGer": ("https://metager.org/meta/meta.ger-en?eingabe={query}", '#results'),
-    "Yandex": ("https://yandex.com/search/?text={query}", '#search-result'),
-    "Baidu": ("https://www.baidu.com/s?wd={query}", '#content_left'),
-    "Perplexity": ("https://www.perplexity.ai/search?q={query}", 'div[class*="prose"]'),
+    "Google": "https://www.google.com/search?q={query}&hl=en",
+    "DuckDuckGo": "https://duckduckgo.com/html/?q={query}",
+    "Bing": "https://www.bing.com/search?q={query}",
+    "Brave": "https://search.brave.com/search?q={query}",
+    "Ecosia": "https://www.ecosia.org/search?q={query}",
+    "Yahoo": "https://search.yahoo.com/search?p={query}",
+    "Startpage": "https://www.startpage.com/sp/search?q={query}",
+    "Qwant": "https://www.qwant.com/?q={query}",
+    "Swisscows": "https://swisscows.com/web?query={query}",
+    "You.com": "https://you.com/search?q={query}",
+    "SearXNG": "https://searx.be/search?q={query}",
+    "MetaGer": "https://metager.org/meta/meta.ger-en?eingabe={query}",
+    "Yandex": "https://yandex.com/search/?text={query}",
+    "Baidu": "https://www.baidu.com/s?wd={query}",
+    "Perplexity": "https://www.perplexity.ai/search?q={query}",
 }
 
 class HTML_TO_MARKDOWN_CONVERTER:
@@ -130,13 +130,11 @@ async def perform_web_browse(query: str, browser_name: str, search_engine_name:
     
     if is_direct_url:
         url = query
-        content_selector = 'body'
     else:
-        engine_data = SEARCH_ENGINES.get(search_engine_name)
-        if not engine_data:
+        url_template = SEARCH_ENGINES.get(search_engine_name)
+        if not url_template:
             return {"status": "error", "query": query, "error_message": f"Invalid search engine: '{search_engine_name}'."}
-        url, content_selector = engine_data
-        url = url.format(query=urllib.parse.quote_plus(query))
+        url = url_template.format(query=urllib.parse.quote_plus(query))
 
     proxy_config = REVOLVER.get_next()
     proxy_server_used = proxy_config["server"] if proxy_config else "Direct Connection"
@@ -148,17 +146,12 @@ async def perform_web_browse(query: str, browser_name: str, search_engine_name:
     page = await context.new_page()
 
     try:
-        response = await page.goto(url, wait_until='commit', timeout=20000)
-
-        await asyncio.wait([
-            page.wait_for_load_state('domcontentloaded', timeout=15000),
-            page.wait_for_selector(content_selector, timeout=15000),
-        ], return_when=asyncio.FIRST_COMPLETED)
+        response = await page.goto(url, wait_until='domcontentloaded', timeout=25000)
         
         html_content = await page.content()
 
-        if any(phrase in html_content for phrase in ["unusual traffic", "CAPTCHA", "prove you are human", "before you continue"]):
-            raise Exception("Anti-bot measure detected. Try another search engine or proxy.")
+        if any(phrase in html_content for phrase in ["unusual traffic", "CAPTCHA", "are you human", "not a robot"]):
+            raise Exception(f"Anti-bot measure detected on {page.url}. Try another search engine or proxy.")
 
         final_url, title = page.url, await page.title() or "No Title"
         soup = BeautifulSoup(html_content, 'lxml')
@@ -170,7 +163,7 @@ async def perform_web_browse(query: str, browser_name: str, search_engine_name:
     except Exception as e:
         error_message = str(e).splitlines()[0]
         if "Timeout" in error_message:
-            return {"status": "error", "query": query, "proxy_used": proxy_server_used, "error_message": f"Page failed to load or find content for '{query}'. The site may be slow, blocking automation, or the content selector '{content_selector}' was not found."}
+            return {"status": "error", "query": query, "proxy_used": proxy_server_used, "error_message": f"Navigation Timeout: The page for '{query}' took too long to load."}
         return {"status": "error", "query": query, "proxy_used": proxy_server_used, "error_message": error_message}
     finally:
         if 'page' in locals() and not page.is_closed(): await page.close()