Update helper_functions_api.py

helper_functions_api.py

@@ -221,13 +221,28 @@ def process_content(data_format, url, query):
         return rephrased_content, url
     return "", url
 
-def fetch_and_extract_content(data_format, urls, query):
-    with ThreadPoolExecutor(max_workers=len(urls)) as executor:
-        future_to_url = {
-            executor.submit(process_content, data_format, url, query): url
-            for url in urls
-        }
-        all_text_with_urls = [future.result() for future in as_completed(future_to_url)]
+def fetch_and_extract_content(
+    data_format: str, query: str, urls: List[str], num_refrences: int = 8
+) -> List[Tuple[str | None, str]]:
+    """
+    Asynchronously makeing request to urls and doing further process
+    """
+    all_text_with_urls = []
+    start_url = 0
+    while (len(all_text_with_urls) != num_refrences) and (start_url < len(urls)):
+        end_url = start_url + (num_refrences - len(all_text_with_urls))
+        urls_subset = urls[start_url:end_url]
+        with ThreadPoolExecutor(max_workers=len(urls_subset)) as executor:
+            future_to_url = {
+                executor.submit(process_content, data_format, url, query): url
+                for url in urls_subset
+            }
+            all_text_with_urls += [
+                future.result()
+                for future in as_completed(future_to_url)
+                if future.result()[0] != ""
+            ]
+        start_url = end_url
 
     return all_text_with_urls