delete async

.gitattributes copy

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

LLM.py

@@ -123,7 +123,13 @@ class openai_llm(base_llm):
                 input=text,
                 timeout= 180
             )
-            return embbeding.data[0].embedding
+            embbeding = embbeding.data
+            if len(embbeding) == 0:
+                return None
+            elif len(embbeding) == 1:
+                return embbeding[0].embedding
+            else:
+                return [e.embedding for e in embbeding]
         except Exception as e:
             print(f"get embbeding failed: {e}")
             print(e)
@@ -147,7 +153,13 @@ class openai_llm(base_llm):
                 input=text,
                 timeout= 180
             )
-            return embbeding.data[0].embedding
+            embbeding = embbeding.data
+            if len(embbeding) == 0:
+                return None
+            elif len(embbeding) == 1:
+                return embbeding[0].embedding
+            else:
+                return [e.embedding for e in embbeding]
         except Exception as e:
             await asyncio.sleep(0.1)
             print(f"get embbeding failed: {e}")
@@ -178,9 +190,32 @@ class openai_llm(base_llm):
 
 
 if __name__ == "__main__":
-    llm = gemini_llm(api_key="")
-    prompt = """
-"""
-    messages = [{"role":"user","content":prompt}]
-    response = asyncio.run(llm.response_async(messages))
-    print(response)
+    import os
+    import yaml
+
+    def cal_cosine_similarity_matric(matric1, matric2):
+        if isinstance(matric1, list):
+            matric1 = np.array(matric1)
+        if isinstance(matric2, list):
+            matric2 = np.array(matric2)
+        if len(matric1.shape) == 1:
+            matric1 = matric1.reshape(1, -1)
+        if len(matric2.shape) == 1:
+            matric2 = matric2.reshape(1, -1)
+        dot_product = np.dot(matric1, matric2.T)
+        norm1 = np.linalg.norm(matric1, axis=1)
+        norm2 = np.linalg.norm(matric2, axis=1)
+
+        cos_sim = dot_product / np.outer(norm1, norm2)
+        scores = cos_sim.flatten()
+        # 返回一个list
+        return scores.tolist()
+    
+    texts = ["What is the capital of France?","What is the capital of Spain?", "What is the capital of Italy?", "What is the capital of Germany?"]
+    text = "What is the capital of France?"
+    llm = openai_llm()
+    embbedings = llm.get_embbeding(texts)
+    embbeding = llm.get_embbeding(text)
+
+    scores = cal_cosine_similarity_matric(embbedings, embbeding)
+    print(scores)

agents.py

@@ -1,7 +1,5 @@
 import json
 import time
-import asyncio
-import os
 from searcher import Result,SementicSearcher
 from LLM import openai_llm
 from prompts import *
@@ -17,10 +15,10 @@ def get_llms():
     cheap_llm = get_llm("gpt-4o-mini")
     return main_llm,cheap_llm
 
-async def judge_idea(i,j,idea0,idea1,topic,llm):
+def judge_idea(i,j,idea0,idea1,topic,llm):
     prompt = get_judge_idea_all_prompt(idea0,idea1,topic)
     messages = [{"role":"user","content":prompt}]
-    response = await llm.response_async(messages)
+    response = llm.response(messages)
     novelty = extract(response,"novelty")
     relevance = extract(response,"relevance")
     significance = extract(response,"significance")
@@ -55,16 +53,16 @@ class DeepResearchAgent:
     def wrap_messages(self,prompt):
         return [{"role":"user","content":prompt}]
 
-    async def get_openai_response_async(self,messages):
-        return await self.llm.response_async(messages)
+    def get_openai_response(self,messages):
+        return self.llm.response(messages)
     
-    async def get_cheap_openai_response_async(self,messages):
-        return await self.cheap_llm.response_async(messages,max_tokens = 16000)
+    def get_cheap_openai_response(self,messages):
+        return self.cheap_llm.response(messages,max_tokens = 16000)
 
-    async def get_search_query(self,topic = None,query=None):
+    def get_search_query(self,topic = None,query=None):
         prompt = get_deep_search_query_prompt(topic,query)
         messages = self.wrap_messages(prompt)
-        response = await self.get_openai_response_async(messages)
+        response = self.get_openai_response(messages)
         search_query = extract(response,"queries")
         try:
             search_query = json.loads(search_query)
@@ -73,17 +71,17 @@ class DeepResearchAgent:
             search_query = [query]
         return search_query
 
-    async def generate_idea_with_chain(self,topic):
+    def generate_idea_with_chain(self,topic):
         self.topic = topic
         print(f"begin to generate search query for {topic}")
-        search_query = await self.get_search_query(topic=topic)
+        search_query = self.get_search_query(topic=topic)
         papers = []
         for query in search_query:
             failed_query = []
             current_papers = []
             cnt = 0
             while len(current_papers) == 0 and cnt < 10:
-                paper = await self.reader.search_async(query,1,paper_list=self.read_papers,llm=self.llm,rerank_query=f"{topic}",publicationDate=self.publicationData)
+                paper = self.reader.search(query,1,paper_list=self.read_papers,llm=self.llm,rerank_query=f"{topic}",publicationDate=self.publicationData)
                 if paper and len(paper) > 0 and paper[0]:
                     self.read_papers.add(paper[0].title)
                     current_papers.append(paper[0])
@@ -91,7 +89,7 @@ class DeepResearchAgent:
                     failed_query.append(query)
                     prompt = get_deep_rewrite_query_prompt(failed_query,topic)
                     messages = self.wrap_messages(prompt)
-                    new_query = await self.get_openai_response_async(messages)
+                    new_query = self.get_openai_response(messages)
                     new_query = extract(new_query,"query")
                     print(f"Failed to search papers for {query}, regenerating query {new_query} to search papers.")
                     query = new_query
@@ -104,67 +102,30 @@ class DeepResearchAgent:
             print(f"failed to generate idea {topic}")
             return None,None,None,None,None,None,None,None,None
 
-        tasks = [self.deep_research_paper_with_chain(paper) for paper in papers]
-        results = await asyncio.gather(*tasks)
-        results = [result for result in results if result]
-        if len(results) ==0:
-            print(f"failed to generate idea {topic}")
-            return None,None,None,None,None,None,None,None,None
-
-        ideas,idea_chains,experiments,entities,trends,futures,humans,years = [[result[i] for result in results] for i in range(8)]
-
-        tasks = []
-        for i,idea_1 in enumerate(ideas):
-            for j,idea_2 in enumerate(ideas):
-                if i != j:
-                    tasks.append(judge_idea(i,j,idea_1,idea_2,topic,self.llm))
-        results = await asyncio.gather(*tasks)
-        elo_scores = [0 for _ in range(len(ideas))]
-        elo_selected = 0
-        def change_winner_to_score(winner,score_1,score_2):
-            try:
-                winner = int(winner)
-            except:
-                return score_1+0.5,score_2+0.5
-            if winner == 0:
-                return score_1+1,score_2
-            if winner == 2:
-                return score_1+0.5,score_2+0.5
-            return score_1,score_2+1
-        for result in results:
-            i,j,novelty,relevance,significance,clarity,feasibility,effectiveness = result
-            for dimension in [novelty,relevance,significance,clarity,feasibility,effectiveness]:
-                elo_scores[i],elo_scores[j] = change_winner_to_score(dimension,elo_scores[i],elo_scores[j])
-            print(f"i:{i},j:{j},novelty:{novelty},relevance:{relevance},significance:{significance},clarity:{clarity},feasibility:{feasibility},effectiveness:{effectiveness}")
-        print(elo_scores)
-        try:
-            elo_selected = elo_scores.index(max(elo_scores))
-        except:
-            elo_selected = 0
+        idea,idea_chain,experiment,entities,trend,future,human,year = self.deep_research_paper_with_chain(papers[0])
         
-        idea,experiment,entities,idea_chain,trend,future,human,year = ideas[elo_selected],experiments[elo_selected],entities[elo_selected],idea_chains[elo_selected],trends[elo_selected],futures[elo_selected],humans[elo_selected],years[elo_selected]
         print(f"successfully generated idea")
-        return idea,experiment,entities,idea_chain,ideas,trend,future,human,year
+        return idea,experiment,entities,idea_chain,idea,trend,future,human,year
     
-    async def get_paper_idea_experiment_references_info(self,paper):
+    def get_paper_idea_experiment_references_info(self,paper):
         article = paper.article
         if not article:
             return None
         paper_content = self.reader.read_paper_content(article)
         prompt = get_deep_reference_prompt(paper_content,self.topic)
         messages = self.wrap_messages(prompt)
-        response = await self.get_cheap_openai_response_async(messages)
+        response = self.get_cheap_openai_response(messages)
         entities = extract(response,"entities")
         idea = extract(response,"idea")
         experiment = extract(response,"experiment")
         references = extract(response,"references")
         return idea,experiment,entities,references,paper.title
 
-    async def get_article_idea_experiment_references_info(self,article):
+    def get_article_idea_experiment_references_info(self,article):
         paper_content = self.reader.read_paper_content_with_ref(article)
         prompt = get_deep_reference_prompt(paper_content,self.topic)
         messages = self.wrap_messages(prompt)
-        response = await self.get_cheap_openai_response_async(messages)
+        response = self.get_cheap_openai_response(messages)
         entities = extract(response,"entities")
         idea = extract(response,"idea")
         experiment = extract(response,"experiment")
@@ -172,7 +133,7 @@ class DeepResearchAgent:
         return idea,experiment,entities,references
 
     
-    async def deep_research_paper_with_chain(self,paper:Result): 
+    def deep_research_paper_with_chain(self,paper:Result): 
         print(f"begin to deep research paper {paper.title}")
         article = paper.article
         if not article:
@@ -183,7 +144,7 @@ class DeepResearchAgent:
         experiments = []
         total_entities = []
         years = []
-        idea,experiment,entities,references = await self.get_article_idea_experiment_references_info(article)
+        idea,experiment,entities,references = self.get_article_idea_experiment_references_info(article)
         try:
             references = json.loads(references)
         except:
@@ -200,7 +161,7 @@ class DeepResearchAgent:
         # search before
         while len(idea_chain)<self.max_chain_length:
             rerank_query = f"{self.topic} {current_title} {current_abstract}"
-            citation_paper = await self.reader.search_related_paper_async(current_title,need_reference=False,rerank_query=rerank_query,llm=self.llm,paper_list=idea_papers)
+            citation_paper = self.reader.search_related_paper(current_title,need_reference=False,rerank_query=rerank_query,llm=self.llm,paper_list=idea_papers)
             if not citation_paper:
                 print(f"failed to find citation paper for {current_title}")
                 break
@@ -208,10 +169,10 @@ class DeepResearchAgent:
             abstract = citation_paper.abstract
             prompt = get_deep_judge_relevant_prompt(current_title,current_abstract,self.topic)
             messages = self.wrap_messages(prompt)
-            response = await self.get_openai_response_async(messages)
+            response = self.get_openai_response(messages)
             relevant = extract(response,"relevant")
             if relevant != "0":
-                result = await self.get_paper_idea_experiment_references_info(citation_paper)
+                result = self.get_paper_idea_experiment_references_info(citation_paper)
                 if not result:
                     break
                 idea,experiment,entities,_,_ = result
@@ -238,13 +199,13 @@ class DeepResearchAgent:
                 references.pop(0)
                 if reference in self.read_papers:
                     continue
-                search_paper = await self.reader.search_async(reference,3,llm=self.llm,publicationDate=self.publicationData,paper_list= idea_papers)
+                search_paper = self.reader.search(reference,3,llm=self.llm,publicationDate=self.publicationData,paper_list= idea_papers)
                 if len(search_paper) > 0:
                     s_p = search_paper[0]
                     if s_p and  s_p.title not in self.read_papers:
                         prompt = get_deep_judge_relevant_prompt(current_title,current_abstract,self.topic)
                         messages = self.wrap_messages(prompt)
-                        response = await self.get_openai_response_async(messages)
+                        response = self.get_openai_response(messages)
                         relevant = extract(response,"relevant")
                         if relevant != "0" or len(idea_chain) < self.min_chain_length:
                             article = s_p.article
@@ -257,7 +218,7 @@ class DeepResearchAgent:
             
             if not article:
                 rerank_query = f"topic: {self.topic} Title: {current_title} Abstract: {current_abstract}"
-                search_paper = await self.reader.search_related_paper_async(current_title,need_citation=False,rerank_query = rerank_query,llm=self.llm,paper_list=idea_papers)
+                search_paper = self.reader.search_related_paper(current_title,need_citation=False,rerank_query = rerank_query,llm=self.llm,paper_list=idea_papers)
                 if not search_paper:
                     print(f"failed to find citation paper for {current_title}")
                     continue
@@ -273,10 +234,10 @@ class DeepResearchAgent:
                     if s_p and s_p.title not in self.read_papers:
                         prompt = get_deep_judge_relevant_prompt(current_title,current_abstract,self.topic)
                         messages = self.wrap_messages(prompt)
-                        response = await self.get_openai_response_async(messages)
+                        response = self.get_openai_response(messages)
                         relevant = extract(response,"relevant")
                         if relevant == "1" or len(idea_chain) < self.min_chain_length:
-                            article = await s_p.article
+                            article = s_p.article
                             if not article:
                                 continue
                             else:
@@ -290,7 +251,7 @@ class DeepResearchAgent:
             paper_content = self.reader.read_paper_content_with_ref(article)
             prompt = get_deep_reference_prompt(paper_content,self.topic)
             messages = self.wrap_messages(prompt)
-            response = await self.get_cheap_openai_response_async(messages)
+            response = self.get_cheap_openai_response(messages)
             idea = extract(response,"idea")
             references = extract(response,"references")
             experiment = extract(response,"experiment")
@@ -317,7 +278,7 @@ class DeepResearchAgent:
         
         prompt = get_deep_trend_idea_chains_prompt(idea_chains,entities,self.topic)
         messages = self.wrap_messages(prompt)
-        response = await self.get_openai_response_async(messages)
+        response = self.get_openai_response(messages)
         trend = extract(response,"trend")
 
         self.deep_research_chains.append({"idea_chains":idea_chains,"trend":trend,"topic":self.topic,"ideas":idea_chain,"experiments":experiments,"entities":total_entities,"years":years})
@@ -326,26 +287,26 @@ class DeepResearchAgent:
     <entities> {{cleaned entities}}</entities>
 """
         messages = self.wrap_messages(prompt)
-        response = await self.get_openai_response_async(messages)
+        response = self.get_openai_response(messages)
         total_entities = extract(response,"entities")
         bad_case = []
         prompt = get_deep_generate_future_direciton_prompt(idea_chain,trend,self.topic,total_entities)
         messages = self.wrap_messages(prompt)
-        response = await self.get_openai_response_async(messages)
+        response = self.get_openai_response(messages)
         future = extract(response,"future")
         human = extract(response,"human")
         
 
         prompt = get_deep_generate_idea_prompt(idea_chains,trend,self.topic,total_entities,future,bad_case)
         messages = self.wrap_messages(prompt)
-        response = await self.get_openai_response_async(messages)
+        response = self.get_openai_response(messages)
         method = extract(response,"method")
         novelty = extract(response,"novelty")
         motivation = extract(response,"motivation")
         idea = {"motivation":motivation,"novelty":novelty,"method":method}
         prompt = get_deep_final_idea_prompt(idea_chains,trend,idea,self.topic)
         messages = self.wrap_messages(prompt)
-        response = await self.get_openai_response_async(messages)
+        response = self.get_openai_response(messages)
         final_idea = extract(response,"final_idea")
 
         idea = final_idea

app.py

@@ -332,7 +332,7 @@ def form_post(topic: str = Form(...)):
         main_llm, cheap_llm = get_llms()
         deep_research_agent = DeepResearchAgent(llm=main_llm, cheap_llm=cheap_llm, improve_cnt=1, max_chain_length=5, min_chain_length=3, max_chain_numbers=1)
         print(f"begin to generate idea of topic {topic}")
-        idea, related_experiments, entities, idea_chain, ideas, trend, future, human, year = asyncio.run(deep_research_agent.generate_idea_with_chain(topic))
+        idea, related_experiments, entities, idea_chain, ideas, trend, future, human, year = deep_research_agent.generate_idea_with_chain(topic)
         idea_md = markdown.markdown(idea)
         # 更新每日回复次数
         reply_count += 1

main.py

@@ -1,8 +1,9 @@
-from agents import DeepResearchAgent,ReviewAgent,get_llms
+from agents import DeepResearchAgent,get_llms
 import asyncio
 import json
 import argparse
 
+
 if __name__ == '__main__':
 
     argparser = argparse.ArgumentParser()
@@ -21,18 +22,12 @@ if __name__ == '__main__':
     topic = args.topic
     anchor_paper_path = args.anchor_paper_path
 
-
-    review_agent = ReviewAgent(save_file=args.save_file,llm=main_llm,cheap_llm=cheap_llm)
     deep_research_agent = DeepResearchAgent(llm=main_llm,cheap_llm=cheap_llm,**vars(args))
 
     print(f"begin to generate idea and experiment of topic {topic}")
-    idea,related_experiments,entities,idea_chain,ideas,trend,future,human,year=  asyncio.run(deep_research_agent.generate_idea_with_chain(topic,anchor_paper_path))
-    experiment = asyncio.run(deep_research_agent.generate_experiment(idea,related_experiments,entities))
-
-    for i in range(args.improve_cnt):
-        experiment = asyncio.run(deep_research_agent.improve_experiment(review_agent,idea,experiment,entities))
+    idea,related_experiments,entities,idea_chain,ideas,trend,future,human,year=  deep_research_agent.generate_idea_with_chain(topic)
         
     print(f"succeed to generate idea and experiment of topic {topic}")
-    res = {"idea":idea,"experiment":experiment,"related_experiments":related_experiments,"entities":entities,"idea_chain":idea_chain,"ideas":ideas,"trend":trend,"future":future,"year":year,"human":human}
+    res = {"idea":idea,"related_experiments":related_experiments,"entities":entities,"idea_chain":idea_chain,"ideas":ideas,"trend":trend,"future":future,"year":year,"human":human}
     with open("result.json","w") as f:
         json.dump(res,f)

searcher/sementic_search.py

@@ -7,7 +7,7 @@ import time
 import aiohttp
 import asyncio
 import numpy as np
-
+import random
 
 def get_content_between_a_b(start_tag, end_tag, text):
     extracted_text = ""
@@ -31,29 +31,6 @@ def extract(text, type):
             return text
     else:
         return ""
-
-
-async def fetch(url):
-    await asyncio.sleep(1)  # 异步的 sleep 而不是 time.sleep
-    try:
-        timeout = aiohttp.ClientTimeout(total=120)
-        connector = aiohttp.TCPConnector(limit_per_host=10)  # 使用连接池
-        async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
-            async with session.get(url) as response:
-                if response.status == 200:
-                    content = await response.read()  # Read the response content as bytes
-                    return content
-                else:
-                    print(f"Failed to fetch the URL: {url} with status code: {response.status}")
-                    return None
-    except aiohttp.ClientError as e:  # 更具体的异常捕获
-        print(f"An error occurred while fetching the URL: {url}")
-        print(e)
-        return None
-    except Exception as e:
-        print(f"An unexpected error occurred while fetching the URL: {url}")
-        print(e)
-        return None
     
 def download(url):
     try:
@@ -103,7 +80,7 @@ class SementicSearcher:
     def __init__(self, ban_paper = []) -> None:
         self.ban_paper = ban_paper
     
-    async def search_papers_async(self, query, limit=5, offset=0, fields=["title", "paperId", "abstract", "isOpenAccess", 'openAccessPdf', "year","publicationDate","citations.title","citations.abstract","citations.isOpenAccess","citations.openAccessPdf","citations.citationCount","citationCount","citations.year"],
+    def search_papers(self, query, limit=5, offset=0, fields=["title", "paperId", "abstract", "isOpenAccess", 'openAccessPdf', "year","publicationDate","citations.title","citations.abstract","citations.isOpenAccess","citations.openAccessPdf","citations.citationCount","citationCount","citations.year"],
                             publicationDate=None, minCitationCount=0, year=None, 
                             publicationTypes=None, fieldsOfStudy=None):
         url = 'https://api.semanticscholar.org/graph/v1/paper/search'
@@ -124,7 +101,6 @@ class SementicSearcher:
         # Load the API key from the configuration file 
         api_key = os.environ.get('SEMENTIC_SEARCH_API_KEY',None)
         headers = {'x-api-key': api_key} if api_key else None
-        await asyncio.sleep(0.5)
         try:
             filtered_query_params = {key: value for key, value in query_params.items() if value is not None}
             response = requests.get(url, params=filtered_query_params, headers=headers)
@@ -135,7 +111,7 @@ class SementicSearcher:
             elif response.status_code == 429:
                 time.sleep(1)  
                 print(f"Request failed with status code {response.status_code}: begin to retry")
-                return await self.search_papers_async(query, limit, offset, fields, publicationDate, minCitationCount, year, publicationTypes, fieldsOfStudy)
+                return self.search_papers(query, limit, offset, fields, publicationDate, minCitationCount, year, publicationTypes, fieldsOfStudy)
             else:
                 print(f"Request failed with status code {response.status_code}: {response.text}")
                 return None
@@ -145,6 +121,23 @@ class SementicSearcher:
                 
     def cal_cosine_similarity(self, vec1, vec2):
         return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
+    
+    def cal_cosine_similarity_matric(self,matric1, matric2):
+        if isinstance(matric1, list):
+            matric1 = np.array(matric1)
+        if isinstance(matric2, list):
+            matric2 = np.array(matric2)
+        if len(matric1.shape) == 1:
+            matric1 = matric1.reshape(1, -1)
+        if len(matric2.shape) == 1:
+            matric2 = matric2.reshape(1, -1)
+        dot_product = np.dot(matric1, matric2.T)
+        norm1 = np.linalg.norm(matric1, axis=1)
+        norm2 = np.linalg.norm(matric2, axis=1)
+
+        cos_sim = dot_product / np.outer(norm1, norm2)
+        scores = cos_sim.flatten()
+        return scores.tolist()
 
     def read_arxiv_from_path(self, pdf_path):
         def is_pdf(binary_data):
@@ -163,97 +156,41 @@ class SementicSearcher:
             return None
         return article_dict
         
-    async def get_paper_embbeding_and_score_async(self,query_embedding, paper,llm):
+    def get_paper_embbeding_and_score(self,query_embedding, paper,llm):
         paper_content = f"""
 Title: {paper['title']}
 Abstract: {paper['abstract']}
 """
-        paper_embbeding = await llm.get_embbeding_async(paper_content)
+        paper_embbeding = llm.get_embbeding(paper_content)
         paper_embbeding = np.array(paper_embbeding)
         score = self.cal_cosine_similarity(query_embedding,paper_embbeding)
         return [paper,score]
 
     
-    async def rerank_papers_async(self, query_embedding, paper_list,llm):
+    def rerank_papers(self, query_embedding, paper_list,llm):
+        if len(paper_list) == 0:
+            return []
+        paper_list = [paper for paper in paper_list if paper]
         if len(paper_list) >= 50:
-            paper_list = paper_list[:50]
-        results = await asyncio.gather(*[self.get_paper_embbeding_and_score_async(query_embedding, paper,llm) for paper in paper_list if paper])
-        reranked_papers = sorted(results,key = lambda x: x[1],reverse = True)
-        return reranked_papers
-    
-    async def get_embbeding_and_score_async(self,query_embedding, text,llm):
-        text_embbeding = await llm.get_embbeding_async(text)
-        text_embbeding = np.array(text_embbeding)
-        score = self.cal_cosine_similarity(query_embedding,text_embbeding)
-        return score
-    
-    async def get_embbeding_and_score_from_texts_async(self,query_embedding, texts,llm):
-        results = await asyncio.gather(*[self.get_embbeding_and_score_async(query_embedding, text,llm) for text in texts])
-        return results
-    
-    async def get_paper_details_async(self, paper_id, fields = ["title", "abstract", "year","citationCount","isOpenAccess","openAccessPdf"]):
-        url = f'https://api.semanticscholar.org/graph/v1/paper/{paper_id}'
-        fields = process_fields(fields)
-        paper_data_query_params = {'fields': fields}
-        try:
-            async with aiohttp.ClientSession() as session:
-                filtered_query_params = {key: value for key, value in paper_data_query_params.items() if value is not None}
-                headers = {'x-api-key': os.environ.get('SEMENTIC_SEARCH_API_KEY',None)}
-                async with session.get(url, params=filtered_query_params, headers=headers) as response:
-                    if response.status == 200:
-                        response_data = await response.json()
-                        return response_data
-                    else:
-                        await asyncio.sleep(0.01)
-                        print(f"Request failed with status code {response.status}: {await response.text()}")
-                        return None
-        except Exception as e:
-            print(f"Failed to get paper details for paper ID: {paper_id}")
-            return None
-    
-    async def batch_retrieve_papers_async(self, paper_ids, fields = semantic_fields):
-        url = 'https://api.semanticscholar.org/graph/v1/paper/batch'
-        paper_data_query_params = {'fields': process_fields(fields)}
-        paper_ids_json = {"ids": paper_ids}
-        try:
-            async with aiohttp.ClientSession() as session:
-                filtered_query_params = {key: value for key, value in paper_data_query_params.items() if value is not None}
-                headers = {'x-api-key': os.environ.get('SEMENTIC_SEARCH_API_KEY',None)}
-                async with session.post(url, json=paper_ids_json, params=filtered_query_params, headers=headers) as response:
-                    if response.status == 200:
-                        response_data = await response.json()
-                        return response_data
-                    else:
-                        await asyncio.sleep(0.01)
-                        print(f"Request failed with status code {response.status}: {await response.text()}")
-                        return None
-        except Exception as e:
-            print(f"Failed to batch retrieve papers for paper IDs: {paper_ids}")
-            return None
-    
-    async def search_paper_from_title_async(self, query,fields = ["title","paperId"]):
-        url = 'https://api.semanticscholar.org/graph/v1/paper/search/match'
-        fields = process_fields(fields)
-        query_params = {'query': query, 'fields': fields}
-        try:
-            async with aiohttp.ClientSession() as session:
-                filtered_query_params = {key: value for key, value in query_params.items() if value is not None}
-                headers = {'x-api-key': os.environ.get('SEMENTIC_SEARCH_API_KEY',None)}
-                async with session.get(url, params=filtered_query_params, headers=headers) as response:
-                    if response.status == 200:
-                        response_data = await response.json()
-                        return response_data
-                    else:
-                        await asyncio.sleep(0.01)
-                        print(f"Request failed with status code {response.status}: {await response.text()}")
-                        return None
-        except Exception as e:
-            await asyncio.sleep(0.01)
-            print(f"Failed to search paper from title: {query}")
-            return None
+            paper_list = random.sample(paper_list,50)
+        paper_contents = []
+        for paper in paper_list:
+            paper_content = f"""
+Title: {paper['title']}
+Abstract: {paper['abstract']}
+"""
+            paper_contents.append(paper_content)
+        paper_contents_embbeding = llm.get_embbeding(paper_contents)
+        paper_contents_embbeding = np.array(paper_contents_embbeding)
+        scores = self.cal_cosine_similarity_matric(query_embedding,paper_contents_embbeding)
+            
+        # 根据score对paper_list进行排序 
+        paper_list = sorted(zip(paper_list,scores),key = lambda x: x[1],reverse = True)
+        paper_list = [paper[0] for paper in paper_list]
+        return paper_list
         
     
-    async def search_async(self,query,max_results = 5 ,paper_list = None ,rerank_query = None,llm = None,year = None,publicationDate = None,need_download = True,fields = ["title", "paperId", "abstract", "isOpenAccess", 'openAccessPdf', "year","publicationDate","citationCount"]):
+    def search(self,query,max_results = 5 ,paper_list = None ,rerank_query = None,llm = None,year = None,publicationDate = None,need_download = True,fields = ["title", "paperId", "abstract", "isOpenAccess", 'openAccessPdf', "year","publicationDate","citationCount"]):
         if rerank_query:
             rerank_query_embbeding = llm.get_embbeding(rerank_query)
             rerank_query_embbeding = np.array(rerank_query_embbeding)
@@ -270,7 +207,7 @@ Abstract: {paper['abstract']}
                 readed_papers = [paper.title for paper in paper_list]
 
         print(f"Searching for papers related to the query: <{query}>")
-        results = await self.search_papers_async(query,limit = 10 * max_results,year=year,publicationDate = publicationDate,fields = fields)
+        results = self.search_papers(query,limit = 10 * max_results,year=year,publicationDate = publicationDate,fields = fields)
         if not results or "data" not in results:
             return []
         
@@ -293,8 +230,7 @@ Abstract: {paper['abstract']}
             paper_candidates = results
         
         if llm and rerank_query:
-            paper_candidates = await self.rerank_papers_async(rerank_query_embbeding, paper_candidates,llm)
-            paper_candidates = [paper[0] for paper in paper_candidates if paper]
+            paper_candidates = self.rerank_papers(rerank_query_embbeding, paper_candidates,llm)
         
         if need_download:
             for result in paper_candidates:
@@ -326,10 +262,10 @@ Abstract: {paper['abstract']}
                     break
         return final_results
     
-    async def search_related_paper_async(self,title,need_citation = True,need_reference = True,rerank_query = None,llm = None,paper_list = []):
-        print(f"Searching for the related papers of <{title}>")
+    def search_related_paper(self,title,need_citation = True,need_reference = True,rerank_query = None,llm = None,paper_list = []):
+        print(f"Searching for the related papers of <{title}>, need_citation: {need_citation}, need_reference: {need_reference}")
         fileds = ["title","abstract","citations.title","citations.abstract","citations.citationCount","references.title","references.abstract","references.citationCount","citations.isOpenAccess","citations.openAccessPdf","references.isOpenAccess","references.openAccessPdf","citations.year","references.year"]
-        results = await self.search_papers_async(title,limit = 3,fields=fileds)
+        results = self.search_papers(title,limit = 3,fields=fileds)
         related_papers = []
         related_papers_title = []
         if not results or "data" not in results:
@@ -367,8 +303,7 @@ Abstract: {paper['abstract']}
         if rerank_query and llm:
             rerank_query_embbeding = llm.get_embbeding(rerank_query)
             rerank_query_embbeding = np.array(rerank_query_embbeding)
-            related_papers = await self.rerank_papers_async(rerank_query_embbeding, related_papers,llm)
-            related_papers = [paper[0] for paper in related_papers]
+            related_papers = self.rerank_papers(rerank_query_embbeding, related_papers,llm)
             related_papers = [[paper["title"],paper["abstract"],paper["openAccessPdf"]["url"],paper["citationCount"],paper['year']] for paper in related_papers]
         else:
             related_papers = [[paper["title"],paper["abstract"],paper["openAccessPdf"]["url"],paper["citationCount"],paper['year']] for paper in related_papers]
@@ -385,13 +320,6 @@ Abstract: {paper['abstract']}
                 return result
         return None
     
-
-    async def download_pdf_async(self, pdf_link):
-        content = await fetch(pdf_link)
-        if not content:
-            return None
-        else:
-            return content
     
     def download_pdf(self, pdf_link):
         content = download(pdf_link)