add utils and the main script

app.py

@@ -0,0 +1,94 @@
+import logging
+import os
+import gradio as gr
+import pandas as pd
+from utils import get_zotero_ids, get_arxiv_papers, get_hf_embeddings, upload_to_pinecone, get_new_papers, recommend_papers
+
+HF_API_KEY = os.getenv('HF_API_KEY')
+PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
+INDEX_NAME = os.getenv('INDEX_NAME')
+NAMESPACE_NAME = os.getenv('NAMESPACE_NAME')
+
+def category_radio(cat):
+    if cat == 'Computer Vision and Pattern Recognition':
+        return 'cs.CV'
+    elif cat == 'Computation and Language':
+        return 'cs.CL'
+    elif cat == 'Artificial Intelligence':
+        return 'cs.AI'
+    elif cat == 'Robotics':
+        return 'cs.RO'
+
+def comment_radio(com):
+    if com == 'CVPR':
+        return 'CVPR'
+    else:
+        return None
+    
+def recommend_link(recs):
+    return recs
+
+with gr.Blocks() as demo:
+
+    zotero_api_key = gr.Textbox(label="Zotero API Key")
+
+    zotero_library_id = gr.Textbox(label="Zotero Library ID")
+
+    zotero_tag = gr.Textbox(label="Zotero Tag")
+
+    arxiv_category_name = gr.State([])
+    radio_arxiv_category_name = gr.Radio(['Computer Vision and Pattern Recognition', 'Computation and Language', 'Artificial Intelligence', 'Robotics'], label="ArXiv Category Query")
+    radio_arxiv_category_name.change(fn = category_radio, inputs= radio_arxiv_category_name, outputs= arxiv_category_name)
+
+    arxiv_comment_query = gr.State([])
+    radio_arxiv_comment_query = gr.Radio(['CVPR', 'None'], label="ArXiv Comment Query")
+    radio_arxiv_comment_query.change(fn = comment_radio, inputs= radio_arxiv_comment_query, outputs= arxiv_comment_query)
+
+    threshold = gr.Slider(minimum= 0.70, maximum= 0.99, label="Similarity Score Threshold")
+
+    init_output = gr.Textbox(label="Project Initialization Result")
+
+    rec_output = gr.Markdown(label = "Recommended Papers")
+
+    init_btn = gr.Button("Initialize")
+
+    rec_btn = gr.Button("Recommend")
+
+    @init_btn.click(inputs= [zotero_api_key, zotero_library_id, zotero_tag], outputs= [init_output])
+    def init(zotero_api_key, zotero_library_id, zotero_tag, hf_api_key = HF_API_KEY,  pinecone_api_key = PINECONE_API_KEY, index_name = INDEX_NAME,  namespace_name = NAMESPACE_NAME):
+
+        logging.basicConfig(filename= '/mnt/c/Users/ankit/Desktop/Portfolio/Paper-Recommendation-System/logs/logfile.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+        logging.info("Project Initialization Script Started (Serverless)")
+        
+        ids = get_zotero_ids(zotero_api_key, zotero_library_id, zotero_tag)
+
+        df = get_arxiv_papers(ids)
+
+        embeddings, dim = get_hf_embeddings(hf_api_key, df)
+
+        feedback = upload_to_pinecone(pinecone_api_key, index_name, namespace_name, embeddings, dim, df)
+
+        logging.info(feedback)
+        if feedback is dict:
+            return f"Retrieved {len(ids)} papers from Zotero. Successfully upserted {feedback['upserted_count']} embeddings in {namespace_name} namespace."
+        else :
+            return feedback
+    
+    @rec_btn.click(inputs= [arxiv_category_name, arxiv_comment_query, threshold], outputs= [rec_output])
+    def recs(arxiv_category_name, arxiv_comment_query, threshold, hf_api_key = HF_API_KEY,  pinecone_api_key = PINECONE_API_KEY, index_name = INDEX_NAME,  namespace_name = NAMESPACE_NAME):
+        logging.info("Weekly Script Started (Serverless)")
+
+        df = get_arxiv_papers(category= arxiv_category_name, comment= arxiv_comment_query)
+
+        df = get_new_papers(df)
+
+        if not isinstance(df, pd.DataFrame):
+            return df
+        
+        embeddings, _ = get_hf_embeddings(hf_api_key, df)
+
+        results = recommend_papers(pinecone_api_key, index_name, namespace_name, embeddings, df, threshold)
+
+        return results
+
+demo.launch(share = True)

utils.py

@@ -0,0 +1,132 @@
+import pandas as pd
+import arxiv
+import requests
+from pinecone import Pinecone, ServerlessSpec
+import logging
+import os
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+os.chdir(script_dir) 
+
+def get_zotero_ids(api_key, library_id, tag):
+
+    base_url = 'https://api.zotero.org'
+    suffix = '/users/'+ library_id +'/items?tag='+ tag
+
+    header = {'Authorization': 'Bearer '+ api_key}
+    request = requests.get(base_url + suffix, headers= header)
+    
+    return [data['data']['archiveID'].replace('arXiv:', '') for data in request.json()]
+
+def get_arxiv_papers(ids = None, category = None, comment = None):
+
+    logging.getLogger('arxiv').setLevel(logging.WARNING)
+
+    client = arxiv.Client()
+
+    if category is None:
+        search = arxiv.Search(
+            id_list= ids,
+            max_results= len(ids),
+        )
+    else :
+        if comment is None:
+            custom_query = f'cat:{category}'
+        else:
+            custom_query = f'cat:{category} AND co:{comment}'
+
+        search = arxiv.Search(
+            query = custom_query,
+            max_results= 15,
+            sort_by= arxiv.SortCriterion.SubmittedDate
+        )
+    if ids is None and category is None:
+        raise ValueError('not a valid query')
+
+    df = pd.DataFrame({'Title': [result.title for result in client.results(search)],
+                'Abstract': [result.summary.replace('\n', ' ') for result in client.results(search)],
+                'Date': [result.published.date().strftime('%Y-%m-%d') for result in client.results(search)],
+                'id': [result.entry_id for result in client.results(search)]})
+    
+    if ids:
+        df.to_csv('arxiv-scrape.csv', index = False)
+    return df
+
+def get_hf_embeddings(api_key, df):
+
+    title_abs = [title + '[SEP]' + abstract for title,abstract in zip(df['Title'], df['Abstract'])]
+
+    API_URL = "https://api-inference.huggingface.co/models/malteos/scincl"
+    headers = {"Authorization": f"Bearer {api_key}"}
+
+    response = requests.post(API_URL, headers=headers, json={"inputs": title_abs, "wait_for_model": False})
+    print(str(response.status_code) + 'This part needs an update, causing KeyError 0')
+    if response.status_code == 503:
+        response = requests.post(API_URL, headers=headers, json={"inputs": title_abs, "wait_for_model": True})
+
+    embeddings = response.json()
+
+    return embeddings, len(embeddings[0])
+
+
+def upload_to_pinecone(api_key, index, namespace, embeddings, dim, df):
+    input = [{'id': df['id'][i], 'values': embeddings[i]} for i in range(len(embeddings))]
+
+    pc = Pinecone(api_key = api_key)
+    if index in pc.list_indexes().names():
+        while True:
+            logging.warning(f'Index name : {index} already exists.')
+            return f'Index name : {index} already exists'
+                
+    pc.create_index(
+        name=index,
+        dimension=dim,
+        metric="cosine",
+        spec=ServerlessSpec(
+            cloud='aws', 
+            region='us-east-1'
+        ) 
+    ) 
+
+    index = pc.Index(index)
+    return index.upsert(vectors=input, namespace=namespace)
+
+
+def get_new_papers(df):
+    df_main = pd.read_csv('arxiv-scrape.csv')
+    df.reset_index(inplace=True)
+    df.drop(columns=['index'], inplace=True)
+    union_df = df.merge(df_main, how='left', indicator=True)
+    df = union_df[union_df['_merge'] == 'left_only'].drop(columns=['_merge'])
+    if df.empty:
+        return 'No New Papers Found'
+    else:
+        df_main = pd.concat([df_main, df], ignore_index= True)
+        df_main.drop_duplicates(inplace= True)
+        df_main.to_csv('arxiv-scrape.csv', index = False)
+        return df
+
+def recommend_papers(api_key, index, namespace, embeddings, df, threshold):
+
+    pc = Pinecone(api_key = api_key)
+    if index in pc.list_indexes().names():
+        index = pc.Index(index)
+    else:
+        raise ValueError(f"{index} doesnt exist. Project isnt initialized properly")
+    
+    results = []
+    score_threshold = threshold
+    for i,embedding in enumerate(embeddings):
+        query = embedding
+        result = index.query(namespace=namespace,vector=query,top_k=3,include_values=False)
+        sum_score = sum(match['score'] for match in result['matches'])
+        if sum_score > score_threshold:
+            results.append(f"Paper-URL : [{df['id'][i]}]({df['id'][i]}) with score: {sum_score / 3} <br />")
+
+    if results:
+        return '\n'.join(results)
+    else:
+        return 'No Interesting Paper'
+
+
+