update embedding model load

app.py

@@ -3,10 +3,11 @@ import os
 sys.path.append("./src")
 import streamlit as st
 from pages import button_interface, step_by_step_generation, one_click_generation
-from utils.hash import check_env
+from utils.hash import check_env, check_embedding
 
 if __name__ == "__main__":
     check_env()
+    check_embedding() 
     backend = button_interface.Backend()
     st.set_page_config(layout="wide")
     def fn1():

configs/datasets.yaml

@@ -3,7 +3,7 @@ DEFAULT:
     ignore_paper_id_list: ./assets/data/ignore_paper_id_list.json
     log_level: "DEBUG"
     log_dir: ./log
-    embedding: "sentence-transformers/all-MiniLM-L6-v2"
+    embedding: ./assets/model/sentence-transformers/all-MiniLM-L6-v2
     device: "cpu" # "cpu"
 
 ARTICLE:
@@ -31,8 +31,4 @@ RETRIEVE:
     s_bg: 0
     s_contribution: 0.5
     s_summary: 0.5
-    similarity_threshold: 0.55
-
-used_llms_apis:
-    summarization: ZhipuAI
-    generation: OpenAI
+    similarity_threshold: 0.55

src/utils/hash.py

@@ -3,7 +3,21 @@ import os
 import hashlib
 import struct
 from collections import Counter
+from huggingface_hub import hf_hub_download
 
+def check_embedding():
+    # Define the repository and files to download
+    repo_id = "sentence-transformers/all-MiniLM-L6-v2"  # "BAAI/bge-small-en-v1.5"
+    files_to_download = [
+        "config.json",
+        "pytorch_model.bin",
+        "tokenizer_config.json",
+        "vocab.txt",
+    ]
+    # Download each file and save it to the /model/bge directory
+    for file_name in files_to_download:
+        print("Checking for file: ", file_name)
+        hf_hub_download(repo_id=repo_id, filename=file_name, local_dir=f"./assets/model/{repo_id}")
 
 def check_env():
     env_name_list = [