Update

app.py

@@ -34,7 +34,7 @@ async def ask_api(request: AskRequest):
     documents = faq.similarity_search(vectordb, request.question, k=request.k)
     df_doc = util.transform_documents_to_dataframe(documents)
     df_filter = util.remove_duplicates_by_column(df_doc, "ID")
-    return util.serialize_dataframe_as_json(df_filter)
+    return util.dataframe_to_dict(df_filter)
 
 
 @app.delete("/api/v1/")

faq.py

@@ -1,3 +1,4 @@
+import util as util
 import pandas as pd
 from langchain.document_loaders import DataFrameLoader
 from langchain.embeddings import HuggingFaceEmbeddings
@@ -10,30 +11,12 @@ import os
 import shutil
 from enum import Enum
 
-SHEET_URL_X = "https://docs.google.com/spreadsheets/d/"
-SHEET_URL_Y = "/edit#gid="
-SHEET_URL_Y_EXPORT = "/export?gid="
 EMBEDDING_MODEL_FOLDER = ".embedding-model"
 VECTORDB_FOLDER = ".vectordb"
 EMBEDDING_MODEL = "sentence-transformers/all-mpnet-base-v2"
 VECTORDB_TYPE = Enum("VECTORDB_TYPE", ["AwaDB", "Chroma"])
 
 
-def faq_id(sheet_url: str) -> str:
-    x = sheet_url.find(SHEET_URL_X)
-    y = sheet_url.find(SHEET_URL_Y)
-    return sheet_url[x + len(SHEET_URL_X) : y] + "-" + sheet_url[y + len(SHEET_URL_Y) :]
-
-
-def xlsx_url(faq_id: str) -> str:
-    y = faq_id.rfind("-")
-    return SHEET_URL_X + faq_id[0:y] + SHEET_URL_Y_EXPORT + faq_id[y + 1 :]
-
-
-def read_df(xlsx_url: str) -> pd.DataFrame:
-    return pd.read_excel(xlsx_url, header=0, keep_default_na=False)
-
-
 def create_documents(df: pd.DataFrame, page_content_column: str) -> pd.DataFrame:
     loader = DataFrameLoader(df, page_content_column=page_content_column)
     return loader.load()
@@ -109,7 +92,7 @@ def create_vectordb_id(
     if embedding_function is None:
         embedding_function = define_embedding_function(EMBEDDING_MODEL)
 
-    df = read_df(xlsx_url(faq_id))
+    df = util.read_df(util.xlsx_url(faq_id))
     documents = create_documents(df, page_content_column)
     vectordb = get_vectordb(
         faq_id=faq_id, embedding_function=embedding_function, documents=documents
@@ -118,7 +101,7 @@ def create_vectordb_id(
 
 
 def load_vectordb(sheet_url: str, page_content_column: str) -> VectorStore:
-    return load_vectordb_id(faq_id(sheet_url), page_content_column)
+    return load_vectordb_id(util.get_id(sheet_url), page_content_column)
 
 
 def delete_vectordb():

util.py

@@ -1,5 +1,25 @@
 import pandas as pd
 
+SHEET_URL_X = "https://docs.google.com/spreadsheets/d/"
+SHEET_URL_Y = "/edit#gid="
+SHEET_URL_Y_EXPORT = "/export?gid="
+
+
+def get_id(sheet_url: str) -> str:
+    x = sheet_url.find(SHEET_URL_X)
+    y = sheet_url.find(SHEET_URL_Y)
+    return sheet_url[x + len(SHEET_URL_X) : y] + "-" + sheet_url[y + len(SHEET_URL_Y) :]
+
+
+def xlsx_url(get_id: str) -> str:
+    y = get_id.rfind("-")
+    return SHEET_URL_X + get_id[0:y] + SHEET_URL_Y_EXPORT + get_id[y + 1 :]
+
+
+def read_df(xlsx_url: str) -> pd.DataFrame:
+    return pd.read_excel(xlsx_url, header=0, keep_default_na=False)
+
+
 def split_page_breaks(df, column_name):
     split_values = df[column_name].str.split("\n")
 
@@ -43,7 +63,7 @@ def remove_duplicates_by_column(df, column):
     return df
 
 
-def serialize_dataframe_as_json(df):
-    json_array = df.to_dict(orient='records')
+def dataframe_to_dict(df):
+    df_records = df.to_dict(orient='records')
 
-    return json_array
+    return df_records