Update app.py

app.py

@@ -5,7 +5,7 @@ from fastapi import FastAPI, Request
 from contextlib import asynccontextmanager
 from transformers import pipeline
 import langid
-from huggingface_hub import login
+from huggingface_hub import InferenceClient, login
 import socket
 import time
 
@@ -20,9 +20,10 @@ if not HF_HUB_TOKEN:
      raise ValueError("Missing Hugging Face API token. Please set HUGGINGFACEHUB_API_TOKEN in environment variables.")
 login(token=HF_HUB_TOKEN)
 
+client = InferenceClient(api_key=HF_HUB_TOKEN)
 
 # Load Hebrew and English text generation models
-lang_generator = pipeline("text-generation", model="microsoft/Phi-3.5-mini-instruct")
+# lang_generator = pipeline("text-generation", model="microsoft/Phi-3.5-mini-instruct")
 
 
 # Function to detect language
@@ -39,12 +40,34 @@ def detect_language(user_input):
 def generate_response(text):
     language = detect_language(text)
     print(f"Detected language: {language}, ", f"current time: {current_time_gmt()}")
-
+    
+    #template = "Use the text in the question as context to answer the question at the end.\
+    #If you don't know the answer, just say that you don't know, don't try to make up an answer."
+    
     if language == "hebrew" or language == "english":
-        # hebrew_generator = pipeline("text-generation", model="onlplab/alephbert-base")
-        output = lang_generator(text, max_length=100, truncation=True)
-        print(f"Model output: {output}, ", f"current time: {current_time_gmt()}")  # Debugging
-        return output[0]["generated_text"]
+        messages = [
+            { "role": "user", "content": text }
+        ]
+        print(f"Messages: {messages}, ", f"current time: {current_time_gmt()}")
+        
+        completion = client.chat.completions.create(    
+            model="microsoft/Phi-3.5-mini-instruct", 
+        	messages=messages, 
+        	max_tokens=400,
+            temperature=0.5,
+            top_p=0.7 
+        )
+        print("\ncompletion: ", completion.choices[0].message, f"\n current time: {current_time_gmt()}")
+        
+    return "Sorry, I only support Hebrew and English."
+
+    
+
+    # if language == "hebrew" or language == "english":
+    #     # hebrew_generator = pipeline("text-generation", model="onlplab/alephbert-base")
+    #     output = lang_generator(text, max_length=250, truncation=True)
+    #     print(f"Model output: {output}, ", f"current time: {current_time_gmt()}")  # Debugging
+    #     return output[0]["generated_text"]
     
     # elif language == "english":
     #     #english_generator = pipeline("text-generation", model="mistralai/Mistral-Nemo-Instruct-2407", max_new_tokens=128)
@@ -53,7 +76,7 @@ def generate_response(text):
     #     print(f"English model output: {output}, ", f"current time: {current_time_gmt()}")  # Debugging
     #     return output[0]["generated_text"]
     
-    return "Sorry, I only support Hebrew and English."
+    # return "Sorry, I only support Hebrew and English."
 
 
 # FastAPI lifespan event