Update app.py

Update to use fireworks...

app.py

@@ -1,21 +1,23 @@
 from time import sleep
 from os import getenv
+from json import dumps
+from requests import post
 
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer
 import spaces
-from openai import OpenAI
-import torch
+# from openai import OpenAI
+# import torch
 from duckduckgo_search import DDGS
 import re
 
-# Load the SmolLM model and tokenizer
-# model_name = "HuggingFaceTB/SmolLM2-360M-Instruct"
-model_name = "HuggingFaceTB/SmolLM3-3B" # "HuggingFaceTB/SmolLM2-1.7B-Instruct"
-model = AutoModelForCausalLM.from_pretrained(model_name)
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-model.to(device)
+# # Load the SmolLM model and tokenizer
+# # model_name = "HuggingFaceTB/SmolLM2-360M-Instruct"
+# model_name = "HuggingFaceTB/SmolLM3-3B" # "HuggingFaceTB/SmolLM2-1.7B-Instruct"
+# model = AutoModelForCausalLM.from_pretrained(model_name)
+# tokenizer = AutoTokenizer.from_pretrained(model_name)
+# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# model.to(device)
 
 
 
@@ -332,26 +334,66 @@ class Applicant:
 #     _output = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=True, temperature=0.6, top_k=40, top_p=0.9, repetition_penalty=1.1)
 #     return _output
 
-def writing_task(prompt: str) -> str:
-    api_key = getenv("HF_TOKEN")
-    if not api_key:
-        raise ValueError("Huggingface token missing. Need to set HF_TOKEN, refer to https://discuss.huggingface.co/t/how-to-manage-user-secrets-and-api-keys/67948")
-    client = OpenAI(
-        base_url="https://router.huggingface.co/v1",
-        api_key = getenv("HF_TOKEN")
-    )
+# def writing_task(prompt: str) -> str:
+#     api_key = getenv("HF_TOKEN")
+#     if not api_key:
+#         raise ValueError("Huggingface token missing. Need to set HF_TOKEN, refer to https://discuss.huggingface.co/t/how-to-manage-user-secrets-and-api-keys/67948")
+#     client = OpenAI(
+#         base_url="https://router.huggingface.co/v1",
+#         api_key = getenv("HF_TOKEN")
+#     )
+
+#     completion = client.chat.completions.create(
+#         model="HuggingFaceTB/SmolLM3-3B:hf-inference",
+#         messages=[
+#             {
+#                 "role": "user",
+#                 "content": prompt
+#             }
+#         ],
+#     )
+
+#     raw_response_content = completion.choices[0].message.content
+#     content_split = raw_response_content.split("</think>")
+#     if len(content_split) > 1:
+#         think = content_split[0]
+#         content = "".join(content_split[1:])
+#     else:
+#         think = content_split[0]
+#         content = "No data found."
+    
+#     return content
 
-    completion = client.chat.completions.create(
-        model="HuggingFaceTB/SmolLM3-3B:hf-inference",
-        messages=[
+def writing_task(prompt: str) -> str:
+    url = "https://api.fireworks.ai/inference/v1/chat/completions"
+    model = "accounts/fireworks/models/qwen3-235b-a22b-thinking-2507"
+    # "accounts/fireworks/models/qwen3-235b-a22b-instruct-2507"
+    payload = {
+        "model": model,
+        "max_tokens": 32768,
+        "top_p": 1,
+        "top_k": 40,
+        "presence_penalty": 0,
+        "frequency_penalty": 0,
+        "temperature": 0.6,
+        "messages": [
             {
                 "role": "user",
                 "content": prompt
             }
-        ],
-    )
-
-    raw_response_content = completion.choices[0].message.content
+        ]
+    }
+    headers = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {FIREWORKS_API_TOKEN}"  # Replace with your actual API key
+    }
+    
+    response = post(url, headers=headers, data=dumps(payload))
+    response.raise_for_status()
+    raw_response_content =\
+            response.json()["choices"][0]["message"]["content"]
+    print(f"Content with reasoning: {raw_response_content}")
     content_split = raw_response_content.split("</think>")
     if len(content_split) > 1:
         think = content_split[0]
@@ -362,6 +404,7 @@ def writing_task(prompt: str) -> str:
     
     return content
 
+
 def smol_lm_jd_process(job_description, system_prompt, max_new_tokens=512):
     prompt = f"""<|im_start|>system
 {system_prompt}<|im_end|>