Update myagent.py

myagent.py

@@ -5,6 +5,7 @@ from tools.fetch import fetch_webpage
 from tools.yttranscript import get_youtube_transcript, get_youtube_title_description
 import myprompts
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+
 import torch
 # --- Basic Agent Definition ---
 class BasicAgent:
@@ -49,18 +50,17 @@ class BasicAgent:
 # )
 
 
-MODEL_NAME = "meta-llama/Llama-3.2-3B"  # 3B isn't released by Meta officially, but use 8B or a 3B variant like TinyLlama if needed
+model_id = "bartowski/Llama-3.2-3B-Instruct-GGUF"
+filename = "Llama-3.2-3B-Instruct-Q4_K_M.gguf"
+
+torch_dtype = torch.float32 # could be torch.float16 or torch.bfloat16 too
+tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
+model_init = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename, torch_dtype=torch_dtype)
 
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-model_init = AutoModelForCausalLM.from_pretrained(
-    MODEL_NAME,
-    device_map="auto",
-    torch_dtype=torch.float16  # or bfloat16
-)
 
 def model(prompt: str, max_new_tokens=512):
     input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to(model.device)
-    output_ids = model.generate(input_ids, max_new_tokens=max_new_tokens)
+    output_ids = model_init.generate(input_ids, max_new_tokens=max_new_tokens)
     output = tokenizer.decode(output_ids[0], skip_special_tokens=True)
     return output