Update app.py

Try using TorchAO for quantization...

app.py

@@ -3,16 +3,16 @@ import gc
 
 import gradio as gr
 # import torch
-# from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
+# from transformers import AutoTokenizer, AutoModelForCausalLM #, HqqConfig
 
-# # quant_config = HqqConfig(nbits=8, group_size=64)
+# # # quant_config = HqqConfig(nbits=8, group_size=64)
 
 # MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
 # DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
 # print("Loading tokenizer & model…")
 # tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-# # model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
+# # # model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
 
 # model =\
 #         AutoModelForCausalLM\
@@ -23,18 +23,38 @@ import gradio as gr
 #                         # quantization_config=quant_config
 #         ).to(DEVICE)
 
-#gc.collect()
+# gc.collect()
 
 #########
 
-from unsloth import FastLanguageModel
-
-model, tokenizer = FastLanguageModel.from_pretrained(
-    "unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
-    max_seq_length=128_000,
-    load_in_4bit=True
+import torch
+from transformers import TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
+from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig
+# quant_config = Float8WeightOnlyConfig()
+quant_config = Float8DynamicActivationFloat8WeightConfig()
+quantization_config = TorchAoConfig(quant_type=quant_config)
+
+MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+quantized_model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype="auto",
+    device_map="auto",
+    quantization_config=quantization_config
 )
 
+
+#########
+
+# from unsloth import FastLanguageModel
+
+# model, tokenizer = FastLanguageModel.from_pretrained(
+#     "unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
+#     max_seq_length=128_000,
+#     load_in_4bit=True
+# )
+
 #########
 
 # import gc