Spaces:
Build error
Build error
Update app.py
Browse filesComment out HQQ config...
app.py
CHANGED
|
@@ -5,7 +5,7 @@ import gradio as gr
|
|
| 5 |
import torch
|
| 6 |
from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
|
| 7 |
|
| 8 |
-
quant_config = HqqConfig(nbits=8, group_size=64)
|
| 9 |
|
| 10 |
MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
|
| 11 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
|
@@ -20,7 +20,7 @@ model =\
|
|
| 20 |
MODEL_ID,
|
| 21 |
torch_dtype=torch.float16,
|
| 22 |
# device_map="cuda",
|
| 23 |
-
quantization_config=quant_config
|
| 24 |
).to(DEVICE)
|
| 25 |
|
| 26 |
#gc.collect()
|
|
|
|
| 5 |
import torch
|
| 6 |
from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
|
| 7 |
|
| 8 |
+
# quant_config = HqqConfig(nbits=8, group_size=64)
|
| 9 |
|
| 10 |
MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
|
| 11 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
|
|
|
| 20 |
MODEL_ID,
|
| 21 |
torch_dtype=torch.float16,
|
| 22 |
# device_map="cuda",
|
| 23 |
+
# quantization_config=quant_config
|
| 24 |
).to(DEVICE)
|
| 25 |
|
| 26 |
#gc.collect()
|