pytorch
/

gemma-3-12b-it-AWQ-INT4

Image-Text-to-Text

text-generation-inference

Model card Files Files and versions

jerryzh168 commited on 21 days ago

Commit

4f826e6

·

verified ·

1 Parent(s): 488fc91

Update README.md

Files changed (1) hide show

README.md +2 -0

README.md CHANGED Viewed

@@ -181,12 +181,14 @@ quantized_model = model
 linear_config = AWQConfig(base_config, step="prepare_for_loading")
 quant_config = get_quant_config(linear_config)
 quantized_model.config.quantization_config = TorchAoConfig(quant_config)
 # Push to hub
 USER_ID = "YOUR_USER_ID"
 MODEL_NAME = model_id.split("/")[-1]
 save_to = f"{USER_ID}/{MODEL_NAME}-AWQ-INT4"
 quantized_model.push_to_hub(save_to, safe_serialization=False)
 tokenizer.push_to_hub(save_to)
 # Manual Testing
 quantized_model = AutoModelForCausalLM.from_pretrained(
     save_to,

 linear_config = AWQConfig(base_config, step="prepare_for_loading")
 quant_config = get_quant_config(linear_config)
 quantized_model.config.quantization_config = TorchAoConfig(quant_config)
 # Push to hub
 USER_ID = "YOUR_USER_ID"
 MODEL_NAME = model_id.split("/")[-1]
 save_to = f"{USER_ID}/{MODEL_NAME}-AWQ-INT4"
 quantized_model.push_to_hub(save_to, safe_serialization=False)
 tokenizer.push_to_hub(save_to)
 # Manual Testing
 quantized_model = AutoModelForCausalLM.from_pretrained(
     save_to,