pytorch
/

Phi-4-mini-instruct-AWQ-INT4

Text Generation

text-generation-inference

Model card Files Files and versions

jerryzh168 commited on Sep 16

Commit

7f81a6b

·

verified ·

1 Parent(s): 5e42abd

Update README.md

Files changed (1) hide show

README.md +4 -1

README.md CHANGED Viewed

@@ -152,7 +152,10 @@ model = AutoModelForCausalLM.from_pretrained(
 )
 tokenizer = AutoTokenizer.from_pretrained(model_id)
-base_config = Int4WeightOnlyConfig(group_size=128, version=2)
 quant_config = AWQConfig(base_config, step="prepare")
 quantize_(
     model,

 )
 tokenizer = AutoTokenizer.from_pretrained(model_id)
+# Note: this is only compatible with H100
+base_config = Int4WeightOnlyConfig(group_size=128)
+# for A100, please use the following for base_config:
+# base_config = Int4WeightOnlyConfig(group_size=128, int4_packing_format="tile_packed_to_4d", int4_choose_qparams_algorithm="hqq")
 quant_config = AWQConfig(base_config, step="prepare")
 quantize_(
     model,