update_readme

eagle/model/ea_model.py

@@ -21,7 +21,6 @@ from .cnets1 import Model as Model1
 from .configs import EConfig
 
 """ Modified to support Eagle-3, marked by <mod> xxx </mod> """
-# from .modeling_minicpm_kv import HackConvertMiniCPMForCausalLM as KVMiniCPMForCausalLM  # <mod> convert opensource impl to llama </mod>
 from .modeling_minicpm_kv import MiniCPMForCausalLM as KVMiniCPMForCausalLM  # <mod> use modified opensource impl </mod>
 
 class EaModel(nn.Module):

eagle/model/modeling_minicpm_kv.py

@@ -2443,45 +2443,3 @@ class MiniCPMForSequenceClassification(MiniCPMPreTrainedModel):
             attentions=transformer_outputs.attentions,
         )
 
-
-
-# hack version
-from .modeling_llama_kv import LlamaForCausalLM as KVLlamaForCausalLM
-
-class HackConvertMiniCPMForCausalLM:
-    def from_pretrained(model_path, **kwargs):
-        model = KVLlamaForCausalLM.from_pretrained(model_path, **kwargs)
-        
-        state_dict = model.state_dict()
-        scale_emb = 12
-        dim_model_base = 256
-        scale_depth = 1.4
-        num_layers = 32
-        hidden_size = 4096
-
-        print(state_dict["model.embed_tokens.weight"])
-        embedding = state_dict["model.embed_tokens.weight"]
-        #model.embed_tokens.weight * scale_emb
-        new_emb = embedding.clone() * scale_emb
-        state_dict["model.embed_tokens.weight"] = new_emb
-
-        #lm_head.weight / (hidden_size / dim_model_base)
-        new_emb = state_dict["lm_head.weight"].clone() / (hidden_size / dim_model_base)
-        state_dict["lm_head.weight"] = new_emb
-
-        #model.layers.34.self_attn.o_proj.weight * (scale_depth / sqrt(num_layers))
-        for i in range(num_layers):
-            attn_out_name = f"model.layers.{i}.self_attn.o_proj.weight"
-            new_weight = state_dict[attn_out_name] * (scale_depth / math.sqrt(num_layers))
-            state_dict[attn_out_name] = new_weight
-
-            ffn_down_proj_name = f"model.layers.{i}.mlp.down_proj.weight"
-            new_weight = state_dict[ffn_down_proj_name] * (scale_depth / math.sqrt(num_layers))
-            state_dict[ffn_down_proj_name] = new_weight
-        
-        print(f"Converting: reload from converted state_dict.\nCheck sd:\n{model}")
-
-        model.load_state_dict(state_dict)
-        print(f"Convert to llama: DONE.")
-
-        return model