Commit 
							
							·
						
						3d19935
	
1
								Parent(s):
							
							72802bf
								
Update modeling_aquila.py
Browse files- modeling_aquila.py +3 -3
    	
        modeling_aquila.py
    CHANGED
    
    | @@ -127,7 +127,7 @@ class AquilaRotaryEmbedding(torch.nn.Module): | |
| 127 |  | 
| 128 | 
             
            # Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->Aquila
         | 
| 129 | 
             
            class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
         | 
| 130 | 
            -
                """ | 
| 131 |  | 
| 132 | 
             
                def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
         | 
| 133 | 
             
                    self.scaling_factor = scaling_factor
         | 
| @@ -145,8 +145,8 @@ class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding): | |
| 145 | 
             
                    self.register_buffer("sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False)
         | 
| 146 |  | 
| 147 | 
             
            # Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->Aquila
         | 
| 148 | 
            -
            class AquilaDynamicNTKScalingRotaryEmbedding( | 
| 149 | 
            -
                """ | 
| 150 |  | 
| 151 | 
             
                def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
         | 
| 152 | 
             
                    self.scaling_factor = scaling_factor
         | 
|  | |
| 127 |  | 
| 128 | 
             
            # Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->Aquila
         | 
| 129 | 
             
            class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
         | 
| 130 | 
            +
                """AquilaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
         | 
| 131 |  | 
| 132 | 
             
                def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
         | 
| 133 | 
             
                    self.scaling_factor = scaling_factor
         | 
|  | |
| 145 | 
             
                    self.register_buffer("sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False)
         | 
| 146 |  | 
| 147 | 
             
            # Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->Aquila
         | 
| 148 | 
            +
            class AquilaDynamicNTKScalingRotaryEmbedding(AquilaRotaryEmbedding):
         | 
| 149 | 
            +
                """AquilaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
         | 
| 150 |  | 
| 151 | 
             
                def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
         | 
| 152 | 
             
                    self.scaling_factor = scaling_factor
         | 
