
    (HJjD                     |   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ e G d de             Z G d	 d
ej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   dZ	eed
<   dZ
eed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizeFbiasqkv_biasi   max_position_embeddingsNnum_key_value_heads'  
rope_thetarope_traditionalrope_scalingtie_word_embeddingsc                       j                    j                   _          j                  rCddh}t         fd|D              st	        d|        j                  d   dvrt	        d      y y )Nfactor	rope_typec              3   :   K   | ]  }|j                   v   y wN)r   ).0keyselfs     a/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/internlm3.py	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>%   s     I=Csd///=s   zrope_scaling must contain keys )lineardynamiczErope_scaling 'rope_type' currently only supports 'linear' or 'dynamic)r   r   r   all
ValueError)r&   required_keyss   ` r'   __post_init__zModelArgs.__post_init__   s    ##+'+'?'?D$%{3MI=II #B=/!RSS  -5JJ [  K     )__name__
__module____qualname__str__annotations__intfloatr   boolr   r   r   r   r   r   r   r   r   r   r.    r/   r'   r   r      s    OOD$Hd#(S(##J"d";?L(4U5#:%6 678? %%r/   r   c                   V     e Zd ZdZ	 	 	 	 ddededededef
 fdZd Zdd	efd
Z	 xZ
S )DynamicNTKScalingRoPEzCImplements the rotary positional encoding with Dynamic NTK scaling.dimsr   traditionalbasescalec                 h    t         |           || _        || _        || _        || _        || _        y r#   )super__init__r   original_baser;   r<   r>   )r&   r;   r   r<   r=   r>   	__class__s         r'   rA   zDynamicNTKScalingRoPE.__init__1   s6     	'>$!	&
r/   c                 l    | j                    d| j                   d| j                   d| j                   S )Nz, traditional=z, max_position_embeddings=z, scaling_factor=)r;   r<   r   scaling_factorr&   s    r'   
extra_reprz DynamicNTKScalingRoPE.extra_repr@   sV    ))N4+;+;*<<VW[WsWsVt  uF  GK  GZ  GZ  F[  \  	\r/   offsetc                    |j                   d   |z   }|| j                  kD  rW| j                  | j                  |z  | j                  z  | j                  dz
  z
  | j                  | j                  dz
  z  z  z  }n| j                  }t
        j                  j                  || j                  | j                  || j                  |      S )Nr      )r<   r=   r>   rH   )	shaper   rB   r>   r;   mxfastroper<   )r&   xrH   seq_lenr=   s        r'   __call__zDynamicNTKScalingRoPE.__call__C   s    ''!*v%T111%%g%(D(DDVWX))tyy1}-)/ /D %%Dww||II((**  
 	
r/   )i   Fr   g      ?)r   )r0   r1   r2   __doc__r5   r7   r6   rA   rG   rQ   __classcell__rC   s   @r'   r:   r:   .   s[    M
 (,! "% 	
  \
# 
r/   r:   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  }|j                  }|j                  x| _        }|j                  x| _        }||j                  z  | _        |j                  |z  x| _	        }|dz  | _
        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        |j"                  $|j"                  d   dk(  rd|j"                  d   z  nd}t%        ||j&                  |j(                  |j*                  |      | _        y )	Ng      ࿩r   r!   r)   r   r    g       @)r   r<   r=   r>   )r@   rA   r   r   r   n_headsr   
n_kv_headsn_kv_groupshead_dimr>   nnLinearq_projk_projv_projo_projr   r:   r   r   r   rN   )	r&   rW   dimr   rZ   r[   r]   
rope_scalerC   s	           r'   rA   zAttention.__init__W   sL   ==!%!9!99w'+'?'??*"d&>&>>#'#3#3w#>>t^
iiWx%7hGiiZ(%:JiiZ(%:Jii( 2ChG   ,!!+.(: !!(++ 	 	 *$($@$@--
	r/   rO   maskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr   rJ   r      )rH   )rg   r>   rf   )rK   r`   ra   rb   reshaperZ   	transposer[   rN   rH   update_and_fetchr   r>   rc   )r&   rO   rf   rg   BLDquerieskeysvaluesoutputs              r'   rQ   zAttention.__call__w   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r/   NNr0   r1   r2   r   rA   rL   arrayr   r   rQ   rS   rT   s   @r'   rV   rV   V   sX    
Y 
F $(#	#88# rxx # }	#
 
#r/   rV   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  |||      | _        t        j                  |||      | _        t        j                  |||      | _        y )NrY   )r@   rA   r^   r_   	gate_proj	down_projup_proj)r&   rd   
hidden_dimr   rC   s       r'   rA   zMLP.__init__   sJ    3
>:s>yyjt<r/   rh   c                 t    | j                  t        | j                  |      | j                  |                  S r#   )r}   r	   r|   r~   )r&   rO   s     r'   rQ   zMLP.__call__   s)    ~~fT^^A%6QHIIr/   )r0   r1   r2   rA   rL   rx   rQ   rS   rT   s   @r'   rz   rz      s    =JRXX Jr/   rz   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockrW   c                 b   t         |           t        |      | _        t	        |j
                  |j                  |j                        | _        t        j                  |j
                  |j                        | _        t        j                  |j
                  |j                        | _        y )Neps)r@   rA   rV   	self_attnrz   r   r   r   mlpr^   RMSNormr   input_layernormpost_attention_layernormr&   rW   rC   s     r'   rA   zTransformerBlock.__init__   sz    "4t'')?)?K!zz$*:*:@Q@QR(*

$"3"3)
%r/   rO   rf   rg   rh   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r#   )r   r   r   r   )r&   rO   rf   rg   rhouts          r'   rQ   zTransformerBlock.__call__   sQ     NN4//2D%@EHHT22156!e
r/   rv   rw   rT   s   @r'   r   r      sW    
Y 
 $(#	
88
 rxx 
 }	

 

r/   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )InternLM2ModelrW   c                 p   t         |           |j                  dkD  sJ t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )Nr   )rW   r   )r@   rA   r   r^   	Embeddingr   embed_tokensranger   r   layersr   r   norm)r&   rW   _rC   s      r'   rA   zInternLM2Model.__init__   s    """LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   'B3inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rg   )r   lenr   r   zipr   )r&   r   rg   r   rf   layercs          r'   rQ   zInternLM2Model.__call__   su    
 f%=FS--E$Qa1DKK/HE1aQ'A 0 yy|r/   r#   )	r0   r1   r2   r   rA   rL   rx   rQ   rS   rT   s   @r'   r   r      s'    HY H r/   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )ModelrW   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFrY   )r@   rA   rW   r   r   modelr   r^   r_   r   r   lm_headr   s     r'   rA   zModel.__init__   sX    	//#D)
''99T%5%5tUSDL (r/   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r#   )r   rW   r   r   	as_linearr   )r&   r   rg   r   s       r'   rQ   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r/   c                 `    |j                         D ci c]  \  }}d|vs|| c}}S c c}}w )Nzattention.rope.inv_freq)items)r&   weightskvs       r'   sanitizezModel.sanitize   s1    !(WA4MUV4V1WWWs   **c                 .    | j                   j                  S r#   )r   r   rF   s    r'   r   zModel.layers   s    zz   r/   r#   )r0   r1   r2   r   rA   rL   rx   rQ   r   propertyr   rS   rT   s   @r'   r   r      sA    TY T 

X ! !r/   r   )dataclassesr   typingr   r   r   r   mlx.corecorerL   mlx.nnr^   activationsr	   r=   r
   r   r   r   Moduler:   rV   rz   r   r   r   r8   r/   r'   <module>r      s    " - -    T T   @%
BII %
P=#		 =#@J")) Jryy .RYY 6!BII !r/   