
    (HJj?                     d   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZ e G d de             Z G d d	e	j                        Z G d
 de	j                        Z G d de	j                        Z G d de	j                        Z G d de	j                        Z G d de	j                        Zy)    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZe	ed<   dZ
eed<   dZeed<   dZeed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headshead_dimrms_norm_eps
vocab_sizenum_key_value_headsi'  
rope_thetaFrope_traditionalg      I@attn_logit_softcappingg      >@final_logit_softcappingg      b@query_pre_attn_scalarN)__name__
__module____qualname__str__annotations__intfloatr   r   boolr   r   r        ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gemma2.pyr
   r
      sk    OMOJ"d"$(E(%)U)#(5(r"   r
   c                   0     e Zd Zddedef fdZd Z xZS )RMSNormdimsepsc                 f    t         |           t        j                  |f      | _        || _        y N)super__init__mxonesweightr'   )selfr&   r'   	__class__s      r#   r+   zRMSNorm.__init__   s'    ggtg&r"   c                 r    t         j                  j                  |d| j                  z   | j                        S )N      ?)r,   fastrms_normr.   r'   r/   xs     r#   __call__zRMSNorm.__call__$   s(    ww3#4dhh??r"   )gh㈵>)r   r   r   r   r   r+   r7   __classcell__r0   s   @r#   r%   r%      s    S u 
@r"   r%   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 r   t         |           |j                  }|j                  x| _        }|j
                  x| _        }||z  | _        |j                  x| _        }d|j                  dz  z  | _
        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        |j"                  | _        t        j$                  ||j&                  |j(                        | _        y )Nr2         ?Fbias)traditionalbase)r*   r+   r   r   n_headsr   
n_kv_headsrepeatsr   r   scalennLinearq_projk_projv_projo_projr   RoPEr   r   rope)r/   r<   dimrC   rD   r   r0   s         r#   r+   zAttention.__init__)   s   !%!9!99w'+'?'??**,#'==0D66;<
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeD&*&A&A#GG--
	r"   r6   maskcachereturnc           	      ,   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }|| j                  z  }| j                  dkD  r_|j	                  || j                  | j                  || j                        }t        j                  |d      }t        j                  |	d      }	||j!                  dd      z  }
t        j"                  |
| j$                  z        }
|
| j$                  z  }
||j&                  t        j(                  k(  r]t        j*                  ||
t        j,                  t        j.                  |
j&                        j0                  |
j&                              }
n|
|z   }
t        j2                  |
dd	      }
|
|	z  }| j                  dkD  r(|j	                  || j
                  || j                        }|j                  dddd      j	                  ||d      }| j5                  |      S )
Nr      r      )offsetT)preciseaxis)shaperI   rJ   rK   reshaperC   	transposerD   rN   rW   update_and_fetchrF   rE   r   r,   expand_dimsswapaxestanhr   dtypebool_wherearrayfinfominsoftmaxrL   )r/   r6   rP   rQ   BLDquerieskeysvaluesscoresoutputs               r#   r7   zAttention.__call__?   s    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?DDJJ&<<!oo4??DLL!T]]G >>$*D^^FA.F4==R00$"="==>$---zzRXX%&"((288FLL+A+E+Ev||"T  $FDr:&<<!^^At||QFF!!!Q1-55aB?{{6""r"   NNr   r   r   r
   r+   r,   re   r   r   r7   r8   r9   s   @r#   r;   r;   (   sW    
Y 
2 $(#	-#88-# rxx -# }	-#
 
-#r"   r;   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y )NFr?   )r*   r+   rG   rH   	gate_proj	down_projup_proj)r/   rO   
hidden_dimr0   s      r#   r+   zMLP.__init__p   sJ    3
?:s?yyju=r"   rR   c                     | j                  t        j                  | j                  |            | j	                  |      z        S r)   )rw   rG   gelu_approxrv   rx   r5   s     r#   r7   zMLP.__call__v   s0    ~~bnnT^^A->?$,,q/QRRr"   )r   r   r   r+   r,   re   r7   r8   r9   s   @r#   rt   rt   o   s    >SRXX Sr"   rt   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr<   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        || _        y )Nr'   )r*   r+   r   r   r;   	self_attnrt   r   mlpr%   r   input_layernormpre_feedforward_layernormpost_feedforward_layernormpost_attention_layernormr<   r/   r<   r0   s     r#   r+   zTransformerBlock.__init__{   s    #'#;#; ++"4t'')?)?@&t'7'7T=N=NO)0$"3"3*
& +2$"3"3+
' )00@0@dFWFW(X%	r"   r6   rP   rQ   rR   c                     | j                  | j                  |      ||      }|| j                  |      z   }| j                  | j	                  |            }|| j                  |      z   }|S r)   )r   r   r   r   r   r   )r/   r6   rP   rQ   rhouts          r#   r7   zTransformerBlock.__call__   si     NN4//2D%@--a00HHT33A67$11!44
r"   rq   rr   r9   s   @r#   r}   r}   z   sW    Y & $(#	
88
 rxx 
 }	

 

r"   r}   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
GemmaModelr<   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        |j                  |j                        | _        y c c}w )Nr   )r<   r   )r*   r+   r<   r   r   rG   	Embeddingr   embed_tokensranger}   layersr%   r   norm)r/   r<   _r0   s      r#   r+   zGemmaModel.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 D,,$2C2CD	
s   Cinputsc                 &   | j                  |      }|| j                  j                  dz  z  }|d gt        | j                        z  }t        ||d   d      }t        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr>   r   T)return_array)r   r<   r   lenr   r   zipr   )r/   r   rQ   r   rP   layercs          r#   r7   zGemmaModel.__call__   s    
 f%&&+,=FS--E$QatDDKK/HE1aq!A 0 yy|r"   r)   )	r   r   r   r
   r+   r,   re   r7   r8   r9   s   @r#   r   r      s'    
EY 
E r"   r   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr<   c                     t         |           |j                  | _        |j                  | _        t	        |      | _        || _        y r)   )r*   r+   r   r   r   modelr<   r   s     r#   r+   zModel.__init__   s:    //'+'C'C$%
	r"   r   c                     | j                  ||      }| j                   j                  j                  |      }t        j                  || j
                  z        }|| j
                  z  }|S r)   )r   r   	as_linearr,   ra   r   )r/   r   rQ   r   s       r#   r7   zModel.__call__   s\    
 jj'jj%%//4ggcD8889D000
r"   c                 .    | j                   j                  S r)   )r   r   )r/   s    r#   r   zModel.layers   s    zz   r"   r)   )r   r   r   r
   r+   r,   re   r7   propertyr   r8   r9   s   @r#   r   r      s9    Y  		 ! !r"   r   )dataclassesr   typingr   r   mlx.corecorer,   mlx.nnrG   rB   r   r   r
   Moduler%   r;   rt   r}   r   r   r!   r"   r#   <module>r      s    "     6 ) ) )"@bii @D#		 D#NS")) Sryy < @!BII !r"   