
    (HJj!                        d dl mZ d dlmZ d dlmZmZmZ d dlm	Z
 d dlmZ ddlmZmZmZ ddlmZmZ ddlmZ e G d	 d
e             Z G d dej.                        Z G d dej.                        Z G d dej.                        Z ee
j6                  d      d        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)partial)AnyDictOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_ropec                       e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed	<   d
Z
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d
Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   y)	ModelArgs
model_typei  hidden_size   num_hidden_layersi   intermediate_size   num_attention_heads   head_dimgư>rms_norm_epsi   
vocab_sizer   num_key_value_headsg    .A
rope_thetag     @rope_local_base_freqquery_pre_attn_scalari   sliding_window   sliding_window_patterni   max_position_embeddingsNrope_scaling)__name__
__module____qualname__str__annotations__r   intr   r   r   r   r   floatr   r   r   r   r   r    r"   r#   r$   r        c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gemma3_text.pyr   r      s    OKs!s!  Hc L% J  #J#"*%*#&5&NC"#C##(S(L$r-   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
	Attentionargs	layer_idxc                 T   t         |           |j                  }|j                  x| _        }|j
                  x| _        }||z  | _        |j                  x| _        }|| _	        |j                  dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t%        ||j&                        | _        t%        ||j&                        | _        |dz   |j,                  z  dk7  | _        | j.                  rt1        ||j2                  d      | _        y t1        ||j6                  d|j8                  |j:                        | _        y )	Ng      Fbias)dimsepsr   r   )r6   basetraditional)r6   r8   r9   r#   scaling_config)super__init__r   r   n_headsr   
n_kv_headsrepeatsr   r2   r   scalennLinearq_projk_projv_projo_projRMSNormr   q_normk_normr"   
is_slidingr   r   roper   r#   r$   )selfr1   r2   dimr=   r>   r   	__class__s          r.   r<   zAttention.__init__$   sh   !%!9!99w'+'?'??**,#'==0"//5
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeD81B1BC81B1BC$q=D,G,GG1L??'..!DI (__!(,(D(D#00DIr-   xmaskcachereturnc                 v   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	| j                  |      }| j                  |      }|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rQ   r@   rP   )shaperC   rD   rE   reshaper=   	transposer>   rH   rI   rK   rW   update_and_fetchr   r@   rF   )rL   rO   rP   rQ   BL_querieskeysvaluesoutputs              r.   __call__zAttention.__call__H   s    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aP++g&{{4 iii=G99T%,,97D 11$?LD&ii(G99T?D .T6djjt
 !!!Q1-55aB?{{6""r-   NNr%   r&   r'   r   r*   r<   mxarrayr   r   rc   __classcell__rN   s   @r.   r0   r0   #   s_    "Y "3 "N $(#	#88# rxx # }	#
 
#r-   r0   c                   0     e Zd Zddedef fdZd Z xZS )rG   r6   r7   c                 f    t         |           t        j                  |f      | _        || _        y N)r;   r<   rf   onesweightr7   )rL   r6   r7   rN   s      r.   r<   zRMSNorm.__init__i   s'    ggtg&r-   c                 r    t         j                  j                  |d| j                  z   | j                        S )Ng      ?)rf   fastrms_normrn   r7   rL   rO   s     r.   rc   zRMSNorm.__call__n   s(    ww3#4dhh??r-   )gh㈵>)r%   r&   r'   r*   r+   r<   rc   rh   ri   s   @r.   rG   rG   h   s    S u 
@r-   rG   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr4   )r;   r<   rA   rB   	gate_proj	down_projup_proj)rL   rM   
hidden_dimrN   s      r.   r<   zMLP.__init__s   sJ    3
?:s?yyju=r-   rR   c                     | j                  t        j                  | j                  |            | j	                  |      z        S rl   )rx   rA   gelu_approxrw   ry   rr   s     r.   rc   zMLP.__call__y   s0    ~~bnnT^^A->?$,,q/QRRr-   )r%   r&   r'   r<   rf   rg   rc   rh   ri   s   @r.   rt   rt   r   s    >SRXX Sr-   rt   T)	shapelessc                    | j                   t        j                  k7  r| |z   S t        j                  t        j                        j                  }t        j
                  | j                  t        j                        |j                  t        j                        z   | |      j                  t        j                        S rl   )dtyperf   float16finfomaxclipastypefloat32)rO   ybounds      r.   clip_residualr   }   s{    ww"**1uHHRZZ $$E77188BJJ'!((2::*>>NUU


 r-   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
TransformerBlockr1   r2   c                    t         |           |j                  | _        |j                  | _        t	        ||      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        y )Nr7   )r;   r<   r   r   r0   	self_attnrt   r   mlprG   r   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormrL   r1   r2   rN   s      r.   r<   zTransformerBlock.__init__   s    #'#;#; ++"43t'')?)?@&t'7'7T=N=NO(/0@0@dFWFW(X%)0$"3"3*
& +2$"3"3+
'r-   rO   rP   rQ   rR   c                     | j                  | j                  |      ||      }t        || j                  |            }| j	                  | j                  |            }t        || j                  |            }|S rl   )r   r   r   r   r   r   r   )rL   rO   rP   rQ   rhouts          r.   rc   zTransformerBlock.__call__   sm     NN4//2D%@!T::1=>HHT33A67At>>qAB
r-   rd   re   ri   s   @r.   r   r      s^    
Y 
3 
$ $(#	
88
 rxx 
 }	

 

r-   r   c                   h     e Zd Zdef fdZ	 	 ddej                  deej                     fdZ xZ	S )Gemma3Modelr1   c                    t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        |j                  |j                         | _        y c c}w )Nr   )r1   r2   r   )r;   r<   r1   r    window_sizer"   r   r   rA   	Embeddingr   embed_tokensranger   layersrG   r   normr   s      r.   r<   zGemma3Model.__init__   s    	..&*&A&A#//!%!7!7"""LL$:J:JK #4#9#9:
:	 $)<:
 D,,$2C2CD		
s   2C5inputsinput_embeddingsc                 x   ||}n| j                  |      }|t        j                  | j                  j                  dz  t        j
                        j                  |j                        z  }|d gt        | j                        z  }t        ||| j                  dz
           }| j                  dkD  rt        ||d   | j                        }nd }t        t        | j                  |            D ]7  \  }\  }}	|| j                  z  | j                  dz
  k(  }
|
r|n|} ||||	      }9 | j                  |      S )Ng      ?r   r   )r   )r   rf   rg   r1   r   bfloat16r   r   lenr   r
   r"   r   	enumeratezipr   )rL   r   rQ   r   r   global_masksliding_window_maskilayerc	is_globalrP   s               r.   rc   zGemma3Model.__call__   s2    ' A!!&)A	RXXdii++S0"++>EEaggNN=FS--E+AuT5P5PST5T/UV&&*"7a ,,# #'&s4;;'>?MAzqD///43N3NQR3RR  #,;1DDaq!A @ yy|r-   rd   )
r%   r&   r'   r   r<   rf   rg   r   rc   rh   ri   s   @r.   r   r      s=    EY E$ /3	   #288,	 r-   r   c                        e Zd Zdef fdZ	 	 d	dej                  deej                     fdZd Z	e
d        Zd Z xZS )
Modelr1   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        d| _        y rv   )r;   r<   r1   r   r   modelrA   rB   r   r   lm_headtie_word_embeddings)rL   r1   rN   s     r.   r<   zModel.__init__   sR    	// &
yy!1!14??O#( r-   r   r   c                     | j                  |||      }| j                  r'| j                   j                  j                  |      }|S | j	                  |      }|S rl   )r   r   r   	as_linearr   )rL   r   rQ   r   r   s        r.   rc   zModel.__call__   sV     jj(89##**))33C8C 
 ,,s#C
r-   c                 >    d|vrd| _         | j                  d       |S )Nzlm_head.weightTr   )r   pop)rL   weightss     r.   sanitizezModel.sanitize   s#    7*'+D$HHYr-   c                 .    | j                   j                  S rl   )r   r   )rL   s    r.   r   zModel.layers   s    zz   r-   c                 J   g }t        | j                  j                        D ]~  }|| j                  j                  z  | j                  j                  dz
  k(  r|j	                  t                      P|j	                  t        | j                  j                                |S )Nr   )max_size)r   r1   r   r"   appendr   r   r    )rL   cachesr   s      r.   
make_cachezModel.make_cache   s{    tyy223ADII4449933a78 gi(otyy7O7OPQ 4 r-   rd   )r%   r&   r'   r   r<   rf   rg   r   rc   r   propertyr   r   rh   ri   s   @r.   r   r      sY    )Y ) /3	 #288,	 ! !
r-   r   ) dataclassesr   	functoolsr   typingr   r   r   mlx.corecorerf   mlx.nnrA   r8   r	   r
   r   rQ   r   r   
rope_utilsr   r   Moduler0   rG   rt   compiler   r   r   r   r,   r-   r.   <module>r      s    "  & &   T T + '   &B#		 B#J@bii @S")) S 	t$ %ryy :0")) 0f*BII *r-   