
    (HJj	                     >   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZmZ e G d de             Z G d d	e	j                        Z G d
 de	j                        Z G d de	j                        Z G d de	j                        Z G d de	j                        Zy)    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   dZeed<   d	Zeed
<   dZ	eed<   dZ
eed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_headsgh㈵>norm_epsiloni   
vocab_sizei 
rope_thetaTtie_word_embeddingsN)__name__
__module____qualname__str__annotations__intr   floatr   r   r   bool     b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/starcoder2.pyr   r      sN    OL%JJ $$r   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 (   t         |           || _        |j                  }|j                  x| _        }|j                  x| _        }|j                  |j                  z  }|dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j                  |d|j                         | _        y )Ng      TbiasF)traditionalbase)super__init__r#   r   r   n_headsr   
n_kv_headsscalennLinearq_projk_projv_projo_projRoPEr   rope)selfr#   dimr+   r,   head_dim	__class__s         r    r*   zAttention.__init__   s    	!%!9!99w'+'?'??*##t'?'??t^
iiWx%7dCiiZ(%:FiiZ(%:Fii( 2CdCGGH%dooN	r   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)r<   r-   r;   )shaper0   r1   r2   reshaper+   	transposer,   r5   rB   update_and_fetchr	   r-   r3   )r6   r:   r;   r<   BLDquerieskeysvaluesoutputs              r    __call__zAttention.__call__,   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r   NNr   r   r   r   r*   mxarrayr   r   rN   __classcell__r9   s   @r    r"   r"      sY    OY O( $(#	#88# rxx # }	#
 
#r   r"   c                   $     e Zd Z fdZd Z xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        y )NTr%   )r)   r*   r.   r/   c_fcc_proj)r6   r7   
hidden_dimr9   s      r    r*   zMLP.__init__L   s6    IIc:D9	ii
Cd;r   c                 h    | j                  t        j                  | j                  |                  S N)rY   r.   gelurX   )r6   r:   s     r    rN   zMLP.__call__Q   s"    {{277499Q<011r   )r   r   r   r*   rN   rS   rT   s   @r    rV   rV   K   s    <
2r   rV   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr#   c                    t         |           |j                  | _        |j                  | _        t        |      | _        t        |j                  |j                        | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)r)   r*   r   r   r+   r"   	self_attnrV   r   mlpr.   	LayerNormr   input_layernormpost_attention_layernormr#   r6   r#   r9   s     r    r*   zTransformerBlock.__init__V   s    ++//"4t'')?)?@!||D,<,<$BSBST(*$"3"3)
% 	r   r:   r;   r<   r=   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r\   )rc   rf   rd   rg   )r6   r:   r;   r<   rhouts          r    rN   zTransformerBlock.__call__c   sQ     NN4//2D%@EHHT22156!e
r   rO   rP   rT   s   @r    r_   r_   U   sW    Y   $(#	
88
 rxx 
 }	

 

r   r_   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )Starcoder2Modelr#   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r#   ra   )r)   r*   r#   r   r   r.   	Embeddingr   embed_tokensranger_   layersre   r   norm)r6   r#   _r9   s      r    r*   zStarcoder2Model.__init__q   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 LL!1!1t7H7HI	
s   Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rq   lenrs   r   ziprt   )r6   rv   r<   rk   r;   layercs          r    rN   zStarcoder2Model.__call__}   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r   r\   )	r   r   r   r   r*   rQ   rR   rN   rS   rT   s   @r    rn   rn   p   s'    
JY 
J r   rn   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr#   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr%   )r)   r*   r#   r   rn   modelr   r.   r/   r   r   lm_headrh   s     r    r*   zModel.__init__   sX    	//$T*
''99T%5%5tUSDL (r   rv   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r\   )r   r#   r   rq   	as_linearr   )r6   rv   r<   rl   s       r    rN   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r   c                 .    | j                   j                  S r\   )r   rs   )r6   s    r    rs   zModel.layers   s    zz   r   r\   )r   r   r   r   r*   rQ   rR   rN   propertyrs   rS   rT   s   @r    r}   r}      s;    TY T 

 ! !r   r}   )dataclassesr   typingr   r   mlx.corecorerQ   mlx.nnr.   r(   r   r   r	   r   Moduler"   rV   r_   rn   r}   r   r   r    <module>r      s    "     T T 
% 
% 
%.#		 .#b2")) 2ryy 6bii >!BII !r   