
    (HJj                     |   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 d dl
ZddlmZ ddlmZmZmZ e G d de             Z G d	 d
e	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                   ~    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   d
Zeed<   dZ	e
ed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_size   n_shared_headi'  
rope_thetaFrope_traditionalN)__name__
__module____qualname__str__annotations__intfloatr   r   r   bool     ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/plamo.pyr   r      sI    OOM3J"d"r!   r   c            	            e Zd Zdeddf fdZ	 	 d	dej                  deej                     dee   dej                  fdZ	 xZ
S )
	AttentionconfigreturnNc                    t         |           || _        |j                  | _        | j                  |j                  z  }|j                  | _        |x| _        | _        t        t        j                  | j
                  |j                  z              x| _        | _        |dz  | _        t        j                   | j                  | j
                  | j                  z  d      | _        t        j                   | j                  | j                  | j                  z  d      | _        t        j                   | j                  | j                  | j                  z  d      | _        t        j                   | j
                  | j                  z  | j                  d      | _        t        j*                  ||j,                  |j.                  d      | _        y )Ng      Fbiasg      ?)traditionalbasescale)super__init__r%   r   r   q_num_headsqk_dimv_dimr   npceilr   k_num_headsv_num_headsr,   nnLinearq_projk_projv_projo_projRoPEr   r   
rotary_emb)selfr%   head_dim	__class__s      r"   r.   zAttention.__init__   sv   !--##v'A'AA!55#++dj.1GGD$$v';';;</
 	
4+ t^
iid..<5
 iid..<5
 iid..;%
 iitzz)4+;+;%
 ''//""	
r!   hidden_statesattention_maskcachec                 &   |j                   \  }}}| j                  |      }| j                  |      }| j                  |      }	|j	                  ||| j
                  | j                        j                  dddd      }|j	                  ||| j                  | j                        j                  dddd      }|	j	                  ||| j                  | j                        j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        j                  |d| j                   j"                  ddg      }t        j                  |	d| j                   j"                  ddg      }	t%        |||	|| j&                  |      }
|
j                  dddd      j	                  ||d      }
| j)                  |
      S )Nr      r      )offset)rC   r,   mask)shaper8   r9   r:   reshaper/   r0   	transposer4   r5   r1   r=   rG   update_and_fetchmxtiler%   r   r
   r,   r;   )r>   rA   rB   rC   bszq_len_querieskeysvaluesoutputs              r"   __call__zAttention.__call__>   s    &++UA++m,{{=)]+ //#ud.>.>LVVq!Q
 ||C(8(8$++FPPq!Q
 UD,<,<djjISSq!Q
 oogelloCG??4?=D 11$?LD&oog.G??4(Dwwta!:!:AqAB!T[[%>%>1!EF-**
 !!!Q1-55c5"E{{6""r!   NNr   r   r   r   r.   rN   arrayr   r   rW   __classcell__r@   s   @r"   r$   r$      s_    
y 
T 
H .2#	+#xx+# !*+# }	+#
 
+#r!   r$   c                   `     e Zd Zdeddf fdZdej                  dej                  fdZ xZS )MLPr%   r&   Nc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        y NFr(   )
r-   r.   r%   r   r   r6   r7   	gate_projup_proj	down_projr>   r%   r@   s     r"   r.   zMLP.__init__m   s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr!   xc                 t    | j                  t        | j                  |      | j                  |                  S N)rc   r   ra   rb   )r>   re   s     r"   rW   zMLP.__call__v   s)    ~~fT^^A%6QHIIr!   )	r   r   r   r   r.   rN   rZ   rW   r[   r\   s   @r"   r^   r^   l   s7    Yy YT YJ"(( Jrxx Jr!   r^   c                   v     e Zd Zdeddf fdZ	 	 d	dej                  deej                     dee   fdZ	 xZ
S )
PlamoDecoderLayerr%   r&   Nc                     t         |           || _        |j                  | _        t	        |      | _        t        |      | _        t        j                  |j                  |j                        | _        y N)eps)r-   r.   r%   r   r$   	self_attnr^   mlpr6   RMSNormr   normrd   s     r"   r.   zPlamoDecoderLayer.__init__{   sX    !--"6*v;JJv11v7J7JK	r!   rA   rB   rC   c                     |}| j                  |      }| j                  |||      }| j                  |      }||z   |z   }|S )N)rA   rB   rC   )rp   rm   rn   )r>   rA   rB   rC   residualhidden_states_sahidden_states_mlps          r"   rW   zPlamoDecoderLayer.__call__   s^     !		-0  >>') * 
 !HH]3 #336GGr!   rX   rY   r\   s   @r"   ri   ri   z   sS    Ly LT L .2#	xx !* }	r!   ri   c                   (     e Zd Zdeddf fdZ xZS )PlamoDecoderr%   r&   Nc                     t         |           t        |j                        D cg c]  }t	        |       c}| _        y c c}w rg   )r-   r.   ranger   ri   layers)r>   r%   rR   r@   s      r"   r.   zPlamoDecoder.__init__   s@    /4V5M5M/N
/N!f%/N
 
s   A)r   r   r   r   r.   r[   r\   s   @r"   rv   rv      s    
y 
T 
 
r!   rv   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )
PlamoModelr%   c                 0   t         |           || _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |      | _	        t	        j                  |j                  |j                        | _        y rk   )r-   r.   r%   r   r6   	Embeddingr   embed_tokensrv   ry   ro   r   rp   rd   s     r"   r.   zPlamoModel.__init__   sk     ++LL):):F<N<NO"6*JJv11v7J7JK	r!   inputsrC   r&   c                 B   | j                  |      }|5t        t        | j                  j                              D cg c]  }d  }}t	        ||d         }t        | j                  j                  |      D ]  \  }} ||||      } | j                  |      S c c}w )Nr   )rC   )r~   rx   lenry   r	   ziprp   )r>   r   rC   hrR   rH   layercs           r"   rW   zPlamoModel.__call__   s    
 f%=#(T[[-?-?)@#AB#AaT#AEB$Qa1DKK..6HE1aQ'A 7 yy| Cs   	Brg   rY   r\   s   @r"   r{   r{      sC    Ly L  $ } 
	r!   r{   c                   ~     e Zd Zdeddf fdZ	 d	dej                  dee   dej                  fdZ	e
d        Z xZS )
Modelargsr&   Nc                     t         |           |j                  | _        t        |      | _        t        j                  |j                  |j                  d      | _	        || _
        y r`   )r-   r.   r   r{   modelr6   r7   r   r   lm_headr   )r>   r   r@   s     r"   r.   zModel.__init__   sO    //%
"$))dooE#
 	r!   r   rC   c                 H    | j                  ||      }| j                  |      S rg   )r   r   )r>   r   rC   outs       r"   rW   zModel.__call__   s#    
 jj'||C  r!   c                 B    | j                   j                  j                  S rg   )r   ry   )r>   s    r"   ry   zModel.layers   s    zz  '''r!   rg   )r   r   r   r   r.   rN   rZ   r   r   rW   propertyry   r[   r\   s   @r"   r   r      s\    Y 4   $!! }! 
	! ( (r!   r   )dataclassesr   typingr   r   mlx.corecorerN   mlx.nnr6   numpyr2   activationsr   r+   r   r	   r
   r   Moduler$   r^   ri   rv   r{   r   r    r!   r"   <module>r      s    "       T T 
# 
# 
#M#		 M#`J")) J		 D
299 
 8(BII (r!   