
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)SuScaledRoPEc                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed<   dZ	eed<   dZ
eed<   dZeeeeeef   f      ed<   dZeed<   y)	ModelArgs
model_typehidden_sizedim_model_basenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_key_value_headsq_lora_rankqk_nope_head_dimqk_rope_head_dimkv_lora_rankscale_depth	scale_embmax_position_embeddingsFattention_biasg    .A
rope_thetarope_traditionalNrope_scalingtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr    boolr!   r"   r#   r   r   r   r$        `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/minicpm3.pyr   r      s    OO   ND !J!"d";?L(4U3:%6 678? %%r.   r   c            
            e Zd Zdef fdZ	 	 ddej                  deej                     deee	ej                  f      fdZ
 xZS )	Attentionargsc                    t         |           || _        | j                  j                  | _        | j                  j                  | _        | j                  j
                  | _        | j                  j                  | _        | j                  j                  | _        | j                  j                  | _	        | j                  j                  | _
        | j                  | j                  j                  z  | _        | j                  | j                  z   | _        | j                  dz  | _        t        j                  | j                  | j                  | j
                        | _        t        j"                  | j                        | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z   | j
                        | _        t        j"                  | j                        | _        t        j                  | j                  | j                  | j                  | j                  z
  | j                  z   z  d      | _        t        j                  | j                  | j                  z  | j                  | j
                        | _        t1        |j                  |j2                  |j4                  |j6                  j9                  dd      |j6                  j9                  dd      |j6                  j9                  dd      	      | _        y )
Ng      ࿩biasF original_max_position_embeddingsi   short_factorg      ?long_factor)dimsbaser   r6   r7   r8   )super__init__r2   r   r   r    r   r   	num_headsr   r   
v_head_dim
q_head_dimsoftmax_scalennLinearq_a_projRMSNormq_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernorm	kv_b_projo_projr   r!   r   r#   getropeselfr2   	__class__s     r/   r<   zAttention.__init__(   s{   	 $		 : : $		 : :"ii66 II226699009900**dii.K.KK//$2G2GG!__6		d..T5H5H
  ZZ(8(89		dnnt>U
 #%)) 5 55$$#
 !jj):):;NN!6!66HJ	
 iiNNT__,$$
 !&&$($@$@-1->->-B-B2D. **..~sC))--mSA	
	r.   xmaskcachec                 4   |j                   \  }}}| j                  | j                  | j                  |                  }|j	                  ||| j
                  d      j                  dddd      }t        j                  || j                  gd      \  }}	| j                  |      }
t        j                  |
| j                  gd      \  }
}|j	                  ||d| j                        j                  dddd      }| j                  | j                  |
            }|j	                  ||| j
                  d      j                  dddd      }t        j                  || j                  gd      \  }}|;| j                  |	|j                         }	| j                  ||j                         }n"| j                  |	      }	| j                  |      }t        j"                  ||| j
                  || j                  f      }t        j$                  ||	gd      }t        j$                  ||gd      }||j'                  ||      \  }}t)        ||||| j*                  |      }|j                  dddd      j	                  ||d      }| j-                  |      S )	Nr      r      )axis)offset)rR   scalerQ   )shaperF   rE   rC   reshaper=   	transposemxsplitr   rG   r   r   rI   rH   rL   rX   broadcast_toconcatenateupdate_and_fetchr   r@   rJ   )rN   rP   rQ   rR   BL_qq_nopeq_pecompressed_kvk_pekvk_nopevaluesk_pe_broadcastedquerieskeysoutputs                      r/   __call__zAttention.__call__a   sV    ''1a MM$,,T]]1-=>?IIaDNNB/99!Q1ExxD$9$9#:D //2 hh}t7H7H6IPRSt||Aq!T%:%:;EEaAqQ^^D//>?ZZ1dnnb1;;Aq!QG"t'<'<&=BG 99T%,,97D99T%,,97D99T?D99T?D ??1dnna)>)>?

 ..&$b9 ~~v'78rB  11$?LD& .T6d6H6Ht
 !!!Q1-55aB?{{6""r.   NN)r%   r&   r'   r   r<   r]   arrayr   r   r(   rq   __classcell__rO   s   @r/   r1   r1   '   sX    7
Y 7
x $(/3	4#884# rxx 4# S"((]+,	4#r.   r1   c                   $     e Zd Z fdZd Z xZS )MLPc                 H   t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        y NFr4   )	r;   r<   rA   rB   r   r   	gate_projup_proj	down_projrM   s     r/   r<   zMLP.__init__   sq    4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr.   c                 t    | j                  t        | j                  |      | j                  |                  S N)r|   r	   rz   r{   )rN   rP   s     r/   rq   zMLP.__call__   s)    ~~fT^^A%6QHIIr.   )r%   r&   r'   r<   rq   rt   ru   s   @r/   rw   rw      s    YJr.   rw   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DecoderLayerr2   c                    t         |           || _        |j                  | _        |j                  | _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        |j                  | _        |j                  | _        y )Neps)r;   r<   r2   r   r   r1   	self_attnrw   mlprA   rD   r   input_layernormpost_attention_layernormr   rM   s     r/   r<   zDecoderLayer.__init__   s    	++!%!7!7"4t9!zz$*:*:@Q@QR(*

$"3"3)
%  ++!%!7!7r.   rP   rQ   rR   returnc                    | j                  | j                  |      ||      }||| j                  | j                  dz  z  z  z   }| j	                  | j                  |            }||| j                  | j                  dz  z  z  z   }|S )Ng      ?)r   r   r   r   r   r   )rN   rP   rQ   rR   rhouts          r/   rq   zDecoderLayer.__call__   s     NN4//2D%@T%%)?)?)DEFFHHT22156!t''4+A+A3+FGHH
r.   rr   )r%   r&   r'   r   r<   r]   rs   r   r   rq   rt   ru   s   @r/   r   r      sW    8Y 8& $(#	
88
 rxx 
 }	

 

r.   r   c                   b     e Zd Zdef fdZ	 	 ddej                  dej                  fdZ xZS )MiniCPM3Modelr2   c                    t         |           || _        |j                  | _        | j                  dkD  sJ t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t	        j                  |j                  |j                        | _        y c c}w )Nr   r   )r;   r<   r2   r   rA   	Embeddingr   embed_tokensranger   r   layersrD   r   norm)rN   r2   rd   rO   s      r/   r<   zMiniCPM3Model.__init__   s    	//"""LL$:J:JK389O9O3PQ3Pa|D)3PQJJt//T5F5FG	 Rs   ?C
inputsrQ   c                    | j                  |      | j                  j                  z  }|t        ||      }|d gt	        | j
                        z  }t        | j
                  |      D ]  \  }} ||||      } | j                  |      S r~   )r   r2   r   r   lenr   zipr   )rN   r   rQ   rR   r   layercs          r/   rq   zMiniCPM3Model.__call__   s     f%		(;(;;<(E2D=FS--EDKK/HE1aq!A 0 yy|r.   rr   )	r%   r&   r'   r   r<   r]   rs   rq   rt   ru   s   @r/   r   r      s8    HY H 	 hhr.   r   c                   r     e Zd Zdef fdZ	 	 ddej                  dej                  fdZed        Z	 xZ
S )Modelr2   c                    t         |           || _        |j                  | _        t	        |      | _        | j                  j                  s2t        j                  |j                  |j                  d      | _        y y ry   )r;   r<   r2   r   r   modelr$   rA   rB   r   r   lm_headrM   s     r/   r<   zModel.__init__   s\    	//"4(
yy,,99T%5%5tUSDL -r.   r   rQ   c                 $   | j                  |||      }| j                  j                  sA| j                  || j                  j                  | j                  j
                  z  z        }|S | j                   j                  j                  |      }|S r~   )r   r2   r$   r   r   r   r   	as_linear)rN   r   rQ   rR   r   s        r/   rq   zModel.__call__   sw     jju-yy,,,,sdii&;&;dii>V>V&VWXC 
 **))33C8C
r.   c                 .    | j                   j                  S r~   )r   r   )rN   s    r/   r   zModel.layers   s    zz   r.   rr   )r%   r&   r'   r   r<   r]   rs   rq   propertyr   rt   ru   s   @r/   r   r      sL    TY T 	 hh ! !r.   r   )dataclassesr   typingr   r   r   r   mlx.corecorer]   mlx.nnrA   activationsr	   r:   r
   r   r   
rope_utilsr   r   Moduler1   rw   r   r   r   r-   r.   r/   <module>r      s    " - -    T T $ & & &0n#		 n#bJ")) J299 <BII >!BII !r.   