
    (HJj                        d dl Z d dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
 ddlmZmZ ddlmZ e G d d             Z G d	 d
e
j"                        Z G d de
j"                        Z G d de
j"                        Z G d de
j"                        Z G d de
j"                        Z G d de
j"                        Z G d de
j"                        Zy)    N)	dataclass)Tuple   )create_attention_maskscaled_dot_product_attention)	SwitchMLPc                       e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed	<   d
Zeed<   dZeed<   ed        Zy)	ModelArgs
model_typei   	num_vocabi 
  	model_dim    	num_heads
num_layers
rotary_dim   num_experts_per_tok   num_local_expertsc                      | di |j                         D ci c])  \  }}|t        j                  |       j                  v r||+ c}}S c c}}w )N )itemsinspect	signature
parameters)clsparamskvs       `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/phixtral.py	from_dictzModelArgs.from_dict   s]     
 #LLN*DAq))#.999 1*
 	
s   .AN)__name__
__module____qualname__str__annotations__r   intr   r   r   r   r   r   classmethodr!   r       r    r
   r
      sa    OIsIsIsJJ  s
 
r)   r
   c                   4     e Zd Zdededef fdZddZ xZS )RoPEAttentiondimsr   r   c                     t         |           || _        t        j                  |d      | _        t        j                  |d|z        | _        t        j                  ||      | _        y )NF)traditional   )	super__init__r   nnRoPEropeLinearWqkvout_proj)selfr,   r   r   	__class__s       r    r1   zRoPEAttention.__init__&   sO    "GGJE:	IIdAH-			$-r)   c                    | j                  |      }t        j                  |dd      \  }}}| j                  }|j                  \  }	}
}|j                  |	|
|d      j                  dddd      }|j                  |	|
|d      j                  dddd      }|j                  |	|
|d      j                  dddd      }|P| j                  ||j                        }| j                  ||j                        }|j                  ||      \  }}n"| j                  |      }| j                  |      }|j                  t        j                        }t        j                  d|j                  d   z        }t        |j                  t        j                        |||||      j                  |j                        }|j!                  dd      j                  |	|
d      }| j#                  |      S )	Nr/   axisr   r   r   )offset)cachescalemask)r6   mxsplitr   shapereshape	transposer4   r>   update_and_fetchastypefloat32mathsqrtr   dtypemoveaxisr7   )r8   xrA   r?   qkvquerieskeysvaluesr   BLDr@   outputs                 r    __call__zRoPEAttention.__call__/   s   iil "ab 9v NN	--1a //!Q	26@@Aq!L||Aq)R0::1aAF1i4>>q!QJ iii=G99T%,,97D 11$?LD&ii(G99T?D.., 		!gmmB//0-NN2::&
 &
 	 A&..q!R8}}V$$r)   NN)r"   r#   r$   r'   r1   rW   __classcell__r9   s   @r    r+   r+   %   s"    .S .S .c .%%r)   r+   c                   d     e Zd Zdededef fdZdej                  dej                  fdZ xZ	S )MOEargsdim
hidden_dimc                 H   t         |           || _        || _        |j                  | _        |j                  | _        t        | j                  | j                  | j
                  d      | _        t        j                  |j                  | j
                  d      | _        y )NT)biasF)r0   r1   r^   r_   r   num_expertsr   r   
switch_mlpr2   r5   r   gate)r8   r]   r^   r_   r9   s       r    r1   zMOE.__init__X   s{    $11#'#;#; #HHdoot'7'7d
 IIdnnd.>.>UK	r)   rN   returnc                 `   | j                  |      }| j                  }t        j                  t        j                  | |dz
  d            dd |f   }t        j
                  ||d      }t        j                  |dd      }| j                  ||      }||d   z  j                  d	      }|S )
Nr   r;   )kthr=   .r<   T)r=   precise).N)	rd   r   rB   stop_gradientargpartitiontake_along_axissoftmaxrc   sum)r8   rN   gatesr   indsscoresys          r    rW   zMOE.__call__c   s    		!$$AE KLSRTSTRTWU##E4b9FT:OOAt$	""''R'0r)   )
r"   r#   r$   r
   r'   r1   rB   arrayrW   rY   rZ   s   @r    r\   r\   W   s<    	LY 	LS 	Lc 	L"(( rxx r)   r\   c                   *     e Zd Zdef fdZd Z xZS )ParallelBlockconfigc                     t         |           |j                  }|dz  }t        ||j                  |j
                        | _        t        j                  |      | _	        t        |||      | _        y )Nr   )r0   r1   r   r+   r   r   mixerr2   	LayerNormlnr\   moe)r8   rv   r,   mlp_dimsr9   s       r    r1   zParallelBlock.__init__r   s^    !8"4)9)96;L;LM
,,t$vtX.r)   c                 |    | j                  |      }| j                  |||      }| j                  |      }||z   |z   S N)rz   rx   r{   )r8   rN   rA   r?   hattn_hff_hs          r    rW   zParallelBlock.__call__z   s=    GGAJAtU+xx{}q  r)   r"   r#   r$   r
   r1   rW   rY   rZ   s   @r    ru   ru   q   s    /y /!r)   ru   c                   *     e Zd Zdef fdZd Z xZS )TransformerDecoderrv   c                     t         |           t        |      | _        t	        |j
                        D cg c]  }t        |       c}| _        y c c}w r~   )r0   r1   Embdembdranger   ru   r   )r8   rv   ir9   s      r    r1   zTransformerDecoder.__init__   sD    L	16v7H7H1IJ1IA-'1IJJs   Ac                     | j                  |      }|d gt        | j                        z  }t        | j                  |      D ]  \  }} ||||      } |S r~   )r   lenr   zip)r8   rN   rA   r?   layercs         r    rW   zTransformerDecoder.__call__   sT    IIaL=FS[(EDFFE*HE1aq!A +r)   r   rZ   s   @r    r   r      s    Ky K
r)   r   c                   *     e Zd Zdef fdZd Z xZS )r   rv   c                     t         |           t        j                  |j                  |j
                        | _        y r~   )r0   r1   r2   	Embeddingr   r   wter8   rv   r9   s     r    r1   zEmbd.__init__   s,    << 0 0&2B2BCr)   c                 $    | j                  |      S r~   )r   )r8   rN   s     r    rW   zEmbd.__call__   s    xx{r)   r   rZ   s   @r    r   r      s    Dy Dr)   r   c                   .     e Zd Zdeddf fdZd Z xZS )
OutputHeadrv   re   Nc                     t         |           t        j                  |j                        | _        t        j                  |j                  |j                        | _        y r~   )	r0   r1   r2   ry   r   rz   r5   r   linearr   s     r    r1   zOutputHead.__init__   sB    ,,v//0ii 0 0&2B2BCr)   c                 B    | j                  | j                  |            S r~   )r   rz   )r8   inputss     r    rW   zOutputHead.__call__   s    {{4776?++r)   r   rZ   s   @r    r   r      s    Dy DT D
,r)   r   c                        e Zd Zdef fdZ	 	 d	dej                  dej                  dej                  fdZd Ze	d        Z
 xZS )
Modelrv   c                     t         |           |j                  | _        t        |      | _        t        |      | _        || _        y r~   )r0   r1   r   r   transformerr   lm_headr]   r   s     r    r1   zModel.__init__   s;     ++-f5!&)	r)   rN   rA   re   c                 f    |t        ||      }| j                  |||      }| j                  |      S r~   )r   r   r   )r8   rN   rA   r?   rr   s        r    rW   zModel.__call__   s7     <(E2DQe,||Ar)   c                 p   d|vr|S t        | j                  j                        D ]  }d| }dD ]|  }dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j	                  | d| d| d|         }}t        j                  |      || d| d| <   w ~  |S c c}w )	Nz$transformer.h.0.moe.mlp.0.fc1.weightztransformer.h.)fc1fc2)weightscalesbiasesra   z.moe.mlp.0..z	.moe.mlp.z.moe.switch_mlp.)r   r]   r   r   poprB   stack)r8   weightslprefixnr   eto_joins           r    sanitizezModel.sanitize   s    1@Ntyy++,A%aS)F#?A QCq4? &+499+F+F%G#%G $KK6()A3as!A3(GH%G   # GIhhwFW6(*:1#Qqc BC @ $ - #s   &#B3
c                 .    | j                   j                  S r~   )r   r   )r8   s    r    layerszModel.layers   s    !!!r)   rX   )r"   r#   r$   r
   r1   rB   rs   rW   r   propertyr   rY   rZ   s   @r    r   r      s]    y  	88 hh
 
 " "r)   r   )r   rJ   dataclassesr   typingr   mlx.corecorerB   mlx.nnr2   baser   r   switch_layersr   r
   Moduler+   r\   ru   r   r   r   r   r   r)   r    <module>r      s      !    E $ 
 
 
*/%BII /%d")) 4!BII !   299 , ,&"BII &"r)   