
    (HJj2                     "   d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ e G d
 de             Z eej2                  d      d        Z G d dej6                        Z G d dej6                        Zej2                  d        Z G d dej6                        Z G d dej6                        Z  G d dej6                        Z! G d dej6                        Z" G d dej6                        Z#y)    )	dataclass)partial)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_rope)	SwitchGLUc                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   dZe	e
eeeef   f      ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZe	e   ed<   dZeed<   dZeed<   dZeed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   dZe	e   ed'<   dZeed(<   y))	ModelArgs
model_typehidden_sizeintermediate_sizemax_position_embeddingsmoe_intermediate_sizenum_expertsnum_shared_expertsnorm_topk_probnum_attention_headsnum_experts_per_toknum_hidden_layersnum_key_value_headsrms_norm_eps
rope_theta
vocab_sizefirst_k_dense_replaceNrope_scalingFuse_biasuse_qkv_bias	norm_headnorm_softmaxuse_qk_normtie_word_embeddingsg      ?partial_rotary_factor
rotary_dimmoe_router_enable_expert_biasT moe_router_enable_routed_scalingrouted_scaling_factorsoftmaxscore_functionr	   n_group   
topk_group#moe_shared_expert_intermediate_sizemoe_router_enable_shared_expert)__name__
__module____qualname__str__annotations__intboolfloatr"   r   r   r   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r/   r0   r2   r3   r4        c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/bailing_moe.pyr   r      s1   O  O;?L(4U5#:%6 678?HdL$ItL$K %%#&5& $J$*/!4/-1$d1#&5&#NC#GSJ9='#=,0#T0r>   r   T)	shapelessc                 d    | |d   z  j                  d      j                  | j                        S )N).Naxis)sumastypedtype)expert_outputsscoress     r?   aggregate_expert_outputsrJ   5   s5     
&+	+00b09@@AUAUVr>   c                   P     e Zd Zddedee   f fdZdej                  fdZ	 xZ
S )BailingMoeMLPargsr   c                    t         |           ||n|j                  | _        t        j                  |j
                  | j                  |j                        | _        t        j                  | j                  |j
                  |j                        | _        t        j                  |j
                  | j                  |j                        | _	        y )Nbias)
super__init__r   nnLinearr   r#   	gate_proj	down_projup_proj)selfrM   r   	__class__s      r?   rR   zBailingMoeMLP.__init__=   s     !, '' 	 d444==
 ""D$4$44==
 yyd444==
r>   returnc                 t    | j                  t        | j                  |      | j                  |                  S N)rV   r
   rU   rW   rX   xs     r?   __call__zBailingMoeMLP.__call__O   s)    ~~fT^^A%6QHIIr>   r\   )r5   r6   r7   r   r   r:   rR   mxarrayr_   __classcell__rY   s   @r?   rL   rL   <   s,    
Y 
8C= 
$JRXX Jr>   rL   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	BailingMoeAttentionrM   c                    t         |           |j                  | _        |j                  | _        |j                  | _        |j
                  | j                  z  | _        | j                  dz  | _        t        j                  |j
                  | j                  d| j                  z  z   | j                  z  |j                        | _        t        j                  | j                  | j                  z  |j
                  |j                        | _        |j                  r`t        j                  | j                  |j                        | _        t        j                  | j                  |j                        | _        |j$                  x}"t'        | j                  |j(                  z        }t+        ||j,                  d|j.                  |j0                        | _        y )Ng         rO   epsF)traditionalscaling_configr   )rQ   rR   r'   r   r   r   head_dimscalerS   rT   r$   query_key_valuer#   denseRMSNormr   key_layernormquery_layernormr*   r:   r)   r   r   r"   r   rope)rX   rM   rope_dimrY   s      r?   rR   zBailingMoeAttention.__init__T   sq   ++#'#;#; #'#;#; ((D,D,DD]]D(
!yy%%D,D,D(DDU"" 

 YY$$t}}4

 !#DMMt?P?P!QD#%::dmmARAR#SD 'H04==4+E+EEFH#OO,,$($@$@
	r>   r^   maskcacherZ   c                 ,   |j                   \  }}}| j                  |      }| j                  | j                  z  }| j                  | j                  z  }	t        j                  ||||	z   gd      \  }
}}|
j                  ||| j                  | j                        j                  dddd      }|j                  ||| j                  | j                        j                  dddd      }|j                  ||| j                  | j                        j                  dddd      }| j                  r"| j                  |      }| j                  |      }|P| j                  ||j                        }| j                  ||j                        }|j                  ||      \  }}n"| j                  |      }| j                  |      }t        ||||| j                   |      }|j                  dddd      j                  ||d      }| j#                  |      S )	NrC   r   rg   r	      )offset)rv   rm   ru   )shapern   r   rl   r   r`   splitreshape	transposer'   rr   rq   rs   rz   update_and_fetchr   rm   ro   )rX   r^   ru   rv   BLDqkvq_sizekv_sizeqkvquerieskeysvaluesoutputs                    r?   r_   zBailingMoeAttention.__call__u   s    ''1a""1%))DMM9**T]]:((3')9 :D1a))Aq$":":DMMJTTq!Q
 yyAt77GQQq!Q
 1a!9!94==ISSq!Q
 **73G%%d+Diii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?zz&!!r>   NNr5   r6   r7   r   rR   r`   ra   r   r   r_   rb   rc   s   @r?   re   re   S   sX    
Y 
H $(#	)"88)" rxx )" }	)"
 
)"r>   re   c                    | j                   }|dk(  r3t        j                  | j                  t        j                              }	n4t        j
                  | j                  t        j                        d      }	|	}
||	|z   }	|dkD  rt        j                  |	d|df      }	t        j                  |	dd      j                  dd      }||z
  }t        j                  ||dz
  d	
      dd |d d f   }t        j                  |	t        j                  |      t        j                  d|	j                         d	      }	t        j                  |	d	d      }	|}t        j                  |	| d
      d| d f   }t        j                  |
|d      }	|dkD  r|r|	j                  dd      dz   }|	|z  }	|	|z  }	||	j                  |      fS )Nsigmoidrx   rC   r	   )rD   r{   rg   TrD   keepdimsrB   )kthrD   .g        g#B;)rG   r`   r   rF   float32r.   	unflattentopkrE   argpartitionput_along_axisstop_gradientra   flattentake_along_axis)gatese_score_correction_biastop_kr0   r2   r-   r   r/   in_typerI   orig_scoresgroup_scoresr   	group_idxindsdenominators                   r?   group_expert_selectr      s    kkG"ELL45ELL42>K*11{f2gr]Cwwvqr266B6Nj OOLa!e"Ec2A2qjQ	""B$$Y/#v||1LSU
 FB+A??6r3C!H=DT;Fqy^jjb4j85@+%++Fw'''r>   c                   *     e Zd Zdef fdZd Z xZS )BailingMoeGaterM   c                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  |j                  |j                  d      | _        |j                  r t!        j"                  |j                  f      nd | _        |j&                  | _        y NFrO   )rQ   rR   r   r   r   r0   r2   r-   r,   enable_routed_scalingrS   rT   r   r   rU   r+   r`   zerosexpert_biasr/   rX   rM   rY   s     r?   rR   zBailingMoeGate.__init__   s    "11--
||//%)%?%?"%)%J%J"4#3#3T5E5EER 11 HHd&&() 	
 #11r>   c           
          t        | j                  |      | j                  | j                  | j                  | j
                  | j                  | j                  | j                        S r\   )	r   rU   r   r   r0   r2   r-   r   r/   r]   s     r?   r_   zBailingMoeGate.__call__   sT    "NN1JJLLOO&&	
 		
r>   r5   r6   r7   r   rR   r_   rb   rc   s   @r?   r   r      s    2Y 2$

r>   r   c                   *     e Zd Zdef fdZd Z xZS )BailingMoeSparseMoeBlockrM   c                    t         |           || _        |j                  | _        t	        |j
                  |j                  |j                  |j                        | _	        t        |      | _        |j                  xs |j                  }|j                  dkD  r,|j                  r t        |||j                  z        | _        y d | _        y )NrO   r   )rM   r   )rQ   rR   rM   r   r   r   r   r   r#   
switch_mlpr   gater3   r   r4   rL   shared_experts)rX   rM   
shared_dimrY   s      r?   rR   z!BailingMoeSparseMoeBlock.__init__   s    	#'#;#; #&&	
 #4(	44R8R8R 	 &&*t/S/S	 ",t/F/F"F 	  	r>   c                     | j                  |      \  }}| j                  ||      }t        ||      }| j                  || j                  |      z   }|S r\   )r   r   rJ   r   )rX   r^   topk_idxtopk_weightouts        r?   r_   z!BailingMoeSparseMoeBlock.__call__  sV     $		!+ooa*&sK8*++A..C
r>   r   rc   s   @r?   r   r      s    
Y 
.r>   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
BailingMoeDecoderLayerrM   	layer_idxc                 n   t         |           t        |      | _        |j                  ||j
                  k\  rt        |      n
t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nrh   )rQ   rR   re   	attentionr   r!   r   rL   mlprS   rp   r   r   input_layernormpost_attention_layernorm)rX   rM   r   rY   s      r?   rR   zBailingMoeDecoderLayer.__init__  s    ,T2
   ,d>X>X1X %T* t$ 	  "zz$*:*:@Q@QR(*

$"3"3)
%r>   r^   ru   rv   rZ   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S r\   )r   r   r   r   )rX   r^   ru   rv   rhs         r?   r_   zBailingMoeDecoderLayer.__call__  sL     NN4//2D%@EHHT221561ur>   r   )r5   r6   r7   r   r:   rR   r`   ra   r   r   r_   rb   rc   s   @r?   r   r     s^    
Y 
3 
& $(#		88	 rxx 	 }		
 
	r>   r   c                   R     e Zd Zdef fdZ	 ddej                  dee   fdZ	 xZ
S )BailingMoeModelrM   c                 P   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )N)r   rh   )rQ   rR   rS   	Embeddingr    r   word_embeddingsranger   r   layersrp   r   norm)rX   rM   irY   s      r?   rR   zBailingMoeModel.__init__)  s    !||DOOT=M=MN 4112
2 #4152
 JJt//T5F5FG		
s   B#inputsrv   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r   zipr   )rX   r   rv   r   ru   layercs          r?   r_   zBailingMoeModel.__call__2  su    
   (=FS--E$Qa1DKK/HE1aq!A 0 yy|r>   r\   r   rc   s   @r?   r   r   (  s5    HY H  $ }r>   r   c                   ~     e Zd Zdef fdZ	 d	dej                  fdZd Ze	d        Z
e	d        Ze	d        Z xZS )
ModelrM   c                    t         |           || _        |j                  | _        |j                  | _        t        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y r   )rQ   rR   rM   r%   r   r   modelr(   rS   rT   r   r    lm_headr   s     r?   rR   zModel.__init__E  sc    	//$T*
''99T%5%5tUSDL (r>   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r\   )r   rM   r(   r   	as_linearr   )rX   r   rv   r   s       r?   r_   zModel.__call__N  sW    
 jj'99((**,,66s;C 
 ,,s#C
r>   c                 b   | j                   j                  r|j                  dd        | j                  rj|d   }|j                  }t
        j                  j                  |j                  t
        j                        dd      dz   }||z  j                  |      |d<   t        | j                   j                        D ]  }d| }|| j                   j                  k\  s"dD ]|  }dD ]u  }| d	| d
| |v st        | j                   j                        D 	cg c]  }	|j                  | d|	 d
| d
|         }
}	t        j                  |
      || d| d
| <   w ~ | d|v r|j                  | d      }||| d<   | d|v s|j                  | d      }||| d<    |S c c}	w )Nzlm_head.weightr   Tr   gHz>zmodel.layers.)rU   rV   rW   )weightscalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.z.mlp.gate.weightz.mlp.gate.gate_proj.weightz.mlp.gate.biasz.mlp.gate.gate_proj.bias)rM   r(   popr%   rG   r`   linalgr   rF   r   r   r   r!   r   stack)rX   weightswrG   weight_normlprefixmr   eto_joingate_weight	gate_biass                r?   sanitizezModel.sanitizeZ  s   99((KK($/>>()AGGE		qxx

3!dKdR  *+[(@(@(GG$%tyy223A$QC(FDII333>A;$X_QCq<G */tyy/D/D)E')EA !(vhmA3as!A3,O P)E $ ' KM(( 'KGvh.>qc1#$FG < ? X-.'9")++8H.I"JKEPGvh&@ABX^,7 'vhn,E FICLGvh&>?@+ 4. !'s   #F,
c                     d }|S )Nc                 0    | j                  d      rdddS y)Nzmlp.gate.gate_proj@      )
group_sizebitsT)endswith)path_s     r?   	predicatez(Model.quant_predicate.<locals>.predicate  s    }}12&(!44r>   r=   rX   r   s     r?   quant_predicatezModel.quant_predicate  s    	
 r>   c                     d }|S )Nc                 
    d| vS )Nr   r=   )r   s    r?   r   z'Model.cast_predicate.<locals>.predicate  s     ))r>   r=   r   s     r?   cast_predicatezModel.cast_predicate  s    	* r>   c                 .    | j                   j                  S r\   )r   r   )rX   s    r?   r   zModel.layers  s    zz   r>   r\   )r5   r6   r7   r   rR   r`   ra   r_   r   propertyr   r   r   rb   rc   s   @r?   r   r   D  si    TY T 

#J     ! !r>   r   )$dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorer`   mlx.nnrS   activationsr
   baser   r   r   
rope_utilsr   switch_layersr   r   compilerJ   ModulerL   re   r   r   r   r   r   r   r=   r>   r?   <module>r     s    "  - -    T T ' $ !1 !1 !1H 	t$ %JBII J.K"")) K"\ %( %(P
RYY 
@ryy BRYY :bii 8M!BII M!r>   