
    (HJjD                        d dl Z d dlmZ d dlmZmZmZmZ d dlm	Z
 d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ e G d de             Z  G d dejB                        Z" G d dejB                        Z# G d dejB                        Z$ G d dejB                        Z% G d dejB                        Z& G d dejB                        Z' G d dejB                        Z(y)    N)	dataclass)AnyDictOptionalTuple)shard_inplaceshard_linearsum_gradients   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)	CacheListKVCache)MultiLinear)initialize_rope)	SwitchGLUc                   .   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed<   dZ	eed<   dZ
ee   ed<   y)	ModelArgs
model_typeattention_methodzero_expert_typehidden_sizeffn_hidden_sizemoe_topkexpert_ffn_hidden_sizen_routed_expertszero_expert_num
num_layers
vocab_sizemax_position_embeddingsnum_attention_headskv_lora_rankq_lora_rankqk_rope_head_dimqk_nope_head_dim
v_head_dimrouted_scaling_factorrms_norm_eps
rope_thetamla_scale_q_loramla_scale_kv_loraattention_biasFnorm_topk_probrouter_biasNrope_scaling)__name__
__module____qualname__str__annotations__intfloatboolr/   r0   r1   r   r        e/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/longcat_flash.pyr   r      s    OMOO  O   ND K#'L(4.'r;   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	LongcatFlashMLAargsc                 ^   t         |           |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _	        | j                  ?t        j                  |j                  | j                  | j                  z  d      | _        nt        j                  |j                  | j                  |j                        | _        t        j                   | j                        | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  |j                  | j
                  | j                  z   |j                        | _        t        j                   | j
                        | _        t+        | j                  | j
                  | j                        | _        t+        | j
                  | j                  | j                        | _        t        j                  | j                  |j                  z  |j                  |j                        | _        |j2                  r!|j                  | j                  z  dz  | _        |j4                  r!|j                  | j
                  z  dz  | _        |j6                  g|j6                  j9                  dd      }|rI|j6                  d   }|dkD  r5d	|z  t;        j<                  |      z  d
z   }| j                  |z  |z  | _	        t?        | j                  |j@                  d|j6                  |jB                        | _"        y )Ng      Fbiasg      ?mscale_all_dimr   factorr   g?g      ?T)dimsbasetraditionalscaling_configr"   )#super__init__r#   r&   r'   r$   r%   r(   qk_head_dimscalennLinearr   q_projr.   q_a_projRMSNormq_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernormr   embed_qunembed_outo_projr,   r-   r1   getmathlogr   r+   r"   rope)selfr?   rC   scaling_factors	__class__s        r<   rJ   zLongcatFlashMLA.__init__1   s   #'#;#;  $ 5 5 $ 5 5 --++//0043H3HH%%t+
#))  ((4+;+;;DK II  $"2"29L9LDM "$D,<,<!=DII  ((4+;+;;DM #%)) 5 55$$#

 !jj):):;"!!4#4#4d6N6N
 't0H0H
 ii$$t6$$
   %)%5%58H8H%HS$PD!!!&*&6&69J9J&Js%RD"(!..223CQGN!%!2!28!<!A%n,txx/GG#MA!%a!!3DJ#&&,,$($@$@
	r;   xmaskcachereturnc           	         |j                   \  }}}| j                  | j                  |      }n/| j                  | j	                  | j                  |                  }|j                  ||| j                  | j                        j                  dddd      }| j                  || j                  z  }t        j                  || j                  gd      \  }}	| j                  |      }
t        j                  |
| j                  gd      \  }
}|j                  ||d| j                         j                  dddd      }| j#                  |
      }| j$                  || j$                  z  }||j&                  nd}| j)                  |	|      }	| j)                  ||      }t        j*                  |d      }||j-                  ||      \  }}|	| j.                  z  |j1                  dd      z  }|\t        j2                  ||t        j4                  t        j6                  |j8                        j:                  |j8                              }|dk(  r| j=                  |      }|x}}n$| j=                  |d	      }| j?                  |      }tA        ||||| j.                  |
      }|dk(  r| j?                  |      }|j                  dddd      j                  ||d      }| jC                  |      S )Nr      r      axisF)	transpose)rc   rL   rb   )"shaper%   rO   rS   rR   rP   reshaper#   rK   rl   r,   mxsplitr'   rT   r$   r&   rU   r-   offsetr\   expand_dimsupdate_and_fetchrL   swapaxeswherearrayfinfodtypeminrV   rW   r   rX   )r]   ra   rb   rc   BL_qq_nopeq_pecompressed_kvk_pe	kv_latentrq   	pe_scoreskvoutputs                     r<   __call__zLongcatFlashMLA.__call__v   s    ''1a#AAd00q1ABCAIIaD44d6F6FGQQq!Q
   ,D)))AxxD$9$9#:D//2 hh}t7H7H6IPRSt||Aq!T%:%:;EEaAqQ''6	!!-!D$:$::I!&!2yyv&yyv&NN915	#44YEOItDJJ&$--B*??	)//266	HI 6\\&)FAY%8A  +A-AqTZZi
 6%%f-F!!!Q1-55aB?{{6""r;   NNr2   r3   r4   r   rJ   ro   rv   r   r   r   __classcell__r`   s   @r<   r>   r>   0   sZ    C
Y C
P $(#	=#88=# rxx =# }	=#
 
=#r;   r>   c                   b     e Zd Zddedef fdZdej                  dej                  fdZ xZ	S )LongcatFlashMLPr?   	is_expertc                 @   t         |           |r|j                  n|j                  }t	        j
                  |j                  |d      | _        t	        j
                  |j                  |d      | _        t	        j
                  ||j                  d      | _	        y NFrA   )
rI   rJ   r   r   rM   rN   r   	gate_projup_proj	down_proj)r]   r?   r   r   r`   s       r<   rJ   zLongcatFlashMLP.__init__   sq    5>d11DDXDX4#3#3[uMyy!1!1;UK;0@0@uMr;   ra   rd   c                 t    | j                  t        | j                  |      | j                  |                  S N)r   r   r   r   )r]   ra   s     r<   r   zLongcatFlashMLP.__call__   s)    ~~fT^^A%6QHIIr;   )F)
r2   r3   r4   r   r9   rJ   ro   rv   r   r   r   s   @r<   r   r      s7    NY N4 NJ"(( Jrxx Jr;   r   c                   z     e Zd Zdef fdZdej                  deej                  ej                  f   fdZ xZ	S )LongcatFlashTopkRouterr?   c                    t         |           || _        |j                  | _        |j
                  |j                  z   | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  |j                  | j
                  | j                        | _        t        j                  | j
                  f      | _        y )NrA   )rI   rJ   configr   top_kr   r   r)   r/   r0   rM   rN   r   
classifierro   zerose_score_correction_biasr]   r?   r`   s     r<   rJ   zLongcatFlashTopkRouter.__init__   s    ]]
 $ 5 58L8L L%)%?%?""11++))d33$:J:J
 (*xx1F1F0H'I$r;   hidden_statesrd   c                    |j                   }| j                  |      }t        j                  |d      }|| j                  z   }t        j
                  || j                   d      d| j                   d f   }t        j                  ||d      }| j                  r t        j                  |dd      dz   }||z  }|| j                  z  }||j                  |      fS )Nrh   ri   )kthrj   .Trj   keepdimsg#B;)rx   r   ro   softmaxr   argpartitionr   take_along_axisr/   sumr)   astype)	r]   r   rx   router_logitsscorescorrected_scorestopk_indicestopk_weightsdenominators	            r<   r   zLongcatFlashTopkRouter.__call__   s    ##6M3!D$@$@@'7djj[rR$**
 ))&,RH&&BFNK'+5L#d&@&@@\00777r;   )
r2   r3   r4   r   rJ   ro   rv   r   r   r   r   s   @r<   r   r      s:    JY J8bhh 852889K3L 8r;   r   c                   *     e Zd Zdef fdZd Z xZS )LongcatFlashMoEr?   c                 F   t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        t        |j                  |j                  |j
                        | _        t        |      | _        d | _        y r   )rI   rJ   r   r   num_experts_per_tokr   r   r   r   r   r   
switch_mlpr   routersharding_groupr   s     r<   rJ   zLongcatFlashMoE.__init__   s    #'==  $ 5 5#33 $ 5 5#''!!
 -T2"r;   c                 R   | j                    t        | j                         |      }| j                  |      \  }}|| j                  k\  }t	        j
                  |d|      }t	        j
                  |d|      }| j                  ||      }||d   z  }t	        j                  |d      }| j                   +t        j                  j                  || j                         }| j                  dk(  sJ t	        j                  t	        j
                  ||d      dd	
      }	|||	z  z   }|S )Nr   g        ).Nrk   ri   groupidentityrh   Tr   )r   r
   r   r   ro   ru   r   r   distributedall_sumr   )
r]   r   r   r   rb   regular_weightsregular_outputsweighted_outputsfinal_outputidentity_weights_sums
             r<   r   zLongcatFlashMoE.__call__   s    *>M$*=*=>}MM%)[[%?"l t444xxa6((4l;//-F*_Y-GGvv.R8*>>11D$7$7 2 L
 $$
222!vvHHT<-B 
 $m6J&JJr;   )r2   r3   r4   r   rJ   r   r   r   s   @r<   r   r      s    #Y #"r;   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	LongcatFlashDecoderLayerr?   c                 F   t         |           |j                  | _        t        |      | _        t        d      D cg c]  }t        |       c}| _        t        d      D cg c]  }t        |d       c}| _	        t        d      D cg c]-  }t        j                  |j                  |j                        / c}| _        t        d      D cg c]-  }t        j                  |j                  |j                        / c}| _        y c c}w c c}w c c}w c c}w )Nrf   F)eps)rI   rJ   r   r   mlpranger>   	self_attnr   mlpsrM   rQ   r*   input_layernormpost_attention_layernorm)r]   r?   r|   r`   s      r<   rJ   z!LongcatFlashDecoderLayer.__init__  s    ++"4(9>qBA/$/B;@8D8a_T518D	INq 
IQABJJt''T->->? 
 JOq)
IQABJJt''T->->?)
% CD 
)
s   D%D2D2Dra   rb   rc   rd   c                 H   |}d }|d}t        d      D ]  }|} | j                  |   |      } | j                  |   ||||         }||z   }|} | j                  |   |      }|dk(  r| j	                  |      } | j
                  |   |      }||z   }|dk(  s||z   } |S )Nr   rf   )rb   rc   r   r   )r   r   r   r   r   r   )r]   ra   rb   rc   r   shortcut_mlp_outputiresiduals           r<   r   z!LongcatFlashDecoderLayer.__call__'  s     "= EqA$H3D003MBM-DNN1-m$eTUhWM$}4M$H<D99!<]KMAv&*hh}&=#(DIIaL7M$}4MAv -0C C# & r;   r   r   r   s   @r<   r   r     sW    
Y 
" $(#	88 rxx  }	
 
r;   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )LongcatFlashModelr?   c                 l   t         |           |j                  | _        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j                  |j                        | _        y c c}w r   )rI   rJ   r    rM   	Embeddingr!   r   embed_tokensr   r   layersrQ   r*   norm)r]   r?   idxr`   s      r<   rJ   zLongcatFlashModel.__init__J  s    //LL$:J:JKAFtAWXAW#/5AWXJJt//1B1BC	 Ys   'B1ra   rc   rd   c                     | j                  |      }|dg| j                  z  }t        ||d   d   d      }t        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   r   T)return_array)rc   )r   r    r   zipr   r   )r]   ra   rc   hrb   layercs          r<   r   zLongcatFlashModel.__call__Q  sv    
 a =!NT__4E$Qa$GDKK/HE1aQ'A 0 yy|r;   r   r   r   s   @r<   r   r   I  sC    DY D  $88 } 
	r;   r   c                        e Zd Zdef fdZ	 ddej                  dee   fdZ	e
d        Ze
d        Ze
d        Zd	 Zd
 Zddeej                   j"                     fdZ xZS )Modelr?   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y r   )rI   rJ   r?   r   r   modelrM   rN   r   r!   lm_headr   s     r<   rJ   zModel.__init__d  sJ    	//&t,
yy!1!14??Or;   inputsrc   c                 H    | j                  ||      }| j                  |      S r   )r   r   )r]   r   rc   outs       r<   r   zModel.__call__k  s#    
 jj'||C  r;   c                 .    | j                   j                  S r   )r   r   )r]   s    r<   r   zModel.layerss  s    zz   r;   c                     d }|S )Nc                 0    | j                  d      rdddS y)Nr   @      )
group_sizebitsT)endswith)pathr|   s     r<   	predicatez(Model.quant_predicate.<locals>.predicatey  s    }}\*&(!44r;   r:   r]   r   s     r<   quant_predicatezModel.quant_predicatew  s    	
 r;   c                     d }|S )Nc                 
    d| vS )Nr   r:   )r   s    r<   r   z'Model.cast_predicate.<locals>.predicate  s    ,A55r;   r:   r   s     r<   cast_predicatezModel.cast_predicate  s    	6 r;   c                    t        | j                  j                        D ]  }d| }dD ]  \  }}dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j	                  | d| d| d|         }}t        j                  |      || d| d| <   w   t        | j                  j                        D ]  }t        d      D ]  }	d| d	|	 }| d
}
|
|v s| j                  j                  }| j                  j                  | j                  j                  z   }| d|v }|j	                  |
      }|r| j                  j                  }|j	                  | d      }|j	                  | d      }|j                  d   dz  |z  }||j                  d   z  }t        j                  |||||      }|j                  ||d      }t        j                  |d d d | j                  j                  d d f   j                  dd            }t        j                  |d d | j                  j                  d d d f         }|rXt        j                   |      \  }}}t        j                   |||      \  }}}||| d<   ||| d<   ||| d<   ||| d<   ||| d<   ||| d<     i }|j#                         D ]  \  }}|j%                  d      r|||<    |S c c}w )Nzmodel.layers.))w1r   )w2r   )w3r   )weightscalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.rf   z.self_attn.z.kv_b_proj.weightz.kv_b_proj.scalesz.kv_b_proj.biasesrh       )r   r   rk   z.embed_q.scalesz.embed_q.biasesz.unembed_out.scalesz.unembed_out.biasesz.embed_q.weightz.unembed_out.weightz	model.mtp)r   r?   r    r   popro   stackr#   r'   r(   r$   rm   
dequantizern   
contiguousrt   quantizeitems
startswith)r]   weightslprefixnmr   eto_joinr   kv_b_key	num_headshead_dim	quantizedr   rE   r   r   r   r   wkwvwk_swk_bwv_swv_bnew_weightss                              r<   sanitizezModel.sanitize  sR   tyy++,A$QC(FU17A 1QC8GC &+499+E+E%F#%F $KK6(-s!A3as(KL%F   # GIhhwFW6(*:1#Qqc BC 8 V - tyy++,A1X(;qc:$X%67w& $		 = =I#yy99DII<P<PPH#)(*; < GIH-A #yy55!(x7H-I!J!(x7H-I!J !b 0T9%)V\\"-=%=
MMvvDZ 		)Xr:A!9tyy9991<=FFr2NB qDII,F,F,H!)K'LMB )+Tj*D$ *,Tj*D$ ?C6(/ :;>B6(/ :;BF6(*= >?BF6(*= >?:<Gvho67>@Gvh&9:;M  -R MMODAq||K(KN $ i#s   ##K>
c                     | j                   j                  D cg c]  }t        t               t                       c}S c c}w r   )r   r   r   r   )r]   r|   s     r<   
make_cachezModel.make_cache  s1    9=9J9JK9JA	')WY/9JKKKs   #?r   c                 x  	
 |xs t         j                  j                         }|j                         }|j	                         }| j
                  j                  D ]  }|j                  D ]  }|j                  t        |j                  d|      |_
        nt        |j                  d|      |_        t        |j                  d|      |_        |xj                  |z  c_        |j                  }||z  

|z   		
fd}|j                  j                  |       |j                   j                  |        |j"                  D ]Y  }t        |j$                  d|      |_        t        |j&                  d|      |_        t        |j(                  d|      |_        [ ||j*                  _        t/        |j*                  j0                  j$                  d|       t/        |j*                  j0                  j&                  d|       t/        |j*                  j0                  j(                  d|        y )Nzall-to-shardedr   zsharded-to-allc                     |  S r   r:   )wehshs    r<   shard_headsz Model.shard.<locals>.shard_heads  s    R8Or;   )ro   r   initsizerankr   r   r   r%   r	   rO   rS   rX   r#   rV   applyrW   r   r   r   r   r   r   r   r   )r]   r   Nr!  r   attnr  r  r   r  r  s            @@r<   shardzModel.shard  s   .,,.JJLzz|ZZ&&E##+".%5U#DK %1'7u%DM +4;;8HPUV((Q.( 44	I%)^$ "";/  &&{3' (* zz ,MM#35! +3;;8HPUV ,MM#35! " (-EII$%))..88:JRWX%))..668HPUV%))..88:JRWXE 'r;   r   )r2   r3   r4   r   rJ   ro   rv   r   r   r   propertyr   r   r   r  r  r   Groupr%  r   r   s   @r<   r   r   c  s    PY P  $!! }! ! !    :xL'Y8BNN$8$89 'Yr;   r   ))rZ   dataclassesr   typingr   r   r   r   mlx.corecorero   mlx.nnrM   mlx.nn.layers.distributedr   r	   r
   activationsr   rF   r   r   r   rc   r   r   mlar   
rope_utilsr   switch_layersr   r   Moduler>   r   r   r   r   r   r   r:   r;   r<   <module>r3     s     ! - -   P P  T T %  ' $ ( ( (<C#bii C#L
Jbii 
J!8RYY !8H.bii .b.ryy .b		 4JYBII JYr;   