
    (HJjJ                     \   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ e G d de             Z G d dej:                        Z G d dej:                        Z G d dej:                        Z  G d dej:                        Z! G d dej:                        Z" G d dej:                        Z# G d dej:                        Z$ G d dej:                        Z% G d dej:                        Z& G d d ej:                        Z'y)!    )	dataclass)AnyListOptionalTupleN   )swiglu)BaseModelArgscreate_attention_maskcreate_ssm_maskscaled_dot_product_attention)ArraysCacheKVCache)initialize_rope)
ssm_update)	SwitchGLUc                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   ee   ed<   eed<   eed<   dZ	e
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZeed<   dZeed<   d Zeed!<   d"Zeeef   ed#<   ed$efd%       Zy)&	ModelArgs
model_type
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersmax_position_embeddingsnum_attention_headsnum_key_value_headsattention_biasembedding_multiplierattention_multiplierlogits_scalingresidual_multiplierlayer_typesrms_norm_eps
rope_thetaNnum_local_expertsnum_experts_per_tokshared_intermediate_sizemamba_n_headsmamba_d_headmamba_proj_biasmamba_d_statemamba_d_convmamba_n_groupsmamba_conv_biasFmlp_biasropeposition_embedding_typeTtie_word_embeddings)gMbP?g      Y@time_step_limitreturnc                 ,    t        | j                        S N)boolr%   selfs    h/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/granitemoehybrid.pyuse_moezModelArgs.use_moeB   s    D**++    )__name__
__module____qualname__str__annotations__intr7   floatr   r%   r   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r1   r2   r3   r   propertyr;    r<   r:   r   r      sK    OO  c (,x}+)-#-.2hsm2 $(M8C='"&L(3-&&*OXd^*#'M8C='"&L(3-&$(NHSM(&*OXd^* Hd $*S) $$+9OU5%<(9 , , ,r<   r   c                   |     e Zd Zddedef fdZd	dej                  dej                  dej                  fdZ xZ	S )
GraniteMoeHybridRMSNormGatedr   epsc                 d    t         |           || _        t        j                  |      | _        y r6   )super__init__rH   mxonesweight)r9   r   rH   	__class__s      r:   rK   z%GraniteMoeHybridRMSNormGated.__init__H   s%    ggk*r<   hidden_statesgater4   c                     |t        ||      }t        j                  j                  || j                  | j
                        S r6   )r	   rL   fastrms_normrN   rH   )r9   rP   rQ   s      r:   __call__z%GraniteMoeHybridRMSNormGated.__call__M   s5    "47Mwwt{{DHHEEr<   )gư>r6   )
r=   r>   r?   rB   rC   rK   rL   arrayrU   __classcell__rO   s   @r:   rG   rG   G   s@    +C +e +
Fbhh Fbhh F"(( Fr<   rG   c                       e Zd Zdef fdZdej                  dee   deej                     dej                  fdZ	dej                  d	ej                  d
ej                  dej                  dee   deej                     dej                  fdZ
	 ddej                  deej                     dee   dej                  fdZ xZS )GraniteMoeHybridMamba2Mixerargsc                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _
        |j                  | _        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  d| j                  z  | j                  z  z   | _        t#        j$                  | j                   | j                   |j                  d| j                   |j&                        | _        | j                  | j                   z   | j                  z   }t#        j*                  | j                  ||j,                        | _        t1        j2                  | j                        | _        t1        j6                  t1        j8                  d| j                  dz   t0        j:                              | _        t1        j2                  | j                        | _        tA        | j                  |jB                        | _"        t#        j*                  | j                  | j                  |j,                        | _#        y )N   r   )in_channelsout_channelskernel_sizepaddinggroupsbiasrc   r   dtyperH   )$rJ   rK   r(   	num_headsr   r+   ssm_state_sizer,   conv_kernel_sizer)   r   r-   n_groupshead_dimr3   heads_per_groupconv_dimnnConv1dr.   conv1dLinearr*   in_projrL   rM   dt_biaslogarangefloat32A_logDrG   r#   normout_proj)r9   r[   projection_sizerO   s      r:   rK   z$GraniteMoeHybridMamba2Mixer.__init__T   s   ++++"00 $ 1 1!%!3!3d6G6G!G++))#33#~~>..T]]1BTEXEX1XXii))==%%
 004==@4>>QyyoD4H4H
 wwt~~.VVBIIa!);2::NO
(0""(9(9
	 		""D$4$44;O;O
r<   
conv_inputcachemaskr4   c                    |t        j                  |d   |d      }||d   It        j                  |j                  d   | j                  dz
  | j
                  f|j                        }n|d   }t        j                  ||gd      }| j                  dz
  }|j                  q|j                  d   }t        j                  |j                  d||z
        }|d d d f   t        j                  |      z   d   }	t        j                  ||	d      |d<   n;|d d | d d d f   |d<   n(t        j                  |d| j                  dz
  dfdg      }| j                  |      }
t        j                  |
      S )N.Nr   r   re   axis)r   r   )rL   wherezerosshaperj   rn   rf   concatenatelengthscliprv   take_along_axispadrq   ro   silu)r9   r}   r~   r   
conv_statepadded_inputn_keeptends	positionsconv_outputs              r:   _convz!GraniteMoeHybridMamba2Mixer._conv{   sh    $y/:qAJQxXX%%a($*?*?!*CT]]S$**

 #1X
>>:z*BKL**Q.F}}( &&q)wwu}}aV<!!T']RYYv->>	J	--lIANa'F78Q7a66Vd&;&;a&?%CVLL kk,/ww{##r<   rP   BCdtc                 <   |j                   \  }}}	|j                  ||| j                  | j                        }|j                  ||| j                  | j
                        }|j                  ||| j                  | j
                        }|r|d   }
|j                  }nd\  }
}t        || j                  ||| j                  j                  |j                        || j                  |
| j                  |
      \  }}
|r|
|d<   |j                  ||| j                        S )Nr   NN)r   reshaperh   rl   rk   ri   r   r   rx   ry   astyperf   rt   r3   r   )r9   rP   r   r   r   r~   r   
batch_sizeseq_len_stater   ys                r:   _ssmz GraniteMoeHybridMamba2Mixer._ssm   s    "/!4!4
GQ%--
 IIj'4==$:M:MNIIj'4==$:M:MN!HEmmG'NE7JJFFMM---.LL  
5 E!HyyWd.D.DEEr<   c                    | j                  |      }t        j                  || j                  | j                  | j                  z   gd      \  }}}| j                  |||      }t        j                  || j                  | j                  | j                  | j                  z  z   gd      \  }	}
}| j                  |	|
||||      }|r|j                  |j                  d          | j                  ||      }| j                  |      S )Nr   r   )rs   rL   splitr   rn   r   rk   ri   r   advancer   rz   r{   )r9   rP   r   r~   	projectedrQ   r}   r   r   hidden_states_ssmr   r   r   s                r:   rU   z$GraniteMoeHybridMamba2Mixer.__call__   s     LL/	!xx##T%;%;dmm%KL 
j"
 jjUD9"$((&&&&9L9L)LL #
1a II'Ar5$?MM!''!*%IIa}}Qr<   r6   )r=   r>   r?   r   rK   rL   rV   r   r   r   r   rU   rW   rX   s   @r:   rZ   rZ   S   s   %
Y %
N $HH $ $ $ rxx 	 $
 
 $D%Fxx%F 88%F 88	%F
 HH%F $%F rxx %F 
%FV (,	 xx  rxx   $	 
 
 r<   rZ   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	GraniteMoeHybridAttentionr[   c                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|j                  | _	        |j                  }t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        |j"                  dk7  }|r3t%        | j                  |j&                  dd |j(                        | _        y d | _        y )Nrd   nopeF)rJ   rK   r   r   n_headsr   
n_kv_headsrl   r   scaler   ro   rr   q_projk_projv_projo_projr1   r   r$   r   r0   )	r9   r[   dimr   r   rl   r   use_roperO   s	           r:   rK   z"GraniteMoeHybridAttention.__init__   s   !%!9!99w'+'?'??*#'#3#3w#>>..
,,iiWx%7nMiiZ(%:PiiZ(%:Pii( 2CnM //69',,DI DIr<   xr   r~   r4   c                 N   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	| j                  _|;| j                  ||j                        }| j                  ||j                        }n"| j                  |      }| j                  |      }||j                  ||	      \  }}	t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr   r   r]   r      )offset)r~   r   r   )r   r   r   r   r   r   	transposer   r0   r   update_and_fetchr   r   r   )r9   r   r   r~   r   Lry   querieskeysvaluesoutputs              r:   rU   z"GraniteMoeHybridAttention.__call__  sy    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aP 99  ))GELL)Ayyelly;))G,yy 11$?LD&-T6djjt
 !!!Q1-55aB?{{6""r<   r   )r=   r>   r?   r   rK   rL   rV   r   r   rU   rW   rX   s   @r:   r   r      sY    Y B $(#'	#88# rxx #  	#
 
#r<   r   c                   L     e Zd Zdededef fdZdej                  fdZ xZS )GraniteMoeHybridTopKGating
input_sizenum_expertstop_kc                     t         |           || _        || _        || _        t        j                  ||d      | _        y NFrd   )rJ   rK   r   r   r   ro   rr   layer)r9   r   r   r   rO   s       r:   rK   z#GraniteMoeHybridTopKGating.__init__#  s8    &$
YYz;UC
r<   rP   c                     | j                  |      }t        j                  || j                   d      d| j                   d f   }t        j                  ||d      }t        j
                  |dd      }||fS )Nr   )kthr   .r   T)preciser   )r   rL   argpartitionr   r   softmax)r9   rP   logits	top_k_idxtop_k_logitstop_k_gatess         r:   rU   z#GraniteMoeHybridTopKGating.__call__*  sp    M*OOF"E$**
	 ))&)"Ejjt"E+%%r<   )	r=   r>   r?   rB   rK   rL   rV   rU   rW   rX   s   @r:   r   r   "  s1    D3 DS D D&bhh &r<   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )GraniteMoeHybridMoEr[   c                 (   t         |           |j                  | _        |j                  | _        t        | j                  | j                  |j                        | _        t        | j                  |j                  |j                        | _
        y )N)r   r   r   )rJ   rK   r   r   r   r   r%   
switch_mlpr   r&   routerr9   r[   rO   s     r:   rK   zGraniteMoeHybridMoE.__init__5  ss    **11#OOT--t/E/E
 1..**
r<   r   r4   c                 ~    | j                  |      \  }}| j                  ||      }||d   z  j                  d      S )Nr   r   )r   r   sum)r9   r   	token_idsgatesr   s        r:   rU   zGraniteMoeHybridMoE.__call__C  sB    ;;q>	5OOAy)E)$$))r)22r<   	r=   r>   r?   r   rK   rL   rV   rU   rW   rX   s   @r:   r   r   4  s*    
Y 
3"(( 3rxx 3r<   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )GraniteMoeHybridSharedMLPr[   c                     t         |           t        j                  |j                  |j
                  dz  d      | _        t        j                  |j
                  |j                  d      | _        y )Nr]   Frd   )rJ   rK   ro   rr   r   r'   input_linearoutput_linearr   s     r:   rK   z"GraniteMoeHybridSharedMLP.__init__J  s]    IId;;a?e
  YY))4+;+;%
r<   r   r4   c                     t        j                  | j                  |      dd      \  }}| j                  t	        ||            S )Nr]   r   r   )rL   r   r   r   r	   )r9   r   rQ   ups       r:   rU   z"GraniteMoeHybridSharedMLP.__call__S  s;    88D--a0!"=b!!&r"233r<   r   rX   s   @r:   r   r   I  s*    
Y 
4"(( 4rxx 4r<   r   c                   D     e Zd Zdef fdZdej                  fdZ xZS )GraniteMoeHybridMLPr[   c                    t         |           |j                  }|j                  }|j                  }t        j                  |||      | _        t        j                  |||      | _        t        j                  |||      | _	        y )Nrd   )
rJ   rK   r   r   r/   ro   rr   	gate_proj	down_projup_proj)r9   r[   r   
hidden_dimr/   rO   s        r:   rK   zGraniteMoeHybridMLP.__init__Y  si    ++
==3
B:sByyjx@r<   r4   c                 t    | j                  t        | j                  |      | j                  |                  S r6   )r   r	   r   r   )r9   r   s     r:   rU   zGraniteMoeHybridMLP.__call__c  s)    ~~fT^^A%6QHIIr<   r   rX   s   @r:   r   r   X  s#    AY AJRXX Jr<   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
GraniteMoeHybridLayerr[   
layer_typec                 "   t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _	        |dk(  rt        |      | _        n$|dk(  rt        |      | _        nt        d|       | j                  r!t        |      | _        t#        |      | _        nt'        |      | _        t        j                  |j                  |j                        | _        y )Nrg   mamba	attentionzUnknown layer type: )rJ   rK   r   r!   r;   ro   RMSNormr   r#   input_layernormrZ   r   r   	self_attn
ValueErrorr   
shared_mlpr   block_sparse_moer   mlppost_attention_layernormr9   r[   r   rO   s      r:   rK   zGraniteMoeHybridLayer.__init__h  s    $#'#;#; ||!zz$*:*:@Q@QR 4T:DJ;&6t<DN3J<@AA <<7=DO$7$=D! +40DH(*

$"3"3)
%r<   r   r   r~   r4   c                    |}| j                  |      }| j                  dk(  r| j                  |||      }n| j                  |||      }||| j                  z  z   }|}| j                  |      }| j                  r(| j                  |      }| j                  |      }||z   }	n| j                  |      }	||	| j                  z  z   }|S )Nr   r   r~   )
r   r   r   r   r!   r   r;   r   r   r   )
r9   r   r   r~   residualrP   normedmoe_out
shared_outmlp_outs
             r:   rU   zGraniteMoeHybridLayer.__call__  s     ,,Q/??g% JJ}4uJMM NN=t5NQM =43K3K#KK !..}=<<++F3G0J
*Ghhv&G 7T-E-E#EEr<   r   )r=   r>   r?   r   r@   rK   rL   rV   r   r   rU   rW   rX   s   @r:   r   r   g  s^    
Y 
C 
< $(#	88 rxx  }	
 
r<   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )GraniteMoeHybridModelr[   c                 8   t         |           || _        t        j                  |j
                  |j                        | _        |j                  D cg c]  }t        ||       c}| _
        t        j                  |j                  |j                        | _        |j                  | _        d|j                  v r|j                  j                  d      nd | _        d|j                  v r!|j                  j                  d      | _        y d | _        y c c}w )Nrg   r   r   )rJ   rK   r[   ro   	Embeddingr   r   embed_tokensr"   r   layersr   r#   rz   r   indexfa_idxssm_idxr   s      r:   rK   zGraniteMoeHybridModel.__init__  s    	LL$:J:JKFJFVFV
FV
!$
3FV
 JJt//T5F5FG	$($=$=!
 d... "";/ 	 07$:J:J/JD""7+ 	PT 	
s   Dinputsr~   r4   c                    | j                  |      | j                  z  }|d gt        | j                        z  }d }d }| j                  t        ||| j                           }| j                  t        ||| j                           }t        | j                  |      D ]#  \  }}|j                  dk(  r|n|} ||||      }% | j                  |      S )Nr   r   )r  r   lenr  r	  r   r
  r   zipr   rz   )	r9   r  r~   rP   	attn_mask
mamba_maskr   cr   s	            r:   rU   zGraniteMoeHybridModel.__call__  s    
 ))&1D4M4MM=FS--E 	
;;"-mU4;;=OPI<<#(dll8KLJDKK/HE1 % 0 0K ?9ZD!-d!DM 0 yy''r<   r6   )r=   r>   r?   r   rK   rL   rV   r   r   rU   rW   rX   s   @r:   r  r    sA    
Y 
.  $(( }( 
	(r<   r  c                        e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	e
d        Zd Zd	 Ze
d
        Z xZS )Modelr[   c                    t         |           || _        |j                  | _        t	        |      | _        |j                  s1t        j                  |j                  |j                  d      | _        |j                  | _        y r   )rJ   rK   r[   r   r  modelr2   ro   rr   r   r   lm_headr    r   s     r:   rK   zModel.__init__  sa    	//*40
''99T%5%5tUSDL"11r<   r  r~   r4   c                     | j                  ||      }| j                  j                  r&| j                   j                  j	                  |      }n| j                  |      }|| j                  z  S )N)r~   )r  r[   r2   r  	as_linearr  r    )r9   r  r~   outs       r:   rU   zModel.__call__  s\    
 jjuj-99((**))33C8C,,s#CT((((r<   c                 .    | j                   j                  S r6   )r  r  r8   s    r:   r  zModel.layers  s    zz   r<   c                     g }| j                   D ]V  }|j                  dk(  r|j                  t        d             .|j                  dk(  s>|j                  t	                      X |S )Nr   r]   )sizer   )r  r   appendr   r   )r9   cachesr   s      r:   
make_cachezModel.make_cache  sW    [[E7*kq12!![0gi(	 !
 r<   c                 <   |j                         D ]2  \  }}d|v s|j                  d   dk7  s|j                  dd      ||<   4 | j                  j                  rd|v rt        | j                  j                        D ]x  }d| d}|j                  | d      }|j                  \  }}}|d d d |dz  d d f   }	|d d |dz  d d d f   }
|	|| d	<   |
|| d
<   |j                  | d      || d<   z |S | j                  j                  sd|v rt        | j                  j                        D ]d  }d| d}|j                  | d      }t        j                  |dd      \  }	}
|	|d| d<   |
|d| d<   |j                  | d      |d| d<   f |S )Nzconv1d.weightr   r   r]   z3model.layers.0.block_sparse_moe.input_linear.weightzmodel.layers.z.block_sparse_moez.input_linear.weightz.switch_mlp.gate_proj.weightz.switch_mlp.up_proj.weightz.output_linear.weightz.switch_mlp.down_proj.weightz-model.layers.0.shared_mlp.input_linear.weightz.shared_mlpr   r   z.mlp.gate_proj.weightz.mlp.up_proj.weightz.mlp.down_proj.weight)
itemsr   moveaxisr[   r;   ranger   poprL   r   )r9   weightskvlprefixinput_weightr   expert_hiddenr   r   s              r:   sanitizezModel.sanitize  s   MMODAq!#q(8ZZ1-
 $ IIEP499667(+<=&{{fX5I+JK&2&8&8#=! ),@mq.@,@!)CD	&q-1*<*>'ABCL6(">?@AH6("<=>CJ;;h34D6(">?@ 8F # 		!!?7J499667(;7  '{{fX5I+JK%'XXlAA%F"	7DM-s*?@ABI-s*=>?DKKKh34E-s*?@A 8 r<   c                       fd}|S )Nc                 ^    j                   j                  r| j                  d      rdddS y)Nzrouter.layer@      )
group_sizebitsT)r[   r;   endswith)pathr   r9   s     r:   	predicatez(Model.quant_predicate.<locals>.predicate*  s)    yy  T]]>%B&(!44r<   rE   )r9   r5  s   ` r:   quant_predicatezModel.quant_predicate(  s    	
 r<   r6   )r=   r>   r?   r   rK   rL   rV   r   r   rU   rD   r  r  r,  r6  rW   rX   s   @r:   r  r    st    2Y 2  $)) }) 
	) ! !.`  r<   r  )(dataclassesr   typingr   r   r   r   mlx.corecorerL   mlx.nnro   activationsr	   baser
   r   r   r   r~   r   r   
rope_utilsr   ssmr   switch_layersr   r   ModulerG   rZ   r   r   r   r   r   r   r  r  rE   r<   r:   <module>rB     s    " - -     ( '  $ -, -, -,`	F299 	FL ")) L ^=#		 =#@& &$3")) 3*4		 4J")) J:BII :z,(BII ,(^\BII \r<   