
    (HJj/0                        d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ e G d	 d
e             Z G d de
j0                        Z G d de
j0                        Z G d de
j0                        Z G d de
j0                        Z G d de
j0                        Z G d de
j0                        Z G d de
j0                        Zy)    )	dataclass)AnyListOptionalN   )swiglu)BaseModelArgscreate_attention_maskcreate_ssm_maskscaled_dot_product_attention)ArraysCacheKVCache)	SwitchGLUc                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed<   dZ	e
e   ed<   dZe
ee      ed<   dZe
ee      ed<   d Zy)	ModelArgs
model_type
vocab_sizehidden_sizeintermediate_sizemoe_intermediate_sizenum_hidden_layersnum_expertsnum_experts_per_toknorm_topk_probnum_attention_headsnum_key_value_headsmax_position_embeddingsuse_expert_biasnum_dense_layersnorm_eps	conv_biasconv_L_cacheg    .A
rope_thetaNrope_parametersfull_attn_idxslayer_typesc                     | j                   "d| j                   v r| j                   d   | _        | j                  3t        | j                        D cg c]  \  }}|dk(  r| c}}| _        y y c c}}w )Nr#   full_attention)r$   r#   r%   	enumerater&   )selfi
layer_types      `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/lfm2_moe.py__post_init__zModelArgs.__post_init__+   s}    +@T@T0T"22<@DO& &/t/?/?%@#%@MAz!11 %@#D '#s   A/)__name__
__module____qualname__str__annotations__intboolfloatr#   r$   r   dictr%   r   r&   r.        r-   r   r      s    OO  OO!J!&*OXd^**.NHT#Y'.'+K$s)$+r9   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|dz  | _        t        j                  ||j                        | _        t        j                  ||j                        | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j&                  | j                  |j(                  d      | _        y )Ng      ࿩epsFbias)basetraditional)super__init__r   r   n_headsr   
n_kv_headshead_dimscalennRMSNormr    q_layernormk_layernormLinearq_projk_projv_projout_projRoPEr#   rope)r*   r<   dimrF   rG   rH   	__class__s         r-   rE   zAttention.__init__7   s   !%!9!99w'+'?'??*#'#3#3w#>>t^
::hDMMB::hDMMBiiWx%7eDiiZ(%:GiiZ(%:G		'H"4cFGGMM
	r9   xmaskcachereturnc                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	||| j                        }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r      )offset)rY   rX   rI   )shaperO   rP   rQ   rL   reshaperF   	transposerM   rG   rT   r_   update_and_fetchr   rI   rR   )r*   rW   rX   rY   BLDquerieskeysvaluesoutputs              r-   __call__zAttention.__call__P   s    ''1a $AAAv""7??1ar#JKUUq!Q
 Q4??B GHRRq!Q
 1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6T
 !!!Q1-55aB?}}V$$r9   NN)r/   r0   r1   r   rE   mxarrayr   r   rk   __classcell__rV   s   @r-   r;   r;   6   sW    
Y 
8 $(#	%88% rxx % }	%
 
%r9   r;   c                   v     e Zd Zdedef fdZ	 	 ddej                  deej                     dee	   fdZ
 xZS )		ShortConvr<   	layer_idxc                    t         |           || _        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  | j
                  |j                  | j                        | _        t        j                  |j                  d|j                  z  | j                        | _        t        j                  |j                  |j                  | j                        | _        y )N)in_channelsout_channelskernel_sizegroupsrA   r^   r@   )rD   rE   r<   rs   r"   L_cacher!   rA   rJ   Conv1dr   convrN   in_projrR   r*   r<   rs   rV   s      r-   rE   zShortConv.__init__r   s    
 		"((NN	II(())##
	 yy!1!11t7G7G3GdiiX		$"2"2D4D4D499Ur9   rW   rX   rY   c                    | j                  |      }t        j                  |dd      \  }}}||z  }|t        j                  |d   |d      }|#|d   St        j                  |j
                  d   | j                  dz
  | j                  j                  f|j                        }n|d   }t        j                  ||gd      }| j                  dz
  }	|j
                  d   }
|j                  _t        j                  |j                  d|
      }|d d d f   t        j                  |	      z   d   }t        j                  ||d      |d<   n|d d |	 d d d f   |d<   |j                  |
       n(t        j                   |d| j                  dz
  dfdg      }| j#                  |      }||z  }| j%                  |      S )	Nr^   r\   axis.Nr   r   )dtype)r   r   )r|   rm   splitwherezerosr`   ry   r<   r   r   concatenatelengthscliparangetake_along_axisadvancepadr{   rR   )r*   rW   rX   rY   BCxrd   CBxstaten_keeptends	positionsconv_outys                  r-   rk   zShortConv.__call__   s    ll1o((3+1aU$y/2q1BQxXXa[$,,"2DII4I4IJ((
 a!4B\\A%F
A}}(wwu}}a3!!T']RYYv->>	J	--b)!Daa&1n-aMM!VdllQ&6%:FCDB99R=L}}Qr9   rl   r/   r0   r1   r   r4   rE   rm   rn   r   r   rk   ro   rp   s   @r-   rr   rr   q   sW    VV V0 $(#	$ 88$  rxx $  }	$ r9   rr   c                   P     e Zd Zddedee   f fdZdej                  fdZ	 xZ
S )MLPconfigr   c                    t         |           |j                  | _        ||j                  n|| _        t	        j
                  | j                  | j                  d      | _        t	        j
                  | j                  | j                  d      | _        t	        j
                  | j                  | j                  d      | _        y NFr@   )	rD   rE   r   r   rJ   rN   	gate_projup_proj	down_proj)r*   r   r   rV   s      r-   rE   zMLP.__init__   s    !--(9(AF$$GX 	 4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr9   rZ   c                 t    | j                  t        | j                  |      | j                  |                  S N)r   r   r   r   )r*   rW   s     r-   rk   zMLP.__call__   s)    ~~fT^^A%6QHIIr9   r   )r/   r0   r1   r   r   r4   rE   rm   rn   rk   ro   rp   s   @r-   r   r      s/    Yy YXc] YJRXX Jr9   r   c                   D     e Zd Zdef fdZdej                  fdZ xZS )Lfm2MoeSparseMoeBlockr<   c                    t         |           |j                  }|j                  }|j                  x| _        }|j
                  | _        |j                  | _        |j                  | _        t        j                  ||d      | _        t        |||      | _        | j                  r&t        j                  | j                  f      | _        y y r   )rD   rE   r   r   r   r   top_kr   r   rJ   rN   gater   
switch_mlprm   r   expert_bias)r*   r<   rU   r   r   rV   s        r-   rE   zLfm2MoeSparseMoeBlock.__init__   s     66)-)9)99;--
"11#33IIc;U;	#C):KH!xx)9)9(;<D  r9   rW   c                 .   | j                  |      j                  t        j                        }t        j                  |d      }| j
                  r|| j                  z  }| j                  }t        j                  || d      d| d f   }t        j                  ||d      }| j                  r|t        j                  |dd      dz   z  }|j                  |j                        }| j                  ||      }||d   z  j                  d	      }|S )
Nr\   r   )kthr   .T)r   keepdimsg#B;r   )r   astyperm   float32softmaxr   r   r   argpartitionr   r   sumr   r   )r*   rW   gateskindsscoresr   s          r-   rk   zLfm2MoeSparseMoeBlock.__call__   s     		!##BJJ/

5r*T%%%EJJu1"26sQBCx@##E4b9bffV"t<uDDFqww'OOAt$	""''R'0r9   )	r/   r0   r1   r   rE   rm   rn   rk   ro   rp   s   @r-   r   r      s     =Y =88r9   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Lfm2DecoderLayerr<   rs   c                    t         |           ||j                  v | _        | j                  rt	        |      | _        nt        ||      | _        ||j                  k  rt        ||j                        n
t        |      | _        t        j                  |j                  |j                         | _        t        j                  |j                  |j                         | _        y )N)r   r   r>   )rD   rE   r%   is_attention_layerr;   	self_attnrr   r{   r   r   r   r   feed_forwardrJ   rK   r   r    operator_normffn_normr}   s      r-   rE   zLfm2DecoderLayer.__init__   s    "+t/B/B"B""&t_DN!$	2DI 4000	 "&"8"8
 't, 	  ZZ(8(8dmmL

4#3#3Gr9   rW   rX   rY   rZ   c                     | j                   r$| j                  | j                  |      ||      }n#| j                  | j                  |      ||      }||z   }|| j	                  | j                  |            z   }|S )N)rX   rY   )r   r   r   r{   r   r   )r*   rW   rX   rY   rhouts          r-   rk   zLfm2DecoderLayer.__call__   s     ""t11!44uMA		""1%  A
 E$##DMM!$455
r9   rl   r   rp   s   @r-   r   r      sa    HY H3 H. $(#	88 rxx  }	
 
r9   r   c                   h     e Zd Zdef fdZ	 	 ddej                  deej                     fdZ xZ	S )	Lfm2Modelr<   c                 V   t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        |j                  d   | _        d| _        t        |j                        D ]'  }||j                  v r| xj"                  dz  c_        ' y  y c c}w )N)rs   r>   r   r   )rD   rE   r<   r   r   rJ   	Embeddingr   embed_tokensranger   layersrK   r    embedding_normr%   fa_idxconv_idx)r*   r<   r+   rV   s      r-   rE   zLfm2Model.__init__  s    	//!%!7!7LL$:J:JK9>t?U?U9V
9VATQ/9V
 !jj)9)9t}}M))!,t--.AD'''"	 /
s   ?D&inputsinput_embeddingsc                 \   ||}n| j                  |      }|d gt        | j                        z  }t        ||| j                           }t        ||| j                           }t        | j                  |      D ]   \  }}|j                  r|n|}	 |||	|      }" | j                  |      S )N)rY   )
r   lenr   r
   r   r   r   zipr   r   )
r*   r   rY   r   r   	attn_mask	conv_masklayercrX   s
             r-   rk   zLfm2Model.__call__#  s     ' A!!&)A=FS--E)!U4;;-?@	#AuT]]';<	DKK/HE1 % 8 89iDaQ'A 0 ""1%%r9   rl   )
r/   r0   r1   r   rE   rm   rn   r   rk   ro   rp   s   @r-   r   r     s;    Y . /3	&& #288,	&r9   r   c                        e Zd Zdef fdZ	 	 ddej                  deej                     fdZd Z	e
d        Zd Ze
d	        Ze
d
        Z xZS )Modelr<   c                 r    t         |           || _        |j                  | _        t	        |      | _        y r   )rD   rE   r<   r   r   model)r*   r<   rV   s     r-   rE   zModel.__init__<  s+    	//t_
r9   r   r   c                 r    | j                  |||      }| j                   j                  j                  |      S r   )r   r   	as_linear)r*   r   rY   r   r   s        r-   rk   zModel.__call__B  s2     jj(89zz&&0055r9   c                 ^   i }|j                         D ]u  \  }}d|v r2|j                  d   |j                  d   kD  r|j                  ddd      }dddd	}|j                         D ]  \  }}||v s|j                  ||      } |||<   w t	        | j
                  j                        D ]{  }d
| }	dD ]o  }
|	 d|
 d|v st	        | j
                  j                        D cg c]  }|j                  |	 d| d|
 d       }}t        j                  |      ||	 d|
 d<   q } |S c c}w )Nzconv.weightr\   r   r   r]   zgate_proj.weightzdown_proj.weightzup_proj.weight)z	w1.weightz	w2.weightz	w3.weightzmodel.layers.)r   r   r   z.feed_forward.experts.0.z.weightz.feed_forward.experts..z.feed_forward.switch_mlp.)itemsr`   rb   replacer   r<   r   r   poprm   stack)r*   weightssanitized_weightsnameparamreplacementsoldnewlprefixneto_joins                r-   sanitizezModel.sanitizeK  sp   "==?KD%$;;r?U[[^3!OOAq!4E//-L
 )..0S$;<<S1D 1 ',d# + tyy223A$QC(F:X5aS@DUU
 "'tyy'<'<!=	 ">A *--%h&<QCq7K ">	   ) &!(";A3gF ; 4 ! s   #!D*c                 .    | j                   j                  S r   )r   r   )r*   s    r-   r   zModel.layersk  s    zz   r9   c                     | j                   D cg c]$  }|j                  r
t               nt        d      & c}S c c}w )Nr   )size)r   r   r   r   )r*   r   s     r-   
make_cachezModel.make_cacheo  sA     [[
  --GI;A3FF 
 	
 
s   );c                     d }|S )Nc                 0    | j                  d      rdddS y)Nzfeed_forward.gate@      )
group_sizebitsT)endswith)path_s     r-   	predicatez(Model.quant_predicate.<locals>.predicatew  s    }}01&(!44r9   r8   r*   r   s     r-   quant_predicatezModel.quant_predicateu  s    	
 r9   c                     d }|S )Nc                 
    d| vS )Nr   r8   )r   s    r-   r   z'Model.cast_predicate.<locals>.predicate  s     ))r9   r8   r   s     r-   cast_predicatezModel.cast_predicate~  s    	* r9   rl   )r/   r0   r1   r   rE   rm   rn   r   rk   r   propertyr   r   r   r   ro   rp   s   @r-   r   r   ;  s    %Y % /3	66 #288,	6!@ ! !
    r9   r   ) dataclassesr   typingr   r   r   mlx.corecorerm   mlx.nnrJ   activationsr   rB   r	   r
   r   r   rY   r   r   switch_layersr   r   Moduler;   rr   r   r   r   r   r   r8   r9   r-   <module>r
     s    ! & &     ( $   D8%		 8%v: 		 : zJ")) J%BII %P&ryy &R*&		 *&ZHBII Hr9   