
    (HJjf*                        d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ dd	lmZ e G d
 de             Ze G d de             Z G d de
j4                        Z G d de
j4                        Z G d de
j4                        Z G d de
j4                        Z G d de
j4                        Z G d de
j4                        Z  G d de
j4                        Z!y)    )	dataclass)AnyOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)ChunkedKVCacheKVCache)initialize_rope)	SwitchGLUc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZ	eed<   dZ
eed<   dZeed<   y)TextArgsattention_biasattention_chunk_sizehead_dimhidden_sizeinterleave_moe_layer_stepintermediate_sizeintermediate_size_mlpmax_position_embeddings
model_typenum_attention_headsnum_experts_per_toknum_hidden_layersnum_key_value_headsnum_local_expertsrms_norm_epsrope_scaling
rope_thetause_qk_norm
vocab_size   attn_temperature_tuningi    floor_scaleg?
attn_scaleN)__name__
__module____qualname__bool__annotations__intstrfloatr   r&   r'   r(        ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/llama4.pyr   r      s    M""  OO#$S$KJr2   r   c                   2    e Zd ZU eeef   ed<   eed<   d Zy)	ModelArgstext_configr   c                 L    t         j                  | j                        | _        y N)r   	from_dictr6   selfs    r3   __post_init__zModelArgs.__post_init__/   s    #--d.>.>?r2   N)	r)   r*   r+   r   r   dictr-   r/   r<   r1   r2   r3   r5   r5   *   s    x~&&O@r2   r5   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
	Attentionargs	layer_idxc                 n   t         |           |j                  }|j                  x| _        }|j
                  x| _        }t        |dz   dz  dk7        | _        |j                  | _	        |j                  | _
        |j                  | _        |j                  xs |j                  |z  x| _        }|dz  | _        t        |d      r|j                  }nd}t!        j"                  |||z  |      | _        t!        j"                  |||z  |      | _        t!        j"                  |||z  |      | _        t!        j"                  ||z  ||      | _        |j,                  xr | j                  | _        | j                  r4t/        ||j0                  d|j2                  |j4                  	      | _        y y )
Nr   r%   r   g      r   FbiasT)traditionalscaling_configr   )super__init__r   r   n_headsr   
n_kv_headsr.   use_roper&   r'   r(   r   scalehasattrr   nnLinearq_projk_projv_projo_projr#   r   r"   r!   r   rope)	r;   r@   rA   dimrI   rJ   r   r   	__class__s	           r3   rH   zAttention.__init__4   sv   !%!9!99w'+'?'??*Y]a/145'+'C'C$++//#'==#OD4D4D4OOt^
4)*!00N"NiiWx%7nMiiZ(%:PiiZ(%:Pii( 2CnM++===' #00(,(D(DDI r2   xmaskcachereturnc           	         |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	||j                  }
nd}
| j                  r&| j                  ||
      }| j                  ||
      }| j                  rDt        j                  j                  |d d      }t        j                  j                  |d d      }| j                  r| j                  st        j                   t        j"                  t        j$                  |
dz   |
|z   dz         | j&                  z        d	z         | j(                  z  d	z   }|d d d f   }||z  j+                  |j,                        }||j/                  ||	      \  }}	t1        |||	|| j2                  |
      }|j                  dddd      j	                  ||d      }| j5                  |      S )Nr      r      )offsetgư>)weightepsg      ?)rY   rL   rX   )shaperP   rQ   rR   reshaperI   	transposerJ   r_   rK   rT   r#   mxfastrms_normr&   logflooraranger'   r(   astypedtypeupdate_and_fetchr   rL   rS   )r;   rW   rX   rY   BLDquerieskeysvaluesr_   attn_scalesoutputs                r3   __call__zAttention.__call__X   s5    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aP\\FF==iii7G99T&91Dgg&&wt&FG77##D4#@D''HHRYYvz6A:>BTEUEUUV //	"
   &ag.K,44W]]CG 11$?LD&-T6djjt
 !!!Q1-55aB?{{6""r2   NNr)   r*   r+   r   r.   rH   re   arrayr   r   rv   __classcell__rV   s   @r3   r?   r?   3   s_    "X "# "N $(#	.#88.# rxx .# }	.#
 
.#r2   r?   c                   J     e Zd Zddedef fdZdej                  fdZ xZ	S )MLPr@   r   c                    t         |           |j                  }|xs |j                  }t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        y NFrC   )	rG   rH   r   r   rN   rO   	gate_proj	down_projup_proj)r;   r@   r   rU   
hidden_dimrV   s        r3   rH   zMLP.__init__   se    &@$*@*@
3
?:s?yyju=r2   rZ   c                 t    | j                  t        | j                  |      | j                  |                  S r8   )r   r   r   r   )r;   rW   s     r3   rv   zMLP.__call__   s)    ~~fT^^A%6QHIIr2   r8   )
r)   r*   r+   r5   r.   rH   re   ry   rv   rz   r{   s   @r3   r}   r}      s(    >Y >3 >JRXX Jr2   r}   c                   >     e Zd Z fdZdej
                  fdZ xZS )MoEc                 t   t         |           |j                  | _        | j                  dk(  sJ d       |j                  | _        t        |j                  |j                  | j
                        | _	        t        j                  |j                  |j                  d      | _        t        |      | _        y )Nr   z!Only 1 expert per token supportedFrC   )rG   rH   r   top_kr   num_expertsr   r   r   expertsrN   rO   routerr}   shared_expertr;   r@   rV   s     r3   rH   zMoE.__init__   s    --
zzQC CC11 d44d6F6F
 ii 0 0$2H2HuU Yr2   rZ   c                    | j                  |      }| j                  }t        j                  | |dz
  d      dd |f   }t        j                  ||d      }t        j
                  |j                  t        j                              j                  |j                        }| j                  ||z  |      j                  d      }|| j                  |      z   S )Nr   r\   )kthaxis.r   r]   )r   r   re   argpartitiontake_along_axissigmoidrk   float32rl   r   squeezer   )r;   rW   logitskindicesscoresouts          r3   rv   zMoE.__call__   s    QJJ//6'q1u2>sBQBwG##FG"=FMM"**56==aggFll1v:w/77:T''***r2   )r)   r*   r+   rH   re   ry   rv   rz   r{   s   @r3   r   r      s    	'+RXX +r2   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
TransformerBlockr@   rA   c                    t         |           |j                  | _        |j                  | _        t	        ||      | _        ||j                  z  |j                  dz
  k(  | _        | j                  rt        |      | _	        nt        ||j                        | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Nr   ra   )rG   rH   r   r   r?   	self_attnr   is_moe_layerr   feed_forwardr}   r   rN   RMSNormr    input_layernormpost_attention_layernormr@   )r;   r@   rA   rV   s      r3   rH   zTransformerBlock.__init__   s    #'#;#; ++"43&)G)GG**Q.
  #D	D #D$*D*D ED!zz$*:*:@Q@QR(*

$"3"3)
% 	r2   rW   rX   rY   rZ   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r8   )r   r   r   r   )r;   rW   rX   rY   rhr   s          r3   rv   zTransformerBlock.__call__   sS     NN4//2D%@Ed;;A>?!e
r2   rw   rx   r{   s   @r3   r   r      s^    X # , $(#	
88
 rxx 
 }	

 

r2   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
LlamaModelr@   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        |j                  | _        y c c}w )Nr   r   )rG   rH   r@   r$   r   rN   	Embeddingr   embed_tokensranger   layersr   r    normr   )r;   r@   irV   s      r3   rH   zLlamaModel.__init__   s    	//!%!7!7"""LL$:J:JK:?@V@V:WX:WQ'a0:WXJJt//T5F5FG	$($=$=! Ys   C-inputsc                    | j                  |      }|Nt        |      D ]!  \  }}|dz   dz  dk7  s|j                          # |d   j                  }|d   j                  }nd}d}||j
                  d   z   }t        j                  ||      }	t        j                  ||      d d d f   }
t        j                  |	| j                  z  |
| j                  z  z
        }|	|
k  }|dk(  |z  }|d gt        | j                        z  }t        ||d         }t        t        | j                  |            D ]$  \  }\  }}|dz   dz  dk7  }|r|n|} ||||      }& | j                  |      S )Nr   r%   r   r^   )rY   )r   	enumeratemaybe_trim_frontstart_positionr_   rb   re   rj   absr   lenr   r
   zipr   )r;   r   rY   r   idxcstartr_   endlindsrinds	block_pos	token_pos
chunk_maskglobal_masklayeruse_chunked_attentionrX   s                     r3   rv   zLlamaModel.__call__   sx   
 f%#E*Q!Gq=A%&&( + !H++E1X__FEFqwwqz!		%%		&#&q$w/FFd///ET=V=V4VW
	 UN	1n	1
=FS--E+AuQx8(T[[%)@AOC%%(1WMQ$6!!6:KDaQ'A  B
 yy|r2   r8   	r)   r*   r+   r   rH   re   ry   rv   rz   r{   s   @r3   r   r      s%    	>X 	> ##r2   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )LanguageModelr@   c                    t         |           || _        |j                  | _        t	        | j                        | _        t        j                  | j                  j                  | j                  j                  d      | _
        y r   )rG   rH   r@   r   r   modelrN   rO   r   r$   lm_headr   s     r3   rH   zLanguageModel.__init__  s[    	//		*
yyII!!499#7#7e
r2   r   c                 H    | j                  ||      }| j                  |      S r8   )r   r   )r;   r   rY   r   s       r3   rv   zLanguageModel.__call__  s#    
 jj'||C  r2   r8   r   r{   s   @r3   r   r     s%    
X 
 !!r2   r   c                   d     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
d Z xZS )	Modelr@   c                     t         |           || _        |j                  | _        t	        |j
                        | _        y r8   )rG   rH   r@   r   r   r6   language_modelr   s     r3   rH   zModel.__init__  s3    	//+D,<,<=r2   r   c                 &    | j                  ||      S r8   )r   )r;   r   rY   s      r3   rv   zModel.__call__  s    
 ""6511r2   c                 2   d }|j                         D ci c]  \  }} ||      r|| }}}t        | j                  j                  j                        D ]  }d| d}| d|v rm|j                  | d      }| d}| d}t        j                  |dd	      \  }	}
t        j                  |	d
d      ||<   t        j                  |
d
d      ||<   | d|v s|j                  | d      }t        j                  |d
d      || d<    |S c c}}w )Nc                     d| v xs d| v S )Nvision_modelmulti_modal_projectorr1   )r   s    r3   	to_removez!Model.sanitize.<locals>.to_remove$  s    !Q&F*AQ*FFr2   zlanguage_model.model.layers.z.feed_forward.expertsz.gate_up_projz.gate_proj.weightz.up_proj.weightr]   r\   r   r   z
.down_projz.down_proj.weight)	itemsr   r@   r6   r   popre   splitswapaxes)r;   weightsr   r   vlprefixgate_kup_kr   r   r   s               r3   sanitizezModel.sanitize#  s4   	G %,MMOHODAq9Q<1a4OH tyy,,>>?A3A36KLF'72KK6(- 89"8#45 1%'XXa%<"	7"$++iA"> "GQ :$/#KK6(*(=>	8:IqRS8T6("345 @  Is
   DDc                 B    | j                   j                  j                  S r8   )r   r   r   r:   s    r3   r   zModel.layers9  s    ""((///r2   c                    | j                   j                  j                  }g }t        t	        | j
                              D ]A  }|dz   dz  dk7  r|j                  t        |             )|j                  t                      C |S )Nr   r%   r   )	r@   r6   r   r   r   r   appendr   r   )r;   
chunk_sizecachesr   s       r3   
make_cachezModel.make_cache=  sj    YY**??
s4;;'(AA{anZ89gi(	 )
 r2   r8   )r)   r*   r+   r5   rH   re   ry   rv   r   propertyr   r   rz   r{   s   @r3   r   r     sC    >Y > 22, 0 0r2   r   )"dataclassesr   typingr   r   r   mlx.corecorere   mlx.nnrN   activationsr   baser	   r
   r   rY   r   r   
rope_utilsr   switch_layersr   r   r5   Moduler?   r}   r   r   r   r   r   r1   r2   r3   <module>r      s    " ' '    T T * ' $ }  2 @ @ @S#		 S#lJ")) J+")) +.ryy B/ /d!BII !&0BII 0r2   