
    (HJjL,                        d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ e G d de             ZdedefdZ G d dej@                        Z! G d dej@                        Z" G d dej@                        Z# G d deej@                        Z$ G d dej@                        Z%y)    )	dataclass)AnyDictListOptionalUnionN)shard_linear   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)PipelineMixin)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZee   ed	<   dZ	ee   ed
<   dZ
ee   ed<   dZeeeeeef   f      ed<   dZeed<   dZeee      ed<   dZee   ed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizeNhead_dimmax_position_embeddingsnum_key_value_headsrope_parametersTtie_word_embeddingslayer_typessliding_windowc                     | j                   | j                  | _         | j                  dg| j                  z  | _        y y )Nfull_attention)r   r   r!   r   selfs    b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/ministral3.py__post_init__zModelArgs.__post_init__"   sC    ##+'+'?'?D$# 01D4J4JJD $    )__name__
__module____qualname__str__annotations__intfloatr   r   r   r   r   r   r   r    boolr!   r   r"   r(    r)   r'   r   r      s    OO"Hhsm"-1Xc]1)-#->BOXd3eSj(9#9:;B $$'+K$s)$+$(NHSM(Kr)   r   betar   c           
      D   t        |t        j                        r|j                  dkD  r	|d d d f   }d|t        j                  dt        j
                  t        j                  |       |z   |z        z         z  z   }|j                  dk(  r|d d d d d d f   S |d d d f   S )Nr   r
      )
isinstancemxarrayndimlogfloorarange)sizeoffsetr3   r   scalings        r'   _get_llama_4_attn_scaler@   *   s    &"((#a4$	BHHbiio.2IIJJ  G ||qq$4'((q$wr)   c                        e Zd Zdef fdZ	 	 d	dej                  dej                  deej                     dee   dej                  f
dZ	 xZ
S )
	Attentionargsc                 `   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        | j                  |j                   d   d|j                   |j"                        | _        y )Ng      Fbias
rope_theta)super__init__r   r   n_headsr   
n_kv_headsr   scalennLinearq_projk_projv_projo_projr   r   r   rope)r&   rC   dimrJ   rK   r   	__class__s         r'   rI   zAttention.__init__8   s   !%!9!99w'+'?'??*#'==#OD4D4D4OOt^
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeD#MM  .  ((
	r)   x
attn_scalemaskcachereturnc                 0   |j                   \  }}}| j                  |      | j                  |      | j                  |      }
}	}|j	                  ||| j
                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|
j	                  ||| j                  d      j                  dddd      }
d}|H|j                  }| j                  ||      }| j                  |	|      }	|j                  |	|
      \  }	}
n"| j                  |      }| j                  |	      }	||z  }t        ||	|
|| j                  |      }|j                  dddd      j	                  ||d      }| j                  |      S )Nr   r5   r
      )r>   )rY   rL   rX   )shaperO   rP   rQ   reshaperJ   	transposerK   r>   rS   update_and_fetchr   rL   rR   )r&   rV   rW   rX   rY   BLDquerieskeysvaluesr>   outputs                r'   __call__zAttention.__call__P   sy    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aP\\Fiii7G99T&91D 11$?LD&ii(G99T?DJ&-T6djjt
 !!!Q1-55aB?{{6""r)   NN)r*   r+   r,   r   rI   r7   r8   r   r   ri   __classcell__rU   s   @r'   rB   rB   7   se    
Y 
8 $(##88# HH# rxx 	#
 }# 
#r)   rB   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPrC   c                     t         |           |j                  }|j                  }t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        y NFrE   )	rH   rI   r   r   rM   rN   	gate_proj	down_projup_proj)r&   rC   rT   
hidden_dimrU   s       r'   rI   zMLP.__init__s   s`    ++
3
?:s?yyju=r)   rZ   c                 t    | j                  t        | j                  |      | j                  |                  S N)rr   r   rq   rs   )r&   rV   s     r'   ri   zMLP.__call__|   s)    ~~fT^^A%6QHIIr)   )	r*   r+   r,   r   rI   r7   r8   ri   rk   rl   s   @r'   rn   rn   r   s!    >Y >JRXX Jr)   rn   c                        e Zd Zd
dedef fdZ	 	 ddej                  dej                  deej                     dee	   dej                  f
d	Z
 xZS )TransformerBlockrC   use_slidingc                    t         |           |j                  | _        |j                  | _        || _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)rH   rI   r   r   ry   rB   	self_attnrn   mlprM   RMSNormr   input_layernormpost_attention_layernormrC   )r&   rC   ry   rU   s      r'   rI   zTransformerBlock.__init__   s    #'#;#; ++&"4t9!zz$*:*:@Q@QR(*

$"3"3)
% 	r)   rV   rW   rX   rY   rZ   c                     | j                  | j                  |      |||      }||z   }| j                  | j                  |            }||z   }|S rv   )r}   r   r~   r   )r&   rV   rW   rX   rY   rhouts           r'   ri   zTransformerBlock.__call__   sS     NN4//2JeLEHHT22156!e
r)   )Frj   )r*   r+   r,   r   r1   rI   r7   r8   r   r   ri   rk   rl   s   @r'   rx   rx      sl    Y T " $(#88 HH rxx 	
 } 
r)   rx   c                   r     e Zd Zdef fdZ fdZ	 	 ddej                  deej                     fdZ	 xZ
S )LanguageModelrC   c                    t         |           || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        j                  |j                  |j                        | _
        | j
                  D cg c]  }t        ||dk(         c}| _        t        j                  |j                  |j                        | _        | j
                  j!                  d      | _        d | _        t'        | j                        D ]  \  }}|j(                  s|| _         y  y c c}w )Nsliding_attention)rC   ry   r{   r$   )rH   rI   rC   r   r   r!   r"   rM   	Embeddingr   embed_tokensrx   layersr   r   normindexfa_idxswa_idx	enumeratery   )r&   rC   
layer_typeelrU   s        r'   rI   zLanguageModel.__init__   s   	//!%!7!7++"11LL$:J:JK #..
.
 $JBU4UV.
 JJt//T5F5FG	&&,,-=>dkk*DAq}}  +
s   Ec                 .   t         |   |       d | _        d | _        t	        | j
                        D ]_  \  }}| j                  |j                  r|| _        n| j                  |j                  s|| _        | j                  R| j                  _ y  y rv   )rH   pipeliner   r   r   pipeline_layersry   )r&   groupr   r   rU   s       r'   r   zLanguageModel.pipeline   s{    d223DAq||# $Q]]{{&4<<+C 4r)   inputsinput_embeddingsc                 :   ||}n| j                  |      }| j                  }| j                  }|d gt        | j                        z  }d}n|d   j
                  }d x}}	| j                  t        ||| j                           }	| j                  %t        ||| j                     | j                        }t        |j                  d   || j                  j                  d   | j                  j                  d         j                  |j                        }
||dz
  k  r#t         j"                  j%                  ||dz         }t'        | j                  |      D ]!  \  }}|j(                  r|n|	} |||
||      }# |dk7  rVt         j"                  j+                  ||dz
  |z        }|d   +t!        j,                  |d   j.                  |      |d   _        |dkD  r/t         j"                  j1                  |      d |j                  d    }| j3                  |      S )Nr   )window_sizer
   llama_4_scaling_beta original_max_position_embeddings)rY   r\   )r   pipeline_rankpipeline_sizelenr   r>   r   r   r   r"   r@   r^   rC   r   astypedtyper7   distributed	recv_likezipry   senddependsrf   
all_gatherr   )r&   r   rY   r   r   r   r   r>   swa_maskfa_maskrW   r   crX   s                 r'   ri   zLanguageModel.__call__   s    ' A!!&)A****=FS!5!566EF1X__F!!7;;"+AuT[[/ABG<<#,5&D4G4GH -LLOII%%&<=II%%&HI	

 &/ 	 =1,,((]Q->@A,,e4DAq }}8'D!ZQ/A 5
 A##A(9]'JKARy$!#E"INNA!>b	 1))!,\qwwqz:Ayy|r)   rj   )r*   r+   r,   r   rI   r   r7   r8   r   ri   rk   rl   s   @r'   r   r      s@    Y (
 /3	55 #288,	5r)   r   c                        e Zd Zdef fdZ	 	 ddej                  deej                     fdZd Z	ddeej                  j                     fdZed	        Zd
 Z xZS )ModelrC   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rp   )rH   rI   rC   r   r   modelr    rM   rN   r   r   lm_head)r&   rC   rU   s     r'   rI   zModel.__init__   sX    	//"4(
''99T%5%5tUSDL (r)   r   r   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rv   )r   rC   r    r   	as_linearr   )r&   r   rY   r   r   s        r'   ri   zModel.__call__   sZ     jj(8999((**))33C8C 
 ,,s#C
r)   c                 T   |j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j                  dd        i }|j                         D ]:  \  }}d|v r"|}|j	                  dd      }||   }||z  ||<   ,d|v r1||vs6|||<   < |}|S c c}}w )Nzself_attn.rotary_emb.inv_freqzlm_head.weightweight_scale_inv
_scale_inv activation_scale)itemsrC   r    popreplace)r&   weightskvnew_weights	scale_invwkweights           r'   sanitizezModel.sanitize  s     %]]_
,TQ0OWX0XAqD_ 	 
 99((KK($/MMODAq!Q&	YY|R0 "(9"4B#q(+%!"A $ '
s
   B$B$r   c                    |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  j                  d|      |j                  _        t        |j                  j                   d|      |j                  _        t        |j                  j"                  d|      |j                  _         y )Nzall-to-sharded)r   zsharded-to-all)r7   r   initr=   r   r   r	   r}   rO   rP   rQ   rR   rJ   rK   r~   rq   rr   rs   )r&   r   Nlayers       r'   shardzModel.shard"  se   .,,.JJLZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO##)#OO&&1,& #/		##%5U#EII #/		##%5U#EII !-		!!#35!EII1 'r)   c                 .    | j                   j                  S rv   )r   r   r%   s    r'   r   zModel.layersA  s    zz)))r)   c                     | j                   D cg c]8  }|j                  r t        | j                  j                        n	t               : c}S c c}w )N)max_size)r   ry   r   r   r"   r   )r&   r   s     r'   
make_cachezModel.make_cacheE  sU     
 % $$  )B)BCY %
 	
 
s   =Arj   rv   )r*   r+   r,   r   rI   r7   r8   r   ri   r   r   Groupr   propertyr   r   rk   rl   s   @r'   r   r      sv    TY T /3	 #288,	.8BNN$8$89 > * *
r)   r   )&dataclassesr   typingr   r   r   r   r   mlx.corecorer7   mlx.nnrM   mlx.nn.layers.distributedr	   activationsr   baser   r   r   rY   r   r   r   r   
rope_utilsr   r   r0   r/   r@   ModulerB   rn   rx   r   r   r2   r)   r'   <module>r      s    " 3 3   2  T T + # ' K K K0
  
 PS 
 8#		 8#vJ")) Jryy 8VM299 VrX
BII X
r)   