
    (HJj#                     ~   d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ e G d
 de             Z G d dej6                        Z G d dej6                        Z G d dej6                        Z G d dej6                        Z G d dej6                        Z y)    )	dataclass)AnyDictListOptionalUnionN)shard_linear   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_ropec                   2   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZee   ed	<   dZ	ee   ed
<   dZ
ee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   dZeee      ed<   dZee   ed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizeNhead_dimmax_position_embeddingsnum_key_value_headsFattention_biasmlp_biasi'  
rope_thetarope_traditionalrope_scalingTtie_word_embeddingslayer_typessliding_windowc                     | j                   | j                  | _         | j                  dg| j                  z  | _        y y )Nfull_attention)r   r   r$   r   selfs    ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/llama.py__post_init__zModelArgs.__post_init__%   sC    ##+'+'?'?D$# 01D4J4JJD $    )__name__
__module____qualname__str__annotations__intfloatr   r   r   r   r   boolr   r    r!   r"   r   r   r#   r$   r   r%   r+    r,   r*   r   r      s    OO"Hhsm"-1Xc]1)-#- ND HdJ"d";?L(4U5#:%6 678? $$'+K$s)$+$(NHSM(Kr,   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        t        |d      r|j                  }nd}t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        t#        | j                  |j$                  |j&                  |j(                  |j*                        | _        y )Ng      r   Fbias)super__init__r   r   n_headsr   
n_kv_headsr   scalehasattrr   nnLinearq_projk_projv_projo_projr   r    r!   r"   r   rope)r)   r8   dimr>   r?   r   r   	__class__s          r*   r=   zAttention.__init__.   s   !%!9!99w'+'?'??*#'==#OD4D4D4OOt^
4)*!00N"NiiWx%7nMiiZ(%:PiiZ(%:Pii( 2CnM#MMOO!!((
	r,   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r
      )offset)rM   r@   rL   )shaperD   rE   rF   reshaper>   	transposer?   rH   rS   update_and_fetchr   r@   rG   )r)   rK   rL   rM   BLDquerieskeysvaluesoutputs              r*   __call__zAttention.__call__J   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r,   NN)r-   r.   r/   r   r=   mxarrayr   r   r_   __classcell__rJ   s   @r*   r7   r7   -   sW    
Y 
> $(#	#88# rxx # }	#
 
#r,   r7   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr8   c                 6   t         |           |j                  }|j                  }t	        |d      r|j
                  }nd}t        j                  |||      | _        t        j                  |||      | _	        t        j                  |||      | _
        y )Nr   Fr:   )r<   r=   r   r   rA   r   rB   rC   	gate_proj	down_projup_proj)r)   r8   rI   
hidden_dimr   rJ   s        r*   r=   zMLP.__init__j   sx    ++
4$}}HH3
B:sByyjx@r,   rN   c                 t    | j                  t        | j                  |      | j                  |                  S N)ri   r   rh   rj   )r)   rK   s     r*   r_   zMLP.__call__x   s)    ~~fT^^A%6QHIIr,   )	r-   r.   r/   r   r=   ra   rb   r_   rc   rd   s   @r*   rf   rf   i   s#    AY AJRXX Jr,   rf   c            	            e Zd Zd	dedef fdZ	 	 d
dej                  deej                     dee	   dej                  fdZ
 xZS )TransformerBlockr8   use_slidingc                    t         |           |j                  | _        |j                  | _        || _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)r<   r=   r   r   rp   r7   	self_attnrf   mlprB   RMSNormr   input_layernormpost_attention_layernormr8   )r)   r8   rp   rJ   s      r*   r=   zTransformerBlock.__init__}   s    #'#;#; ++&"4t9!zz$*:*:@Q@QR(*

$"3"3)
% 	r,   rK   rL   rM   rN   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rm   )rt   rw   ru   rx   )r)   rK   rL   rM   rhouts          r*   r_   zTransformerBlock.__call__   sQ     NN4//2D%@EHHT22156!e
r,   )Fr`   )r-   r.   r/   r   r4   r=   ra   rb   r   r   r_   rc   rd   s   @r*   ro   ro   |   s^    Y T   $(#	
88
 rxx 
 }	

 

r,   ro   c                   h     e Zd Zdef fdZ	 	 ddej                  deej                     fdZ xZ	S )
LlamaModelr8   c                    t         |           || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _
        | j
                  D cg c]  }t        ||dk(         c}| _        t        j                  |j                  |j                        | _        | j
                  j!                  d      | _        d | _        t'        | j                        D ]  \  }}|j(                  s|| _         y  y c c}w )Nr   sliding_attention)r8   rp   rr   r'   )r<   r=   r8   r   r   r$   r%   rB   	Embeddingr   embed_tokensro   layersrv   r   normindexfa_idxswa_idx	enumeraterp   )r)   r8   
layer_typeelrJ   s        r*   r=   zLlamaModel.__init__   s   	//!%!7!7++"11"""LL$:J:JK #..
.
 $JBU4UV.
 JJt//T5F5FG	&&,,-=>dkk*DAq}}  +
s   )Einputsinput_embeddingsc                    ||}n| j                  |      }|d gt        | j                        z  }t        ||| j                           }| j
                  %t        ||| j
                     | j                        }t        | j                  |      D ]   \  }}|j                  rn|} ||||      }" | j                  |      S )N)window_size)rM   )
r   lenr   r   r   r   r%   ziprp   r   )	r)   r   rM   r   r{   fa_maskswa_masklayerrL   s	            r*   r_   zLlamaModel.__call__   s     ' A!!&)A=FS--E'5+=><<#,5&D4G4GH  U3LE5$008gDaU+A 4 yy|r,   r`   )
r-   r.   r/   r   r=   ra   rb   r   r_   rc   rd   s   @r*   r~   r~      s;    Y 0 /3	 #288,	r,   r~   c                        e Zd Zdef fdZ	 	 ddej                  deej                     fdZd Z	ddeej                  j                     fdZed	        Zd
 Z xZS )Modelr8   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr:   )r<   r=   r8   r   r~   modelr#   rB   rC   r   r   lm_head)r)   r8   rJ   s     r*   r=   zModel.__init__   sX    	//%
''99T%5%5tUSDL (r,   r   r   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rm   )r   r8   r#   r   	as_linearr   )r)   r   rM   r   r|   s        r*   r_   zModel.__call__   sZ     jj(8999((**))33C8C 
 ,,s#C
r,   c                     |j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j                  dd        |S c c}}w )Nzself_attn.rotary_emb.inv_freqzlm_head.weight)itemsr8   r#   pop)r)   weightskvs       r*   sanitizezModel.sanitize   s_     %]]_
,TQ0OWX0XAqD_ 	 
 99((KK($/
s
   AAgroupc                    |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  j                  d|      |j                  _        t        |j                  j                   d|      |j                  _        t        |j                  j"                  d|      |j                  _         y )Nzall-to-sharded)r   zsharded-to-all)ra   distributedinitsizer   r   r	   rt   rD   rE   rF   rG   r>   r?   ru   rh   ri   rj   )r)   r   Nr   s       r*   shardzModel.shard   se   .,,.JJLZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO##)#OO&&1,& #/		##%5U#EII #/		##%5U#EII !-		!!#35!EII1 'r,   c                 .    | j                   j                  S rm   )r   r   r(   s    r*   r   zModel.layers  s    zz   r,   c                     | j                   D cg c]8  }|j                  r t        | j                  j                        n	t               : c}S c c}w )N)max_size)r   rp   r   r   r%   r   )r)   r   s     r*   
make_cachezModel.make_cache
  sU     
 % $$  )B)BCY %
 	
 
s   =Ar`   rm   )r-   r.   r/   r   r=   ra   rb   r   r_   r   r   Groupr   propertyr   r   rc   rd   s   @r*   r   r      sv    TY T /3	 #288,	8BNN$8$89 > ! !
r,   r   )!dataclassesr   typingr   r   r   r   r   mlx.corecorera   mlx.nnrB   mlx.nn.layers.distributedr	   activationsr   baser   r   r   rM   r   r   
rope_utilsr   r   Moduler7   rf   ro   r~   r   r5   r,   r*   <module>r      s    " 3 3   2  T T + ' K K K89#		 9#xJ")) J&ryy 6. .bJ
BII J
r,   