
    (HJjL                        d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z
 d dlmZ ddlmZmZmZ e G d de             Zd	 Zdd
Z G d dej(                        Z G d dej(                        Z G d dej(                        Z G d dej(                        Z ee
j2                  d      dd       Z G d dej(                        Zy)    N)	dataclass)partial)AnyOptional   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed	<   d
Zeed<   dZeed<   dZeed<   dZeed<   y)	ModelArgsnanochat
model_typei   hidden_size   num_hidden_layers
   num_attention_headsnum_key_value_headsi   
vocab_sizei   max_position_embeddingsi   intermediate_size     @
rope_thetaN)__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r   r   float     `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/nanochat.pyr   r      sa     J Ks!!!!J#'S'!s!Jr"   r   c                 D    t         j                  j                  | dd      S )z0Functional RMSNorm with no learnable parameters.Ngh㈵>)mxfastrms_norm)xs    r#   r'   r'      s    77AtT**r"   c           	          | j                   d   }|X|dz  }t        j                  t        j                  d|t        j                        t        j                  |      |z  z         }t        j                  j                  | |dd|d|      S )	a@  Apply RoPE with blocked layout.


    Args:
        x: Input tensor in (B, H, T, D) format
        offset: Position offset for KV caching
        base: RoPE base frequency (default 10000.0)
        freqs: Precomputed negated frequencies (optional)

    Returns:
        Tensor with RoPE applied, same shape as input
    N           dtypeFg      ?)dimstraditionalbasefreqsscaleoffset)	shaper%   exparangefloat32mathlogr&   rope)r(   r4   r1   r2   head_dimhalf_Ds         r#   apply_rotary_embr>       s     wwr{H}QIIc648OP
 

 77<<	   r"   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  | _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  dz  | _        |j                  | _	        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  d      | _        | j                  dz  }t!        j"                  t!        j$                  d|t         j&                        t)        j*                  | j                        |z  z         | _        y )Ng      Fbiasr+   r,   r-   )super__init__r   r   	num_headsr   num_kv_headsr<   r3   r   nnLinearc_qc_kc_vc_projr%   r6   r7   r8   r9   r:   _rope_freqs)selfrA   r=   	__class__s      r#   rF   zAttention.__init__C   sd   ++11 44((DNN:]]D(
//99dnnt}}<5
 99d//$--?e
 99d//$--?e
 ii 0 0$2B2BO !#FFIIc64xx(613
 
r"   r(   maskcachereturnc                    |j                   \  }}}| j                  |      }| j                  |      }| j                  |      }	|j	                  ||| j
                  | j                        j                  dddd      }|j	                  ||| j                  | j                        j                  dddd      }|	j	                  ||| j                  | j                        j                  dddd      }	||j                  nd}
t        ||
| j                  | j                        }t        ||
| j                  | j                        }t        |      }t        |      }||j                  ||	      \  }}	t        |||	|| j                   |      }|j                  dddd      j	                  ||| j"                        }| j%                  |      S )Nr   r+   r      )r4   r1   r2   )rS   r3   rR   )r5   rK   rL   rM   reshaperG   r<   	transposerH   r4   r>   r   rO   r'   update_and_fetchr
   r3   r   rN   )rP   r(   rR   rS   BL_querieskeysvaluesr4   outputs               r#   __call__zAttention.__call___   s    ''1a((1+xx{! //!QFPPq!Q
 ||Aq$"3"3T]]CMMq!Q
 1d&7&7GQQq!Q

 "'!2"F@P@P
  dooT=M=M

 7#~  11$?LD&-T6djjt

 !!!Q1-55aD<L<LM{{6""r"   NNr   r   r   r   rF   r%   arrayr   r   ra   __classcell__rQ   s   @r#   r@   r@   B   sW    
Y 
> $(#	.#88.# rxx .# }	.#
 
.#r"   r@   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )MLPrA   c                     t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        y NFrC   )rE   rF   rI   rJ   r   r   c_fcrN   rP   rA   rQ   s     r#   rF   zMLP.__init__   sN    IId..0F0FUS	ii 6 68H8HuUr"   r(   rT   c                 p    | j                  |      }t        j                  |      }| j                  |      S N)rk   rI   relu2rN   )rP   r(   s     r#   ra   zMLP.__call__   s*    IIaLHHQK{{1~r"   	r   r   r   r   rF   r%   rd   ra   re   rf   s   @r#   rh   rh      s,    VY V
"(( rxx r"   rh   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockrA   c                 b    t         |           t        |      | _        t	        |      | _        y rn   )rE   rF   r@   attnrh   mlprl   s     r#   rF   zTransformerBlock.__init__   s$    dO	t9r"   r(   rR   rS   rT   c                     || j                  t        |      ||      z   }|| j                  t        |            z   }|S )NrR   rS   )rt   r'   ru   )rP   r(   rR   rS   houts         r#   ra   zTransformerBlock.__call__   s=     		(1+D	>>$((8A;''
r"   rb   rc   rf   s   @r#   rr   rr      sW    Y  $(#		88	 rxx 	 }		
 
	r"   rr   c                   `     e Zd Zdef fdZ	 ddej                  dej                  fdZ xZS )NanoChatModelrA   c                     t         |           || _        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        y c c}w rn   )rE   rF   rA   rI   	Embeddingr   r   wteranger   rr   rx   )rP   rA   r\   rQ   s      r#   rF   zNanoChatModel.__init__   s]    	<<1A1AB278N8N2OP2OQ"4(2OPPs   A8inputsrT   c                     | j                  |      }t        |      }|d gt        | j                        z  }t	        ||d         }t        | j                  |      D ]  \  }} ||||      } t        |      }|S )Nr   rw   )r~   r'   lenrx   r	   zip)rP   r   rS   rx   rR   layercs          r#   ra   zNanoChatModel.__call__   s{    
 HHVQK=FS[(E$Qa1DFFE*HE1ad!,A + QKr"   rn   rp   rf   s   @r#   r{   r{      s5    QY Q  
	r"   r{   T)	shapelessc                 8    |t        j                  | |z        z  S rn   )r%   tanh)logitscaps     r#   softcapr      s    #&&&r"   c                   p     e Zd Zdef fdZ	 ddej                  dej                  fdZed        Z	 xZ
S )ModelrA   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y rj   )rE   rF   rA   r   r{   transformerrI   rJ   r   r   lm_headrl   s     r#   rF   zModel.__init__   sK    	//(.yy!1!14??Or"   r   rT   c                 d    | j                  ||      }| j                  |      }t        |      }|S )N)rS   )r   r   r   )rP   r   rS   ry   r   s        r#   ra   zModel.__call__   s7    
 vU3c" r"   c                 .    | j                   j                  S rn   )r   rx   )rP   s    r#   layerszModel.layers   s    !!!r"   rn   )r   r   r   r   rF   r%   rd   ra   propertyr   re   rf   s   @r#   r   r      sI    PY P  
	 " "r"   r   )r   N)g      .@)r9   dataclassesr   	functoolsr   typingr   r   mlx.corecorer%   mlx.nnrI   r1   r   r	   r
   r   r'   r>   Moduler@   rh   rr   r{   compiler   r   r!   r"   r#   <module>r      s     !      T T 	  	  	 +
DK#		 K#\
")) 
ryy $BII < 	t$' %'"BII "r"   