
    (HJj"                        d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Zy)    )	dataclass)AnyDictOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)	SwitchGLUc                   "   e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZeed <   y)!	ModelArgsdeepseek
model_typei  
vocab_sizei   hidden_sizei +  intermediate_sizei  moe_intermediate_size   num_hidden_layers    num_attention_headsnum_key_value_headsNn_shared_expertsn_routed_expertsnum_experts_per_tokr   moe_layer_freqr   first_k_dense_replacei   max_position_embeddingsgư>rms_norm_epsg     @
rope_thetarope_scalingFattention_bias)__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r   r   r   r   r   r   r   r   r    floatr!   r"   r   r#   bool     `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/deepseek.pyr   r      s     J JK"s"!%3%s!!!!&*hsm*&*hsm*)-#-NC!"3"#'S'L%J#'L(4.' ND r-   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DeepseekAttentionconfigc                    t         |           || _        |j                  | _        |j                  | _        |j
                  | _        |j                  |j                  z  | _        | j                  dz  | _        t        |dd      }t        j                  | j                  |j                  | j                  z  |      | _        t        j                  | j                  |j
                  | j                  z  |      | _        t        j                  | j                  |j
                  | j                  z  |      | _        t        j                  | j                  |j                  | j                  z  |      | _        d}|j                   rC|j                   d   dk(  r1t#        |j                   d   t$              sJ d	|j                   d   z  }t        j&                  | j                  |j(                  |
      | _        y )Ng      r#   Fbiasg      ?typelinearfactorr   )basescale)super__init__r1   r   r   r   num_kv_headshead_dimr9   getattrnnLinearq_projk_projv_projo_projr"   
isinstancer*   RoPEr!   rope)selfr1   r#   
rope_scale	__class__s       r.   r;   zDeepseekAttention.__init__#   s   !--#)#=#= "66**f.H.HH]]D(
 )95Aii&&6

 ii&&6

 ii&&6

 ii&&6
 
6#6#6v#>(#Jf11(;UCCCV00::JGGMM""
	r-   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rM   r9   rL   )shaperA   rB   rC   reshaper   	transposer<   rG   rS   update_and_fetchr   r9   rD   )rH   rK   rL   rM   BL_querieskeysvaluesoutputs              r.   __call__zDeepseekAttention.__call__M   ss    ''1a $AAAv//!Q(@(@"EOOq!Q
 ||Aq$"3"3R8BB1aAN1d&7&7<FFq!QPQRiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r-   NNr$   r%   r&   r   r;   mxarrayr   r   r_   __classcell__rJ   s   @r.   r0   r0   "   sX    (
y (
Z $(#	#88# rxx # }	#
 
#r-   r0   c                   v     e Zd Z	 	 ddedee   dee   f fdZdej                  dej                  fdZ	 xZ
S )	DeepseekMLPr1   r   r   c                    t         |           || _        |xs |j                  | _        |xs |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        y NFr3   )
r:   r;   r1   r   r   r?   r@   	gate_projup_proj	down_proj)rH   r1   r   r   rJ   s       r.   r;   zDeepseekMLP.__init__m   s     	&<&*<*<!2!Nf6N6N4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr-   rK   rN   c                 t    | j                  t        | j                  |      | j                  |                  S N)rl   r   rj   rk   )rH   rK   s     r.   r_   zDeepseekMLP.__call__{   s)    ~~fT^^A%6QHIIr-   r`   )r$   r%   r&   r   r   r)   r;   rb   rc   r_   rd   re   s   @r.   rg   rg   l   sV     &*+/	YY c]Y $C=	YJ"(( Jrxx Jr-   rg   c                   *     e Zd Zdef fdZd Z xZS )MoEGater1   c                     t         |           || _        |j                  | _        |j
                  | _        t        j                  | j
                  |j                  f      | _	        y rn   )
r:   r;   r1   r   top_kr   rb   zerosr   weightrH   r1   rJ   s     r.   r;   zMoEGate.__init__   sQ    //
 & 7 7hh 5 5v7I7IJKr-   c                     || j                   j                  z  }t        j                  |dd      }| j                  }t        j
                  t        j                  | |dz
  d      dd |f         }t        j                  ||d      }||fS )NrP   T)axispreciser   )kthrw   .rw   )rt   Trb   softmaxrr   stop_gradientargpartitiontake_along_axis)rH   rK   gatesscoreskindss         r.   r_   zMoEGate.__call__   s    DKKMM!ED9JJQU LSRTSTRTW UV##FDr:V|r-   r$   r%   r&   r   r;   r_   rd   re   s   @r.   rp   rp      s    Ly Lr-   rp   c                   *     e Zd Zdef fdZd Z xZS )DeepseekMoEr1   c                     t         |           || _        t        |j                  |j
                  |j                        | _        t        |      | _	        |j                  ,|j
                  |j                  z  }t        ||      | _        y y )N)r1   r   )r:   r;   r1   r   r   r   r   
switch_mlprp   gater   rg   shared_experts)rH   r1   r   rJ   s      r.   r;   zDeepseekMoE.__init__   s    # < <f>U>U
 FO	"". & < <v?V?V V"-1B#D /r-   c                     | j                  |      \  }}| j                  ||      }||d   z  j                  d      }| j                  j                  || j                  |      z   }|S )N).Nrz   )r   r   sumr1   r   r   )rH   rK   r   r   ys        r.   r_   zDeepseekMoE.__call__   sj    yy|fOOAt$	""''R'0;;''3D''**Ar-   r   re   s   @r.   r   r      s    y r-   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
DeepseekDecoderLayerr1   	layer_idxc                    t         |           t        |      | _        |j                  ,||j
                  k\  r||j                  z  dk(  rt        |      n
t        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   eps)r:   r;   r0   	self_attnr   r   r   r   rg   mlpr?   RMSNormr   r    input_layernormpost_attention_layernorm)rH   r1   r   rJ   s      r.   r;   zDeepseekDecoderLayer.__init__   s    *62 ''3!=!== 5 55:	  V$ 	  "zz&*<*<&BUBUV(*

F$7$7)
%r-   rK   rL   rM   rN   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rn   )r   r   r   r   )rH   rK   rL   rM   rhouts          r.   r_   zDeepseekDecoderLayer.__call__   sQ     NN4//2D%@EHHT22156!e
r-   r`   )r$   r%   r&   r   r)   r;   rb   rc   r   r   r_   rd   re   s   @r.   r   r      s^    
y 
S 
( $(#	
88
 rxx 
 }	

 

r-   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )DeepseekModelr1   c                 \   t         |           || _        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r:   r;   r1   r?   	Embeddingr   r   embed_tokensranger   r   layersr   r    norm)rH   r1   idxrJ   s      r.   r;   zDeepseekModel.__init__   s    LL):):F<N<NO9>v?W?W9X
9X# -9X
 JJv11v7J7JK	
s   B)rK   rM   rN   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r
   zipr   )rH   rK   rM   r   rL   layercs          r.   r_   zDeepseekModel.__call__   su    
 a =FS--E$Qa1DKK/HE1aq!A 0 yy|r-   rn   ra   re   s   @r.   r   r      sC    Ly L  $88 } 
	r-   r   c                   h     e Zd Zdef fdZ	 ddej                  dee   fdZ	d Z
ed        Z xZS )	Modelr1   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y ri   )r:   r;   argsr   r   modelr?   r@   r   r   lm_headru   s     r.   r;   zModel.__init__   sN    	 ++"6*
yy!3!3V5F5FUSr-   inputsrM   c                 H    | j                  ||      }| j                  |      S rn   )r   r   )rH   r   rM   r   s       r.   r_   zModel.__call__   s#    
 jj'||C  r-   c                 d   t        | j                  j                        D ]  }d| }dD ]|  }dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j	                  | d| d| d|         }}t        j                  |      || d| d| <   w ~  |S c c}w )Nzmodel.layers.)rj   rl   rk   )rt   scalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.)r   r   r   r   poprb   stack)rH   weightslprefixmr   eto_joins           r.   sanitizezModel.sanitize   s    tyy223A$QC(F:7A 1QC8GC &+499+E+E%F#%F $KK6(-s!A3as(KL%F   # GIhhwFW6(*:1#Qqc BC 8 ; 4 #s    #B-
c                 .    | j                   j                  S rn   )r   r   )rH   s    r.   r   zModel.layers  s    zz   r-   rn   )r$   r%   r&   r   r;   rb   rc   r   r   r_   r   propertyr   rd   re   s   @r.   r   r      sN    Ty T  $!! }! ! !r-   r   )dataclassesr   typingr   r   r   mlx.corecorerb   mlx.nnr?   activationsr   r8   r	   r
   r   switch_layersr   r   Moduler0   rg   rp   r   r   r   r   r,   r-   r.   <module>r      s    ! & &    T T $ ! ! !*G#		 G#TJ")) J&bii "")) 2299 >BII 8!BII !r-   