
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zee   ed<   dZ	e
ed<   dZeed<   dZe
ed<   y
)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizehead_dimnum_key_value_headsNmax_position_embeddingsFattention_biasi'  
rope_thetaTtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   r   r   boolr   r        [/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/glm.pyr   r      sc    OOM-1Xc]1 ND J $$r&   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	GLMAttentionargsc                    t         |           |j                  | _        |j                  | _        |j                  | _        |j
                  xs |j                  | j                  z  | _        | j
                  dz  | _        t        j                  | j                  | j                  | j
                  z  |j                        | _
        t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j
                  z  | j                  d      | _        t        j                  | j
                  d|j                        | _        y )Ng      ࿩biasFT)dimstraditionalbase)super__init__r   r   r   r   scalennLinearr   q_projk_projv_projo_projRoPEr   ropeselfr*   	__class__s     r'   r2   zGLMAttention.__init__    sU   ++#'#;#; #'#;#; U)9)9T=U=U)U]]D(
ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4d6F6FU
 GGDtW	r&   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rA   r3   r@   )shaper6   r7   r8   reshaper   	transposer   r;   rG   update_and_fetchr   r3   r9   )r=   r?   r@   rA   BLDquerieskeysvaluesoutputs              r'   __call__zGLMAttention.__call__=   sy    ''1a $AAAv//!Q(@(@"EOOq!Q
 ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r&   NNr   r   r   r   r2   mxarrayr   r   rS   __classcell__r>   s   @r'   r)   r)      sZ    XY X@ $(#	#88# rxx # }	#
 
#r&   r)   c                   D     e Zd Zdef fdZdej                  fdZ xZS )GLMMLPr*   c                     t         |           t        j                  |j                  d|j
                  z  d      | _        t        j                  |j
                  |j                  d      | _        y )NrE   Fr,   )r1   r2   r4   r5   r   r   gate_up_proj	down_projr<   s     r'   r2   zGLMMLP.__init__`   sZ    IIa$"8"88u
 4#9#94;K;KRWXr&   rB   c                     | j                  |      }t        j                  |dd      \  }}| j                  t	        ||            S )NrE   rD   )axis)r]   rV   splitr^   r	   )r=   r?   gates      r'   rS   zGLMMLP.__call__g   s=    a ((1ab)a~~fT1o..r&   )	r   r   r   r   r2   rV   rW   rS   rX   rY   s   @r'   r[   r[   _   s!    YY Y/RXX /r&   r[   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	GLMBlockr*   c                 "   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Neps)r1   r2   r)   	self_attnr[   mlpr4   RMSNormr   r   input_layernormpost_attention_layernormr<   s     r'   r2   zGLMBlock.__init__n   sf    %d+$<!zz$*:*:@Q@QR(*

$"3"3)
%r&   r?   r@   rA   rB   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S N)rh   rk   ri   rl   )r=   r?   r@   rA   rhouts          r'   rS   zGLMBlock.__call__w   sQ     NN4//2D%@EHHT22156!e
r&   rT   rU   rY   s   @r'   rd   rd   m   sW    
Y 
 $(#	
88
 rxx 
 }	

 

r&   rd   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )GLMModelr*   c                 N   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )N)r*   rf   )r1   r2   r4   	Embeddingr   r   embed_tokensranger   rd   layersrj   r   norm)r=   r*   _r>   s      r'   r2   zGLMModel.__init__   sw    LL$:J:JK49$:P:P4QR4QqxT*4QRJJt//T5F5FG	 Ss   B"inputsrA   rB   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rv   lenrx   r   zipry   )r=   r{   rA   rp   r@   layercs          r'   rS   zGLMModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r&   rn   rU   rY   s   @r'   rs   rs      sC    HY H  $ } 
	r&   rs   c                        e Zd Zdef fdZ	 d	dej                  dee   dej                  fdZ	d Z
ed        Z xZS )
Modelr*   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr,   )r1   r2   r*   r   rs   modelr   r4   r5   r   r   lm_headr<   s     r'   r2   zModel.__init__   sW    	//d^
''99T%5%5tUSDL (r&   r{   rA   rB   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rn   )r   r*   r   rv   	as_linearr   )r=   r{   rA   rq   s       r'   rS   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r&   c                     |j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j                  dd        |S c c}}w )Nzself_attn.rotary_emb.inv_freqzlm_head.weight)itemsr*   r   pop)r=   weightskvs       r'   sanitizezModel.sanitize   s]    $]]_
,TQ0OWX0XAqD_ 	 
 99((KK($/
s
   AAc                 .    | j                   j                  S rn   )r   rx   )r=   s    r'   rx   zModel.layers   s    zz   r&   rn   )r   r   r   r   r2   rV   rW   r   r   rS   r   propertyrx   rX   rY   s   @r'   r   r      s\    TY T  $

 }
 
	
 ! !r&   r   )dataclassesr   typingr   r   r   r   mlx.corecorerV   mlx.nnr4   activationsr	   r0   r
   r   r   
rope_utilsr   r   Moduler)   r[   rd   rs   r   r%   r&   r'   <module>r      s    " - -    T T ' % % % =#299 =#@/RYY /ryy .ryy 2!BII !r&   