
    (HJj|                     j   d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZmZmZ ddlmZ ddlmZ e G d	 d
e             Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)partial)AnyDictOptionalUnionN   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)	BitLinear)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	Zee   ed
<   d	Z	ee   ed<   dZ
eed<   dZeed<   dZeed<   dZeed<   d	Zeeeeeef   f      ed<   dZeed<   y	)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_headsrms_norm_eps
vocab_sizeNhead_dimmax_position_embeddingsFattention_biasmlp_biasi'  
rope_thetarope_traditionalrope_scalingTtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   r   r   r   boolr   r   r   r   r   r   r         ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/bitnet.pyr   r      s    OO"Hhsm"-1Xc]1 ND HdJ"d";?L(4U5#:%6 678? $$r*   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        |j                  }t        |||z  |      | _        t        |||z  |      | _        t        |||z  |      | _        t        ||z  ||      | _        t        | j                  |j                   |j"                  |j$                  |j&                        | _        t+        j,                  |j                  |j.                        | _        y )Ng      ࿩biaseps)super__init__r   r   n_headsr   
n_kv_headsr   scaler   r   q_projk_projv_projo_projr   r   r   r   r   ropennRMSNormr   attn_sub_norm)selfr.   dimr6   r7   r   r   	__class__s          r+   r5   zAttention.__init__$   s   !%!9!99w'+'?'??*#'==#OD4D4D4OOt^
,,Wx%7nMZ(%:PZ(%:P( 2CnM#MMOO!!((
	  ZZ(8(8d>O>OPr*   xmaskcachereturnc                 X   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      }
| j                  |
      }
|
S )Nr      r	      )offset)rF   r8   rE   )shaper9   r:   r;   reshaper6   	transposer7   r=   rL   update_and_fetchr   r8   r@   r<   )rA   rD   rE   rF   BLDquerieskeysvaluesoutputs              r+   __call__zAttention.__call__>   s}    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?##F+V$r*   NNr!   r"   r#   r   r5   mxarrayr   r   rX   __classcell__rC   s   @r+   r-   r-   #   sY    QY Q: $(#	88 rxx  }	
 
r*   r-   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr.   c                 Z   t         |           |j                  }|j                  }t	        |d      r|j
                  }nd}t        |||      | _        t        |||      | _        t        |||      | _	        t        j                  |j                  |j                        | _        y )Nr   Fr0   r2   )r4   r5   r   r   hasattrr   r   	gate_proj	down_projup_projr>   r?   r   ffn_sub_norm)rA   r.   rB   
hidden_dimr   rC   s        r+   r5   zMLP.__init__a   s    ++
4$}}HH"3
B":sB jx@JJt'='=4CTCTUr*   rG   c                     t        j                  | j                  |            | j                  |      z  }| j	                  |      }| j                  |      }|S N)r>   relu2rc   re   rf   rd   )rA   rD   s     r+   rX   zMLP.__call__q   sI    HHT^^A&'$,,q/9a NN1r*   	r!   r"   r#   r   r5   r[   r\   rX   r]   r^   s   @r+   r`   r`   `   s!    VY V RXX r*   r`   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr.   c                 f   t         |           |j                  | _        |j                  | _        t	        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr2   )r4   r5   r   r   r-   	self_attnr`   mlpr>   r?   r   input_layernormpost_attention_layernormrA   r.   rC   s     r+   r5   zTransformerBlock.__init__y   s    #'#;#; ++"4t9!zz$*:*:@Q@QR(*

$"3"3)
%r*   rD   rE   rF   rG   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S ri   )ro   rq   rp   rr   )rA   rD   rE   rF   rhouts          r+   rX   zTransformerBlock.__call__   sQ     NN4//2D%@EHHT22156!e
r*   rY   rZ   r^   s   @r+   rm   rm   x   sW    	
Y 	
 $(#	88 rxx  }	
 
r*   rm   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
LlamaModelr.   c                    t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )N)r.   r2   )r4   r5   r.   r   r   r>   	Embeddingr   embed_tokensrangerm   layersr?   r   norm)rA   r.   _rC   s      r+   r5   zLlamaModel.__init__   s    	//!%!7!7LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   ?Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rF   )r|   lenr~   r   zipr   )rA   r   rF   rv   rE   layercs          r+   rX   zLlamaModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aQ'A 0 yy|r*   ri   rk   r^   s   @r+   ry   ry      s'    	HY 	H r*   ry   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr.   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr0   )r4   r5   r.   r   ry   modelr    r>   Linearr   r   lm_headrs   s     r+   r5   zModel.__init__   sX    	//%
''99T%5%5tUSDL (r*   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S ri   )r   r.   r    r|   	as_linearr   )rA   r   rF   rw   s       r+   rX   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r*   c                     |j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j                  dd        |S c c}}w )Nzself_attn.rotary_emb.inv_freqzlm_head.weight)itemsr.   r    pop)rA   weightskvs       r+   sanitizezModel.sanitize   s_     %]]_
,TQ0OWX0XAqD_ 	 
 99((KK($/
s
   AAc                 .    | j                   j                  S ri   )r   r~   )rA   s    r+   r~   zModel.layers   s    zz   r*   ri   )r!   r"   r#   r   r5   r[   r\   rX   r   propertyr~   r]   r^   s   @r+   r   r      s@    TY T 

 ! !r*   r   )dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorer[   mlx.nnr>   baser
   r   r   bitlinear_layersr   
rope_utilsr   r   Moduler-   r`   rm   ry   r   r)   r*   r+   <module>r      s    "  - -   T T ' ' % % %&:		 :z")) 0ryy 4 < !BII  !r*   