
    (HJj&              	       V   d dl mZ d dlmZ d dlmZmZmZmZm	Z	m
Z
 d dlmZ d dlmZ d dlmZ ddlmZmZmZ ddlmZmZ dd	lmZ  eej6                  d
      dej8                  dej8                  dej8                  dej8                  fd       Z eej6                  d
      dej8                  dej8                  dej8                  fd       Z eej6                  d
      dej8                  dej8                  dej8                  dej8                  fd       Ze G d de             Z  G d dejB                        Z" G d dejB                        Z# G d dejB                        Z$ G d d ejB                        Z% G d! d"ejB                        Z& G d# d$ejB                        Z'y)%    )	dataclass)partial)AnyDictListOptionalTupleUnionN)shard_linear   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_ropeT)	shapelessqueryweightbiasreturnc                 z    | |d d d d d f   j                  dd      z  }||d   z   }t        j                  |      S )N).NN)swapaxesmxsigmoid)r   r   r   gate_logitss       g/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/iquestloopcoder.py_compute_gater       sB    &D!,55b"==K_ 55K::k""    gateupc                 2    t        j                  |       |z  S N)nnsilu)r"   r#   s     r   	_silu_mulr(      s    774=2r!   attn_global
attn_localc                     | |z  d| z
  |z  z   S )Nr    )r"   r)   r*   s      r   _mix_attentionr-      s     +TZ 777r!   c                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZe	ed<   dZ
e	ed<   dZeed<   dZeeeeeef   f      ed<   dZe	ed<   dZeed<   dZeed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizehead_dimnum_key_value_headsi   max_position_embeddingsFattention_biasmlp_biasg    A
rope_thetaNrope_scalingtie_word_embeddings   loop_num@   loop_window_size)__name__
__module____qualname__str__annotations__intfloatr9   r:   boolr;   r<   r=   r   r   r
   r>   r@   rB   r,   r!   r   r/   r/   #   s    OOM#)S) ND Hd J ;?L(4U5#:%6 678? %%Hccr!   r/   c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZS )LoopGateProjection	num_headsr7   c                     t         |           || _        || _        t	        j
                  ||f      | _        t	        j
                  |f      | _        y r%   )super__init__rM   r7   r   zerosr   r   )selfrM   r7   	__class__s      r   rP   zLoopGateProjection.__init__9   sC    " hh	845HHi\*	r!   r   r   c                 D    t        || j                  | j                        S r%   )r    r   r   )rR   r   s     r   __call__zLoopGateProjection.__call__@   s    UDKK;;r!   )	rC   rD   rE   rH   rP   r   arrayrU   __classcell__rS   s   @r   rL   rL   8   s1    +# + +<bhh <288 <r!   rL   c                   .    e Zd Zdef fdZ	 ddej                  dedeej                  ej                  ej                  f   fdZ		 	 ddej                  dej                  d	ej                  d
e
ej                     de
e   dej                  fdZ xZS )	Attentionargsc                 v   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  x| _        }|dz  | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  ||j                        | _        t!        ||j"                  d|j$                  |j&                        | _        y )Ng      ࿩r   F)traditionalscaling_configr9   )rO   rP   r1   r4   n_headsr8   
n_kv_headsr7   scaler&   Linearr:   q_projk_projv_projo_projr   r<   r=   r9   rope)rR   r[   dimr`   ra   r7   rS   s         r   rP   zAttention.__init__E   s    !%!9!99w'+'?'??*#'==0t^
iiWx%7d>Q>QRiiZ(%:ATATUiiZ(%:ATATUii( 2Cd>Q>QR#OO,,$($@$@
	r!   xoffsetr   c                    |j                   \  }}}| j                  |      j                  ||| j                  d      j	                  dddd      }| j                  |      j                  ||| j                  d      j	                  dddd      }| j                  |      j                  ||| j                  d      j	                  dddd      }| j                  ||      }| j                  ||      }|||fS )Nr   r   r?   r      )rk   )	shaperd   reshaper`   	transposere   ra   rf   rh   )	rR   rj   rk   BL_querieskeysvaluess	            r   get_qkvzAttention.get_qkv[   s     ''1a++a.((At||R@JJ1aQRTUV{{1~%%aDOOR@JJ1aQRTUVQ''1doorBLLQPQSTVWX))GF)3yyfy-f$$r!   rt   ru   rv   maskcachec                 8    t        ||||| j                  |      S )N)ry   rb   rx   )r   rb   )rR   rt   ru   rv   rx   ry   s         r   	attentionzAttention.attentionh   s"     ,T6djjt
 	
r!   )r   )NN)rC   rD   rE   r/   rP   r   rV   rH   r	   rw   r   r   r{   rW   rX   s   @r   rZ   rZ   D   s    
Y 
. *+%%#&%	rxx288+	,%$ $(#



 hh

 	


 rxx 

 }

 


r!   rZ   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )MLPr[   c                 <   t         |           |j                  }|j                  }t	        j
                  |||j                        | _        t	        j
                  |||j                        | _        t	        j
                  |||j                        | _	        y )Nr]   )
rO   rP   r1   r3   r&   rc   r;   	gate_proj	down_projup_proj)rR   r[   ri   
hidden_dimrS   s       r   rP   zMLP.__init__v   sl    ++
3
G:sGyyjt}}Er!   rj   r   c                 t    | j                  t        | j                  |      | j                  |                  S r%   )r   r(   r   r   )rR   rj   s     r   rU   zMLP.__call__~   s)    ~~iq(94<<?KLLr!   )	rC   rD   rE   r/   rP   r   rV   rU   rW   rX   s   @r   r}   r}   u   s/    FY FM"(( Mrxx Mr!   r}   c                   $     e Zd Zdef fdZ xZS )TransformerBlockr[   c                 "   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Neps)rO   rP   rZ   	self_attnr}   mlpr&   RMSNormr1   r5   input_layernormpost_attention_layernormrR   r[   rS   s     r   rP   zTransformerBlock.__init__   se    "4t9!zz$*:*:@Q@QR(*

$"3"3)
%r!   )rC   rD   rE   r/   rP   rW   rX   s   @r   r   r      s    
Y 
 
r!   r   c                   X     e Zd Zdef fdZ	 ddej                  deee	      fdZ
 xZS )IQuestLoopCoderModelr[   c                    t         |           |j                  dk(  sJ d|j                          || _        |j                  | _        t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        t        |j                        D cg c]"  }t!        |j"                  |j$                        $ c}| _        |j                  | _        |j(                  | _        y c c}w c c}w )Nr?   z"Only loop_num=2 is supported, got )r[   r   )rO   rP   r@   r[   r6   r&   	Embeddingr1   embed_tokensranger2   r   layersr   r5   normrL   r4   r7   gate_projectionsrB   )rR   r[   rs   rS   s      r   rP   zIQuestLoopCoderModel.__init__   s   }}!W%G#WW!	//LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	 4112!
2 t77G2!
  $ 5 5
!
s   E3'E
inputsry   c           	         |j                   d d \  }}| j                  |      }|d gdt        | j                        z  z  }t	        ||d         }t	        ||t        | j                           | j
                        }g }t        | j                  |      D ]  \  }	}
|	j                  |      }|
|
j                  nd}|	j                  j                  ||      \  }}}|
|
j                  ||      \  }}|j                  ||f       |	j                  j                  |||||
      }|	j                  j                  |j                  dddd      j!                  ||d            }||z   }|	j#                  |	j%                  |            }||z   } t        | j                  | j&                  |t        | j                        d  |      D ]   \  }	}}
\  }}|	j                  |      }|
|
j                  nd}|	j                  j                  ||      \  }}} ||      }|	j                  j                  |||||
      }|
|
j                  ||      \  }}|	j                  j                  |||||
      }t)        |||      }|	j                  j                  |j                  dddd      j!                  ||d            }||z   }|	j#                  |	j%                  |            }||z   }# | j+                  |      S )Nr?   r   )window_size)ry   r   rm   r   )rn   r   lenr   r   rB   zipr   rk   r   rw   update_and_fetchappendr{   rg   rp   ro   r   r   r   r-   r   )rR   r   ry   rq   rr   hrx   window_maskloop1_kvlayerch_normrk   q1k1v1outrr   q2k2v2r"   r)   r*   mixeds                             r   rU   zIQuestLoopCoderModel.__call__   s   
 ||BQ1f%=Fa#dkk"223E$Qa1+uS%&D4I4I
 DKK/HE1**1-F!"QXXAF00@JBB}++B3BOORH%//++BBA+FC&&s}}Q1a'@'H'HAr'RSAAA		%88;<AAA 0 .1KK..c$++6F6H0I8.
)E9a"b **1-F!"QXXAF00@JBBR=D//33BBA3NK}++B3B22 3 J #4jAE&&uq!Q'B'J'J1aQS'TUAAA		%88;<AAA/.
2 yy|r!   r%   )rC   rD   rE   r/   rP   r   rV   r   r   r   rU   rW   rX   s   @r   r   r      s8    6Y 6( &*99 S	"9r!   r   c                        e Zd Zdef fdZ	 d	dej                  fdZd	deej                  j                     fdZed        Zd Z xZS )
Modelr[   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr]   )rO   rP   r[   r0   r   modelr>   r&   rc   r1   r6   lm_headr   s     r   rP   zModel.__init__   sX    	//)$/
''99T%5%5tUSDL (r!   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r%   )r   r[   r>   r   	as_linearr   )rR   r   ry   r   s       r   rU   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r!   groupc                    |xs t         j                  j                         }|j                         }|j	                         }t        | j                  j                        D ]  \  }}t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                   j"                  d|      |j                   _        t        |j                   j$                  d|      |j                   _        t        |j                   j&                  d|      |j                   _        | j                  j(                  |   }|j*                  |z  }||z  }||z   }	|j,                  ||	d d f   |_        |j.                  ||	 |_        ||_         y )Nzall-to-sharded)r   zsharded-to-all)r   distributedinitsizerank	enumerater   r   r   r   rd   re   rf   rg   r`   ra   r   r   r   r   r   rM   r   r   )
rR   r   Nr   ir   r   heads_per_rankstartends
             r   shardzModel.shard   s   .,,.JJLzz|!$**"3"34HAu%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO##)#OO&&1,&".		##%5U#EII #/		##%5U#EII !-		!!#35!EII 

33A6I&00A5N>)E.(C(//c	1=I&^^E#6IN"0IA 5r!   c                 .    | j                   j                  S r%   )r   r   )rR   s    r   r   zModel.layers  s    zz   r!   c                     | j                   D cg c]  }t                c}| j                   D cg c]"  }t        | j                  j                        $ c}z   S c c}w c c}w )N)max_size)r   r   r   r[   rB   )rR   rs   s     r   
make_cachezModel.make_cache  sV    #';;/;a	;/JN++3
JUQOTYY%?%?@+3
 
 	
/ 3
s
   A'A!r%   )rC   rD   rE   r/   rP   r   rV   rU   r   r   Groupr   propertyr   r   rW   rX   s   @r   r   r      s\    TY T 

%18BNN$8$89 %1N ! !
r!   r   )(dataclassesr   	functoolsr   typingr   r   r   r   r	   r
   mlx.corecorer   mlx.nnr&   mlx.nn.layers.distributedr   baser   r   r   ry   r   r   
rope_utilsr   compilerV   r    r(   r-   r/   ModulerL   rZ   r}   r   r   r   r,   r!   r   <module>r      s   "  : :   2 T T + ' 	t$# #288 #288 # # %# 	t$BHH "(( rxx  % 	t$8
((8!#879xx8XX8 %8   (	< 	<.
		 .
b
M")) 
M
ryy 
K299 K\C
BII C
r!   