
    (HJj\                     J   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ e G d de             Z G d	 d
e	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                   v    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   y)	ModelArgs
model_type
block_sizelayer_norm_epsn_embdn_head
n_kv_headsn_layer
rope_theta
vocab_sizeTtie_word_embeddingsN)	__name__
__module____qualname__str__annotations__intfloatr   bool     b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/lille-130m.pyr   r      s<    OOKKOLO $$r    r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Lille130mAttentionargsc                    t         |           |j                  | _        |j                  | _        |j                  |j                  z  | _        | j
                  dz  | _        t        j                  |j                  |j                  d|j                  z  z   | j
                  z  d      | _	        t        j                  |j                  | j
                  z  |j                  d      | _
        t        j                  |j                  |j                        | _        t        j                  |j                  |j                  z  d|j                        | _        y )Ng         FbiasepsT)super__init__r   r   r   head_dimscalennLinearqkv_projout_projRMSNormr   normRoPEr   ropeselfr$   	__class__s     r!   r,   zLille130mAttention.__init__   s    kk//t{{2]]D(
		KK$++DOO(;;t}}LSX
 		$++"=t{{QVWJJt{{0C0CD	GGDKK4;;6dooN	r    xmaskcachereturnc                    |j                   \  }}}| j                  | j                  |            }| j                  | j                  z  }| j
                  | j                  z  }	t        j                  ||||	z   gd      \  }
}}|
j                  ||| j                  d      j                  dddd      }
|j                  ||| j
                  d      j                  dddd      }|j                  ||| j
                  d      j                  dddd      }|P| j                  |
|j                        }
| j                  ||j                        }|j                  ||      \  }}n"| j                  |
      }
| j                  |      }t        |
|||| j                  |      }|j                  dddd      j                  ||d      }| j                  |      S )	N)axisr   r&   r      )offset)r<   r.   r;   )shaper1   r4   r   r-   r   mxsplitreshape	transposer6   rB   update_and_fetchr
   r.   r2   )r8   r:   r;   r<   BLDqkvq_sizekv_sizequerieskeysvaluesoutputs                 r!   __call__zLille130mAttention.__call__,   s    ''1ammDIIaL)t}},//DMM1 "vv7G.Hr Rv//!QR8BB1aAN||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?}}V$$r    NNr   r   r   r   r,   rD   arrayr   r   rS   __classcell__r9   s   @r!   r#   r#      sY    OY O& $(#	 %88 % rxx  % }	 %
 
 %r    r#   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Lille130mMLPr$   c                    t         |           dt        t        d|j                  z  dz        dz        z  }t        j                  |j                  |j                        | _        t        j                  |j                  |d      | _
        t        j                  |j                  |d      | _        t        j                  ||j                  d      | _        y )N      rA   r)   Fr'   )r+   r,   roundr   r   r/   r3   r   r4   r0   	gate_projup_proj	down_proj)r8   r$   
hidden_dimr9   s      r!   r,   zLille130mMLP.__init__P   s    5Q_q%8!9C!?@@
JJt{{0C0CD	4;;
GyyjuE:t{{Gr    r:   r=   c                     | j                  |      }| j                  t        | j                  |      | j	                  |                  S N)r4   ra   r   r_   r`   )r8   r:   hs      r!   rS   zLille130mMLP.__call__Y   s6    IIaL~~fT^^A%6QHIIr    )	r   r   r   r   r,   rD   rV   rS   rW   rX   s   @r!   rZ   rZ   O   s/    HY HJ"(( Jrxx Jr    rZ   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Lille130Blockr$   c                 b    t         |           t        |      | _        t	        |      | _        y rd   )r+   r,   r#   	attentionrZ   feed_forwardr7   s     r!   r,   zLille130Block.__init___   s'    +D1(.r    r:   r;   r<   r=   c                 Z    || j                  |||      z   }|| j                  |      z   }|S rd   )ri   rj   )r8   r:   r;   r<   re   outs         r!   rS   zLille130Block.__call__d   s5     q$..$##A&&
r    rT   rU   rX   s   @r!   rg   rg   ^   sW    /Y / $(#	88 rxx  }	
 
r    rg   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )Lille130r$   c                 N   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )N)r$   r)   )r+   r,   r/   	Embeddingr   r   tok_embeddingsranger   rg   layersr3   r   r4   )r8   r$   _r9   s      r!   r,   zLille130.__init__p   sq     ll4??DKKH9>t||9LM9LA}$/9LMJJt{{0C0CD	 Ns   B"inputsr<   r=   c                    | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                   j                  | j                  |            S )Nr   r<   )rq   lenrs   r	   zip	as_linearr4   )r8   ru   r<   re   r;   layercs          r!   rS   zLille130.__call__v   s    
 '=FS--E$Qa1DKK/HE1aQ'A 0 "",,TYYq\::r    rd   rU   rX   s   @r!   rn   rn   o   sC    EY E  $;; }; 
	;r    rn   c                        e Zd Zdef fdZ	 d	dej                  dee   dej                  fdZ	e
d        Zd Z xZS )
Modelr$   c                 r    t         |           || _        |j                  | _        t	        |      | _        y rd   )r+   r,   r$   r   rn   transformerr7   s     r!   r,   zModel.__init__   s,    	//#D>r    ru   r<   r=   c                 (    | j                  ||      S )Nrw   )r   )r8   ru   r<   s      r!   rS   zModel.__call__   s    
 e44r    c                 .    | j                   j                  S rd   )r   rs   )r8   s    r!   rs   zModel.layers   s    &&&r    c                 `    |j                         D ci c]  \  }}d|vs|| c}}S c c}}w )N
rotary_emb)items)r8   weightskvs       r!   sanitizezModel.sanitize   s/    !(JAL4I1JJJs   **rd   )r   r   r   r   r,   rD   rV   r   r   rS   propertyrs   r   rW   rX   s   @r!   r~   r~      s[    *Y *  $55 }5 
	5 ' 'Kr    r~   )dataclassesr   typingr   r   mlx.corecorerD   mlx.nnr/   activationsr   baser   r	   r
   r   Moduler#   rZ   rg   rn   r~   r   r    r!   <module>r      s    "      T T 
% 
% 
%1% 1%hJ299 JBII ";ryy ;2KBII Kr    