
    (HJjV                         d dl mZ d dlmZ d dlmZ d dlmZ ddl	m
Z
 e G d de
j                               Z G d d	ej                        Z G d
 dej                        Zy)    )	dataclass)OptionalN   )llamac                   R     e Zd ZU eed<   dZeed<   dZee	e      ed<    fdZ
 xZS )	ModelArgs
model_type   no_rope_layer_intervalNno_rope_layersc                 (   t         |           | j                  Et        | j                        D cg c]   }t        |dz   | j                  z  dk7        " c}| _        y t        | j                        | j                  k7  rt        d      y c c}w )Nr   r   z `no_rope_layers` length mismatch)	super__post_init__r   rangenum_hidden_layersintr   len
ValueError)selfi	__class__s     _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/smollm3.pyr   zModelArgs.__post_init__   s    & t556#6A QUd999Q>?6#D $$%)?)???@@ @	#s   %B)__name__
__module____qualname__str__annotations__r   r   r   r   listr   __classcell__r   s   @r   r   r      s2    O"#C#*.NHT#Y'.A A    r   c                       e Zd ZdZddefdZy)NoPEz;No-op used to disable rotary embeddings in selected layers.offsetc                     |S N )r   xr$   s      r   __call__zNoPE.__call__    s    r!   N)r   )r   r   r   __doc__r   r)   r'   r!   r   r#   r#      s    E# r!   r#   c                        e Zd ZdZdef fdZ	 	 d
dej                  deej                     fdZ	e
d        Zdefd	Z xZS )Modelz;Wrapper around Llama that respects NoPE layers in SmolLM-3.argsc                    t         |           || _        |j                  | _        t	        j
                  |      | _        |j                  s1t        j                  |j                  |j                  d      | _        t        |j                        D ]8  \  }}|r	t               | j                  j                   |   j"                  _        : y )NF)bias)r   __init__r-   r	   r   
LlamaModelmodeltie_word_embeddingsnnLinearhidden_size
vocab_sizelm_head	enumerater   r#   layers	self_attnrope)r   r-   idxuse_roper   s       r   r0   zModel.__init__'   s    	#%%d+
''99T%5%5tUSDL&t':':;MC8<

!!#&005 <r!   inputsinput_embeddingsc                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r&   )r2   r-   r3   embed_tokens	as_linearr8   )r   r?   cacher@   outs        r   r)   zModel.__call__4   sZ     jj(8999((**))33C8C 
 ,,s#C
r!   c                 .    | j                   j                  S r&   )r2   r:   )r   s    r   r:   zModel.layersA   s    zz   r!   weightsc                     |j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j                  dd        |S c c}}w )Nzself_attn.rotary_emb.inv_freqzlm_head.weight)itemsr-   r3   pop)r   rG   kvs       r   sanitizezModel.sanitizeE   s]    $]]_
,TQ0OWX0XAqD_ 	 
 99((KK($/
s
   AA)NN)r   r   r   r*   r   r0   mxarrayr   r)   propertyr:   dictrM   r   r    s   @r   r,   r,   $   s^    E?Y ?  /3	 #288,	 ! ! r!   r,   )dataclassesr   typingr   mlx.corecorerN   mlx.nnr4    r   r   Moduler#   r,   r'   r!   r   <module>rY      sX    "     A A A 299 'BII 'r!   