
    (HJj                        d dl Z d dlmZ d dlmZ d dlmZ ddlm	Z	 ddl
mZ ddlmZ e G d de             Z G d	 d
ej                        Z G d dej                        Z G d dej                        Z G d dej                        Zy)    N)	dataclass   )swiglu)BaseModelArgs)ArraysCachec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   dZeed<   dZeed<   dZ	e
ed<   d Zy)	ModelArgs
model_type
vocab_sizehidden_sizeintermediate_size
state_sizenum_hidden_layersconv_kerneluse_biasuse_conv_biastime_step_rankTtie_word_embeddingsFuse_bcdt_rmsgư>mixer_rms_epsc                 .   t        | d      st        | d      r| j                  | _        t        | d      st        | d      r| j                  | _        t        | d      st        | d      r| j
                  | _        t        | d      st        | d      r| j                  | _        t        | d      st        | d	      r| j                  | _        t        | d
      st        | d      r| j                  | _        t        | d      st        | d      r| j                  | _        t        | d      st        | d      r| j                  | _        | j                   dk(  r't#        j$                  | j                  dz        | _        | j&                  dk(  rd| _        y y )Nr   d_modelr   d_innerr   d_stater   n_layern_layersr   d_convr   biasr   	conv_biasauto   falcon_mambaT)hasattrr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   mathceilr
   r   selfs    ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/mamba.py__post_init__zModelArgs.__post_init__   s>   t]+i0H#||Dt01gdI6N%)\\D"t\*wtY/G"llDOt01gdI6N%)\\D"t01gdJ6O%)]]D"t]+h0G#{{DtZ(WT6-B IIDMt_-'$2L!%D&("&))D,<,<r,A"BD??n, $D -    N)__name__
__module____qualname__str__annotations__intboolr   r   r   floatr)    r*   r(   r	   r	      s^    OOON $$L$M5%r*   r	   c                   8     e Zd Zdef fdZddZd Zd Z xZS )
MambaBlockargsc                    t         |           | _        j                  | _        j                  | _        j                  | _        j                  | _        t        j                        | _
        j                  | _        j                  | _        | j                  r
fd| _        t        j                  | j                  | j                  dz  j                         | _        t        j$                  | j                  | j                  | j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d| j
                  z  z   d      | _        t        j                  | j                  | j                  d      | _        t-        j.                  t-        j0                  d| j
                  dz         j3                  d	| j
                  g      | j                  d
      }t-        j4                  |      | _        t-        j8                  | j                  g      | _        t        j                  | j                  | j                  j                         | _        y )Nc                     t         j                  j                  | t        j                  | j                  d   | j
                        j                        S )N)eps)mxfastrms_normonesshapedtyper   )xr6   s    r(   <lambda>z%MambaBlock.__init__.<locals>.<lambda>C   s=    (8(82771772;0d6H6H )9 )r*      r   r   )in_channelsout_channelskernel_sizegroupsr   paddingFTg      ?r   )repeatsaxis)super__init__r6   r   r   ssm_state_sizer   conv_kernel_sizer   r0   r   r   r   
mixer_normnnLinearr   in_projConv1dconv1dx_projdt_projr;   repeatarangereshapelogA_logr>   Dout_proj)r'   r6   A	__class__s    ` r(   rM   zMambaBlock.__init__7   s   	++"oo $ 0 0!%!7!7!$"5"56!// --DO yyd44q8t}}
 ii..//--))##
 ii""!d&9&9"99

 yy!4!4d6L6LSWXIIIIc4..45==q$BUBU>VW**

 VVAY
$0012		""D$4$44==
r*   c           	         | j                   }| j                  |      }t        | j                  r| j                  nd t        j                  || j                  | j                  | j                  z   gd            \  }}}| j                  rt        | j                  |||f      \  }}}t        j                  | j                  |            }t        j                  ||z  d      t        j                  |d      z  }	|2|	|t        j                  t        j                  |d      |z        z  z  }	|	t        j                  |d      z  j                  d      }
|
||z  z   }
|
|	fS )Nc                     | S Nr3   )rA   s    r(   rB   z%MambaBlock.ssm_step.<locals>.<lambda>k   s    r*   r9   rK   r   rC   )r]   rV   mapr   rP   r;   splitr   rN   rQ   softplusrW   expand_dimsexpsqueeze)r'   rA   r_   stater]   deltaBCdeltaBC	new_stateys              r(   ssm_stepzMambaBlock.ssm_stepg   s1   FF++a.#00DOOkHH$$d&9&9D<O<O&OP
q! dooq!}=KE1aDLL/0NN519b1BNN1a4HH	ub(AA(E!FFFI2..77:AI)|r*   c                    |j                   \  }}}| j                  |      }|j                  dd      \  }}| j                  }	|t	        j
                  ||gd      }
nt	        j                  |d|	dz
  dfdg      }
| j                  |
      }|
d d |	dz
   d d d f   }t        j                  |      }t	        j                  | j                         }|}g }t        |      D ]0  }| j                  |d d |f   ||      \  }}|j                  |       2 t	        j                  |d      }| j!                  t#        ||            }|||ffS )NrC   r9   )indices_or_sectionsrK   r   rd   )r   r   r   )r?   rS   rf   rO   r;   concatenatepadrU   rQ   siluri   r\   rangerr   appendstackr^   r   )r'   rA   
conv_cachestate_cachern   Tr]   xzzKx_fullconv_outnew_conv_cacher_   current_staterq   ty_ts                     r(   _process_sequencezMambaBlock._process_sequence|   s:   ''1a\\!_xxABx71!!!^^ZO!<FVVAQ
F;<F;;v&QU8:q 01GGHVVDJJ#qA!%qAw=!ICHHSM  HHQQMM&A,'>=111r*   c                     |d\  }}n
|d   |d   }}| j                  |||      \  }\  }}t        |t              r
||d<   ||d<   |S )N)NNr   r   )r   
isinstancer   )r'   rA   cacher{   r|   outputr   new_state_caches           r(   __call__zMambaBlock.__call__   sf    =&0#J&+AhaJ484J4Jz;5
11 e[)%E!H&E!Hr*   rc   )	r+   r,   r-   r	   rM   rr   r   r   __classcell__r`   s   @r(   r5   r5   6   s    .
Y .
`*2,r*   r5   c                   D     e Zd Zdef fdZdej                  fdZ xZS )ResidualBlockr6   c                     t         |           t        |      | _        t	        j
                  |j                        | _        y rc   )rL   rM   r5   mixerrQ   RMSNormr   normr'   r6   r`   s     r(   rM   zResidualBlock.__init__   s0    %
JJt//0	r*   rA   c                 J    | j                  | j                  |      |      |z   S rc   )r   r   )r'   rA   r   s      r(   r   zResidualBlock.__call__   s     zz$))A,.22r*   	r+   r,   r-   r	   rM   r;   arrayr   r   r   s   @r(   r   r      s    1Y 1
3"(( 3r*   r   c                   D     e Zd Zdef fdZdej                  fdZ xZS )Mambar6   c                 4   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                        | _        y c c}w rc   )rL   rM   rQ   	Embeddingr   r   
embeddingsrx   r   r   layersr   norm_f)r'   r6   _r`   s      r(   rM   zMamba.__init__   sn    ,,t8H8HI49$:P:P4QR4Qq}T*4QRjj!1!12 Ss   BrA   c                     | j                  |      }|d gt        | j                        z  }t        | j                  |      D ]  \  }} |||      } | j	                  |      S rc   )r   lenr   zipr   )r'   rA   r   layercs        r(   r   zMamba.__call__   s\    OOA=FS--EDKK/HE1aA 0{{1~r*   r   r   s   @r(   r   r      s    3Y 3"(( r*   r   c                   b     e Zd Zdef fdZddej                  fdZd Ze	d        Z
d Z xZS )	Modelr6   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFrD   )rL   rM   r6   r
   r   backboner   rQ   rR   r   r   lm_headr   s     r(   rM   zModel.__init__   sW    	//d''99T%5%5tUSDL (r*   inputsc                     |j                   \  }}| j                  ||      }| j                  j                  r'| j                  j                  j                  |      }|S | j                  |      }|S rc   )r?   r   r6   r   r   	as_linearr   )r'   r   r   rn   r}   rA   logitss          r(   r   zModel.__call__   sa    ||1MM&%(99((]]--77:F  \\!_Fr*   c                 x    t        t        | j                              D cg c]  }t        d       c}S c c}w )NrC   )size)rx   r   r   r   )r'   r   s     r(   
make_cachezModel.make_cache   s/    -23t{{3C-DE-D#-DEEEs   7c                 .    | j                   j                  S rc   )r   r   r&   s    r(   r   zModel.layers   s    }}###r*   c                     |j                         D ]2  \  }}d|v s|j                  d   dk7  s|j                  dd      ||<   4 |S )Nzconv1d.weightr9   r   rC   )itemsr?   moveaxis)r'   weightskvs       r(   sanitizezModel.sanitize   sH    MMODAq!#q(8ZZ1-
 $ r*   rc   )r+   r,   r-   r	   rM   r;   r   r   r   propertyr   r   r   r   s   @r(   r   r      s@    TY T
rxx 
F $ $r*   r   )r$   dataclassesr   mlx.corecorer;   mlx.nnrQ   activationsr   baser   r   r   r	   Moduler5   r   r   r   r3   r*   r(   <module>r      s{     !      $% $% $%Nj jZ3BII 3BII   BII  r*   