
    (HJjs/                        d dl Z d dlmZ d dlmZ d dlmZ d dlmZm	Z	m
Z
mZ d dlmZ d dlmZ ddlmZ ddlmZmZmZ dd	lmZmZ dd
lmZ e G d de             Z G d dej:                        Z G d dej>                        Z  G d dejB                        Z" eejF                  d      d        Z$ G d dej:                        Z% G d dej:                        Z& G d dej:                        Z' G d dej:                        Z( G d dej:                        Z) G d  d!ej:                        Z* G d" d#ej:                        Z+y)$    N)	dataclass)partial)
accumulate)AnyDictOptionalUnion   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)ConcatenateKVCacheKVCache)initialize_ropec                   ~    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   d
Zeed<   dZ	eed<   y)	ModelArgs
model_type
vocab_size
hidden_dim
num_layersnum_kv_reuse_layers	num_headsnum_kv_headsg      
@hidden_dim_scale_factoriP  
rope_thetagh㈵>rms_norm_epsN)
__name__
__module____qualname__str__annotations__intr   floatr   r        \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/afm7.pyr   r      sF    OOOON%)U)JL%r&   r   c                   V     e Zd Z	 	 	 d
dedee   dededef
 fdZddefdZd	 Z	 xZ
S )FusedLoRALinear
input_dimsoutput_dimsrdropoutscalec           	         t         |           t        ||      | _        t	        j
                  |      | _        || _        dt        j                  |      z  }|D cg c]'  }t        j                  j                  | |||f      ) c}| _        |D cg c]  }t        j                  ||f       c}| _        y c c}w c c}w )N)pr
   )lowhighshape)super__init__FusedLinearlinearnnDropoutr-   r.   mathsqrtmxrandomuniformlora_azeroslora_b)	selfr*   r+   r,   r-   r.   _od	__class__s	           r'   r5   zFusedLoRALinear.__init__!   s     	!*k:zzG,
DIIj)) !
  II5&uZOL 
 4??;RrxxB(;?	
 @s   ,B>C
dequantizec                    | j                   }|j                  }t        |t              }|j                  }|rW|j
                  j                  }t        j                  ||j
                  |j                  |j                  |j                        }|j                  d   }|j                  }t        ||      }||_        t        | j                  | j                         D 	
cg c]:  \  }	}
| j"                  |
j$                  z  |	j$                  z  j'                  |      < }}	}
t        j(                  |d      }||z   |_        |r(|s&|j+                  |j                  |j                        }|S c c}
}	w )Nr   axis)r7   weight
isinstanceFusedQuantizedLineardtypescalesr<   rF   biases
group_sizebitsr3   r+   r6   zipr?   rA   r.   Tastypeconcatenateto_quantized)rB   rF   r7   rK   is_quantizedrN   r*   r+   fused_linearabdeltasdeltas                r'   fusezFusedLoRALinear.fuse6   s<   !&*>? MM''E]]!!F \\"%
((":{;$ DKK5
51 jj133!##%--e45 	 
 vA.$un
'44V5F5FTL
s   ?E.c                     |j                    j                  |      } j                  |      }t         j                   j
                        D cg c]  \  }}||z  |z   }}}t         fdt        ||      D              S c c}}w )Nc              3   f   K   | ](  \  }}|j                   |z  j                        z    * y wN)r.   rU   ).0yizidtrB   s      r'   	<genexpr>z+FusedLoRALinear.__call__.<locals>.<genexpr>]   s.     OY62rR4::?22266Ys   .1)rN   r7   r-   rS   r?   rA   tuple)rB   xyrZ   r[   zre   s   `     @r'   __call__zFusedLoRALinear.__call__X   st    WWKKNLLO%(dkk%BC%BTQa!eq[%BCOSAYOOO Ds   B   g        g      4@)F)r   r   r    r#   listr$   r5   boolr^   rk   __classcell__rE   s   @r'   r)   r)       s`    
 @@ #Y@ 	@
 @ @* t  DPr&   r)   c                   j     e Zd Zddedef fdZed        Zed        Z fdZddede	d	e	fd
Z
 xZS )rM   rQ   rR   c                 X    t        |      ^ }}|| _        t        |   ||d||       y )NF)biasrQ   rR   r   indicesr4   r5   )rB   r*   r+   rQ   rR   rv   rE   s         r'   r5   zFusedQuantizedLinear.__init__a   s6     *; 7+%JT 	 	
r&   c                 N    | j                   j                  d   | j                  z  S NrH   )rO   r3   rQ   rB   s    r'   r*   zFusedQuantizedLinear.input_dimsh   s     {{  $t66r&   c                     dg| j                   z   | j                  j                  d   gz   }t        dt	        |            D cg c]  }||   ||dz
     z
   c}S c c}w Nr   r
   rv   rK   r3   rangelenrB   rv   is      r'   r+   z FusedQuantizedLinear.output_dimsl   _    #$(9(9!(<'==5:1c'l5KL5K
WQU^+5KLLL   Ac                 \    t         |   |      }|j                  | j                  d      S NrH   rI   r4   rk   splitrv   rB   rh   rE   s     r'   rk   zFusedQuantizedLinear.__call__q   )    GQwwt||"w--r&   r,   r-   r.   c                 Z    t        | j                  | j                  |||      }| |_        |S ra   r)   r*   r+   r7   rB   r,   r-   r.   lora_lins        r'   to_lorazFusedQuantizedLinear.to_lorau   +    "4??D4D4DaRWXr&   @      rl   )r   r   r    r#   r5   propertyr*   r+   rk   r$   r   rp   rq   s   @r'   rM   rM   `   s_    
C 
C 
 7 7 M M. 5 u r&   rM   c                   p     e Zd Z fdZed        Zed        Z fdZddedefdZ	dded	e
d
e
fdZ xZS )r6   c                 T    t        |      ^ }}|| _        t        |   ||d       y NFrt   ru   )rB   r*   r+   rv   rE   s       r'   r5   zFusedLinear.__init__|   s-     *; 7+[u=r&   c                 4    | j                   j                  d   S rx   )rK   r3   ry   s    r'   r*   zFusedLinear.input_dims   s    {{  $$r&   c                     dg| j                   z   | j                  j                  d   gz   }t        dt	        |            D cg c]  }||   ||dz
     z
   c}S c c}w r{   r|   r   s      r'   r+   zFusedLinear.output_dims   r   r   c                 \    t         |   |      }|j                  | j                  d      S r   r   r   s     r'   rk   zFusedLinear.__call__   r   r&   rQ   rR   c                     | j                   }| j                  }t        ||||      }t        j                  | j
                  ||      \  |_        |_        |_        |S ra   )r*   r+   rM   r<   quantizerK   rO   rP   )rB   rQ   rR   r*   r+   qls         r'   rW   zFusedLinear.to_quantized   sN    __
&&!*k:tL*,++dkk:t*T'	29bi	r&   r,   r-   r.   c                 Z    t        | j                  | j                  |||      }| |_        |S ra   r   r   s        r'   r   zFusedLinear.to_lora   r   r&   r   rl   )r   r   r    r5   r   r*   r+   rk   r#   rW   r$   r   rp   rq   s   @r'   r6   r6   {   sd    >
 % % M M.s s  5 u r&   r6   T)	shapelessc                     | j                   }| j                  t        j                        } | |z  j	                         } t        j
                  | dd      } | |z  j                  |      S )Ni   )rN   rU   r<   float32roundclip)rh   r.   re   s      r'   fake_8bit_quantr      sV    	
B	A	
UA
4AIb!!r&   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 p   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|dz  | _        |d|z  z   |z  }t        |||z  gd||z  gz  z         | _
        t        j                  ||d      | _        t        | j                  |j                  d      | _        t        j"                  |      | _        t        j"                  |      | _        t)        j*                  d      | _        t)        j*                  d      | _        y )N         Fr   Tg      ?)r4   r5   r   r   n_headsr   
n_kv_headshead_dimr.   r6   qkv_projr8   Linearout_projr   r   ropeRMSNormq_normk_normr<   arrayquant_key_scalequant_value_scale)rB   r   dimr   r   r   qkv_dimrE   s          r'   r5   zAttention.__init__   s   oo!%/w'+'8'88*#'??g#==t^
Q^+x7#'H$%Z(-B,C(CC
 		#s7#MMOO
	
 jj*jj*!xx}!##r&   rh   maskcachereturnc                    |j                   \  }}}| j                  |      \  }}}	|j                  ||| j                  d      j	                  dddd      }|j                  ||| j
                  d      j	                  dddd      }|	j                  ||| j
                  d      j	                  dddd      }	|| j                  | j                  ||j                              }| j                  | j                  ||j                              }t        || j                        }t        |	| j                        }	|j                  ||	      \  }}	nl| j                  | j                  |            }| j                  | j                  |            }t        || j                        }t        |	| j                        }	t        |||	|| j                  |      }
|
j	                  dddd      j                  ||d      }
| j!                  |
      S NrH   r   r   r
      )offset)r   r.   r   )r3   r   reshaper   	transposer   r   r   r   r   r   r   r   update_and_fetchr   r.   r   )rB   rh   r   r   BLDquerieskeysvaluesoutputs              r'   rk   zAttention.__call__   s    ''1a !%a 0v //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPkk$))GELL)"IJG;;tyyellyCDD"4)=)=>D$VT-C-CDF 11$?LD&kk$))G"45G;;tyy/D"4)=)=>D$VT-C-CDF-T6djjt
 !!!Q1-55aB?}}V$$r&   NNr   r   r    r   r5   r<   r   r   r   rk   rp   rq   s   @r'   r   r      sW    /Y /6 $(#	!%88!% rxx !% }	!%
 
!%r&   r   c                        e Zd Zdef fdZ	 d	dej                  dej                  dej                  deej                     dej                  f
dZ xZ	S )
KVReuseAttentionr   c                    t         |           |j                  }|j                  x| _        }|j                  |z  x| _        }|dz  | _        t        j                  ||d      | _	        t        j                  ||d      | _
        t        | j
                  |j                  d      | _        t        j                  |      | _        y )Nr   Fr   T)r4   r5   r   r   r   r   r.   r8   r   q_projr   r   r   r   r   r   )rB   r   r   r   r   rE   s        r'   r5   zKVReuseAttention.__init__   s    oo!%/w#'??g#==t^
iiSu5		#s7#MMOO
	
 jj*r&   rh   r   r   r   r   c                    |j                   \  }}}|j                   \  }}}	}| j                  |      }
|
j                  ||| j                  d      j	                  dddd      }
| j                  | j                  |
|	|z
              }
t        |
||d | j                  |      }|j	                  dddd      j                  ||d      }| j                  |      S r   )
r3   r   r   r   r   r   r   r   r.   r   )rB   rh   r   r   r   r   r   r   rC   Sr   r   s               r'   rk   zKVReuseAttention.__call__   s     ''1aZZ
1a++a.//!Qb9CCAq!QO++diiAi>?-T6TZZd
 !!!Q1-55aB?}}V$$r&   ra   
r   r   r    r   r5   r<   r   r   rk   rp   rq   s   @r'   r   r      sb    +Y +, $(%88% hh% 	%
 rxx % 
%r&   r   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr   c                    t         |           |j                  }t        ||j                  z        }t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _	        y r   )
r4   r5   r   r#   r   r8   r   	gate_proj	down_projup_proj)rB   r   r   r   rE   s       r'   r5   zMLP.__init__  sh    oot;;;<
3
?:s?yyju=r&   r   c                 |    | j                  |      }| j                  |      }| j                  t        ||            S ra   )r   r   r   r   )rB   rh   gs      r'   rk   zMLP.__call__  s2    NN1LLO~~fQl++r&   	r   r   r    r   r5   r<   r   rk   rp   rq   s   @r'   r   r   
  s    >Y >,RXX ,r&   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr   c                 "   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y N)eps)r4   r5   r   	self_attnr   mlpr8   r   r   r   input_layernormpost_attention_layernormrB   r   rE   s     r'   r5   zTransformerBlock.__init__  sa    "4t9!zz$//t?P?PQ(*

OO!2!2)
%r&   rh   r   r   r   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S ra   r   r   r   r   )rB   rh   r   r   r,   houts          r'   rk   zTransformerBlock.__call__%  sQ     NN4//2D%@EHHT22156!e
r&   r   r   rq   s   @r'   r   r     sW    
Y 
 $(#	
88
 rxx 
 }	

 

r&   r   c                        e Zd Zdef fdZ	 d	dej                  dej                  dej                  deej                     dej                  f
dZ xZ	S )
KVReuseTransformerBlockr   c                 "   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y r   )r4   r5   r   r   r   r   r8   r   r   r   r   r   r   s     r'   r5   z KVReuseTransformerBlock.__init__3  sb    )$/t9!zz$//t?P?PQ(*

OO!2!2)
%r&   rh   r   r   r   r   c                     | j                  | j                  |      |||      }||z   }| j                  | j                  |            }||z   }|S ra   r   )rB   rh   r   r   r   r,   r   r   s           r'   rk   z KVReuseTransformerBlock.__call__<  sS     NN4//2D&$GEHHT22156!e
r&   ra   r   rq   s   @r'   r   r   2  sb    
Y 
 $(88 hh 	
 rxx  
r&   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )AFMModelr   c                    t         |           || _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |j                  |j                  z
        D cg c]  }t        |       c}| _        t        |j                        D cg c]  }t        |       c}| _        t	        j                  |j                  |j                         | _        y c c}w c c}w r   )r4   r5   r   r   r8   	Embeddingr   	embeddingr}   r   r   r   layersr   kv_reuse_layersr   r   output_norm)rB   r   rC   rE   s      r'   r5   zAFMModel.__init__K  s    	//dootG 4??T-E-EEF
F T"F

 499Q9Q3R 
3Ra#D)3R 
 ::doo4;L;LM
 
s   ;C7,C<inputsc                 b   | j                  |      }|&d gt        | j                        z  }t               |d<   t	        ||d         }t        | j                  |      D ]  \  }} ||||      } |d   j                  \  }}| j                  D ]  } |||||      } | j                  |      S )NrH   r   )r   )	r   r~   r   r   r   rS   stater   r   )	rB   r   r   r   r   layercr   r   s	            r'   rk   zAFMModel.__call__Z  s    
 NN6"=FS--E*,E"I$Qa1DKK/HE1aQ'A 0 Ryf))Eavt,A * ""r&   ra   r   rq   s   @r'   r   r   J  s'    NY N$ ##r&   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr   c                 r    t         |           || _        |j                  | _        t	        |      | _        y ra   )r4   r5   r   r   r   modelr   s     r'   r5   zModel.__init__r  s+    	//d^
r&   r   c                 t    | j                  ||      }| j                   j                  j                  |      }|S ra   )r   r   	as_linear)rB   r   r   r   s       r'   rk   zModel.__call__x  s3    
 jj'jj"",,S1
r&   c                     t        t        | j                  j                              D cg c]  }t	                c}S c c}w ra   )r}   r~   r   r   r   )rB   rC   s     r'   
make_cachezModel.make_cache  s2    #(TZZ->->)?#@A#@a	#@AAAs   ?c                 \    | j                   j                  | j                   j                  z   S ra   )r   r   r   ry   s    r'   r   zModel.layers  s!    zz  4::#=#===r&   ra   )r   r   r    r   r5   r<   r   rk   r  r   r   rp   rq   s   @r'   r   r   q  s?    $Y $ B > >r&   r   ),r:   dataclassesr   	functoolsr   	itertoolsr   typingr   r   r   r	   mlx.corecorer<   mlx.nnr8   activationsr   baser   r   r   r   r   r   
rope_utilsr   r   Moduler)   QuantizedLinearrM   r   r6   compiler   r   r   r   r   r   r   r   r%   r&   r'   <module>r     s    !    - -    T T . ' 
 
 
=Pbii =P@2-- 6")) B 	t$" %":%		 :%z%%ryy %%P,")) ,"ryy .bii 0$#ryy $#N>BII >r&   