
    hJJj!                     d   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ	 d dl
mZ d dlZddlmZ ddlmZ e G d d             Zdd	Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    N)	dataclass)Union   )decode)detect_languagec                   r    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   y)ModelDimensionsn_melsn_audio_ctxn_audio_staten_audio_headn_audio_layern_vocab
n_text_ctxn_text_staten_text_headn_text_layerN)__name__
__module____qualname__int__annotations__     ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_whisper/whisper.pyr	   r	      s;    KLOr   r	   c                 z   |dz  dk(  sJ t        j                  |      |dz  dz
  z  }t        j                  | t        j                  |dz        z        }t        j                  |       dddf   |dddf   z  }t        j
                  t        j                  |      t        j                  |      gd      S )z*Returns sinusoids for positional embedding   r   r   Naxis)mathlogmxexparangeconcatenatesincos)lengthchannelsmax_timescalelog_timescale_incrementinv_timescalesscaled_times         r   	sinusoidsr.      s    a<1"hh}5Q9JKVV44ryyQ7OOPN))F#AtG,~dAg/FFK>>266+.{0CD1MMr   c                   >     e Zd Zdedef fdZ	 	 	 ddZddZ xZS )MultiHeadAttentionn_staten_headc                    t         |           || _        t        j                  ||      | _        t        j                  ||d      | _        t        j                  ||      | _        t        j                  ||      | _        y )NF)bias)	super__init__r2   nnLinearquerykeyvalueout)selfr1   r2   	__class__s      r   r6   zMultiHeadAttention.__init__)   s_    YYw0
99WgE:YYw0
99Wg.r   c                    | j                  |      }|]| j                  |      }| j                  |      }|ct        j                  |d   |gd      }t        j                  |d   |gd      }n*|#| j                  |      }| j                  |      }n|\  }}| j                  ||||      \  }}	| j                  |      ||f|	fS )Nr   r   r   )r9   r:   r;   r"   r%   qkv_attentionr<   )
r=   xxamaskkv_cacheqkvwvqks
             r   __call__zMultiHeadAttention.__call__1   s     JJqM:A

1A#NNHQK#3!<NNHQK#3!<A

2ADAq##Aq!T2Bxx|aVR''r   c                    |j                   \  }}}|| j                  z  dz  } |j                  g |j                   d d | j                  d j                  dddd      |z  } |j                  g |j                   d d | j                  d j                  dddd      |z  } |j                  g |j                   d d | j                  d j                  dddd      }||z  }	||	|d |d |f   z   }	t	        j
                  |	dd      }
|
|z  j                  dddd      }|j                  |||      }||	fS )	Ng      пr   r   r      T)r   precise)shaper2   reshape	transposer"   softmax)r=   rE   rF   rG   rC   n_batchn_ctxr1   scalerI   wr<   s               r   r@   z MultiHeadAttention.qkv_attentionI   sT   "#''DKK'E1AII4qwwr{4DKK44>>q!QJURAII4qwwr{4DKK44>>q!QJURAII4qwwr{4DKK44>>q!QJUd6E66E6>**BJJrD11u1a+kk'5'2Bwr   NNNN)r   r   r   r   r6   rJ   r@   __classcell__r>   s   @r   r0   r0   (   s+    / /S / (0r   r0   c                   6     e Zd Zddededef fdZddZ xZS )ResidualAttentionBlockr1   r2   cross_attentionc                    t         |           t        ||      | _        t	        j
                  |      | _        |rt        ||      nd | _        |rt	        j
                  |      nd | _        |dz  }t	        j                  ||      | _
        t	        j                  ||      | _        t	        j
                  |      | _        y )N   )r5   r6   r0   attnr7   	LayerNormattn_ln
cross_attncross_attn_lnr8   mlp1mlp2mlp_ln)r=   r1   r2   r]   n_mlpr>   s        r   r6   zResidualAttentionBlock.__init__[   s    &w7	||G, 4Cw/ 	 7FR\\'24!IIgu-	IIeW-	ll7+r   c           
      x   |r|nd\  }}| j                  | j                  |      ||      \  }}}||z  }d }	| j                  r,| j                  | j                  |      ||      \  }}}	||z  }|| j	                  t        j                  | j                  | j                  |                        z   }|||f|	fS )N)NNrC   rD   )rD   )	r`   rb   rc   rd   rf   r7   gelure   rg   )
r=   rA   rB   rC   rD   kvcross_kvy_cross_qks
             r   rJ   zResidualAttentionBlock.__call__k   s    #+xH99T\\!_4"9E2q	Q??$(OO""1%rH %4 %!Ax FA		"''$))DKKN";<==2x.(**r   )FrW   )r   r   r   r   boolr6   rJ   rY   rZ   s   @r   r\   r\   Z   s"    , ,S ,4 , +r   r\   c                   j     e Zd Zej                  fdedededededej                  f fdZd Z xZ	S )	AudioEncoderr
   rT   r1   r2   n_layerdtypec                 j   t         |           t        j                  ||dd      | _        t        j                  ||ddd      | _        t        ||      j                  |      | _        t        |      D cg c]  }t        ||       c}| _        t        j                  |      | _        y c c}w )NrM   r   )kernel_sizepaddingr   )rw   striderx   )r5   r6   r7   Conv1dconv1conv2r.   astype_positional_embeddingranger\   blocksra   ln_post)	r=   r
   rT   r1   r2   rt   ru   ro   r>   s	           r   r6   zAudioEncoder.__init__z   s     	YYvwAqI
YYwQqRST
%.ug%>%E%Ee%L"HMgW1-gv>W||G, Xs   :B0c                 j   t        j                  | j                  |            }t        j                  | j                  |            }|j                  dd  | j
                  j                  k(  sJ d       || j
                  z   }| j                  D ]  } ||      \  }}} | j                  |      }|S )Nr   zincorrect audio shape)r7   rk   r{   r|   rO   r~   r   r   )r=   rA   blockro   s       r   rJ   zAudioEncoder.__call__   s    GGDJJqM"GGDJJqM"wwqr{d88>>>W@WW>***[[EAhGAq! ! LLOr   
r   r   r   r"   float16r   Dtyper6   rJ   rY   rZ   s   @r   rs   rs   y   sT     **-- - 	-
 - - xx-"
r   rs   c                   l     e Zd Zej                  fdedededededej                  f fdZd	dZ xZ	S )
TextDecoderr   rT   r1   r2   rt   ru   c                    t         |           t        j                  ||      | _        t        j                  ||f      | _        t        |      D cg c]  }t        ||d       c}| _
        t        j                  |      | _        t        j                  j                  |      j                  |      | _        y c c}w )NT)r]   )r5   r6   r7   	Embeddingtoken_embeddingr"   zerospositional_embeddingr   r\   r   ra   lnr0   create_additive_causal_maskr}   _mask)	r=   r   rT   r1   r2   rt   ru   ro   r>   s	           r   r6   zTextDecoder.__init__   s     	!||GW=$&HHeW-=$>! 7^
# #7FDI#
 ,,w'**FFuMTT


s   B?c                    |r|d   d   d   j                   d   nd}| j                  |      | j                  |||j                   d   z    z   }|dgt        | j                        z  }dgt        | j                        z  }t        | j                        D ](  \  }} |||| j                  ||         \  }||<   ||<   * | j                  |      }| j                  j                  |      ||fS )z
        x : mx.array, shape = (batch_size, <= n_ctx)
            the text tokens
        xa : mx.array, shape = (batch_size, n_audio_ctx, n_audio_state)
            the encoded audio features to be attended on
        r   r   rL   Nrj   )	rO   r   r   lenr   	enumerater   r   	as_linear)r=   rA   rB   rD   offsetrp   er   s           r   rJ   zTextDecoder.__call__   s     08!Q"((+Q  #''!''"+1EFG 	

 vDKK 00H6C,,!$++.HAu*/2DJJ!+'Ax{HQK /
 GGAJ##--a0(HDDr   rX   r   rZ   s   @r   r   r      sU     **

 
 	

 
 
 xx
.Er   r   c                        e Zd Zej                  fdedej                  f fdZdee	e
j                  f   fdZd Zd Zd Zd	 Zed
        Zed        ZeZeZ xZS )Whisperdimsru   c                 `   t         |           || _        t        | j                  j                  | j                  j
                  | j                  j                  | j                  j                  | j                  j                  |      | _	        t        | j                  j                  | j                  j                  | j                  j                  | j                  j                  | j                  j                  |      | _        t#        j$                  | j                  j                  | j                  j                  ft&              }d|| j                  j                  dz  d  t)        j*                  t#        j,                  |j/                               j0                        | _        y )Nru   Tr   )r5   r6   r   rs   r
   r   r   r   r   encoderr   r   r   r   r   r   decodernpr   rq   r"   arrayasarraynonzeroTalignment_heads)r=   r   ru   	all_headsr>   s       r   r6   zWhisper.__init__   s,   	#IIII!!II##II""II##
 #IIII  II""II!!II""
 HHYY##TYY%:%:;4
	 48	$))((A-/0!xx

93D3D3F(G(I(IJr   dumpc                 Z   t        |t        j                        rt        j                  |      | _        y t        |t              rt        j                  t        j                  t        j                  |            t              j                         }|j                  | j                  j                   | j                  j"                        }t        j                  t        j$                  |j'                               j(                        | _        y t+        dt-        |       d      )Nr   zInvalid type for `dump`: zU. Expected a np.ndarray or base85-encoded bytes containing alignment_head information)
isinstancer   ndarrayr"   r   r   bytes
frombuffergzip
decompressbase64	b85decoderq   copyrP   r   r   r   r   r   r   
ValueErrortype)r=   r   r   rC   s       r   set_alignment_headszWhisper.set_alignment_heads   s    dBJJ'#%88D>D e$MM 0 0 67tdf  ==!7!79N9NOD#%88BJJt||~,F,H,H#ID +DJ< 8. . r   c                 $    | j                  |      S rX   )r   )r=   mels     r   embed_audiozWhisper.embed_audio   s    ||C  r   c                 ,    | j                  ||      d   S Nr   )r   )r=   tokensaudio_featuress      r   logitszWhisper.logits   s    ||FN3A66r   c                 T    | j                  || j                  |            \  }}}||fS rX   r   r   )r=   r   r   r   ro   rp   s         r   forward_with_cross_qkzWhisper.forward_with_cross_qk   s-    "ll64<<3DE8xr   c                 J    | j                  || j                  |            d   S r   r   )r=   r   r   s      r   rJ   zWhisper.__call__   s!    ||FDLL$56q99r   c                 4    | j                   j                  dk\  S )Ni  )r   r   r=   s    r   is_multilingualzWhisper.is_multilingual  s    yy  E))r   c                 `    | j                   j                  dz
  t        | j                        z
  S )Ni5  )r   r   r   r   r   s    r   num_languageszWhisper.num_languages  s'    yy  5(3t/C/C+DDDr   )r   r   r   r"   r   r	   r   r6   r   r   r   r   r   r   r   r   rJ   propertyr   r   detect_language_functionr   decode_functionr   rY   rZ   s   @r   r   r      s    @B

 K_ KRXX K6eRZZ.?(@ !7 : * * E E /OFr   r   )i'  )r   r   r    dataclassesr   typingr   mlx.corecorer"   mlx.nnr7   numpyr   decodingr   r   r   r   r	   r.   Moduler0   r\   rs   r   r   r   r   r   <module>r      s       !     / A 
 
 
N/ /d+RYY +>299 >.E")) .EbAbii Ar   