
    (HJj&1              	       $   d dl Z d dlmZmZ d dlmZ d dlmZ d dlm	Z
 d dlmZ d dlZd dlmZ d dlmZmZ d dlmZ dd	lmZ dd
lmZ defdZd Ze G d d             Zd Z	 	 	 ddZdeed fdededefdZ d e       eedfdedededefdZ!y)    N)	dataclassfield)partial)Path)average_gradients)tree_flattentree_map)tqdm   )TrainingCallback)CacheDataset	thresholdc                 \    t        j                         | kD  rt        j                          y y N)mxget_cache_memoryclear_cache)r   s    ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/tuner/trainer.py_clear_cacher      s"    	y(
 )    c                 Z    t        |       j                  fd}|t        |       _        y)zL
    Update all instances of type(layer) to use gradient checkpointing.
    c                 l      fd} t        j                  |       j                         g|i |S )Nc                 >    j                  |         g|i |S r   )update)paramsargskwargsfnmodels      r   inner_fnz:grad_checkpoint.<locals>.checkpointed_fn.<locals>.inner_fn    s$    LL e-d-f--r   )r   
checkpointtrainable_parameters)r   r   r   r    r   s   `   r   checkpointed_fnz(grad_checkpoint.<locals>.checkpointed_fn   s3    	. 'r}}X&u'A'A'CUdUfUUr   N)type__call__)layerr#   r   s     @r   grad_checkpointr'      s(     
e		BV +DKr   c                      e Zd ZU  edddi      Zeed<    edddi      Zeed<    ed	dd
i      Zeed<    edddi      Z	eed<    edddi      Z
eed<    edddi      Zeed<    edddi      Zeed<    edddi      Zeed<    edddi      Zeed<    edddi      Zeed<    ed dd!i      Zeed"<   y#)$TrainingArgs   helpzMinibatch size.)defaultmetadata
batch_sized   zIterations to train for.iters   z@Number of validation batches, -1 uses the entire validation set.val_batches
   z0Number of training steps between loss reporting.steps_per_report   z-Number of training steps between validations.steps_per_evalz!Save the model every number stepssteps_per_save   zMaximum sequence length.max_seq_lengthzadapters.safetensorsz/Save/load path for the trained adapter weights.adapter_fileFz0Use gradient checkpointing to reduce memory use.r'   r   zLNumber of steps to accumulate gradients before applying an optimizer update.grad_accumulation_stepsr   z>Clear the allocator cache between steps if it grows too large.clear_cache_thresholdN)__name__
__module____qualname__r   r.   int__annotations__r0   r2   r4   r6   r7   r9   r:   strr'   boolr;   r<    r   r   r)   r)   )   sK   A9J0KLJLsf6P-QRE3RV
K  "LMc   v'VWNC   v'JKNC   (BCNC  &KLL#  "LMOT  $)b
$S  "'T
"3 r   r)   c                    |d d d df   }|d d dd f   } | |      }t        j                  d|j                  d   dz         }t        j                  ||d d ddf   k\  ||d d dd f   k        }t        j
                  j                  ||      |z  }|j                         }	|j                  t         j                        j                         |	z  }||	fS )Nr   r   )
r   arangeshapelogical_andnnlossescross_entropysumastypefloat32)
r   batchlengthsinputstargetslogitsstepsmaskcentokss
             r   default_lossrY   V   s    1crc6]FAqrElG6]FIIaq)A-.E>>%71ac6?2EWQU^4KLD		 	 	1D	8BHHJE	2::		"	"	$u	,Bu9r   c           
   #      K   t         t              r fd}n fd}t        t        t	                     |      }t	               |k  rt        d| dt	                d      |!|j                         }|j                         }	nd}d}	||	z  dk7  rt        d	      t        dt	        |      |z
  dz   |      D 
cg c]  }
||
|z   |
|z   |z   |	    }}
|rt        j                  j                  |       	 t        j                  j                  t	        |            }|D ]P  }
||
   D cg c]  } |   	 }}t	        |d         d
k(  rt        | \  }}ndgt	        |      z  }|D cg c]  }t	        |       }}t        |      |kD  rt        d| dt        |       d| d       d}d|t        |      |z   dz
  |z  z  z   }t        ||      }t        j                   ||	z  |ft        j"                        }t        ||	z        D ]%  }t        ||   |      }||   d | ||d |f<   |||<   ' t%        j&                  |      }|t%        j&                  t)        t        ||                  f S |sy c c}
w c c}w c c}w w)Nc                 &    j                  |       S r   )itemlenidxdatasets    r   <lambda>z!iterate_batches.<locals>.<lambda>p   s    W__S1r   c                 &    t        |    d         S )Nr   )lenr]   s    r   r`   z!iterate_batches.<locals>.<lambda>r   s    Sa1r   )keyz&Dataset must have at least batch_size=z examples but only has .r   r   z9The batch size must be divisible by the number of workers   z)[WARNING] Some sequences are longer than z tokens. The longest sentence z will be truncated to z2. Consider pre-splitting your data to save memory.    )
isinstancer   sortedrangerb   
ValueErrorranksizenprandomseedpermutationzipmaxprintminzerosint32r   arraylist)r_   r.   r9   loopro   
comm_grouplen_fnr^   offsetstepi	batch_idxindicesjrP   offsetsxrQ   pad_tomax_length_in_batch	batch_arrtruncated_lengths   `                     r   iterate_batchesr   f   s     '<(11
s7|$&
1C
7|j 4ZL%c'l^16
 	
 " DATUU
 q#c(Z/!3Z@@A 	AJVj0478@   
		t
))''I7A)216AWQZE658}!!$ew#E
*',-u!s1vuG-7|n,??O P,,/L>9OP^O_ `GG F"#fW1F1Jv0U&V"V"%&9>"J*"46I!JBHHUI:-.#&wqz>#B 27(;L<L2M	!..../$ 
 / HHY'E$s7G'<"=>>>9 < C  7
 .s,   B<J?I>AJ0J<2J.J DJr8   lossr   r<   c                    | j                          t        j                  d      }t        j                  d      }	|dk7  rt        t	        |            nt        t
        d      }
t        t        |
 ||||t        j                  j                                     dt        t        |      |z  |            D ]?  \  }} || g| \  }}|||z  z  }|	|z  }	t        j                   ||	       t        |       A t        j                  j                  |t        j                        }t        j                  j                  |	t        j                        }	||	z  j                         }|S )	Ng        r   rF   r   )r_   r.   r9   rz   zCalculating loss...)desctotalstream)evalr   rw   iterri   r@   r
   rq   distributedinitrt   rb   r   all_sumcpuitem)r   r_   r.   num_batchesr9   r   r   r<   
all_lossesntokensindex_iterator_rP   rK   toksavg_losss                   r   evaluater      s4    
JJL#JhhqkG1<1BT%,-SRSN%->>..0		
 ##g,*,k:5 E*E*ftm#
4

G$*+#& ''
266'BJnn$$WRVV$<GW$**,HOr   r   training_callbackc                 X   #$ t         j                  j                         r*t        j                  t        j                         d          t        d|j                          t         j                  j                         }|j                         }	|j                         }
|	dkD  rt        d|
 d|	        |j                  rt         j                  d          t        j                   |      $|j                  ##dk  rt!        d       j"                  j"                  t         j$                  j"                  g}t'        t         j(                  ||      #$ fd	       } j+                          d}d}d}d}d}d }t-        t/        d|j                  dz          |||j0                  |j2                  d
|            D ]x  \  }}t5        j6                         }|r|dk(  s!||j8                  z  dk(  s||j                  k(  rt5        j6                         }t;         |||j0                  |j<                  |j2                  |      } j+                          t5        j6                         |z
  }|
dk(  rt        d| d|dd|ddd
       ||dz
  ||d}|j?                  |       t5        j6                         } ||||#z  dk(        \  }}}||z  }||z  }|dz  }t        j@                  ||||       tC        |jD                         |t5        j6                         |z
  z  }||jF                  z  dk(  s||j                  k(  r't         j                  jI                  |t         jJ                        jM                         }|||	z  z  }t         j                  jI                  |t         jJ                        jM                         }jN                  jM                         }|jF                  |z  }tQ        |      |z  }||z  }t        jR                         dz  }|
dk(  r(t        d| d|dd|dd|dd|dd| d|ddd
       ||||||||d} |jU                  |        d}d}d}d}||jV                  z  dk(  s|
dk(  stY        t[         j]                                     }!t        j^                  ta        |jb                        |!       te        |jb                        jf                  |dd z  }"t        j^                  ta        |"      |!       t        d| d!|jb                   d"|" d#       { |
dk(  retY        t[         j]                                     }!t        j^                  ta        |jb                        |!       t        d$|jb                   d#       y y )%N max_recommended_working_set_sizezStarting training..., iters: r   zNode z of r   z*grad_accumulation_steps must be at least 1)rR   outputsc                      g|  \  \  }}}|t        d ||      }|r3t        |      }dkD  rt        fd|      }	j                  |       d }|||fS )Nc                     | |z   S r   rD   )r   ys     r   r`   z%train.<locals>.step.<locals>.<lambda>   s    Qr   r   c                     | z  S r   rD   )r   grad_accum_stepss    r   r`   z%train.<locals>.step.<locals>.<lambda>  s    !.>*>r   )r	   r   r   )
rP   	prev_grad	do_updatelvaluer   gradr   loss_value_and_gradr   	optimizers
         r   r}   ztrain.<locals>.step   st    25A5A .i@D$T*D!# >EUD)DtT!!r   T)r_   r.   r9   ry   rz   )r   r_   r   r.   r   r9   r   zIter z: Val loss z.3fz, Val took s)flush)	iterationval_lossval_timer   g    eAz: Train loss z, Learning Rate z.3ez	, It/sec z, Tokens/sec z, Trained Tokens z, Peak mem z GB)r   
train_losslearning_rateiterations_per_secondtokens_per_secondtrained_tokenspeak_memory07dz_adapters.safetensorsz: Saved adapter weights to z and rd   zSaved final weights to )4r   metalis_availableset_wired_limitdevice_infors   r0   r   r   rl   rk   r'   layersrJ   value_and_gradr;   rj   statern   r   compiletrainrq   ri   r.   r9   timeperf_counterr6   r   r2   on_val_loss_reportr   r   r<   r4   r   r   r   r   floatget_peak_memoryon_train_loss_reportr7   dictr   r"   save_safetensorsrB   r:   r   parent)%r   r   train_datasetval_datasetr   r   r   r   world
world_sizerk   r   r}   rK   n_tokensrU   r   
train_time
grad_accumitrP   ticr   r   val_infor   r   r   r   it_sec
tokens_secpeak_mem
train_infoadapter_weightsr!   r   r   s%   ``                                 @@r   r   r      s    
xx
2>>+,NOP	)$**
67NN!EJ::<DA~dV4
|,-Q(++E4833!EFF[[)//299??;ERZZu5" 6" 
KKMFHENJJ aa !..	
		E ! !GrD///14djj8H##%C#?? ,,#22 /H KKM((*S0HqyB4    (~ .  (~Q0 	 !,!#a ( (
 "44X>##%C#'!!Q&$
 j 	&D

vx4T//0d'')C//
 %%%*bDJJ.>//rvv/FKKMJ%*,,J~~--hrvv-FKKMH%3388:M**Z7Fx:5Jh&N))+c1HqyB4}Z,< =%%23$7 8$S\ *"",S!1 2&&4%5 6  (~S2  !,!#",%2-3)3&4#+
 "66zBFHEJ ###q(TQY"<0J0J0L#MNOD$5$5 6HT&&'..Bs8;P1QQ  JAt6$$%U:,a9S	^ qy|E,F,F,HIJ
C 1 12OD'(9(9':!<= r   )FNN)"r   dataclassesr   r   	functoolsr   pathlibr   mlx.corecorer   mlx.nnrJ   numpyrm   mlx.nn.utilsr   	mlx.utilsr   r	   r
   	callbacksr   datasetsr   r@   r   r'   r)   rY   r   callabler   r   rD   r   r   <module>r      s     (      * ,  ' "C 
+  ) ) )X( 
	G^ ! /!"' ' ' '\ %! /*.i>
 i> i> i> (i>r   