(HJjL.dZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z mZddlZddlmZddlmZddlZddlmZddlmZddlmZd d lmZd d lmZd d l m!Z!d d l"m#Z#dZ$dZ%dZ&dZ'dZ(edGddeZ)dZ*y)z9 Adapted from a PyTorch implementation by David Grangier N)version)Path)AnyCallableOptional)LM)register_model)tqdm)batch_generate)make_prompt_cache) make_sampler)loadi ct|}|Dcgc]}|j|}}|Dcgc] }|dkr|n| }}|dt|Scc}wcc}w)zFLimit a string to the first occurrence of any substring in untils.rN)lenfindmin)suntilslufxs Y/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/evaluate.py _rstrip_untilr!s^ AA"#FqFA#$%&Aqa!e AA& Xs1v; $&s AAcT|j|x}dk7r||t|zdS|S)zHTruncate the prefix of the string after the first occurrence of pattern.N)rr)rpatternidxs r_lstripr )s3vvg2%s7|#%&& Hc ptj|Dcgc] }t|c}}|j}tj|Dcgc]&}tj |d|t|z f(c}d}t j|t j|fScc}wcc}w)Nraxis)nparrayrmaxstackpadmx)inputsrlengthsmaxlenpaddeds r _pad_inputsr/0shh/1A/0G [[]F XX289&QAvA' (&9 F 88F RXXg. .. 0 :s B.+B3c "ddtffd }|S)NreturncH|jj|fd|| dS)NF)tokenizeadd_generation_promptcontinue_final_message) tokenizerapply_chat_template)self chat_historyr4 extra_kwargss rr7z-chat_template_fn..apply_chat_template;s:1t~~11  "7'<#<     r!)T)str)r:r7s` rchat_template_fnr<:s s  r!mlxlmceZdZeZ ddedeededeededee e jge jfddffd Z dd efd Z dd eed efd ZdZedefdZdeeeeffdZdeefdZdeefdZxZS)MLXLMNpath_or_hf_repo max_tokens batch_sizeuse_chat_templatetrust_remote_codesamplerr1ct|d|rdndi}t||\|_|_||_||_||_||jjdu|_||_ y)NrDT)tokenizer_config) super__init__r_modelr6 _max_tokens _batch_sizerC chat_template_sampler) r8r@rArBrCrDrErG __class__s rrIzMLXLM.__init__Lsx /9JPTU&* .>' # T^&%!2  $%)^^%A%A%MD " r! step_sizectj|d}t|j}t d|j d|D]b}|j|dd|||zf|}tj |Dcgc]}|jc}tjdtjdddddfjtj}||fScc}w)Nrr cacher) r*r&r rJrangeshapeevalstate clear_cachenn log_softmaxastypefloat32)r8promptrPrSilogitsclogprobss r_process_promptzMLXLM._process_promptas&!$'!$++.q&,,q/95A[[1q9}+<(>&B"2"9"9"**"EF-s=C- rSc t|\}}|dddf|dddf}}|xst|j}d}gg}}td|jd|D]J} |dd| | |zf} | jd} |j| |} t j | jtj} tj| |dd| | |ztjfdd}|dd| | |zftj| dk(}tjtj|| |z|dddfk|d}tj||tj |j#||j#||| z }Mtj$|d}tj$|d}|||fS) N.rr rrRr#).rF)r/r rJrTrUrYrZr[r*r\take_along_axisnewaxisargmaxwherearangerVrXappend concatenate)r8r+rSrPr,targetsoffsetscores is_greedyr^inpTr_ log_probsscoreigs r _score_fnzMLXLM._score_fnks%f- crc*F37O7*4;;7 q&,,q/95AAI --.C ! A[[E[2Fv}}RZZ'@AI&&71a!i-&7#CD2EAI --."))F2LLB"))FAJ7'!T':JJBPUVB GGE2  NN    R MM% aKF'6*Q/NN915 w ))r!c |Dcgc]3}t|jj||j 5c}Scc}w)Nadd_special_tokens)tupler6encoderC)r8textsts r _tokenizezMLXLM._tokenizesR    %%a@V@Vz&MLXLM.loglikelihood..s!A.Q#a&.infFrRrzPrefix eliminated for z requests with zcompletion longer than context.stream)*logginginforr* distributedinit collections defaultdictlist enumerateargsrikeysvalueszipranksizer r|r'rKDEFAULT_MAX_TOKENS all_scoresextendfloat all_is_greedyrbrfitemrtr&copydeepcopysumallall_maxcpu all_gatherrVrTargsorttolist)'r8requestsgroup group_reqsrreq questions responsesindicesvresplong_completionsrmrnqrsprefixrfull_sequencesmax_completed_lrA truncation orig_prefix_lprefix_lrarSmax_idxrr+rr_rs num_results per_group all_indicesrquestion rank_indicesinv_sorts' r loglikelihoodzMLXLM.loglikelihoods(  =H MN##%!,,T2 !(+HC sxx{ # * *C!+= >,*+  ""$AQIC NN3    T "% ejjl:ejjl:; ejjl:ejjl:;  #i33y>JJEAr^^QC(+F!^^B,?BqQUB,?@N!!A.!AAO))?-?JQ* Wctjdt|z|j|Dcgc]}|jdc}}g}t t dt||jD]}||||jz}|j|\}}} tj|jd|dddfk} |j| |zjdj|Scc}w)aCompute full log-likelihood of a string, with no truncation, for perplexity computation - We will use the full max context length of the model. - For inputs that exceed the max context length, we divide the tokenized string into chunks of up to the max context length. - IMPORTANT: Each document's loglikelihood/perplexity is computed *separately*, unlike other implementations which may simply concatenate multiple documents together. - IMPORTANT: We maximize the amount of context for each prediction. Specifically, for inputs that we break into multiple chunks, the last input will still a full-sized context. Example: Input tokens: [ 0 1 2 3 4 5 6 7 8 9 ] Prefix: EOT Max context length: 4 Resulting input/prediction pairs: INPUT: EOT 0 1 2 PRED: 0 1 2 3 INPUT: 3 4 5 6 PRED: 4 5 6 7 INPUT: 5 6 7 8 PRED: 8 9 Observe that: 1. Each token is predicted exactly once 2. For the last pair, we provide the full context, but only score the last two tokens :param requests: list[Instance] A list of Instance objects with property `args` which returns a tuple (context,). string: str String for which we are computing overall loglikelihood :return: list[tuple[float]] A list of tuples (logprob,) logprob: float The log probability of `context` conditioned on the EOT token. z2Estimating loglikelihood rolling for %d sequences.rrNr#)rrrr|rr rTrLrtr*rhrUrrr) r8rrr+rr^batchrmr,rmasks rloglikelihood_rollingzMLXLM.loglikelihood_rollings@  @3x= P  A! AB eAs6{D,<,<=>A1q4#3#334E!%!6 FGQ99V\\"-.D1AAD   tf}11r1:AAC D ? !BsDc Jtjj}t|}||j d|j }t jdt|zt|Dcgc]}|jc}\}}|Dcgc]*}|jj||j ,}}|Dcgc]&}|jxs|jdt(} }t!|j"|j|| d|j$j&} t)t| |D]T\} \} }t+| |d| | <|jj,s2t/| |jj0| | <V|j dkDrAtj2tj45||j zdz |j z} | t| z }| Dcgc]}t7|jd } }tj8t;d | D}tjj=|j?}tj8| Dcgc] }t|c}d g|zz}tj8| Dcgc]}|d g|t|z zzc}d g|zg|zztj@} tjjC| djEd djGd djI} tjjC|djEd djGd djI}| d|} |d|}t| |Dcgc]!\}}tK|d|jM#} }}ddd| S| Scc}wcc}wcc}wcc}wcc}wcc}wcc}}w#1swY| SxYw) aCGenerate greedily until a stopping sequence :param requests: list[Instance] A list of Instance objects with property `args` which returns a tuple (context, until). context: str Context string until: [str] The string sequences to generate until. These string sequences may each span across multiple tokens, or may be part of one token. :return: list[str] A list of strings continuation continuation: str The generated continuation. Nz)Generating continuation for %d sequences.rvmax_gen_tokensT)modelr6promptsrAverboserEuntilr zutf-8c32K|]}t|ywrr)rr`s rrz'MLXLM.generate_until..ps&C{!s1v{rr)'r*rrrrrrrrrr6ryrCrKgetrr rJrNrzrr has_thinkingr think_endrrrr&r'rruint8rswapaxesflattenr bytearraydecode)r8rrtotal_requestsrcontextsoptionscontextoptrA completionsetextpad_tor)r`max_lenr,rs rgenerate_untilzMLXLM.generate_until5s##%XEJJL8EJJL89 @3x=PQh!?hs#((h!?@'$  $ NN ! !0F0F,F " $       M(8:L M M  %++nn!MM   % (K(ABNA{c*4W>KN~~**!(t~~/G/G!H AC ::>@((K#@KqCFK#@A39#LM hh;FG;aQ!#a& 011;GsW}o+,HH NN--k$.?@Xa^WQ]VX NN--gdm<Xa^WQ]VX  */>: !/>2:=k7:S:S$!QIae$++-:S5#<{A"@   0M$AG$5#<sV8O4/O9 +O>:P!P/A'PP (!P P %CP?&P%PPP")NNFN))Nr)__name__ __module__ __qualname__r<r7r;rintboolrr*r&rIrbrrtr|propertyrrrxrrrr __classcell__)rOs@rr?r?Gs +, %),0"'<@  SM   $D>    (BHH:rxx#789   **x}**B >>>n>eE4K.@)An>`+e+ZU$s)Ur!r?c d tjd}|jddd|jddd|jd d d |jd tdd|jdtdd|jdtdd|jdddt|jdtdd|jdddd |jd!tjd"d |jd#t j d$d%|jd&dd'd |jd(dd)*|jd+td,d-|jd.td/d0|jd1td2d3|j}t|j}|jdd4d5tjd6<tjj!|j tj"j%}tj&tj"j)d7tj*8|j-d7kDr0|j/d2k(rt1d9|j-d:t3|j4|j6|j8;}t;|j<|j>|j@|jB|jD|<}tGdGi|jHt:_!tKjL||jN|jP|jR|jT|jV|j |j |j |j |jX= }d>|j<j[d?d@t]dAg}|jT||jTdBgz }||jNz }d@j_|}|j/d2k(rr||z } | jat jb|dCdDEt1dF|dCjeD]"} t1t jb| dDE$yy)HNz2Evaluate an MLX model using lm-evaluation-harness.z--modelzModel to evaluateT)helprequiredz--tasks+)nargsrz --output-dir.z"Output directory for result files.)defaultrz --batch-sizez Batch size)typerrz --num-shotszNumber of shotsz --max-tokenszhMaximum number of tokens to generate. When set, this value takes precedence over task specific defaults.)rrrz--limitz&Limit the number of examples per task.)rrrz--seed{z Random seed.z--fewshot-as-multiturn store_truezZWhether to provide the fewshot examples as a multiturn conversation or a single user turn.F)actionrrz--apply-chat-templatezSpecifies whether to apply a chat template to the prompt. If the model has a chat template, this defaults to `True`, otherwise `False`.z--chat-template-argszvA JSON formatted string of arguments for the tokenizer's apply_chat_template, e.g. '{"enable_thinking":false}'z{}z--confirm-run-unsafe-codez?Confirm that you want to run tasks that execute untrusted code.z--trust-remote-codez)Enable trusting remote code for tokenizer)rrz--tempgzSampling temperaturez--top-pg?zSampling top-pz--top-krzSampling top-k)parentsexist_okfalseTOKENIZERS_PARALLELISMr rzEvaluating with z nodes)temptop_ptop_k)rArBrCrDrE) rtasksfewshot_as_multiturnr7 num_fewshotlimit random_seednumpy_random_seedtorch_random_seedfewshot_random_seedconfirm_run_unsafe_coderVr~rlm_eval02dresults)indentzResults:)3argparseArgumentParser add_argumentrBooleanOptionalActionjsonloadsr parse_argsr output_dirmkdirosenvironr*randomseedrrrVall_sumrrrprintrrrrr?rrArBr7rDr<chat_template_argsr simple_evaluaterrrC num_shotsrr rrjoin write_textdumpsr) parserrrworldrElmr  file_keysfilename output_pathresults rmainr,s?  $ $<F  (;dK  t< *N S"<P  CDUV  3   5    sCnM  .  --   ZZA  # N    8  uc@VW  sAQR  Q=MN    Ddoo&JTD1,3BJJ'(IINN499 NN   !EGGBNN " "1RVV " 45 zz|aEJJLA-  f56 YYjjjjG  ????2200  B!1 K43J3J KE%%jj!6600NNjjII)))) II $ < < G++C5wy7IJI ~~!,..  Ixx "H zz|q 8+ tzz')*r;s  &** /$+& / BBBBJ t0r!