(HJj 8ddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddlm Z ddl m Z mZmZmZmZmZmZmZddlmZddlmZddlmZddlmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+dd l,m-Z-dd l.m/Z/m0Z0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:dZ;dZej~ejZAejdRdejdeeejfdZEe GddZFdZGdddddddddddd d ejdejd!eId"eeejgejfd#eeeejejgejfd$eeId%ee d&eId'eeId(eId)eId*eeeIeIgdfd+eejd,eeejejfddffd-ZJd.ddddd/dddd0 d ejdejd1ejd2eId!eId"eeejgejfd#eeeejejgejfd%ee d&eId'eeId(eId)eId,eeejejeKfddffd3ZL dSdejd4eee-fd eeMejeeIfd!eId1eejd,eeFddff d5ZN dTdejd4eee-fd eeMeeIfd6eKd,eMf d7ZOdRd8ZPdRd9ZQe Gd:d;ZRd<ZSd=ZTd>ZUd?ZVd@ZWGdAdBZXGdCdDZYGdEdFZZGdGdHZ[e GdIdJZ\ dUdKeeeIdLeeee d!eeIeeIfd6eKdMeKd,e\f dNZ]dOZ^e_dPk(re`dQe^yy)VN)deque) dataclass)partial)AnyCallable GeneratorListOptionalSequenceTupleUnion) tree_reduce)PreTrainedTokenizercache) ArraysCache BatchKVCacheBatchRotatingKVCache CacheListKVCacheQuantizedKVCacheRotatingKVCache TokenBufferload_prompt_cache) make_sampler)TokenizerWrapper)#does_model_support_input_embeddingsloadhellodg?z(mlx-community/Llama-3.2-3B-Instruct-4bitic&|jdvS)N)falsef)lower)strings Y/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/generate.pystr2boolr);s <<> //ctjd}|jdtdtdd|jdd d |jd td |jdtddd|jddd|jddt d|jddd|jddt td|jd ttd!|jd"ttd#|jd$ttd%|jd&t td'|jd(ttd)|jd*td+d,|jd-t td.|jd/t td0|jd1d d2 |jd3d d4 |jd5d6d7|jd8t d9d:|jd;t d|jd?d@d dA |jdBt dCd|jdDt dEdF|jdGdHt t"I|jdJtdKd|jdLt dMdN|S)Oz&Set up and return the argument parser.zLLM inference script) descriptionz--modelz[The path to the local model directory or Hugging Face repo. If no model is specified, then z is used.N)typehelpdefaultz--trust-remote-code store_truez)Enable trusting remote code for tokenizer)actionr.z--adapter-pathz9Optional path for the trained adapter weights and config.)r-r.z--extra-eos-token+z:Add tokens in the list of eos tokens that stop generation.)r-r/nargsr.z--system-promptz.System prompt to be used for the chat template)r/r.z--promptz-pz;Message to be processed by the model ('-' reads from stdin)z--prefill-responsez1Prefill response to be used for the chat templatez --max-tokensz-mz$Maximum number of tokens to generate)r-r/r.z--tempzSampling temperaturez--top-pzSampling top-pz--min-pzSampling min-pz--top-kzSampling top-kz--xtc-probabilityz5Probability of XTC sampling to happen each next tokenz--xtc-thresholdr"zDThresold the probs of each next token candidate to be sampled by XTCz--min-tokens-to-keepz*Minimum tokens to keep for min-p sampling.z--seedz PRNG seedz--ignore-chat-templatez9Use the raw prompt without the tokenizer's chat template.z--use-default-chat-templatezUse the default chat templatez--chat-template-configzAdditional config for `apply_chat_template`. Should be a dictionary of string keys to values represented as a JSON decodable string.)r.r/z --verboseTzTLog verbose output when 'True' or 'T' or only print the response when 'False' or 'F'z --max-kv-sizez$Set the maximum key-value cache sizez--prompt-cache-filez;A file containing saved KV caches to avoid recomputing themz--quantize-activationsz-qazSQuantize activations using the same quantization config as the corresponding layer.z --kv-bitszFNumber of bits for KV cache quantization. Defaults to no quantization.z--kv-group-sizez%Group size for KV cache quantization.@z--quantized-kv-startzLWhen --kv-bits is set, start quantizing the KV cache from this step onwards.)r.r-r/z --draft-modelz,A model to be used for speculative decoding.z--num-draft-tokensz:Number of tokens to draft when using speculative decoding.)argparseArgumentParser add_argumentstr DEFAULT_MODELDEFAULT_PROMPTintDEFAULT_MAX_TOKENSfloat DEFAULT_TEMP DEFAULT_TOP_P DEFAULT_MIN_P DEFAULT_TOP_KDEFAULT_XTC_PROBABILITYDEFAULT_MIN_TOKENS_TO_KEEP DEFAULT_SEEDr)DEFAULT_QUANTIZED_KV_START)parsers r(setup_arg_parserrI?s  $ $1G HF   ..;_I G  8    H    I   =    J    @   " 3  ul9O };K };K ]9I  ' D     S    * 9          H  % ,   I     c     3     J      b     U     4   " *    ;     I  Mr*modelstreamsc#NKtjjs dytd|d}tjd}|d|zkDr|dz}|dz}t d|d|d tj |} d||D]}tj|ntjtj |y#wxYw#||D]}tj|ntjtj |wxYww) a A context manager to temporarily change the wired limit. Note, the wired limit should not be changed during an async eval. If an async eval could be running pass in the streams to synchronize with prior to exiting the context manager. NcXt|tjr||jzS|SN) isinstancemxarraynbytes)accxs r(zwired_limit..s!Z288-D3>M#Mr*r max_recommended_working_set_sizeg?iz0[WARNING] Generating with a model that requires z6 MB which is close to the maximum recommended size of z MB. This can be slow. See the documentation for possible work-arounds: https://github.com/ml-explore/mlx-lm/tree/main#large-models)rPmetal is_availabler device_infoprintset_wired_limit synchronize)rJrK model_bytes max_rec_sizemodel_mb max_rec_mb old_limitss r( wired_limitrcs! 88 "   ! MuVW ~~'(JK |+ +"e+H%.J B8*MEEOLQNN  &&|4  * " ANN1%!   y )1 &" ANN1%!   y )s6D%CAD%C A D%CD%A D""D%ceZdZUdZeed<eed<ejed<e ed<eed<e ed<eed<e ed <e ed <d Z e eed <y ) GenerationResponsea The output of :func:`stream_generate`. Args: text (str): The next segment of decoded text. This can be an empty string. token (int): The next token. from_draft (bool): Whether the token was generated by the draft model. logprobs (mx.array): A vector of log probabilities. prompt_tokens (int): The number of tokens in the prompt. prompt_tps (float): The prompt processing tokens-per-second. generation_tokens (int): The number of generated tokens. generation_tps (float): The tokens-per-second for generation. peak_memory (float): The peak memory used so far in GB. finish_reason (str): The reason the response is being sent: "length", "stop" or `None` texttokenlogprobs from_draft prompt_tokens prompt_tpsgeneration_tokensgeneration_tps peak_memoryN finish_reason) __name__ __module__ __qualname____doc__r:__annotations__r=rPrQboolr?ror r2r*r(rere sO I Jhh#'M8C='r*rec|yt|D]8\}}t|ds|j|k\s#|j||||<:y)N to_quantized) group_sizebits) enumeratehasattroffsetrw) prompt_cachequantized_kv_start kv_group_sizekv_bitsecs r(maybe_quantize_kv_cacher+sI,'1 1n %!((6H*Hnn GnTLO(r*r5) max_tokenssamplerlogits_processors max_kv_sizer}prefill_step_sizerrr~prompt_progress_callbackinput_embeddingspromptrrrrr}rrrr~rrreturnc #K| _ts tdt|dkDrTt|t| k7r=tdt| dt|dt|dk(r tddtj|| xsd } t j t| | | xsd d tjd ttjffd dd tjd ttjffd } tjt5| t| n t|}d}| ||||z dkDr||z dz }t||}|d|d| | d|dndtjDcgc]}|jc}||z }| ||||d}| | |dn| } tj ||z dkDr| || \}}dddtj"d} ||k7r!| |\}}tj"|||dk(rtj|| |||k(ry|j%|f|dzdk(rtj }}|dz }cc}w#1swYxYww)a A generator producing token ids based on the given prompt from the model. Args: prompt (mx.array): The input prompt. model (nn.Module): The model to use for generation. max_tokens (int): The maximum number of tokens. Use``-1`` for an infinite generator. Default: ``256``. sampler (Callable[mx.array, mx.array], optional): A sampler for sampling a token from a vector of log probabilities. Default: ``None``. logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional): A list of functions that take tokens and logits and return the processed logits. Default: ``None``. max_kv_size (int, optional): Maximum size of the key-value cache. Old entries (except the first 4 tokens) will be overwritten. prompt_cache (List[Any], optional): A pre-computed prompt cache. Note, if provided, the cache will be updated in place. prefill_step_size (int): Step size for processing the prompt. kv_bits (int, optional): Number of bits to use for KV cache quantization. None implies no cache quantization. Default: ``None``. kv_group_size (int): Group size for KV cache quantization. Default: ``64``. quantized_kv_start (int): Step to begin using a quantized KV cache. when ``kv_bits`` is non-None. Default: ``0``. prompt_progress_callback (Callable[[int, int], None]): A call-back which takes the prompt tokens processed so far and the total number of prompt tokens. input_embeddings (mx.array, optional): Input embeddings to use instead of or in conjunction with prompt tokens. Default: ``None``. Yields: Tuple[mx.array, mx.array]: One token and a vector of log probabilities. Nz(Model does not support input embeddings.rz8When providing input_embeddings, their sequence length (z0) must match the sequence length of the prompt (z), or the prompt must be empty.z=Either input_embeddings or prompt (or both) must be provided.)rcyrNr2)_s r(rUzgenerate_step..ystr*r~rrc0tj|dSNaxisrPargmaxrTs r(rUzgenerate_step..BIIab$9r* input_tokensrc2| ||S|S)N)rrrr2)rrrJr}s r( _model_callz"generate_step.._model_calls-  'LCS \: :r*ctjt5|d||dnd}|dddddf}r9t|dkDr+ tj |gn| D] }| |}  |tj |dz } |}||j dfcdddS#1swYyxYw)NrrrrT)keepdims)rPstreamgeneration_streamlenconcat logsumexpsqueeze) rrlogits processorrhsampledrrr}quantize_cache_fnrtokenss r(_stepzgenerate_step.._stepsYY( ) )$/.>.J$T*PTFAr1H%F S%6%:)IIv|45% "3I&vv6F"3 l + Vd CCHh'GH,,Q///* ) )s BB==CrrrrN)r ValueErrorrrmake_prompt_cache functoolsrrrPrQr rrminevalstate clear_cache async_evalitem)rrJrrrrr}rrrr~rrrtotal_prompt_tokensprompt_processed_tokens remaining n_to_processryrhnnext_y next_logprobsrrrs ` `` ` @@@r( generate_stepr3s^#259GH H [1_V4D0E!EJ3O_K`JabAADV N((  V  K  F.. # 8LO!))-# :9G;"((;hrxx>P;0BHH08J008 $ %%5%AC !s6{ #$ !8:MN!$;;a?,/FF!KI0)Q RLM*F$/!/%  NN +"$;;a?.BRS 8; &>MM!X A  ?$)!H !FM MM&- 0 6 GGAJ $%8:M N ? ffh  s7a< NN m8 Q 4% & %s8D;KA5K9K ?K KB*KKK K) num_draft_tokensrrrr}rrrr~ draft_modelrc # !K|jtj} d |+tj|tjn0|dt |j |t |j dtjsBD chc](} | jrt| j*}} td|dxsdtjt| | | !fdd !fd !fd }fd }fd }tjt 5|| }||| } dddd }d }d } t#||z |}||} " d j$| j$z |z dz tj&| |g} || |dz\}}tj(|||j+}|j+}d }||kr2||||||}}}||k7rn|dz }|dz }||d f||k(rn||kr2||kr|dz }||||df||k(rntj,||gtj} | }||k(r.rr*rcvrD] }|||} |tj|ddz }|}||fS)NrTrr)rPr)rrrrhrrrs r(_process_and_samplez6speculative_generate_step.._process_and_sample sI . "662/BLLb4HH H ({r*rc 6tjt5||d|}|dd| dddf} | rgg}}|dkDr |d|dz }t|D]U} tj |gn|  |dd|ddf\}}|j ||j |Wtj|dtj|dfcdddS d|j dcdddS#1swYyxYw)Nrrrr)rPrrrange concatenateappendr) rJrr n_predictrout_y out_logprobsirhrr prev_tokensrs r(rz(speculative_generate_step.._step)s YY( )1T7%0FA z{A-.F e $ &("|q=,i!m,-Ay)A'2 Q'78 #6k6!QPQ'?"SKAxLLO ''1*~~e!4bnn q7'* ).+41BC/* ) )sC D.DDc>|jdkDrt|jdz }||d|d||tj|Dcgc]}|jc}||d}tj |jdkDr|Scc}w)Nrr)sizerrPrrr)rJrrrrrrs r(_prefillz+speculative_generate_step.._prefillCsffqj0!&&1*=L !M\"4( 6 e $ GGe,eQWWe, -,- A NN  ffqj-sBctj||z tjt||z dz dy)Nrr)rtrim_prompt_cachemax) num_draft num_accept draft_cache model_caches r( _rewind_cachez0speculative_generate_step.._rewind_cacheMs9  Y-CD  SZ1G!1KQ-OPr*c |dk(r$tjgtjSg}t|D]5}|\}}tj||j |7tj |SNr)rPrQuint32rrrr)rrysrrrrs r(_draft_generatez2speculative_generate_step.._draft_generateQsm >88B * * y!Ak15DAq MM!  IIaL"~~b!!r*rTFr)r)astyperPrrrrlayerscan_trim_prompt_cache is_trimmabler-rprrrrrrrrrrtolistrQr)"rrJrrrrrr}rrrr~rrtypesrrrdraft_yntoksrr draft_tokensrrhtndtnlpnrrrrrrs" ` `` ` @@@@@@r(speculative_generate_steprsAZ  bii AK--e4 --k: "#6S%67 "3u||#4#67  & &{ 3+6O;aann>Na!!;ONugUW X  :9G!))-# DD4Q" $ %; Q7 UK + & EI A+$J.0@AI*7I>L&)*UK,<,9B(K>"L 1K;6L > L  L  tokenizerc+HKt|ts t|}t|tjsdt|tr?|j duxs|j |j  }|j||}tj|}|j}||d<|)|jddt||fi|}d|D}n2|jdd|jddt|||fi|}t|tg5tj} t!|D]\} \} } } | dk(r:tj| z }|j"|z }tj} | |j$vrn{|j'| | d z|k(rn`t)|j*| | | |j"| d z| d ztj| z z tj,d z d |j/t)|j*   |j" d z| d ztj| z z tj,d z | |j$vrd nd dddy#1swYyxYww)a A generator producing text based on the given prompt from the model. Args: model (nn.Module): The model to use for generation. tokenizer (PreTrainedTokenizer): The tokenizer. prompt (Union[str, mx.array, List[int]]): The input prompt string or integer tokens. max_tokens (int): The maximum number of tokens to generate. Default: ``256``. draft_model (Optional[nn.Module]): An optional draft model. If provided then speculative decoding is used. The draft model must use the same tokenizer as the main model. Default: ``None``. kwargs: The remaining options get passed to :func:`generate_step`. See :func:`generate_step` for more details. Yields: GenerationResponse: An instance containing the generated text segment and associated metadata. See :class:`GenerationResponse` for details. Nadd_special_tokensrrc3,K|] \}}||dfyw)FNr2).0rgrhs r( z"stream_generate..s 5 6  x--.i""3' 8  8556i&&s+? <   h2237s;< Kr*c |td|D}tj|Dcgc]}dg|t|z z|zc}Scc}w)Nc32K|]}t|ywrNrrps r(rz$_left_pad_prompts..$1AQrrrPrQrprompts max_lengthrs r(_left_pad_promptsr "sN111 88gFgaSJQ/014gF GGFA c |td|D}tj|Dcgc]}|dg|t|z zzc}Scc}w)Nc32K|]}t|ywrNrrs r(rz%_right_pad_prompts..*rrrrrs r(_right_pad_promptsr$(sN111 88gFgQ! SV 344gF GGFr!cteZdZUdZdZeed<dZeed<dZ eed<dZ eed<dZ eed<dZ eed<dZ eed <y ) BatchStatsa% An data object to hold generation stats. Args: prompt_tokens (int): The number of prompt tokens processed. prompt_tps (float): The prompt processing tokens-per-second. prompt_time (float): The time in seconds spent in prompt processing. generation_tokens (int): The number of generated tokens. generation_tps (float): The tokens-per-second for generation. generation_time (float): The time in seconds spent in generation . peak_memory (float): The peak memory used so far in GB. rrjrkrrlrmgeneration_timernN)rprqrrrsrjr=rtrkr?rrlrmr'rnr2r*r(r&r&.sP M3JKsNEOUKr*r&c&fdt|dr&|j}|Dcgc] }| c}S|$|jDcgc]}t|c}S|jDcgc] }t c}Scc}wcc}wcc}w)z[ Convert a list of regular caches into their corresponding batch-aware caches. ct|tur tSt|trt j |_|St|tr0|jdkDr tdt|jSt|trtfd|jDStt|d)Nrz2RotatingKVCache with keep tokens is not supported.c3.K|] }|ywrNr2)rsub_cto_batch_caches r(rz6_make_cache..to_batch_cache..WsK(~e4(sz does not yet support batching)r-rrrOrrPrQ left_paddingrkeeprrmax_sizercaches)rr-r,s r(r,z#_make_cache..to_batch_cacheLs 7g  - - ; 'XXl3ANH ? +vvz !UVV' LA A 9 %K!((KL LQy(FGH Hr* make_cache)r{r1rrr)rJr-rrrrr,s ` @r( _make_cacher2Fs Iul#  "+015aq!511  "INIUA$[,? 5:LLALq \*LAA 2BsB B /Bc  g}|s|Stt|dD]h}t|d|dr8|j|d|j |Dcgc]}|| c}Mt t |d|d|Scc}w)Nrmergez+ does not yet support batching with history)rrr{rr4rr-)r0 batch_cacherrs r( _merge_cachesr6fsK  3vay> " 6!9Q< )   vay|112HA1Q42HI Jq ! %&&QR  #  3Is B c`|s|S|s|St||D]\}}|j||SrN)zipextend)cache_acache_bcacbs r( _extend_cacher>vs7  gw'B " ( Nr*c`i}t|D]1\}}|} |D]}|j|i}t||f|d<3t }|j D]!\}}|dk(r ||d<|j |#|r|j} | j D]\\}}|dvr |j || d} || vr| |ur| d} || vr| |ur|| vr| |n||d<d|vsJd|dvsR|dd|d<^|r|S#t$r|j|i}|f|f|d<YwxYw)u}Build an Aho-Corasick trie from the provided sequences See https://en.wikipedia.org/wiki/Aho–Corasick_algorithm . __match____fail__)rAr@)rz setdefaulttuple TypeErrorritemsrpopleft) sequencestrieidxseqnodetokqueuekeychildparentfails r( _build_trierRsz Di(S .sB/!&sS 1D  ) GEjjl U +   j U #  ,,.JC// LL *%DT/d$&6J'T/d$&6-0D[S dE* %'K5;L,L%*:%6{%Ck")  K/ .??3+D"% D  .s)D#D-,D-cD||vr||ur|d}||vr||ur||vr||}|S)z"One step in the Aho-Corasick trie.rAr2)rKrHrTs r( _step_trierTs? 4-D,J 4-D,DyAw Kr*c8eZdZdZidfdZdZdZedZy)SequenceStateMachineaA state machine that uses one Aho-Corasick trie per state to efficiently track state across a generated sequence. The transitions are provided as state -> [(sequence, new_state)]. Example: sm = SequenceStateMachine( transitions={ "normal": [ (think_start_tokens, "reasoning"), (tool_start_tokens, "tool"), (eos, None), ], "reasoning": [ (think_end_tokens, "normal"), (eos, None), ], "tool": [ (tool_end_tokens, None), (eos, None) ], }, initial="normal" ) normalc||_i|_|jD]*\}}t|\}}t ||f|j|<,|jst ggf|j|<yyrN)_initial_statesrEr8rR)self transitionsinitialsrcedgesrGdsts r(__init__zSequenceStateMachine.__init__sq  %++-JC %[NIs!,Y!7 =DLL .||%0_b$9DLL !r*c|tjt}|j|_|j|_|SrN)object__new__rVrYrZ)r[memonews r( __deepcopy__z!SequenceStateMachine.__deepcopy__s,nn12}} ll  r*ch|j|j|jd|jfSr)rYrZr[s r( make_statezSequenceStateMachine.make_states) t||DMM:1=t||LLr*c|\}}}t|||d|}d}|jd}||d}||d|d}|||dnd}|||f||fS)Nrr@r)rTget)rrTrbrstatesrJmatchs r(rnzSequenceStateMachine.matchs 1f q&)A, *k"  (Cq ! U1X&A ! q ! 4A1f~sA%%r*N) rprqrrrsrargrj staticmethodrnr2r*r(rVrVs26$&x: M & &r*rVczeZdZdZeGddZ ddejdee deee de eee d e d e ee e jge jfd e e e jge jfd e eee e je jge jfd e eede ee fdZdZde dee fdZdZdZdee fdZdee fdZdeee fdZdeee fdZe ddejd e e jge jfd e fdZy)PromptProcessingBatchz A batch processor for prompt tokens with support for incremental processing. This class handles batched prompt processing, managing KV caches and preparing tokens for generation. It supports extending, filtering, and splitting batches. c6eZdZUeed<eed<eed<eed<y)PromptProcessingBatch.Responseuidprogressend_of_segment end_of_promptN)rprqrrr=rtrCrur2r*r(Responserss r*rxNrJuidsr0rrsamplersfallback_samplerrstate_machinesrc f||_||_t||_||n|D cgc]} gc} |_||_||ng|_|xsd|_||ng|_| | ntgt|z|_ | | |_ ytgt|jz|_ ycc} w)Nc0tj|dSrrrs r(rUz0PromptProcessingBatch.__init__..sryyQS?Tr*)rJryr6r}rrrzr{rrVrr|r>r) r[rJryr0rrrzr{rr|rrs r(razPromptProcessingBatch.__init__s  )&1 & 2fT8JTT8J !2$,$8b 0 U5T!2!> B  ) &()CI5 %  %%DII6 9Ks B.c,t|jSrNrryris r(__len__zPromptProcessingBatch.__len__ 499~r*rIrc^|jDcgc]}|j|c}Scc}wrNr}extractr[rIrs r( extract_cachez#PromptProcessingBatch.extract_cache#*(,(9(9:(91 #(9:::*ct|jsdgt|jz|_t|jsdgt|jz|_t|jr |jndgt|jz}t|jr |jndgt|jz}|jj |jt |j|j|_|jj |j|jj ||jj ||jj |j|jj |jyrN) anyrzrryrr9r>r}rrr|)r[batchrzrs r(r9zPromptProcessingBatch.extend&sB4==!!FS^3DM4))*&*Vc$))n%>tfs5::>V5**+  # ##ejj/)  $)$*;*;U=O=OP 5<<( X& %%&78 u//0 ""5#7#78r*c6|jj|j}|j|_t|j|_t j |j|_t|j|_|j|_ t|j|_ |j|_ t|j|_ t|j|_ t|j|_|SrN) __class__rdrJlistrycopydeepcopyr}rrrzr{rr|r)r[ new_batchs r(_copyzPromptProcessingBatch._copy:sNN**4>>: ** dii !%t/@/@!A  , &*&<&< #!$--0 %)%:%: "&*4+A+A&B ##'(;(;#<  #DOO4 r*indicesc t|}tttt|jt|z }|j }|j ||j ||SrN)sortedsetrrryrfilter)r[r indices_leftrs r(splitzPromptProcessingBatch.splitHs[/c%DII"783w<GH JJL  L!!r*r.c|Dcgc]}|j|c}|_|s|jjn"|jD]}|j||Dcgc]}|j|c}|_t |j r#|Dcgc]}|j |c}|_ndgt|z|_t |jr#|Dcgc]}|j|c}|_nggt|z|_|Dcgc]}|j|c}|_ |Dcgc]}|j|c}|_ ycc}wcc}wcc}wcc}wcc}wcc}wrN) ryr}clearrrrrzrrrr|r[r.rIrs r(rzPromptProcessingBatch.filterQsL/34tTYYs^t4     # # %&&'3784Ct{{3'48 t}} ;?@4CT]]3/4@DM!FSY.DM t%% &MQ%RTcd&<&S[ (RS S 4;;/GC 2IC0$**6a3q66*\ +237a:>73'l  ?':FF&& ' A'XXf%Fll1o!t55v||AGL JJva,./t7H7HJ I GGd&7&78&7QWW&78 9 NN A|},-F ll1o! ?&& ' GGd&7&78&7QWW&78 9 NN  /+3 99sH: HH &H%c td|Dr"|j|Dcgc]}|dd c}tj|Dcgc]}|d c}}t |j |j ||j|j|j|j|j|j|j }g|_g|_g|_g|_ g|_ g|_ |Scc}wcc}w)z Transition from prompt processing to generation. Args: tokens: Final tokens for each sequence to start generation. Returns: A GenerationBatch ready for token generation. c38K|]}t|dkDyw)rNr)rts r(rz1PromptProcessingBatch.generate..s*6as1vz6sNr)rrrPrQGenerationBatchrJryr}rrzr{rr|r)r[rr last_token generations r(rzPromptProcessingBatch.generates *6* * KK0A3B0 1XXf5fquf56 $ JJ II     KK MM  ! !  " "    OO     !#/15s C5 C:c &||||ggggggg S)N) rJr{rryr0rrzrrr|r2)clsrJr{rs r(emptyzPromptProcessingBatch.emptys/-/   r*)NrNNNNN)r)rprqrrrsrrxnnModuler r=rr rrPrQrVrarrr9rrrrr classmethodrr2r*r(rqrqs-1!%CGEI ?C*.# yy# 3i# T#Y # d3i) #  # 4"((RXX)= >?@# #8RXXJ,@#AB# $ h"((3RXX=>? @ # !&:!;<# T#Y'# J;;c;9( T#YI49I&.T$s)_.`"tDI"H "&  yy #BHH:rxx#78    r*rqcXeZdZdZeGddZdejdee de jdee deee d e eee jge jfd ee jge jfd e eeee je jge jfd eed ee fdZdZdZdeee ee jffdZde dee fdZdee fdZdeefdZedejd ee jge jffdZy)rz A batched token generator that manages multiple sequences in parallel. This class handles the generation phase after prompt processing, managing KV caches, sampling, and stop sequence detection for multiple sequences. ceZdZUeed<eed<ej ed<eeed<eeed<ee eed<ee e ed<ee eed<y ) GenerationBatch.Responsertrgrhro current_statematch_sequencer} all_tokensN) rprqrrr=rtrPrQr r:r rr2r*r(rxrsW  ((}$}$ c++tCy))T#Y''r*rxrJryinputsr}rrzr{rr|rc ||_||_||_||_||_||_||_| |_| |_|jr6t|jt|jk7r td|j r6t|j t|jk7r tdd|_ g|_ ||_ g|_|D cgc] } t| c} |_dgt|jz|_| D cgc]} | j%c} |_|jr|j)yycc} wcc} w)Nz(Insufficient number of samplers providedz1Insufficient number of logits_processors providedr)rJryr}rrzr{rr|rrr_current_tokens_current_logprobs _next_tokens_next_logprobsr_token_context _num_tokensrj_matcher_statesr) r[rJryrr}rrzr{rr|rrms r(razGenerationBatch.__init__s-  (   0!2,$ ==S/3tyy>AGH H  ! !c$*@*@&AS^&SPQ Q#!#" 7=>v!{1~v>3TYY/8FG1 G 99 JJL  ?Gs $E E"c,t|jSrNrris r(rzGenerationBatch.__len__ rr*c(|jj|jt|j|j|_|jj|j|j j|j |j j|j |jj|j|jj|j|j#|j|_ |j|_ na|jUtj|j|jg|_ |jj|j|j#|j|_ |j|_na|jUtj|j|jg|_ |jj|j|jj|j|j j|j |j"j|j"y)z0Extend this batch with another generation batch.N)ryr9r>r}rrzrrr|rrrPrrrrrr)r[rs r(r9zGenerationBatch.extend s $)$*;*;U=O=OP 5<<( U^^, %%e&=&=> u//0 ""5#7#78    '#(#8#8D %*%<%>%%u'<'<=$D   " " ) )%*A*A B    $ % 2 2D "'"6"6D     + "0A0A5CUCU/V WD     & &u';'; < ""5#7#78  1 12 ##E$9$9:r*rc ,|j|_|j|_|j}|j |dddf|j }|dddddf}g}t |jrt|jDcgc]\}}|j|||dz}}}g}tt|jD];}|||dz}|j|D]} | |||}|j|=tj |d}|tj"|ddz } t |j$ryg} tt|jD]>}|j$|xs |j&} | | ||dz} | j| @tj | d} n|j'| } | |_t)| |_tj*|j|j|tj,||j|j/}t1|j2|D]\}}|j|||jfScc}}w) zx Perform a single generation step. Returns: Tuple of token list and logprobs list. NrrrrrTr)rrrrrJr}rrrzrupdate_and_fetchrrryrrPrrrzr{rrrrr8r)r[rr token_contextrtcprocessed_logitsr sample_logitsrrh all_samplessample_samplerrrrs r(rzGenerationBatch._step(s^ $00!%!4!4%%F1d7O43D3DE2q! t%% &'t':':;;EAr##F1q1u$56;  " 3tyy>* &q1q5 1 !%!7!7!:I$-mA.> $NM"; '' 6 + ^^$41=FBLLb4HH t}} K3tyy>*!%q!1!JT5J5J(!a!e)<=""7++nn[q9G++H5G$"8n d'')<)"   A  MM IIaL$Qi!)!&*'5&3%)#'"  ;'T t9s499~ % KK r*c l|||gtjgtjgggggg S)N)dtype) rJr{ryrr}rrzrrr|)rPrQr)rrJr{s r(rzGenerationBatch.emptys> -88Bbii0   r*N)rprqrrrsrrxrrr r=rPrQrr rrVrarr9r rrrrrrr2r*r(rrs((((yy(3i( ( 3i ( T#Y (4"((RXX)= >?@(#BHH:rxx#78($ h"((3RXX=>? @ (12(I(T;6:.uT#YRXX67:.x;;c;K49K,;d8n;z yy #BHH:rxx#78  r*rceZdZdZdddddddddd dej d ed eeeed ee e jge jfd ee e e je jge jfd edededeefdZ edZdZdZej&d(dZ d)de e ed ee edee e edee e edee e e jge jfd ee e e e je jge jfdee efdZ d)de e e ed ee edee e edee e edee e e jge jfd ee e e e je jge jfdee efdZdZdZd Zd*d!Zed"Zd#efd$Zd%Zd&Z d'Z!y)+BatchGeneratoraW A batch generator implements continuous batching. This class provides automatic management of prompt processing and generation batches, handling the transition between the two. It also allows for segmented prompt processing which guarantees that the generator will stop at these boundaries when processing an input. N r) r stop_tokensrrcompletion_batch_sizeprefill_batch_sizerrrrJrrrrrrrrc ||_||_|xsd|_|xsg|_d|_||_||_t|||_| |_ | xst|_ t|rd|D cgc]} | dfc} inid|_ d|_tj!|j|j||_t$j!|j|j|_t)|_g|_d|_d|_d|_d|_t6j8j;r0t7j<t7j>d|_ yd|_ ycc} w)Nc0tj|dSrrrs r(rUz)BatchGenerator.__init__..sRYYqr-Br*rrW)r])rrV)!rJrrr uid_countrrrrrr_streamrV_default_state_machine _uid_countrqr _prompt_batchr_generation_batchr_unprocessed_sequences_currently_processing_prompt_tokens_counter_prompt_time_counter_gen_tokens_counter_steps_counterrPrWrXr[rY_old_wired_limit) r[rJrrrrrrrrrrJs r(razBatchGenerator.__init__s^  $C#B !2!8b!2"4%()>@R%S"&2!2 &:@KX{;{d {; #> >    !:!: : $22U5F5FFE   # #t'?'? ? #  ! !X - !#(#:#:U=R=R#RE #E$5$5r7I7I7Kc7Q RE ##%CsJ!D$=$==H   4#>#> >    !:!: : $22U5F5FFE   # #t'?'? ? #  ! !X - !#(#:#:U=R=R#RE #E$5$5r7I7I7Kc7Q RE s8IEDIDIIrr0rrzr|c X|j|Dcgc]}|gc}||||||Scc}wrN)insert_segments) r[rrr0rrzrr|rs r(insertzBatchGenerator.insert1s@##! "'QaS' "        "s 'segmentsc  g}|xs|jgt|z}|xs|D cgc]} gc} }|xsdgt|z}|xs|jgt|z}|xs|jgt|z}|xsdgt|z}t t|D]} ||  |j || <t |||||||D]\} } } }}}}t| } t| ddk7r"| j| ddd| ddd| d<|jj|j| | | ||||f|j|j|xjdz c_ |Scc} w)Nrr) rrrrr_make_new_cacher8rrrr)r[rrr0rrzrr|ryrrrJrratrblpsms r(rzBatchGenerator.insert_segmentsGsDDOO#4s8}#D 9#91B#9 5vH 5-  # # $s8} 4 (  ( ( )CM 9 1D6CM1s8}%Aay  002q &),       ) $CAr1b"s)C3r7|q  3r723<(b'#2,B  ' ' . .#q!RB;  KK ( OOq O#) & C$:s Fc|jtj|jStj|jDcgc]*}t |t rt |jn|,c}Scc}w)N)r/)rrrrJrOrr)r[cis r(rzBatchGenerator._make_new_cacheys|    #**4::6 6--djj9  :b'* )9)9::    s /A?cJt|}i}t|jjD]\}}||vs d|f||<t|jjD]\}}||vs d|f||<t|j D]\}}|d|vsd|f||d<|S)Nrrr)rrzrryrr)r[ryresultsruid_irJs r( _find_uidszBatchGenerator._find_uidss4y!$"8"8"="=>HAu}"#Q?"$"4"4"9"9:HAu}"#Q;  ; ; C%z#::3?!D !&&44S9&&--c2  **88=**11#6  "?r*c i}|r|j|}ttt|jttt|j ttt|j f|j|jD]\}}|j|tdt|jkr,tfdt|jD|_tdt|j kr\|j jtdt|jDcgc]\}}|dvs|c}}|_tdt|j kr'|j jtd|Scc}}w)Nrc38K|]\}}|dvs|yw)rNr2)rrrTr.s r(rz(BatchGenerator.remove..s%0DdaTRSW Dsrr)rrrrrrrrvaluesremoverrzrrr) r[ryreturn_prompt_cachesr0rrIrrTr.s @r(r zBatchGenerator.removes ''-F c$5567 8 c$,,-. / c$0012 3  //$/668JE3 K  s #9 tAw<#d99: :*/0'(C(CD0+D ' tAw<#d001 1    % %fT!Wo 6'(B(BC*CdaqDQRG|C*D & tAw<#d445 5  " " ) )&a/ : *s 9G# G#ctd|jD}|td|jjDz }|td|jjDz }|S)Nc3HK|]}|dD]}|jyw)r6NrR)rrrs r(rz5BatchGenerator.prompt_cache_nbytes..s#P&A1Q4aAHH4H&As "c34K|]}|jywrNrrrs r(rz5BatchGenerator.prompt_cache_nbytes..sG'F!QXX'Fc34K|]}|jywrNrrs r(rz5BatchGenerator.prompt_cache_nbytes..sK'J!QXX'Jr)sumrrr}r)r[totals r(prompt_cache_nbytesz"BatchGenerator.prompt_cache_nbytess_Pd&A&APP Gt'9'9'F'FGGG Kt'='='J'JKKK r*rc Xg}g}g}g}g}g}g}t|D]} |jj} |j| d|j| d|j| d|j| d|j| d|j| d|j| d|jj| ddt d | dDgt |j||||j||j||| S) Nrr6rrrc32K|]}t|ywrNr)rrbs r(rz-BatchGenerator._make_batch..s$A[SV[r) rJryr0rrrzr{rr|r) rrrFrrrrqrJrr) r[rryr0rrzrrr|rsequences r( _make_batchzBatchGenerator._make_batchs* qA22::||z z c_||fScc}wcc}wcc}w) NrrrrTFrc32K|]}t|ywrNrrs r(rz'BatchGenerator._next../s*C7a3q67r) rrrrrrPrrrrrrr9rrzrrrrrqrxryrrrvrurrrrrr)r[generation_responsesprompt_responsesrr.rrrJr last_inputsru gen_batchrrrrrs r(_nextzBatchGenerator._nexts! t%% & *#'#9#9#>#>#@  $ $,@(A A $   1 $ ""S(A-  t%% &$*D*D D#%99 9   # #c$*<*<&= =  & &T-C-C)D D ++ ,  q5    % %d&6&6q&9 : : :;FAs1vH8}!c(1+&6!&; Q A < HMN1455a8;A>KNINOA33A6q9;a?HOQU)VQUA$*D*DQ*GQU)VD &**007@@MI!(+1 '')22!q) ,  " " ) )) 4 : :;FAs,55""''*AueH1vHC $d&<&<=A NN8A;r? +"1+ab/HQK8A;1$ Q*.' Fc'"+& &F!$QQ 0H   # #H -< ##s*C7*C'CC#! !!'*! !!S3Y.!!555OOO)Vs3PP$8P)ctj|j5|jcdddS#1swYyxYw)z Get the next batch of responses. Returns: Tuple of prompt processing responses and generation responses. NrPrrr%ris r(rzBatchGenerator.next7s)YYt|| $::<% $ $s :Actj|j5 |j\}}|s|r|cdddS#1swYyxYw)z Return only generated tokens ignoring batch generation responses. Returns: List of GenerationBatch.Response objects Nr')r[r"r!s r(next_generatedzBatchGenerator.next_generatedAsAYYt|| $9=6 "6+0@+ % $ $s AArN)NNNNNNF)"rprqrrrsrrr=r r rrPrQr rapropertyrrr contextlibcontextmanagerrrrVrrrrrr rrr%rr)r2r*r(rrs!9=<@ %'"#!%%)4)yy4) 4) hx}56 4) (BHH:rxx#789 4)$ 288RXX.89 : 4) #4) 4)4)c]4)l) SS.+/,004CG ?C d3i T#Y' d3i)  T$s)_-  4"((RXX)= >?@  $ h"((3RXX=>? @  !&:!;< 2+/,004CG ?C0tDI'0T#Y'0d3i) 0 T$s)_- 0 4"((RXX)= >?@ 0$ h"((3RXX=>? @ 0!&:!;<0d   "6  S  DL6\  ,r*rcHeZdZUdZeeed<eed<eeee ed<y) BatchResponsez A data object to hold a batch generation response. Args: texts: (List[str]): The generated text for each prompt. stats (BatchStats): Statistics about the generation. textsrr0N) rprqrrrsr r:rtr&r rr2r*r(r/r/Ps, 9  T$s)_ %%r*r/r prompt_cachesr c |t|fd|jDcgc]}|gc}i|} t|} d} |rtd| ddt |t r|gt|z}| j |||} | D cic]} | g}} i}| j5}| jx}r|D]|}|j6|r|j||j<|r| d z } td | d | dd|jd k7sU||jj|j~| jx}rddd| j|rtd | d | | D cgc]} |j|| }} |r| D cgc]} ||  c} nd}|rhtd j d|j"ddtd|j$d|j&ddtd|j(ddt+||Scc}wcc} w#1swYxYwcc} wcc} w)a| Generate responses for the given batch of prompts. Args: model (nn.Module): The language model. tokenizer (PreTrainedTokenizer): The tokenizer. prompts (List[List[int]]): The input prompts. prompt_caches (List[List[Any]], optional): Pre-computed prompt-caches for each input prompt. Note, unlike ``generate_step``, the caches won't be updated in-place. verbose (bool): If ``True``, print tokens and timing information. Default: ``False``. max_tokens (Union[int, List[int]): Maximum number of output tokens. This can be per prompt if a list is provided. return_prompt_caches (bool): Return the prompt caches in the batch responses. Default: ``False``. kwargs: The remaining options get passed to :obj:`BatchGenerator`. See :obj:`BatchGenerator` for more details. rrz'[batch_generate] Finished processing 0/z ... )r )r0Nrz%[batch_generate] Finished processing /rz[batch_generate] Prompt: rrrz[batch_generate] Generation: z[batch_generate] Peak memory: r)rrrrZrOr=rrr)ror}rtrrgrdecoderjrkrlrmrnr/)rJrrr1rr r rrgen num_samplesfinryrtrrrrr0r0s r(batch_generater:_sk<   "+"9"9:"9QaS"9:   C g,K C 7 }DItT*c" \CL0 ::gz-: @D"&'$3sBw$G'M --//i/??.+/0~~ aee,qCC5+VZ[ $??f,AEEN))!''2--//i/ IIK 5cU!K=IJ8< r|j@d|j@dg} ng} | jCd| d|jDdu} | r| jCd|jDd|jF| fd| | d| } |r2d| dd<|jG| d| | }| |jIdd} |jK| d} n|jK| } |jLrrGrz5Draft model tokenizer does not match model tokenizer.)top_kxtc_probability xtc_thresholdxtc_special_tokens) rr rrr}rrr~rr)9rI parse_argsseedrPrandomprompt_cache_filerrOrrryrrrxjsonloadsr>rJr;rr@r?extra_eos_token add_eos_tokenchat_template_configrreplacesysstdinreadignore_chat_templatehas_chat_template system_promptrprefill_responseapply_chat_templateindexrr vocab_sizertemptop_pmin_pmin_tokens_to_keeprMrNrOrrrrr rr~rrZ)rHargs using_cacher}metadatar= model_pathrJr eos_tokentemplate_kwargsrmessages has_prefill test_promptrdraft_tokenizerrrs r(mainrssA  F    D yy tyy!((4K!2  " " "  h l1o'7 8||'DLLLO%>? [[  - 5 5eT BF!'3SYY^^ FF  $ $)D)D    )!)d6H6HIJHHF;<++47  OO[T=R=RS T...  #.&1/     &/HRL ##77'2*5o 8K K--i8:;F!!&U!C!!&) #'+D,<,<'=$ _  % %)=)= =TU U     jj,,(($++D1D9P9P4QQ G ?? $$%0\d ((22..H << h r*__main__zCalling `python -m mlx_lm.generate...` directly is deprecated. Use `mlx_lm.generate...` or `python -m mlx_lm generate ...` instead.rN)rNr*)NrFF)ar7r,rrrUr[r collectionsr dataclassesrrtypingrrrr r r r r mlx.corecorerPmlx.nnr mlx.utilsr transformersrmodelsr models.cacherrrrrrrrr sample_utilsrtokenizer_utilsrutilsrrr<r>r@rArBrCrDDEFAULT_XTC_THRESHOLDrErFr;rGr)rInew_thread_local_streamdefault_devicerr-rStreamrcrerrQr=rrurr:r rr r$r&r2r6r>rRrTrVrqrrr/r:rsrprZr2r*r(rs  !   !,   '-<     : !0_F/B../@r/@/@/BC $*ryy$*8DO+D$*$*N (( (:U8