
    (HJj                    4   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# d dl$m%Z& d d
l'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8 d Z9 G d d      Z:d;dee;   de e;   fdZ<d Z=e G d d             Z>e G d d             Z?e G d d             Z@e G d d             ZAe G d  d!             ZBe G d" d#             ZCe G d$ d%             ZDd& ZE G d' d(      ZF G d) d*      ZGd+ ZHd, ZId-e"eeJef      fd.ZK G d/ d0      ZL G d1 d2e      ZMeeMfd3eJd4eNfd5ZOeeMfd3eJd4eNd6eGfd7ZPd8 ZQeRd9k(  r eSd:        eQ        yy)<    N)deque)	dataclassreplace)BaseHTTPRequestHandlerThreadingHTTPServer)Path)Empty)Queue)Thread)
AnyCallableDictListLiteral
NamedTupleOptionalSequenceTupleUnion)scan_cache_dir   )__version__)BatchGeneratorSequenceStateMachinestream_generate)LRUPromptCachemake_prompt_cache)make_logits_processorsmake_sampler)_parse_sizeloadsharded_loadc                      t        j                         d   } t         dt         j                   dt        j                          d|  S )Narchitecture-)mxdevice_infor   platform)gpu_archs    W/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/server.pyget_system_fingerprintr+   1   s?    ~~/H]!BNN+1X->->-@,A8*MM    c                        e Zd ZddZd Zd Zy)ToolCallFormatterc                 <    d| _         || _        || _        || _        y Nr   )_idx_tool_parser_tools
_streaming)selftool_parsertools	streamings       r*   __init__zToolCallFormatter.__init__7   s    	'#r,   c                    |j                  dd       xs t        t        j                               }t	        j
                  |d   d      |d<   |d|d}| j                  r$| j                  |d<   | xj                  dz  c_        |S )	Nid	argumentsF)ensure_asciifunction)r>   typer;   indexr   )popstruuiduuid4jsondumpsr4   r1   )r5   tctc_idouts       r*   _formatzToolCallFormatter._format=   sr    tT"7c$**,&7**R_5I;

 ??99CLIINI
r,   c           	      `    |sg S g }|D ]L  }	  j                  | j                        }t        |t              s|g}|j                   fd|D               N |S # t        t        j                  f$ r9}t        j                  dt        |      j                   d| d       Y d }~d }~ww xY w)NzFailed to parse tool call (: u4   ) — tool text was likely truncated mid-generation.c              3   @   K   | ]  }j                  |        y wN)rJ   ).0rG   r5   s     r*   	<genexpr>z-ToolCallFormatter.__call__.<locals>.<genexpr>Z   s     <Vr$,,r*Vs   )r2   r3   
ValueErrorrE   JSONDecodeErrorloggingwarningr?   __name__
isinstancelistextend)r5   
tool_callsresult	tool_textparsedes   `     r*   __call__zToolCallFormatter.__call__J   s    I#I**9dkkB fd+ MM<V<< $   4 45 1$q'2B2B1C2aS IE F s   AB-4/B((B-N)F)rU   
__module____qualname__r9   rJ   r^    r,   r*   r.   r.   6   s    $r,   r.   messagesrole_mappingc                    dddddd}|xs |}d}| D ]F  }|j                  |d   d      }|j                  d	d      }|j                  d
d      }|| | | z  }H ||j                  dd      z  }|j                         S )Nz}A chat between a curious user and an artificial intelligence assistant. The assistant follows the given rules no matter what.zASSISTANT's RULE: zUSER: zASSISTANT: 
)system_promptsystemuser	assistantstop rolerj   contentri   )getrstrip)rb   rc   default_role_mappingpromptlinerole_prefixrj   rm   s           r*   convert_chatrt   ^   s    O '"	  7#7LF"&&tF|R8+((9b)[M'4&11	  l{B//F==?r,   c                    | D ]  }|j                  d      }t        |t              rR|D cg c]  }|d   dk(  s|d    }}t        |      t        |      k7  rt	        d      dj                  |      |d<   n|d|d<   |j                  d      x}s|D ]B  }|j                  d      x}s|j                  d	      x}s+t        j                  |      |d	<   D  yc c}w )
a  
    Convert message content to a format suitable for `apply_chat_template`.

    The function operates on messages in place. It converts the 'content' field
    to a string instead of a list of text fragments.

    Args:
        message_list (list): A list of dictionaries, where each dictionary may
          have a 'content' key containing a list of dictionaries with 'type' and
          'text' keys.

    Raises:
        ValueError: If the 'content' type is not supported or if 'text' is missing.

    rm   r?   textz&Only 'text' content type is supported.rk   NrY   r>   r<   )rn   rV   rW   lenrQ   joinrE   loads)	rb   messagerm   fragmenttext_fragmentsrY   	tool_callfuncargss	            r*   process_message_contentr   v   s      ++i(gt$1818XHV<LPV<V    >"c'l2 !IJJ!#!8GI_!#GI \22:2'	$==4444#xx44t4,0JJt,<[) ( s
   CCc                   ,    e Zd ZU eed<   eed<   eed<   y)ModelDescriptionmodeldraftadapterN)rU   r_   r`   rB   __annotations__ra   r,   r*   r   r      s    JJLr,   r   c                   J    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   y)SamplingArgumentstemperaturetop_ptop_kmin_pxtc_probabilityxtc_thresholdN)rU   r_   r`   floatr   intra   r,   r*   r   r      s#    LJLr,   r   c                   d    e Zd ZU eeeef      ed<   eed<   eed<   eed<   eed<   eed<   eed<   y)	LogitsProcessorArguments
logit_biasrepetition_penaltyrepetition_context_sizepresence_penaltypresence_context_sizefrequency_penaltyfrequency_context_sizeN)rU   r_   r`   r   r   r   r   r   ra   r,   r*   r   r      s:    c5j)**  r,   r   c                       e Zd ZU eed<   eed<   eed<   ee   ed<   e	ed<   e	ed<   e
ed<   e	ed<   ee	   ed	<   eeeef      ed
<   y)GenerationArgumentsr   samplinglogits
stop_words
max_tokensnum_draft_tokenslogprobstop_logprobsseedchat_template_kwargsN)rU   r_   r`   r   r   r   r   r   rB   r   boolr   r   r   ra   r,   r*   r   r      sQ    $$S	ON
3-"4S>22r,   r   c                   h    e Zd ZU ed   ed<   eed<   ee   ed<   eee      ed<   ee	eef      ed<   y)CompletionRequest)chatrv   request_typerq   rb   r7   rc   N)
rU   r_   r`   r   r   rB   r   r   r   r   ra   r,   r*   r   r      s>    .))K3iDI4S>**r,   r   c                       e Zd ZU eed<   eed<   eeegef   ed<   ee	e
   ef   ed<   ee
   ed<   dZe
ed<   dZeed	<   d
 Zy)GenerationContexthas_tool_callinghas_thinkingr6   	sequencesrq   prompt_cache_countF_should_stopc                     d| _         y NT)r   r5   s    r*   rj   zGenerationContext.stop   s
     r,   N)rU   r_   r`   r   r   r   rB   r   r   r   r   r   r   r   rj   ra   r,   r*   r   r      sY    3*d*++E#JO$$I  L$!r,   r   c                   p    e Zd ZU eed<   eed<   eed<   ee   ed<   eed<   ee   ed<   ee	ee
f      ed<   y)	Responserv   tokenstatematchlogprobfinish_reason
top_tokensN)rU   r_   r`   rB   r   r   r   r   r   r   r   ra   r,   r*   r   r      s=    
IJJ:NC= d38n%%r,   r   c              #     K   t        d | j                  D              }t               }|D ]  }|j                  |       |j                  S|j                  D cg c]  }|j                          }}t        |      D ]  }|j                  t        |d               t        |      |k\  s|j                           t        |      dkD  r"|j                          t        |      dkD  r!y y c c}w w)Nc              3   2   K   | ]  }t        |        y wrN   )rw   )rO   ss     r*   rP   z*_process_control_tokens.<locals>.<genexpr>   s     4mc!fms   rk   rv   r   )
maxr   r   appendr   rA   reversedr   rw   popleft)ctxtoken_streambuffer_sizebuffered_streamtok_poppedts           r*   _process_control_tokensr      s     4cmm44KgOs#99 58YY?Yo))+YF?f%&&wqr':; &;.!))++  o

"%%'' o

" @s   AC7C20<C7-AC70C7c                        e Zd ZddZd Zd Zy)
TimeBudgetc                     t         j                  j                         j                         dkD  | _        || _        || _        || _        d | _        d | _	        d| _
        d| _        y Nr   r   )r&   distributedinitsize_is_distributed_budget_iterations_sync_frequency_start_current_iterations_loops_time_spent)r5   budget
iterationssync_frequencys       r*   r9   zTimeBudget.__init__   sZ    !~~22499;a?%-#' r,   c                 F    t        j                          | _        d| _        | S r0   )timer   r   r   s    r*   __iter__zTimeBudget.__iter__  s    iik#$ r,   c                 6   | j                   s9t        j                         | j                  z
  | j                  kD  r
t	               y | xj
                  dz  c_        | j
                  | j                  k  ry | xj                  dz  c_        | xj                  t        j                         | j                  z
  z  c_        | j                  | j                  z  dk(  rt        j                  j                  | j                        j                         }|t        j                  j                         j                         | j                  z  z  }| j                  |z  }t!        t#        | j                  |z        d      | _        d| _        d| _        t	               r   )r   r   r   r   StopIterationr   r   r   r   r   r&   r   all_sumitemr   r   r   round)r5   	loop_timeavg_loop_timefactors       r*   __next__zTimeBudget.__next__  s5   ##yy{T[[(4<<7#o%  A% ##t'7'77qDIIK$++55;;---2..t/?/?@EEGI%##%**,t/C/CCM \\M1F"5)9)9F)B#CQGDDK Dor,   N)g      ?   
   )rU   r_   r`   r9   r   r   ra   r,   r*   r   r      s    
r,   r   c                   B    e Zd Zdej                  fdZddZd ZddZy)ModelProvidercli_argsc                    || _         d| _        d| _        d| _        d| _        d| _        t        j                  j                         }|j                         dkD  r|j                  r|nd| _        |j                         dkD  r|j                  s|nd| _        |j                         dkD  | _        i | _        i | _        i | _        | j                   j                  | j                  d<   | j                   j"                  | j                  d<   | j                   j                  | j                   d<   d|j$                  rdndi| _        |j(                  r|j(                  | j&                  d<   yy)z@Load models on demand and persist them across the whole process.NFr   default_modeltrust_remote_codeTchat_template)r   	model_keyr   	tokenizerdraft_modelis_batchabler&   r   r   r   pipelinepipeline_grouptensor_groupis_distributed
_model_map_adapter_map_draft_model_mapadapter_pathr   _tokenizer_configr   )r5   r   groups      r*   r9   zModelProvider.__init__%  s=    
!##%',zz|a'7H<M<MeSWZZ\A%h.?.?ET 	 $jjlQ.
  "+/==+>+>(-1]]-G-G/*151J1Jo.  )C)C"
 !!6>6L6LD""?3 "r,   Nc                    | j                   r||t        d      d | _        d | _        d | _        d | _        | j                   r1t        || j                  | j                  | j                        \  }}nt        ||| j                        \  }}| j                  j                  r|j                  |j                  |_        d }|<t        |      \  }}|j                  |j                  k7  rt!        j"                  d       |d u }|xr t%        d t'        |      D              }|||f| _        || _        || _        || _        || _        y )NzGLoading with adapters or draft models not supported in distributed mode)r   r   tokenizer_config)r   r  zdDraft model tokenizer does not match model tokenizer. Speculative decoding may not work as expected.c              3   4   K   | ]  }t        |d         yw)mergeN)hasattr)rO   cs     r*   rP   z&ModelProvider._load.<locals>.<genexpr>t  s      ,
)AAGAw)A   )r   rQ   r   r   r   r   r"   r   r   r  r!   r   use_default_chat_templater   default_chat_template
vocab_sizerS   rT   allr   r   )	r5   
model_pathr   draft_model_pathr   r   r   draft_tokenizerr   s	            r*   _loadzModelProvider._loadF  sh   $(8(DY 
 
 +#22!..!%!7!7	 E9  $)!%!7!7 E9 ==22&&.*3*I*I	' '+/0@+A(K))Y-A-AAE #d*# 
 ,
):5)A,
 )

 %l4DE
"&(r,   c                 J    | j                   d   | j                  dd d       y y )Nr   )r   r!   r   s    r*   load_defaultzModelProvider.load_default  s&    ???+7IIot_= 8r,   c                     | j                   j                  ||      }| j                  j                  ||      }| j                  j                  ||      }|||f}| j                  |k7  r | j
                  |  | j                  | j                  fS rN   )r   rn   r   r   r   r  r   r   )r5   r  r   r  r   s        r*   r!   zModelProvider.load  s    __((Z@
((,,ZF00445EGWX/?@	>>Y&DJJ	"zz4>>))r,   )NN)	rU   r_   r`   argparse	Namespacer9   r  r  r!   ra   r,   r*   r   r   $  s'    M!3!3 MB7)r>	*r,   r   c                 L   t        | j                  j                  | j                  j                  | j                  j                  | j                  j
                  | j                  j                  | j                  j                  |j                  |j                  d      g      S )Nre   )r   r   r   r   r   xtc_special_tokens)
r   r   r   r   r   r   r   r   eos_token_idencode)r   r   s     r*   _make_samplerr    sy    !!mm!!mm!!mm!!55mm11""T"
 r,   c           	      <   t        | j                  j                  | j                  j                  | j                  j                  | j                  j
                  | j                  j                  | j                  j                  | j                  j                        S rN   )	r   r   r   r   r   r   r   r   r   )r   s    r*   _make_logits_processorsr    sj    !&&++$$))%%** r,   returnc                     |dk  ryt        j                  |  |dz
        }|d| j                         }| |   j                         }|j                  |      }t	        d t        |||      D              S )z=Returns info dicts for the top `top_n` tokens from `logprobs`r   ra   r   )kthNc              3   0   K   | ]  \  }}}|||d   yw))r;   r   r   Nra   )rO   ir   gs       r*   rP   z'_format_top_logprobs.<locals>.<genexpr>  s%      8GAq! 1+8s   )r&   argpartitiontolistconvert_ids_to_tokenstuplezip)r   top_nr   sorted_indicestop_indices	top_probstxtss          r*   _format_top_logprobsr.    s}    z__hYEAI>N %(//1K%,,.I**;7D ;i8  r,   c            
           e Zd ZdedefdZd Zd Zd ZddZ	d	 Z
d
 Zd Z	 ddZd Zd Zd Z	 ddededeeeegdf      fdZed        Zy)ResponseGeneratormodel_providerprompt_cachec                    || _         || _        t               | _        i | _        t               | _        t        j                  j                         j                         dkD  | _        t        j                  j                         j                         | _        d| _        t        | j                         | _        | j"                  j%                          y )Nr   F)target)r1  r2  r
   requests_state_machine_cacher   _time_budgetr&   r   r   r   r   rank_rank_stopr   	_generate_generation_threadstart)r5   r1  r2  s      r*   r9   zResponseGenerator.__init__  s    ,($&!&L!~~22499;a?^^((*//1

"("?%%'r,   c                 F    d| _         | j                  j                          y r   )r:  r<  rx   r   s    r*   stop_and_joinzResponseGenerator.stop_and_join  s    
$$&r,   c                 8    | j                   j                          y rN   )r<  rx   r   s    r*   rx   zResponseGenerator.join  s    $$&r,   c           
      T   t        | j                        }| j                  j                  }t        j                  d| d|dz  dd       | j                  j                         j                         D ]2  \  }}|d   }|d   }t        j                  d| d	| d|dz  dd       4 y )
NzPrompt Cache: z sequences, g    eAz.2fz GBn_sequencesn_bytesz- rL   )rw   r2  nbytesrS   infostats_by_typeitems)r5   rB  rC  
cache_typestatss        r*   _log_cache_statsz"ResponseGenerator._log_cache_stats  s    $++,##**~k],w}S>QQTUV!%!2!2!@!@!B!H!H!JJ.KI&GLLZL;-|GcM#;NcR "Kr,   Nc                     d }| j                   r| j                  dk(  r:	 || j                  j                  |      }n| j                  j	                         }| j                  |      S # t
        $ r Y w xY w)Nr   timeout)r   r9  r5  rn   
get_nowait
QueueEmpty_share_request)r5   rM  requests      r*   _next_requestzResponseGenerator._next_request  su    ##tzzQ&"mm///@G"mm668G ""7++  s   9A) )	A54A5c                    | j                   s|S | j                  dk(  r|3t        j                  t        j                  j                  d             y t        j                  t        j                  |            }t        j                  t        j                  j                  |j                               t        j                  t        j                  j                  |             |S t        j                  j                  d      j                         }|dk(  ry t        j                  |t        j                        }t        j                  j                  |      }t        j                  |      S )Nr   )dtype)r   r9  r&   evalr   r   arraypicklerF   r   r   zerosuint8ry   )r5   objdatar   s       r*   _share_objectzResponseGenerator._share_object  s    ##J::?{..q1288FLL-.DGGBNN**49956GGBNN**401J>>))!,113Dqy88D1D>>))$/D<<%%r,   c                     | j                   s|S ||dd  nd }| j                  |      }|y ||d   n	t               }|g|S r   )r   r\  r
   )r5   rQ  	shareablerqs       r*   rP  z ResponseGenerator._share_request  sY    ##N#*#6GABKD	&&y1	".WQZEG~I~r,   c                 v   |j                   dk(  r|j                  }|j                  }|j                  }|j                  rt        |       |r!|j                  st        j                  d       | j                  j                  j                  }|j                  r+|j                         }|j                  |j                         t        d|dd|} |j                   |fddi|}	nF|j#                  t%        ||            }	|	|	gdgdfS |j#                  |j&                        }	|	|	gdgdfS d}
|j(                  r)|j+                  |	      }|j-                  |	      }||kD  rd}
|d	   d
   dk7  r|	|	gdg|
fS g }g }d}d}|D ]  }|d
   dk(  r|dz  } n |dkD  r |j                   |d| dddgz   fddi|}t/        t1        ||	            D ]  \  }\  }}||k7  s|} n |dkD  r3|t3        |	      k  r%|j5                  |	d|        |j5                  d       t3        |	      }|j(                  r|j+                  |	|dz
        }|dk\  r|}||k  r%|j5                  |	||        |j5                  d       |t3        |	      k  r%|j5                  |	|d        |j5                  d       |s|	g}dg}|	|||
fS )aJ  Tokenize a request and split the prompt into segments.

        Returns a tuple

          * prompt - Full list of tokens
          * segments - A list of lists of tokens. Up to 3 segments that
            correspond to system prompt, context, thinking tail.
          * segment_types - A string per segment indicating if the segment is a
            system prompt or a user prompt or nothing special.
          * initial state - A string that contains the initial state of the
            state machine (normal or thinking depending on whether we have tail
            or not)
        r   zReceived tools but model does not support tool calling. If you think this is an error, file an issue here: https://github.com/ml-explore/mlx-lm/issuesT)r7   tokenizeadd_generation_promptri   normal	reasoningr   rl   rh   r   rg   r   Nrk   )rl   rm   F   )r=  ra   )r   rb   r7   rc   has_chat_templater   r   rS   rT   r1  r   chat_template_argsr   copyupdatedictapply_chat_templater  rt   rq   r   rfind_think_startrfind_think_end	enumerater(  rw   r   )r5   r   rQ  r   rb   r7   rc   rg  template_kwargsrq   initial_statethink_start	think_endsegmentssegment_types
num_systemsys_endm
sys_tokensr"  ab
tail_starts                          r*   	_tokenizezResponseGenerator._tokenize  s7    6)''HMME"//L**'1!;!;OOF &*%8%8%A%A%T%T",,);)@)@)B&&--d.G.GH"& #!# )#
 766*. & #)),x*NOx+@@%%gnn5FF8k]H<< !!!#55f=K!11&9IY& + B<6)F8k]MAA 
AyH$a
	 
 >666*%&R)H(II&+ "J
 's:v'>?	6Aq6G @ {wV4x 01$$X. [
!!#55fJQSO5TKa(
 ZOOF7:67  (F#OOF:;/0  -xH(MMx==r,   c                    |t        |      |f}| j                  j                  |      }||S i }i }g }	|j                  D ]+  }
|j	                  |
      ||
f<   |	j                  |
fdf       - |D ]6  }t        |j                  |d            }
|||
<   |	j                  |
df       8 t        |	      |d<   |j                  rh|j                  }|j                  }|d   j                  |df       |dfg|d<   |d   j                  |	       |j                  ||<   |j                  ||<   |j                  rn|j                  }|j                   }|d   j                  |df       |r|dfgng |d<   |d   j                  |	       |j"                  ||<   |r|j$                  ||<   t'        ||      }t)        | j                        dkD  r| j                  j+                          ||f| j                  |<   ||fS )	zMake a new SequenceStateMachine or fetch it if we 've made it before.

        Return also a dictionary that maps the token sequences in the state
        machine to their strings.
        NF)add_special_tokensrc  rd  tool)initiald   )r'  r6  rn   eos_token_idsr&  r   r  rW   r   think_start_tokensthink_end_tokensrX   rq  rr  r   tool_call_start_tokenstool_call_end_tokenstool_call_starttool_call_endr   rw   clear)r5   r   r   r   rp  	cache_keyrstransitionsr   common_stopsr   wtstesms                  r*   _make_state_machinez%ResponseGenerator._make_state_machiner  s    j 1=A	&&**95>I 	 ((A'==a@IqdO!t- ) Ai&&qU&CDAIaLD	*  !%\ 2H !!--B++B!(("k):;)+X'7K$$++L9%11IbM%//IbM %%11B//B!(("f668B>"2bK&&|4%55IbM ) 7 7	"!+}Et(()C/%%++-02I!!),9}r,   c                 N    | j                   j                  xr |j                  d u S rN   )r1  r   r   )r5   r   s     r*   _is_batchablezResponseGenerator._is_batchable  s"    ""//EDII4EEr,   c                    ) t        j                  t        j                               } j                  j	                          d }d }d }d }d }d}i }g )d )fd	}	 j
                  rt         j                  j                  t         j                  j                  d         j                  t         j                        j                         }
t         j                  j                  |
        j                  sd }|s|t        |      dkD  rd nd} |	|      }|S|\  }}}|#||j                   k(  r j#                  |      r	  j%                  |||      \  }}}} j+                   j                  j,                  |j.                  |      \  }} j1                           j2                  j5                  ||      \  }}t        |      t        |      z
  }|}|dkD  rR|t        |d         k\  r/|t        |j7                  d            z  }|j7                  d       n|d   |d  |d<   n|dkD  rRt9        |j:                  |j<                  |j>                  |||      }|j)                  |       |jA                  |g|jB                  g|g|d | gtE        ||      gtG        |      g|g      \  }|||jH                  |d d d   |jJ                  d	||<   ~ j                  jL                  jN                  K j                  jL                  jN                  }|jP                  } j2                  jS                  ||z
  
       ]|	  j                  jU                  |j                   j                   |j                   jV                  |j                   jX                        \  }} j#                  |      s j[                  |||f       |j                   }|} j                  j,                  }i }t]        | jL                  j^                   jL                  j`                   jL                  jb                  |      })je                  |||f       ld})je                  |||f       |t        |      dk(  r |rd }d }d }d }|jg                          d }d}g }  jh                  D ]  }!|jk                         \  }"}#|"s|#s nv|"D ]Z  }$||$jl                     }%|%d   j)                  |$jn                         |%d   jp                  s@| je                  |$jl                         \ |"D $cg c]8  }$|$jr                  r*|$jt                  s||$jl                     d   r|$jl                  : }&}$|jw                  |&      }'|'jy                         D ]R  \  }\  }}( j2                  j{                   j                  j,                  |(d d  |||   d   j7                                T ~'|#D ]T  }$||$jl                     }%|%d   j}                  |$j~                         |%d   j)                  t        |%d   j                  |$j~                  |$j                  |$j                  |$j                  |$j~                     j                         |$j                  t        |$j                  |%d   |                   |$j                  W|%d   j)                  d         j2                  j{                  ||$j                  d d  |$j2                  d       ||$jl                  = |%d   jp                  s:| je                  |$jl                         W   j                  |       } | r*|j                  |        | D ]  }|j7                  |d          j                  sy y # t&        $ r}|j)                  |       Y d }~d }~ww xY w# t&        $ r}|j)                  |       Y d }~d }~ww xY wc c}$w )NFc                 J    rj                         S j                  |       S rN   )rA   rR  )rM  r5   unprocessed_requestss    r*   get_next_requestz5ResponseGenerator._generate.<locals>.get_next_request  s&    #+//11))'22r,   r   g?rL  )r   r   r6   r   rq   r   )rs  r   caches
all_tokenssamplerslogits_processorsstate_machinesr   )r   rqueuedetokenizerrt  r   )rC  )completion_batch_sizeprefill_batch_sizeprefill_step_sizestreamTr  r   rt  )rH  r  r   ri   rN   )Jr&   default_streamdefault_devicer1  r  r   r   r   randomr   viewuint64r   r   r:  rw   r   r  r|  	Exceptionputr  r   r   rJ  r2  fetch_nearest_cacherA   r   r   r   r6   insert_segmentsr   r  r  r  r   r   prompt_cache_bytesprompt_cache_nbytestrim_tor!   r   r   _serve_singler   decode_concurrencyprompt_concurrencyr  r   closer7  nextuidprogressr   end_of_segmentend_of_promptextract_cacherG  insert_cache	add_tokenr   r   last_segmentcurrent_statematch_sequencer   r   r.  r  r\  remove)*r5   generation_streamcurrent_modelcurrent_samplingcurrent_tokenizercurrent_model_keybatch_generatordrain_batchbatch_resultsr  r   rQ  rM  r  r   rq   rs  rt  rp  r]   r   r  r   cacherestr   Nr   r  totalactiver   uids_to_remover   prompt_responsesgen_responsesrrZ   eos_idsr  r  r  s*   `                                        @r*   r;  zResponseGenerator._generate  s    --b.?.?.AB 	((*  !	3 >>))"))//!*<=BB299MRRTDIINN4 **G (3M8JQ8N  
 +7; "(/% $/%3**40!IM-wJF- %)$<$<++55!%	%MB	 ))+"&"3"3"G"G)6#KE4 *-Vs4y)@&*Aa%HQK 00X\\!_!55A)--a0*21+ab/HQK! a% ,)2)C)C%.%;%;$-$9$9"+%+=C JJsO,<<"*$(OO#4 %w$*+>,>$?#@"/i"@!A+B4+H*I(*t = FS  #"('0'<'<)6tt)<(,(9(9*M#& **33FFR $ 3 3 < < O O!0!D!D))11%&.1I
 %,!+/+>+>+C+C JJ,,djj.@.@$**BRBR,(y  --d3**FGT+BC $(JJM(1%(,(;(;(E(E%$&M&4.2mm.N.N+/==+K+K*.--*I*I0'O )//$0GH
 #'K(//$0GH !,}%*"(,+/(,0),0)'--/*.&+!#**A6E6J6J6L3$m+M .!.quu!5x(,,QZZ8!%=55*11!%%8	 . "2!1A++ !)!%%0A	 !1   -::7CF39<<>//eY))66 //99%aL!'4S'9/'J'N'N'P	 7  4B *!.quu!5}-77@x(,,$ &} 5 B B ! ! ! 0 0 !

177 3 8 8 : ! 4$%JJ$*>$:$5!"  ??6"8,006 --:: 1 !Q !+6	 ;  !.aee 4!%=55*11!%%8= += +| "&!3!3N!C!#**>:- &))#t4  .Q **0 % !

1 !H % !

1 !ps7   ^ A^7 /=_	^4^//^47	_ __c                     |\  }}fd}	 | j                   j                  }| j                   j                  }| j                   j                  }| j	                  |||      \  }}}}	| j                  | j                   j                  ||j                  |	      \  }
}|
j                         }t        |j                  |j                  |j                  ||      }j                  |       |j                  )t        j                   j                  |j                         t#        ||      }t%        |      }| j'                          | j(                  j+                  | j                   j                  |      \  }}t-        |      t-        |      z
  |_        |d d  }|Wt1        | j                   j                        }| j                   j                  "|t1        | j                   j                        z  }t3        ||||j4                  |||||j6                  || j8                  j:                        D ]  }|j<                  }|
j?                  ||j@                        \  }}}||d}j                  tC        |jD                  |j@                  |||jF                  |j@                     jI                         |tK        |jF                  |jL                  |                   |jO                  |j@                         |jP                  r| jR                  r
tU                n| n j                  d        | j(                  jW                  | j                   j                  ||       y # tX        $ r}j                  |       Y d }~y d }~ww xY w)Nc                 ,    j                  | |f       y rN   )r  )tokens_processedtokens_totalr  s     r*   r  z1ResponseGenerator._serve_single.<locals>.progress  s    JJ(,78r,   )rp  )r   r   r6   r   rq   )r   r   rq   r   samplerr  r2  r   r   prompt_progress_callbackr  rj   )-r1  r   r   r   r|  r  r   r   
make_stater   r   r   r6   r  r   r&   r  r  r  rJ  r2  r  rw   r   r   r   r   r   r   r  r   r   r   r   rv   r   r   r.  r   r   r   r   NotImplementedErrorr  r  )r5   rQ  r   r  r   r   r   rq   r   rp  r  r   sm_stater   r  r  r  r  r  genr   r  r  r]   r  s                           @r*   r  zResponseGenerator._serve_single  s/    '	9_	''--E++55I--99K +/..GT*R'FAq- 44##--+	 5 MB	 }}H $&33!*!;!;%11#C JJsO yy$		tyy) $D)4G 7 = !!#++??##--vKE4 &)[3t9%<C"q	I})$*=*=*C*CD&&22>.t/B/B/N/NOOE '#??"3"'!%!6!6)1"&--"A"A !$ 1 1:<((8SYY:W7.-!--2G$*M

		%&SYY/446%,LL$*;*;Y
   +##++133 ,MP JJt **##--y%  	JJqMM	s   LM) #AM) )	N2NNrQ  generation_argsprogress_callbackc                     t               | j                  j                  ||f       fd}j                         }t	        |t
              r||t        | |             fS )Nc               3      K   	 j                         } | y t        | t              r| t        | t              r |   >|  BwrN   )rn   rV   r  r'  )responser  response_queues    r*   _innerz*ResponseGenerator.generate.<locals>._inner  sU     )--/#h	2"Nh.(4)84 s   AA)r
   r5  r  rn   rV   r  r   )r5   rQ  r  r  r  r   r  s      `  @r*   generatezResponseGenerator.generate  s^     >7ODE	   "c9%I+C:::r,   c                 .    | j                   j                  S rN   )r1  r   r   s    r*   r   zResponseGenerator.cli_args  s    ""+++r,   rN   )rc  )rU   r_   r`   r   r   r9   r?  rx   rJ  rR  r\  rP  r|  r  r  r;  r  r   r   r   r   r   r  propertyr   ra   r,   r*   r0  r0    s    (} (N (''	
,&(
l>^ ?G9vFh5TfX CG	;"; -; $Hc3Z-=$>?	;8 , ,r,   r0  c                       e Zd Zdddedee   f fdZd Zd$defdZ	d$defd	Z
d
 Zd Z	 	 	 	 d%dZd Z	 	 	 	 	 	 	 	 d&dedeed   df   dee   dee   dee   deee      deeeeeef            deee      deee      dee   defdZdedee   fdZ	 	 	 d'dee   dee   dee   fdZdefdZdefd Zd! Zd" Zd# Z xZ S )(
APIHandlerN)system_fingerprintresponse_generatorr  c                    t        t        j                               | _        || _        |xs
 t	               | _        t        |   |i | y)z9
        Create static request specific metadata
        N)r   r   createdr  r+   r  superr9   )r5   r  r  r   kwargs	__class__s        r*   r9   zAPIHandler.__init__$  sB     499;'"4"4"P8N8P$)&)r,   c                 @   | j                   j                  j                  }| j                  j	                  d      }d|v r| j                  dd       n(||v r$| j                  d|       | j                  dd       | j                  dd       | j                  dd       y )NOrigin*zAccess-Control-Allow-OriginVaryzAccess-Control-Allow-MethodszAccess-Control-Allow-Headers)r  r   allowed_originsheadersrn   send_header)r5   r  origins      r*   _set_cors_headerszAPIHandler._set_cors_headers3  s    11::JJ!!(+/!:C@&:FCVX.7=7=r,   status_codec                 j    | j                  |       | j                  dd       | j                          y )NContent-typezapplication/jsonsend_responser  r  r5   r  s     r*   _set_completion_headersz"APIHandler._set_completion_headers>  s-    ;');< r,   c                     | j                  |       | j                  dd       | j                  dd       | j                          y )Nr  ztext/event-streamzCache-Controlzno-cacher  r  s     r*   _set_stream_headerszAPIHandler._set_stream_headersC  s=    ;')<=*5 r,   c                 F    | j                  d       | j                          y )N   )r  end_headersr   s    r*   
do_OPTIONSzAPIHandler.do_OPTIONSI  s    $$S)r,   c                    | j                   | j                  | j                  d}| j                  |vr=| j                  d       | j	                          | j
                  j                  d       y| j                  j                  d      }|`| j                  d       | j	                          | j
                  j                  t        j                  ddi      j                                y	 t        |      }| j                  j                  |      }	 t        j                   |j#                               | _        t)        j,                         j/                  t(        j0                        r9t        j                  | j$                  d      }t)        j2                  d|        t5        | j$                  t6              st        j                  | j$                  d      }t)        j*                  d|        | j                  d	       | j	                          | j
                  j                  t        j                  ddi      j                                y| j$                  j                  dd      | _        | j$                  j                  dd      | _        | j$                  j                  dd      | _        | j$                  j                  dd      | _        | j$                  j                  d| j@                  jB                  jD                        | _"        | j$                  j                  dd      | _#        | j$                  j                  dd      | _$        | jH                  ?| j$                  j                  d| j@                  jB                  jH                        | _$        | j$                  j                  d| j@                  jB                  jJ                        | _&        | j$                  j                  d| j@                  jB                  jN                        | _'        | j$                  j                  d| j@                  jB                  jP                        | _(        | j$                  j                  d | j@                  jB                  jR                        | _)        | j$                  j                  d!d"      | _*        | j$                  j                  d#d$      | _+        | j$                  j                  d%d"      | _,        | j$                  j                  d&d$      | _-        | j$                  j                  d'd"      | _.        | j$                  j                  d(d$      | _/        | j$                  j                  d)d"      | _0        | j$                  j                  d*d"      | _1        | j$                  j                  d+d      | _2        | j$                  j                  d,d      | _3        | j$                  j                  d-d.      | _4        | j$                  j                  d/d      | _5        | j$                  j                  d0      | _6        | jo                          | j$                  j                  d1      }|xs g }t5        |tp              r|gn|} || j                            }| js                  ||       y# t        $ rb | j                  d	       | j	                          | j
                  j                  t        j                  dd
i      j                                Y yw xY w# t        j&                  $ r}t)        j*                  d| d|j#                                 | j                  d	       | j	                          | j
                  j                  t        j                  dd| i      j                                Y d}~yd}~ww xY w)2z:
        Respond to a POST request from a client.
        )z/v1/completionsz/v1/chat/completionsz/chat/completions  	   Not FoundNContent-Lengthi  errorz!Content-Length header is requiredi  zInvalid Content-Length headerzJSONDecodeError: z - Raw body: zInvalid JSON in request body: 	indentzIncoming Request Body: zInvalid Request Body: z#Request should be a JSON dictionaryr  Fstream_optionsr   r   r   r   adaptersmax_completion_tokensr   r   r   r   r   r           r      r   r   r   r   r   r   r   r   r   r   r   r   rj   ):handle_text_completionshandle_chat_completionspathr  r  wfilewriter  rn   rE   rF   r  r   rQ   rfilereadry   decodebodyrR   rS   r	  	getLoggerisEnabledForDEBUGdebugrV   rj  r  r  requested_modelrequested_draft_modelr  r   r   r   r   tempr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   validate_model_parametersrB   handle_completion)r5   request_factoriescontent_lengthraw_bodyr]   
debug_bodyr   rQ  s           r*   do_POSTzAPIHandler.do_POSTM  s   
  $;;$($@$@!%!=!=
 99--((-JJ\* ))*:;!((-JJ

G%HIJQQS 	 0N ::??>2		

8??#45DI ++GMM:DIId;JMM3J<@A$))T*DIId;JMM2:,?@((-JJ

G%JKLSSU  iimmHe4"iimm,<dC#yy}}WoF%)YY]]=/%R" $		 7 7 @ @ Q Q!
 yy}}Z6))--(?F??""iimmd55>>IIDO  99==422;;@@
 YY]]7D,C,C,L,L,R,RS
YY]]7D,C,C,L,L,R,RS
YY]]7D,C,C,L,L,R,RS
"&))--0Dc"J'+yy}}5NPR'S$ $		.@# F%)YY]]3JB%O"!%/BC!H&*iimm4Lb&Q##yy}}->D!YY]]?C@))--d;		j%8 IIMM."=IIMM&$/	$(IIMM2H$I!&&( YY]]6*
%2
%/
C%@j\j
 /#DII.0w
3U  	((-JJ

G%DEFMMO 	 ## 	MM-aShoo>O=PQR((-JJ

G'EaS%IJKRRT 	s,   .Z? (\- ?A(\*)\*-_ B__c                 &   t        | |      }|r|y t        ||      s#	 t        d |D              }t        | d|       |||v ry |||k  rt        | d|       |||kD  rt        | d|       y y # t        $ r |j                  }Y ^w xY w)Nc              3   4   K   | ]  }|j                     y wrN   )rU   )rO   ets     r*   rP   z'APIHandler._validate.<locals>.<genexpr>  s     Dmmr	  z must be of type z must be at least z must be at most )getattrrV   r'  	TypeErrorrU   rQ   )	r5   nameexpected_typemin_valmax_valoptional	whitelistvaluealloweds	            r*   	_validatezAPIHandler._validate  s     d#%/1DmDD v%6wi@AA Ui%757?v%7yABB57?v%6wi@AA $3  1'001s   A8 8BBc                    | j                  dt               | j                  dt        d       | j                  dt        t        fd       | j                  dt        t        fdd       | j                  d	t        d       | j                  d
t        t        fdd       | j                  dt        d       | j                  dt        t        fd       | j                  dt        d       | j                  dt        t        f       | j                  dt        d       | j                  dt        t        f       | j                  dt        d       | j                  dt               | j                  dt        dddg       | j                  dt        dd       | j                  dt        dd       | j                  dt               | j                  dt        d       | j                  dt        d       | j                  dt
        d       | j                  G	 | j                  j                         D ci c]  \  }}t        |      t        |       c}}| _        yyc c}}w # t        $ r t        d       w xY w)!zHValidate that the passed model parameters have correct types and values.r  r   r   )r0  r   r   r   )r0  r1  r   r   r   r   r   r   r   r   r   r   r   re  r   )r0  r1  r3  r   r   r  r   T)r2  r   r   Nz)logit_bias must be a dict of int to float)	r6  r   r   r   rB   rj  r   rG  rQ   )r5   kvs      r*   r"  z$APIHandler.validate_model_parameters  s   x&|S!4}uclA>waCwQ/waC)3:+eS\1E0#qA)E3<8.Q?*UCL9/a@z4(~sArbTR(%AFq!D(#.y#5vsT2|TD9??&N@D@U@U@W"X@W13q658#3@W"X '"X N !LMMNs   'I4 I.#	I4 .I4 4J	rv   r   )lengthrj   prompt_token_countcompletion_token_countr   token_logprobsr   tokensrY   reasoning_textr  c           	      Z   |xs g }|xs g }|	xs g }	| j                   | j                  | j                  | j                  | j                  d|dgd}|r.d|D cg c]  }|rt        |d   |      ni  c}i|d   d   d<   n7|r5dt        ||      D cg c]  \  }}t        ||       c}}i|d   d   d<   | j                  sHt        |t              rt        |t              st        d	      ||||z   d
|d<   ||dk\  r
d|i|d   d<   |d   d   }| j                  j                  d      r7| j                  rdnd}ddi||<   |r|||   d<   |
r|
||   d<   |	r|	||   d<   |S | j                  dk(  r|j                  |       |S t        d| j                         c c}w c c}}w )a  
        Generate a single response packet based on response type (stream or
        not), completion type and parameters.

        Args:
            text (str): Text generated by model
            finish_reason (Union[Literal["length", "stop"], None]): The reason the
              response is being sent: "length", "stop" or `None`.
            prompt_token_count (Optional[int]): The number of tokens in the prompt,
              used to populate the "usage" field (not used when stream).
            completion_token_count (Optional[int]): The number of tokens in the
              response, used to populate the "usage" field (not used when stream).
            prompt_cache_count (Optional[int]): The portion of prompt_token_count
              that was found in the cache when servicing the request.
            token_logprobs (Optional[List[float]]): The log probabilities per token,
              in token order.
            top_tokens (Optional[List[Tuple[Dict[str, Any]]]]): List of outputs from
              _format_top_logprobs, giving info on the top N tokens at each token position.
            tokens (Optional[List[int]]): List of tokens to return with logprobs structure
            tool_calls (Optional[List[str]]): List of tool calls.
            reasoning_text (Optional[str]): The reasoning text generated by the model.

        Returns:
            dict: A dictionary containing the response, in the same format as
              OpenAI's API.
        r   )r@   r   )r;   r  objectr   r  choicesrm   )r   rB  r   )r;   r   z8Response type is complete, but token counts not providedprompt_tokenscompletion_tokenstotal_tokensusagecached_tokensprompt_tokens_detailschat.completiondeltarz   rl   ri   rd  rY   text_completionr   zUnsupported response type: )
request_idr  object_typer  r  rj  r(  r  rV   r   rQ   
startswithri  )r5   rv   r   r;  r<  r   r=  r   r>  rY   r?  r   r  r"  r#  choicekey_names                    r*   generate_responsezAPIHandler.generate_response  s8   N (-2!'R%2
 //"&"9"9&&))|| %2
 EQEQ!D1A.;\2HY":.
 69&.6Q6QdaDAq)6Q2HY":. {{-s35s; N 
 "4%; 25K K!HW
 "-2D2I#%7>!"9: )$Q' &&'89"&++w9H &4F8.2x +0>x -1;x .  !22MMtM$  :4;K;K:LMNN[s   F"F'rQ  r   c                 N    t        t         j                   j                   j                        t         j                   j                   j                   j                   j                   j                        t         j                   j                   j                   j                    j"                   j$                   j&                        | j(                   j*                   j,                   j.                   j0                   j2                  
      } fd}	  j4                  j7                  |||      \  }} jJ                  r7 jM                  d
        j=                          tO        jP                  d       n& j;                  d
       tO        jP                  d       tS        |jT                  |jV                   jJ                        }d	}	d}
d}d}d}g }d}g }g }g }	 |D ]  }tO        jP                  |jX                         |jZ                  dk(  r||jX                  z  }nW|jZ                  dk(  r||jX                  z  }n8|jZ                  dk(  r)|	dk(  r|j]                  |       d}d}||jX                  z  }|j]                  |j^                         |j,                  r|j]                  |j`                         |j.                  dkD  r|j]                  |jb                          jJ                  r|jZ                  dk7  r|s|s|r{ je                  |d	 ||      |      } j>                  jA                  dtC        jD                  |       djI                                 j>                  jg                          d}d}g }|jh                  |jh                  }
|jZ                  }	 |	dk(  r|r|j]                  |       d}|
dk(  r|rd}
 jJ                  rh je                  ||
 ||      |      } j>                  jA                  dtC        jD                  |       djI                                 j>                  jg                           jj                   jj                  d   r jm                  to        |jp                        to        |      |jr                        } j>                  jA                  dtC        jD                  |       djI                                 j>                  jg                           j>                  jA                  djI                                 j>                  jg                          n3 je                  ||
to        |jp                        to        |      |jr                  |||| ||      
      }tO        jt                         jw                  tN        jx                        r/tC        jD                  |d      }tO        jP                  d|        tC        jD                  |      jI                         } j{                  dtG        to        |                    j=                           j>                  jA                  |        j>                  jg                          |j}                          y	# t8        $ rr} j;                  d        j=                           j>                  jA                  tC        jD                  dtG        |      i      jI                                Y d	}~y	d	}~ww xY w# |j}                          w xY w) z
        Generate a response to a prompt and send it to the client in a single batch.

        Args:
            prompt (List[int]): The tokenized prompt.
            stop_words (List[str]): A list of stop words
        )r   r   r   )r   r   r   r   r   r   )r   r   r   r   r   r   r   )
r   r   r   r   r   r   r   r   r   r   c                     t        j                  d|  d|        j                  rMd|  d| dj                         }j                  j                  |       j                  j                          y y )NzPrompt processing progress: /z: keepalive 

)rS   rE  r  r  r  r  flush)	processedr  msgr5   s      r*   keepalive_callbackz8APIHandler.handle_completion.<locals>.keepalive_callback  sh    LL7	{!E7KL{{$YKqt<CCE

  %

  " r,   )r  r  r	  N   zStarting stream:zStarting completion:rj   rk   Frd  r  rc  Tr   )rY   r?  zdata: rV  rY   include_usagezdata: [DONE]

)r=  r   r>  r?  rY   r
  r  zOutgoing Response: r  )?r   r   r  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  rE   rF   rB   r  r  r   rS   r  r.   r6   r7   rv   r   r   r   r   r   rR  rW  r   r  completion_usage_responserw   rq   r   r  r  r  r  rj   )r5   rQ  r   r   rZ  r   r  r]   tool_formatter
prev_stater   r?  made_tool_callr[   rY   rv   r>  r=  r   r  respresponse_debugresponse_jsons   `                      r*   r#  zAPIHandler.handle_completionX  s    #"**00
 ' ,,jjjjjj $ 4 4"00 ,??#'#:#:(,(D(D!%!6!6&*&@&@"&"8"8'+'B'B "!22]]**!%!:!:;
D	#
	 33<<"4 = MC ;;$$S)MM,-((-MM01 +3??GMM4;;W 
	

b	chh' 99+"chh.NYY&()IYY(*!V+")))4$&	)-CHH$D cii(=="))#++6$$q(%%cnn5 KK		V+~11#1*#='5	 2 D JJ$$vdjj.>-?t%D%K%K%MNJJ$$&%'ND!#J$$0$'$5$5M YY
W  Z V#	!!),!%&> ,{{--!-j9#1	 .  

  6$**T*:);4!@!G!G!IJ

  "''3++O<99CJJF..D
 JJ$$vdjj.>-?t%D%K%K%MNJJ$$&

  !3!:!:!<=

  "--!

OK**#1)!#1-j9 .  $$&33GMMB%)ZZT%BNMM$77G"HI $

4 0 7 7 9  !13s=7I3JK  "

  /

  "HHJA  	((-JJTZZ#a&(9:AACD		@ HHJs&   $!Z .R\ 	\A(\

\\$c           
          | j                   | j                  d| j                  | j                  g ||||z   dd}||dk\  r
d|i|d   d<   |S )NrJ  rC  )r;   r  rA  r   r  rB  rG  r   rH  rG  rI  )rM  r  r  r  )r5   r;  r<  r   r  s        r*   r]  z$APIHandler.completion_usage_response  st     //"&"9"9'))||!3%; 25K K
 ).@A.E!3:HW56 r,   c           	          | j                   }d|v sJ d       dt        j                          | _        | j                  rdnd| _        t        dd|d   |j                  d      xs d	|j                  d
            S )z
        Handle a chat completion request.

        Returns:
            mx.array: A mx.array of the tokenized prompt from the request body
        rb   z Request did not contain messagesz	chatcmpl-zchat.completion.chunkrJ  r   rk   r7   Nrc   )r  rC   rD   rM  r  rN  r   rn   )r5   r  s     r*   r  z"APIHandler.handle_chat_completions+  s     yyT!E#EE! &djjl^46:kk2GX HHW%HH^$
 	
r,   c                     dt        j                          | _        d| _        d| j                  v sJ d       t        d| j                  d   g dd      S )z
        Handle a text completion request.

        Returns:
            mx.array: A mx.array of the tokenized prompt from the request body
        zcmpl-rL  rq   z Request did not contain a promptrv   N)rC   rD   rM  rN  r  r   r   s    r*   r  z"APIHandler.handle_text_completionsA  s]     "$**,0,499$H&HH$ IIh
 	
r,   c                    | j                   j                  d      r| j                          y| j                   dk(  r| j                          y| j	                  d       | j                          | j                  j                  d       y)z9
        Respond to a GET request from a client.
        z
/v1/modelsz/healthr  r  N)r  rO  handle_models_requesthandle_health_checkr  r  r  r  r   s    r*   do_GETzAPIHandler.do_GETT  sf     99-&&(YY)#$$&((-JJ\*r,   c                     | j                  d       | j                          | j                  j                  dj	                                | j                  j                          y)z@
        Handle a GET request for the /health endpoint.
        r[  z{"status": "ok"}N)r  r  r  r  r  rW  r   s    r*   ri  zAPIHandler.handle_health_checka  sJ     	$$S)

+2245

r,   c                 r   | j                  d       | j                          g d| j                  j                  d      }dt	        |      dkD  rdj                  |dd       fd}t               }|j                  D cg c]  } ||      s| }}|D cg c]  }|j                  d| j                  d }}| j                  j                  j                  rqt        | j                  j                  j                        }|j                         r8t        |j!                               }|j#                  |d| j                  d       d	|d
}	t%        j&                  |	      j)                         }
| j*                  j-                  |
       | j*                  j/                          yc c}w c c}w )zC
        Handle a GET request for the /v1/models endpoint.
        r[  )zconfig.jsonzmodel.safetensors.index.jsonztokenizer_config.jsonrU  N   c                    | j                   dk7  ryd| j                  vry| j                  k7  ry| j                  d   j                  D ch c]  }|j                  j
                   c}t        fdD              S c c}w )Nr   Fmainc              3   &   K   | ]  }|v  
 y wrN   ra   )rO   f
file_namess     r*   rP   zLAPIHandler.handle_models_request.<locals>.probably_mlx_lm.<locals>.<genexpr>  s     61qJs   )	repo_typerefsrepo_idfiles	file_pathr.  r  )reporq  rr  rv  filter_repo_ids     @r*   probably_mlx_lmz9APIHandler.handle_models_request.<locals>.probably_mlx_lmy  s{    ~~(TYY&)dlln.L48IIf4E4K4KL4Kq!++**4KLJ6666 Ms   Br   )r;   rA  r  rW   )rA  r[  )r  r  r  splitrw   rx   r   reposru  r  r  r   r   r   existsrB   resolver   rE   rF   r  r  r  rW  )r5   partsrz  hf_cache_inforx  downloaded_modelsmodelsr  model_idr  rc  rv  ry  s              @@r*   rh  z APIHandler.handle_models_requestk  s    	$$S)X		$u:> XXeABi0N	7 '(*00
0TOD4ID0 	 
 *
 *	 ll!<<
 * 	 
 ""++11d55>>DDEJ  "z1134&")#'<< %f5

8,335

'

=


s   F/F/!F4)r[  )NNFN)NNNNNNNN)NNN)!rU   r_   r`   r0  r   rB   r9   r  r   r  r   r  r(  r6  r"  r   r   r   r   r   r   r   rj  rR  r   r#  r]  r  r  rj  ri  rh  __classcell__)r  s   @r*   r  r  #  s   
 -1	*-* %SM	*	>!3 !
!s !e4V B2ND -104,004<@&**.(,kk W%56<=k %SM	k
 !)k %SMk !e-k T%S#X"789k c#k T#Y'k !k 
kZx): xS	 xx -104,0	$SM !) %SM	2
): 
,
): 
&+8r,   r  hostportc                    | |f}t        j                  |t         j                  t         j                  d}t	        t        |            \  |_        }}}} ||fd      }t        j                  d       t        j                  d|  d| d       	 |j                          y # t        $ r# |j                          j                          Y y w xY w)N)r?   flagsc                  0     g| dt               i|S )Nr  )r+   )r   r  handler_classr  s     r*   <lambda>z"_run_http_server.<locals>.<lambda>  s,    !
 !
57!
 	!
r,   z\mlx_lm.server is not recommended for production as it only implements basic security checks.zStarting httpd at z	 on port z...)socketgetaddrinfoSOCK_STREAM
AI_PASSIVEr  iteraddress_familywarningswarnrS   rE  serve_foreverKeyboardInterruptshutdownr?  )	r  r  r  server_classr  server_addressinfosr   httpds	     ` `    r*   _run_http_serverr    s     D\N	f008I8IE <@U;L8LAq.	
E MM	4 LL%dV9TF#>?+ +((*+s   B( ()CCr1  c                     t         j                  j                         }t        |j                  j
                        }t        ||      }|j                         dk(  rt        | ||       y |j                          y r0   )
r&   r   r   r   r   prompt_cache_sizer0  r8  r  rx   )r  r  r1  r  r  r  r2  r  s           r*   runr    s`     NN!E!."9"9"K"KLL*><Hzz|qt%78!r,   c                  $   t        j                  d      } | j                  dt        d       | j                  dt        d       | j                  dt        d	d
       | j                  dt        dd       | j                  dd dd       | j                  dt        dd        | j                  dt        dd       | j                  ddd       | j                  dt        dg dd !       | j                  d"t        d#d$d%&       | j                  d'dd(       | j                  d)t
        d*d+       | j                  d,t
        d-d.       | j                  d/t        d0d1       | j                  d2t
        d*d3       | j                  d4t        d5d6       | j                  d7t        j                  d8d9       | j                  d:t        d;d<       | j                  d=t        d>d?       | j                  d@t        dAdB       | j                  dCt        dDdE       | j                  dFt        dG       | j                  dHddI       | j                         }t        j                  j                         r,t        j                         dJ   }t        j                  |       t        j                   t#        t        |j$                  j'                         d       dKL       t)        |j*                  |j,                  t/        |             y )MNzMLX Http Server.)descriptionz--modelz8The path to the MLX model weights, tokenizer, and config)r?   helpz--adapter-pathz9Optional path for the trained adapter weights and config.z--hostz	127.0.0.1z-Host for the HTTP server (default: 127.0.0.1))r?   defaultr  z--porti  z(Port for the HTTP server (default: 8080)z--allowed-originsc                 $    | j                  d      S )N,)r{  )xs    r*   r  zmain.<locals>.<lambda>  s    qwws|r,   r  zAllowed origins (default: *)z--draft-modelz,A model to be used for speculative decoding.)r?   r  r  z--num-draft-tokensz:Number of tokens to draft when using speculative decoding.rm  z--trust-remote-code
store_truez)Enable trusting remote code for tokenizer)actionr  z--log-levelINFO)r  r  WARNINGERRORCRITICALz%Set the logging level (default: INFO))r?   r  rB  r  z--chat-templaterk   z)Specify a chat template for the tokenizerF)r?   r  r  requiredz--use-default-chat-templatezUse the default chat templatez--tempr  z+Default sampling temperature (default: 0.0)z--top-pg      ?z-Default nucleus sampling top-p (default: 1.0)z--top-kr   z3Default top-k sampling (default: 0, disables top-k)z--min-pz5Default min-p sampling (default: 0.0, disables min-p)z--max-tokensi   z;Default maximum number of tokens to generate (default: 512)z--chat-template-argsznA JSON formatted string of arguments for the tokenizer's apply_chat_template, e.g. '{"enable_thinking":false}'z{}z--decode-concurrency    zFWhen a request is batchable then decode that many requests in parallelz--prompt-concurrency   zFWhen a request is batchable then process that many prompts in parallelz--prefill-step-sizei   z0Step size for prefill processing (default: 2048)z--prompt-cache-sizer   z@Maximum number of distinct KV caches to hold in the prompt cachez--prompt-cache-bytesz&Maximum size in bytes of the KV cachesz
--pipelinez,Use pipelining instead of tensor parallelism max_recommended_working_set_sizez)%(asctime)s - %(levelname)s - %(message)s)levelformat)r  ArgumentParseradd_argumentrB   r   r   rE   ry   r    
parse_argsr&   metalis_availabler'   set_wired_limitrS   basicConfigr,  	log_levelupperr  r  r  r   )parserr   wired_limits      r*   ro  ro    s   $$1CDF
G  
 H  
 <	   7	   #+	   ;	   I	   8  
 A4   8   %,  
 :	   <	   B	   D	   J	   ZZ B	   U	   U	   ?	   O	   5  
 ;  
 D	xxnn&'IJ
;'gt~~335t<: 		499mD12r,   __main__z}Calling `python -m mlx_lm.server...` directly is deprecated. Use `mlx_lm.server...` or `python -m mlx_lm server ...` instead.rN   )Tr  rE   rS   rW  r(   r  r   rC   r  collectionsr   dataclassesr   r   http.serverr   r   pathlibr   queuer	   rO  r
   	threadingr   typingr   r   r   r   r   r   r   r   r   r   mlx.corecorer&   huggingface_hubr   _versionr   r  r   r   r   models.cacher   r   sample_utilsr   r   utilsr    r!   r"   r+   r.   rj  rt   r   r   r   r   r   r   r   r   r   r   r   r  r  rB   r.  r0  r  r   r  r  ro  rU   printra   r,   r*   <module>r     s             * C  %      * ! 
 ? 2 2N
% %P4: Xd^ 0 =F             3 3 3 + + + ! ! !  & & &( % %Ph* h*V	d38n8M h	, h	,V@
' @
N %+
+
+J %"
"
" "" Q3h z		L 	F r,   