
    (HJjQ                     f   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
 d dlmZmZ  G d d      Z G d d	e      Z G d
 de      Z G d de      Zd Z G d d      Z G d de      Z ej*                  de       d Zd Zd Zd Zd Z	 	 d de
eeef      defdZde	dedede	fdZy)!    N)partial)JSONDecodeError)AnyDictListOptional)AutoTokenizerPreTrainedTokenizerFastc                   6    e Zd ZdZdZd Zd Zd Zed        Z	y)StreamingDetokenizerab  The streaming detokenizer interface so that we can detokenize one token at a time.

    Example usage is as follows:

        detokenizer = ...

        # Reset the tokenizer state
        detokenizer.reset()

        for token in generate(...):
            detokenizer.add_token(token.item())

            # Contains the whole text so far. Some tokens may not be included
            # since it contains whole words usually.
            detokenizer.text

            # Contains the printable segment (usually a word) since the last
            # time it was accessed
            detokenizer.last_segment

            # Contains all the tokens added so far
            detokenizer.tokens

        # Make sure that we detokenize any remaining tokens
        detokenizer.finalize()

        # Now detokenizer.text should match tokenizer.decode(detokenizer.tokens)
    )texttokensoffsetc                     t               NNotImplementedErrorselfs    `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/tokenizer_utils.pyresetzStreamingDetokenizer.reset+       !##    c                     t               r   r   r   tokens     r   	add_tokenzStreamingDetokenizer.add_token.   r   r   c                     t               r   r   r   s    r   finalizezStreamingDetokenizer.finalize1   r   r   c                 \    | j                   }|| j                  d }t        |      | _        |S )zTReturn the last segment of readable text since last time this property was accessed.N)r   r   len)r   r   segments      r   last_segmentz!StreamingDetokenizer.last_segment4   s-     yyt{{}%$ir   N)
__name__
__module____qualname____doc__	__slots__r   r   r   propertyr#    r   r   r   r      s1    : -I$$$  r   r   c                   8    e Zd ZdZd Zd Zd Zd Zed        Z	y)NaiveStreamingDetokenizera  NaiveStreamingDetokenizer relies on the underlying tokenizer
    implementation and should work with every tokenizer.

    Its complexity is O(T^2) where T is the longest line since it will
    repeatedly detokenize the same tokens until a new line is generated.
    c                 j    || _         | j                   j                  dg       | j                          y )Nr   )
_tokenizerdecoder   )r   	tokenizers     r   __init__z"NaiveStreamingDetokenizer.__init__E   s&    #s#

r   c                 J    d| _         g | _        d| _        g | _        d| _        y Nr    )r   r   _text_current_tokens_current_textr   s    r   r   zNaiveStreamingDetokenizer.resetJ   s'    
!r   c                 p    | j                   j                  |       | j                  j                  |       y r   )r6   appendr   r   s     r   r   z#NaiveStreamingDetokenizer.add_tokenQ   s(    ##E*5!r   c                     | xj                   | j                  j                  | j                        z  c_         g | _        d| _        y )Nr4   )r5   r.   r/   r6   r7   r   s    r   r   z"NaiveStreamingDetokenizer.finalizeU   s5    

doo,,T-A-ABB
!r   c                 :   | j                   r| j                  j                  | j                         | _        | j                  j	                  d      s@| j                  j
                  r>t        | j                        dkD  r&| j                  d   dk(  r| j                  d d | _        | j                  rR| j                  d   dk(  r@| xj                  | j                  z  c_        | j                   j                          d| _        | j                  | j                  z   S )N   �r    
r4   )	r6   r.   r/   r7   endswithclean_up_tokenization_spacesr!   r5   clearr   s    r   r   zNaiveStreamingDetokenizer.textZ   s    !%!7!78L8L!MD!!**84<<**+a/&&r*c1%)%7%7%<"$"4"4R"8D"@JJ$,,,J  &&(!#DzzD....r   N)
r$   r%   r&   r'   r1   r   r   r   r)   r   r*   r   r   r,   r,   =   s/    
 " 
 / /r   r,   c                   2    e Zd ZdZddZd Zd	dZd Zd Zy)
SPMStreamingDetokenizerzA streaming detokenizer for SPM models.

    It adds tokens to the text if the next token starts with the special SPM
    underscore which results in linear complexity.
    c                    || _         dj                         | _        dgt        |j                  j                               dz   z  | _        |j                  j                         D ]Z  \  }}|j                  d      r't        t        |dd d      g      | j                  |<   >|j                         | j                  |<   \ | j                          y )N   ▁r4      z<0x         )
trim_spaceencode_sepmaxvocabvaluestokenmapitems
startswithbytesintr   )r   r0   rK   valuetokenids        r   r1   z SPMStreamingDetokenizer.__init__r   s    $OO%	 IOO$:$:$< = AB'oo335NE7&).E!AJ0C/D)Eg&).g& 6 	

r   c                 <    d| _         d| _        d| _        g | _        y )Nr   r   r4   r   
_unflushedr   r   r   s    r   r   zSPMStreamingDetokenizer.reset   s    	r   c                    | j                   j                  | j                  d      j                  dd      }|s|j	                  d      ry | j
                  s| j                  r|r|d   dk(  r|dd  }| xj
                  |z  c_        d| _         y )	N    utf-8replacer<   r   r>   rG   r   )rZ   r^   rM   r/   r@   r   rK   )r   forcer   s      r   
_try_flushz"SPMStreamingDetokenizer._try_flush   ss    &&tyy$7>>w	Rx0yyT__$q'S.8D		T	r   c                     | j                   j                  |       | j                  |   }| xj                  |z  c_        | j	                          y r   )r   r9   rQ   rZ   r`   )r   r   vs      r   r   z!SPMStreamingDetokenizer.add_token   s:    5!MM% 1r   c                 6    | j                  d       d| _        y )NT)r_   r   )r`   rZ   r   s    r   r   z SPMStreamingDetokenizer.finalize   s    d#r   N)T)F)	r$   r%   r&   r'   r1   r   r`   r   r   r*   r   r   rD   rD   k   s     r   rD   c                   L    e Zd ZdZdZdZd Zd Zd Zd Z	d Z
d	 Zed
        Zy)BPEStreamingDetokenizerzA streaming detokenizer for OpenAI style BPE models.

    It adds tokens to the text if the next token starts with a space similar to
    the SPM detokenizer.
    N)	.?!,zn'tz'mz'sz'vez'rec                    |j                   | _        d gt        |j                        z  | _        |j                  j                         D ]  \  }}|| j                  |<    | j                          | j                          y r   )rA   clean_spacesr!   rO   rQ   rR   r   make_byte_decoder)r   r0   rV   rW   s       r   r1   z BPEStreamingDetokenizer.__init__   sk    %BB Y__!55'oo335NE7%*DMM'" 6 	

 	 r   c                 <    d| _         d| _        d| _        g | _        y r3   rY   r   s    r   r   zBPEStreamingDetokenizer.reset   s    	r   c                     t               }|D ]M  }| j                  j                  |d      }|r|j                  |       3|j	                  t        |d             O |j                  dd      S )NFr]   r^   )	bytearray_byte_decodergetr9   extendrT   r/   )r   seqbarrcress        r   _decode_bytesz%BPEStreamingDetokenizer._decode_bytes   s`    {A$$((E2CC E!W-.  {{7I..r   c                     t        |      dk(  r|S |d   dk7  r|S | j                  s|dd  S | j                  r#|dd  j                  | j                        r|dd  S |S )Nr   r>   rG   )r!   r   rk   rS   _space_matchesr   current_texts     r   _maybe_trim_spacez)BPEStreamingDetokenizer._maybe_trim_space   sq    |!!_###<#3#>#>t?R?R#S##r   c                    | j                   j                  |       |t        | j                        k  r| j                  |   nd}| xj                  |z  c_        | j                  | j                        }|j                  d      s\t        |      dk(  r!| j                  j                  |d         dk(  s,| xj                  | j                  |      z  c_	        d| _        y y y )Nrh   r<   rG   r       r4   )r   r9   r!   rQ   rZ   rw   r@   rp   rq   r   r|   )r   r   rb   r   s       r   r   z!BPEStreamingDetokenizer.add_token   s    5!$)C,>$>DMM% C1!!$//2 }}X&FaKD..221Q48B>II//55I DO ? 'r   c                      t         fd j                  D              j                  dd      } xj                   j	                  |      z  c_        d _        y )Nc              3   <   K   | ]  }j                   |     y wr   )rp   ).0ru   r   s     r   	<genexpr>z3BPEStreamingDetokenizer.finalize.<locals>.<genexpr>   s      P1!3!3A!6s   r]   r^   r4   )ro   rZ   r/   r   r|   rz   s   ` r   r   z BPEStreamingDetokenizer.finalize   sJ      P PPWW
 			T++L99	r   c           	         | j                   yi }dt        d      t        d      dz   t        d      t        d      dz   t        d      t        d	      dz   g}d}t        t        ||dd             D ]W  \  }\  }}|d
z  dk(  r(t	        ||      D ]  }||t        d|z         <   |dz  } 9t	        ||      D ]  }||t        |      <    Y || _         y)zQSee https://github.com/openai/gpt-2/blob/master/src/encoder.py for the rationale.Nr   rh   ~rG      ¡   ¬   ®   ÿ      )rp   ord	enumerateziprangechr)clschar_to_byteslimitsnistartstopbs           r   rl   z)BPEStreamingDetokenizer.make_byte_decoder   s     (HHqLIIMIIM
  )#ffQRj*A BA}t1uzud+A34M#dQh-0FA , ud+A,-M#a&) , !C *r   )r$   r%   r&   r'   rp   ry   r1   r   rw   r|   r   r   classmethodrl   r*   r   r   re   re      sE     MJN!/	! * *r   re   c           	          | j                         }ddg}|D ]  \  }}||v s||v s||||   f||   ffc S  d|v rBd|v r>d}d}||t        | j                  |d            t        | j                  |d            fS y)	N)z<think>z</think>)z<longcat_think>z</longcat_think>z
<|channel>z
<channel|>z<|channel>thoughtFadd_special_tokens)NNNN)	get_vocabtuplerL   )r0   rO   THINK_TOKENSthink_start	think_ends        r   _infer_thinkingr      s    !E/L #/Y%I$6{#%y!#	  #/ u!6) 	)"";5"IJ)""9"GH	
 	
 $r   c                   \    e Zd ZdZedddddfdZdddZdefdZdd	Z	dd
Z
ddZddZddZed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zd Z fdZ xZS ) TokenizerWrapperzA wrapper that combines an HF tokenizer and a detokenizer.

    Accessing any attribute other than the ``detokenizer`` is forwarded to the
    huggingface tokenizer.
    Nc                    || _         || _        |t        |      n|j                  h| _        t        |      \  | _        | _        | _        | _	        || _
        |j                  d uxs |d u| _        || _        || _        || _        d | _        d | _        |Ct%        |j'                  |d            | _        t%        |j'                  |d            | _        y y )NFr   )r.   _detokenizer_classseteos_token_id_eos_token_idsr   _think_start
_think_end_think_start_tokens_think_end_tokens_chat_templatechat_templatehas_chat_template_tool_parser_tool_call_start_tool_call_end_tool_call_start_tokens_tool_call_end_tokensr   rL   )r   r0   detokenizer_classeos_token_idsr   tool_call_starttool_call_endtool_parsers           r   r1   zTokenizerWrapper.__init__&  s     $"3 ( (() 	 I&	
O$" ,##4/L=3L 	 ( /+'+$%)"&+0  U K,D( */  5 I*D&	 'r   T)tokenizec                    d|vr| j                   |d<   | j                  3 | j                  |i |}|r| j                  j                  |d      }|S d|d<    | j                  j                  |d|i|S )Nenable_thinkingFr   return_dictr   )has_thinkingr   r.   rL   apply_chat_template)r   r   argskwargsouts        r   r   z$TokenizerWrapper.apply_chat_templateO  s    F*(,(9(9F$%*%$%%t6v6Coo,,SU,KJ %}2t22DV8VvVVr   r   c                     d }	 t        |      }|t        d| d      | j                  j                  |       y # t        $ r | j                  j                  |      }Y Sw xY w)N'z#' is not a token for this tokenizer)rU   
ValueErrorr.   convert_tokens_to_idsr   add)r   r   token_ids      r   add_eos_tokenzTokenizerWrapper.add_eos_token\  sn    	D5zH q'JKLL)  	D<<UCH	Ds   < $A#"A#c           
      &   |xs d}|xs t              }|rt        |t              z
  |dz
  d      nt        ||t              z
  dz         }|D ];     d   k(  st        fdt        dt                    D              s9c S  y)Nr   rG   r=   c              3   :   K   | ]  }|z      |   k(    y wr   r*   )r   jr   sequencer   s     r   r   z)TokenizerWrapper._find.<locals>.<genexpr>r  s%     U=Tva!e}3=Ts   )r!   r   all)r   r   r   r   endreverse
outer_loopr   s    ``    @r   _findzTokenizerWrapper._findh  s    
 S[  #H%uqy"5ucCM1A56 	
 AayHQK'UU1c(m=TUUH  r   c                 @    | j                  || j                  ||      S N)r   r   r   r   r   r   r   r   s       r   find_think_startz!TokenizerWrapper.find_think_startv  s    zz&$":":%SzQQr   c                 B    | j                  || j                  ||d      S NT)r   r   r   r   r   s       r   rfind_think_startz"TokenizerWrapper.rfind_think_starty  s)    zzD,,EsD  
 	
r   c                 @    | j                  || j                  ||      S r   r   r   r   s       r   find_think_endzTokenizerWrapper.find_think_end~  s    zz&$"8"83zOOr   c                 B    | j                  || j                  ||d      S r   r   r   s       r   rfind_think_endz TokenizerWrapper.rfind_think_end  s)    zzD**%S$  
 	
r   c                     | j                   d uS r   r   r   s    r   r   zTokenizerWrapper.has_thinking  s      ,,r   c                     | j                   S r   r   r   s    r   r   zTokenizerWrapper.think_start         r   c                     | j                   y t        | j                         dkD  rt        d      | j                   d   S )NrG   z0The start thinking sequence is more than 1 tokenr   )r   r!   r   r   s    r   think_start_idzTokenizerWrapper.think_start_id  sA    ##+t''(1,OPP''**r   c                     | j                   S r   )r   r   s    r   think_start_tokensz#TokenizerWrapper.think_start_tokens  s    '''r   c                     | j                   S r   )r   r   s    r   r   zTokenizerWrapper.think_end  s    r   c                     | j                   y t        | j                         dkD  rt        d      | j                   d   S )NrG   z.The end thinking sequence is more than 1 tokenr   )r   r!   r   r   s    r   think_end_idzTokenizerWrapper.think_end_id  sA    !!)t%%&*MNN%%a((r   c                     | j                   S r   )r   r   s    r   think_end_tokensz!TokenizerWrapper.think_end_tokens  s    %%%r   c                     | j                   d uS r   r   r   s    r   has_tool_callingz!TokenizerWrapper.has_tool_calling  s    $$D00r   c                     | j                   S r   r   r   s    r   r   z TokenizerWrapper.tool_call_start  s    $$$r   c                     | j                   S r   )r   r   s    r   tool_call_start_tokensz'TokenizerWrapper.tool_call_start_tokens  s    +++r   c                     | j                   S r   )r   r   s    r   r   zTokenizerWrapper.tool_call_end  s    """r   c                     | j                   S r   )r   r   s    r   tool_call_end_tokensz%TokenizerWrapper.tool_call_end_tokens  s    )))r   c                     | j                   S r   )r   r   s    r   r   zTokenizerWrapper.tool_parser  r   r   c                 $    | j                  |       S )z7
        Get a stateful streaming detokenizer.
        )r   r   s    r   detokenizerzTokenizerWrapper.detokenizer  s    
 &&t,,r   c                     |dk(  r| j                   S |dk(  r| j                  S |j                  d      r| j                  |      S t	        | j
                  |      S )Nr   r   _)_detokenizerr   rS   __getattribute__getattrr.   )r   attrs     r   __getattr__zTokenizerWrapper.__getattr__  sX    = $$$_$&&&__S!((..4??D11r   c                     |dv r3|dk(  rt        d      |dk(  r|t        |      n	t               | _        y y |j                  d      rt        |   ||       y t        | j                  ||       y )N>   r   r   r   zCannot set the detokenizer.r   r   )AttributeErrorr   r   rS   super__setattr__setattrr.   )r   r   rV   	__class__s      r   r  zTokenizerWrapper.__setattr__  sm    33}$$%BCC(494Ec%j35# )__S!Ge,DOOT51r   )NNFNN) r$   r%   r&   r'   r,   r1   r   strr   r   r   r   r   r   r)   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  __classcell__r  s   @r   r   r     s{    4'R 37 W
*3 
*R

P

 - - ! ! + + ( (   ) ) & & 1 1 % % , , # # * * ! ! - -2	2 	2r   r   c                   V     e Zd ZdZ fdZd Zd Z fdZ fdZ fdZ	 fdZ
 xZS )	NewlineTokenizerzBA tokenizer that replaces newlines with <n> and <n> with new line.c                 $    t        |   |i | y r   )r  r1   r   r   r   r  s      r   r1   zNewlineTokenizer.__init__  s    $)&)r   c                 &    |j                  dd      S )Nr?   <n>r^   r   r   s     r   _preprocess_textz!NewlineTokenizer._preprocess_text  s    ||D%((r   c                 &    |j                  dd      S )Nr  r?   r  r  s     r   _postprocess_textz"NewlineTokenizer._postprocess_text  s    ||E4((r   c                 B    t        |   | j                  |      fi |S r   )r  rL   r  )r   r   r   r  s      r   rL   zNewlineTokenizer.encode  s"    w~d33D9DVDDr   c                 h    t        |   |D cg c]  }| j                  |       c}fi |S c c}w r   )r  encode_batchr  )r   textsr   tr  s       r   r  zNewlineTokenizer.encode_batch  s6    w#u$Mu!T%:%:1%=u$MXQWXX$Ms   /c                 @    | j                  t        |   |i |      S r   )r  r  r/   r  s      r   r/   zNewlineTokenizer.decode  s"    %%egnd&Ef&EFFr   c                 j    t        |   |i |}|D cg c]  }| j                  |       c}S c c}w r   )r  batch_decoder  )r   r   r   decodeddr  s        r   r  zNewlineTokenizer.batch_decode  s:    '&773:;7a&&q)7;;;s   0)r$   r%   r&   r'   r1   r  r  rL   r  r/   r  r  r	  s   @r   r  r    s1    L*))EYG< <r   r  )fast_tokenizer_classc                 F    t               t              k7  ryt         t              r.t               t              k(  xr t	         fd D              S t         t
              r5t               t              k(  xr t	        d t               D              S  k(  S )NFc              3   N   K   | ]  }|v xr t        |   |           y wr   _match)r   kar   s     r   r   z_match.<locals>.<genexpr>   s-     'Qq!Q(E6!A$!3E(Eqs   "%c              3   :   K   | ]  \  }}t        ||        y wr   r"  )r   aibis      r   r   z_match.<locals>.<genexpr>  s     'OY62rr2Ys   )type
isinstancedictr!   r   listr   )r%  r   s   ``r   r#  r#    s    Aw$q'!T1vQQC'Qq'Q$QQ!T1vQOC'OSAY'O$OO6Mr   c           	      J    ddddiddddidd	id
ddddgd}t        ||       S )NSequenceReplaceStringrF   r>   r)  patterncontentr)  ByteFallbackFuseStriprG   r   )r)  r3  r   r   r)  decodersr"  decoder_target_descriptions     r   _is_spm_decoderr<    sM    He+<M^$Vq!D	
 %w//r   c                 >    ddddiddddidd	igd
}t        ||       S )Nr.  r/  r0  rF   r>   r1  r)  r4  r5  r7  r"  r9  s     r   _is_spm_decoder_no_spacer>    sA    He+<M^$V
 %w//r   c                 P    t        | t              xr | j                  dd       dk(  S )Nr)  	ByteLevel)r*  r+  rq   )r:  s    r   _is_bpe_decoderrA     s$    gt$QVT)Bk)QQr   c                     t        | t              syd| v ryd| v rd| v ryd| v ryd	| v ry
d| v ryd| v ryd| v sd| v ryd| v ryd| v ryd| v rd| v ryy)z;Attempt to auto-infer a tool parser from the chat template.Nz<minimax:tool_call>
minimax_m2z<|tool_call>z<tool_call|>gemma4z<start_function_call>function_gemmaz<longcat_tool_call>longcatz	<arg_key>glm47z<|tool_list_start|>pythonicz<tool_call>\n<function=z<tool_call>
<function=qwen3_coderz<|tool_calls_section_begin|>kimi_k2z[TOOL_CALLS]mistralz<tool_call>ztool_call.name
json_tools)r*  r  )r   s    r   _infer_tool_parserrM  $  s    mS)	-	/	=	(^}-L	 M	1	-	/		%	-	/"m3$5	'=	8	=	(	-	',<,Mr   tokenizer_config_extrareturnc           	         t         }| dz  }|j                         r{t        |dd      5 }	 t        j                  |      }	 ddd       dv rIt        |d         rt        }n4t        |d         rt        t        d	      }nt        |d         rt        }t        |t              r|g}| d
z  }d}	t!        j"                  | fi |xs i }
|
j$                  }|j'                  dd      x}r"t)        j*                  d|       j,                  }	|j'                  dt/        |
j0                              }|Bt)        j*                  d|       }|j2                  }|j4                  }|j6                  }||d<   nd}d}d}t9        |
|||	|||      S # t
        $ r&}t        d|j                  |j                        d}~ww xY w# 1 sw Y   xY w)zLoad a huggingface tokenizer and try to infer the type of streaming
    detokenizer to use.

    Note, to use a fast streaming tokenizer, pass a local file path rather than
    a Hugging Face repo ID.
    ztokenizer.jsonrr]   )encodingzFailed to parse tokenizer.jsonNr:  F)rK   ztokenizer_config.jsonchat_template_typezmlx_lm.chat_templates.tool_parser_typezmlx_lm.tool_parsers.)r   r   r   r   r   )r,   existsopenjsonloadr   docposr<  rD   r>  r   rA  re   r*  rU   r	   from_pretrainedinit_kwargsrq   	importlibimport_moduler   rM  r   parse_tool_callr   r   r   )
model_pathrN  r   r   tokenizer_filefidtokenizer_contentetokenizer_config_filer   r0   tokenizer_configrS  rT  tool_moduler   r   r   s                     r   rX  rX  B  s    2"22N.#8CV$(IIcN! 9 ))0;<$;!)*;I*FG$+,CPU$V! !29!=>$;!-%&&)@@M---3I !,,-112FNNN!//$%7$89


 	 (++.y/F/FG #--0DEUDV.WX!11%55#11/?+,##'# W # V%&FquuUUV 98s(   G F	F=!F88F==G  G
r   boseosc                 >    | d   |k7  r| n| dd  }|d   |k(  r|d d S |S )Nr   rG   r=   r*   )r   rh  ri  removed_boss       r   no_bos_or_eosrl    s7    &qkS0(hqrlK*2#5;sF;Fr   r  )r]  rW  warnings	functoolsr   r   typingr   r   r   r   transformersr	   r
   r   r,   rD   re   r   r   r  registerr#  r<  r>  rA  rM  r  rX  rU   rl  r*   r   r   <module>rr     s          , , ?/ /d+/ 4 +/\-2 -`b*2 b*J$>}2 }2@<. <4   )@P Q
0	0R@ 8<F$T#s(^4F 	FRGD Gs G G Gr   