
    (HJj,                         d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
 d dlmZ d dlmZ  G d de      Z G d d	e      Z G d
 d      Zd ZddZd Zde
eef   dededefdZy)    N)IntEnum)Path)IterableOptionalSetTupleUnion)AutoTokenizerc                   $    e Zd ZdZdZdZdZdZdZy)	TokenType                  N)	__name__
__module____qualname__NORMALUNKNOWNCONTROLUSER_DEFINEDUNUSEDBYTE     U/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/gguf.pyr   r   
   s     FGGLFDr   r   c                       e Zd ZdZy)GGMLFileTyper   N)r   r   r   GGML_TYPE_F16r   r   r   r    r       s    Mr   r    c                       e Zd Z	 ddedeeedf      ddfdZdeee	e
ef      fdZdede	d	ee   defd
Zdede
fdZdeee	e
ef      fdZd Zdeee	e
ef      fdZdefdZededd fd       Zy)HfVocabNfname_tokenizerfname_added_tokensreturnc                 :   t        j                  ||d      | _        g | _        t	               | _        t               | _        t        | j                  j                         j                         d       D ]d  \  }}|| j                  j                  k\  s | j                  j                  |       || j
                  |<   | j                  j                  |       f | j                  j                  D ci c]   }|| j                  j                         |   " c}| _        t        | j                  j"                        | _        | j                  j                  | _        | j&                  t)        | j                        z   | _        || _        || _        y c c}w )NT)	cache_dirlocal_files_onlyc                     | d   S )Nr   r   )xs    r   <lambda>z"HfVocab.__init__.<locals>.<lambda>'   s    AaDr   )key)r
   from_pretrained	tokenizeradded_tokens_listdictadded_tokens_dictsetadded_tokens_idssortedget_added_vocabitems
vocab_sizeappendaddall_special_tokens	get_vocabspecialsall_special_idsspecial_idsvocab_size_baselenr$   r%   )selfr$   r%   toktokidxs        r   __init__zHfVocab.__init__   sV   
 '66%!

 "$!% #!NN**,224.
KC 222&&--c2.4&&s+%%))&1
 ~~88
8 ))+C008
 t~~==>#~~88..T5K5K1LL."4
s   6%Fc              #   L  K   | j                   j                         j                         D ci c]  \  }}||
 }}}t        | j                        D ]G  }|| j
                  v r||   }|| j                  |      | j                  ||| j                        f I y c c}}w wN)	r/   r<   r7   ranger@   r4   get_token_scoreget_token_typer?   )rB   encoded_tokidreverse_vocabtoken_id
token_texts         r   	hf_tokenszHfVocab.hf_tokens7   s     37>>3K3K3M3S3S3U
3URBO3U 	 
 d223H4000&x0Jd228<d>Q>Q*d&6&6?  	 4
s   ,B$BA)B$rN   rO   r?   c                     t        j                  d|      rt        j                  S ||v rt        j                  S t        j
                  S )Nz<0x[0-9A-Fa-f]{2}>)re	fullmatchr   r   r   r   )rB   rN   rO   r?   s       r   rJ   zHfVocab.get_token_typeC   s;     <<-z:>>!$,$;y  QAQAQQr   c                      y)N     @r   )rB   rN   s     r   rI   zHfVocab.get_token_scoreJ   s    r   c              #     K   | j                   D ]r  }|| j                  v rI| j                  | j                  |   d| j                        }| j	                  | j                  |         }nt
        j                  }d}|||f t y w)N rU   )r0   r=   rJ   r?   rI   r   r   )rB   texttoktypescores       r   added_tokenszHfVocab.added_tokensM   s{     **Dt}}$--dmmD.A2tGWGWX,,T]]4-@A#00w&& +s   BBc                 f    d| j                   j                  v xs d| j                   j                  v S )Nz<0x0A>
)r/   vocabrB   s    r   has_newline_tokenzHfVocab.has_newline_tokenW   s+    4>>///O44>>;O;O3OOr   c              #   t   K   | j                         E d {    | j                         E d {    y 7 7 wrG   )rP   r[   r_   s    r   
all_tokenszHfVocab.all_tokensZ   s1     >>###$$&&& 	$&s   848688c                 N    d| j                    dt        | j                         dS )Nz<HfVocab with z base tokens and z added tokens>)r@   rA   r0   r_   s    r   __repr__zHfVocab.__repr__^   s,     4 455Fs4KaKaGbFccqrrr   pathc                 d    | j                   dz  }t        | |j                         r|      S d       S )Nzadded_tokens.json)parentr#   exists)re   added_tokens_paths     r   loadzHfVocab.loada   s5     KK*==t2C2J2J2L.WWRVWWr   rG   )r   r   r   r   r   r	   rE   r   r   bytesfloatr   rP   intr   rJ   rI   r[   r`   rb   strrd   staticmethodrj   r   r   r   r#   r#      s    ;?55 %U4:%675 
	5<
8E%	*A$BC 
RR).R=@XR	R  'huUE9-D'EF 'P'HU5%+B%CD 's# s X4 XI X Xr   r#   c                    | j                  dd      } | j                  dd      } d}d}t        j                  |||       } d}d}t        j                  |||       } d	}d
}t        j                  |||       } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd      } | j                  dd       } | j                  d!d"      } | S )#Nzmodel.layers.zblk.zblock_sparse_moe.gateffn_gate_inpz,block_sparse_moe\.experts\.(\d+)\.w1\.weightzffn_gate.\1.weightz,block_sparse_moe\.experts\.(\d+)\.w2\.weightzffn_down.\1.weightz,block_sparse_moe\.experts\.(\d+)\.w3\.weightzffn_up.\1.weightzmlp.gate_projffn_gatezmlp.down_projffn_downzmlp.up_projffn_upzself_attn.q_projattn_qzself_attn.k_projattn_kzself_attn.v_projattn_vzself_attn.o_projattn_outputinput_layernorm	attn_normpost_attention_layernormffn_normzmodel.embed_tokens
token_embdz
model.normoutput_normlm_headoutput)replacerR   sub)namepatternreplacements      r   translate_weight_namesr   g   sC   <<0D<</@D=G'K66';-D=G'K66';-D=G%K66';-D<<4D<<4D<<x0D<<*H5D<<*H5D<<*H5D<<*M:D<<);7D<<2J?D<<,l;D<<m4D<<	8,DKr   c                     |||k7  r|} | j                   |d| j                  d   |z  dz  g| j                  dd   }|j                  dd      }| j                  }|j                  |      S )Nr   r   r   )reshapeshapeswapaxes)weightsn_head	n_head_kvreshapedswappedfinal_shapes         r   permute_weightsr      s}    9!4w7==#v-25<]]125FH 1%G--K??;''r   c                    d| j                  d      (t        j                  | d   t        j                        nd | j                  d      (t        j                  | d   t        j                        nd | j                  d      (t        j                  | d   t        j                        nd | j                  d      (t        j                  | d   t        j                        nd | j                  d      ?| j                  d      .t        j                  | d   | d   z  t        j                        nd | j                  d      (t        j                  | d   t        j                        nd | j                  d      8t        j                  | j                  d| d         t        j                        nd | j                  d	      5t        j                  | j                  d	d       t        j                        nd | j                  d
      5t        j                  | j                  d
d       t        j                        nd | j                  d      %t        j                  | j                  dd            nd | j                  d      5t        j                  | j                  dd      t        j                        nd d}| j                  d      }|J|j                  d      x}r7|j                  d      }|}|dk(  rd}||d<   t        j                  |      |d<   t        j                  t
        j                  j                  t        j                        |d<   t        j                  t
        j                  j                  t        j                        |d<   | j                  dd      j                  d      d   |d<   d|d<   t        j                  dt        j                        |d<   d|d<   g }g }	g }
|j                         D ]C  \  }}}|j                  |       |	j                  |       |
j                  |j                         E t        |      |j                  k(  sJ ||d <   t        j                  |	t        j                        |d!<   t        j                  |
t        j                        |d"<   |j                  j                  <t        j                  |j                  j                  t        j                        |d#<   |j                  j                  <t        j                  |j                  j                  t        j                        |d$<   |j                  j                   <t        j                  |j                  j                   t        j                        |d%<   |j#                         D ci c]  \  }}|	|| }}}|S c c}}w )&Nllamamax_position_embeddings)dtypehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_headsnum_local_expertsnum_experts_per_tokrms_norm_epsgh㈵>
rope_thetai'  )general.namezllama.context_lengthzllama.embedding_lengthzllama.block_countzllama.feed_forward_lengthzllama.rope.dimension_countzllama.attention.head_countzllama.attention.head_count_kvzllama.expert_countzllama.expert_used_countz&llama.attention.layer_norm_rms_epsilonzllama.rope.freq_baserope_scalingtypefactorlinearzllama.rope.scaling.typezllama.rope.scaling.factorzgeneral.file_typezgeneral.quantization_version_name_or_path/r   zgeneral.architecture    zgeneral.alignmentztokenizer.ggml.modelztokenizer.ggml.tokensztokenizer.ggml.scoresztokenizer.ggml.token_typeztokenizer.ggml.bos_token_idztokenizer.ggml.eos_token_idztokenizer.ggml.unknown_token_id)getmxarrayuint32float32r    r!   valuesplitrb   r9   rA   r8   r/   bos_token_ideos_token_idunk_token_idr7   )configr^   metadatar   typrope_factorf_rope_scalerope_scaling_typetokensscorestoktypesrX   rZ   rY   kvs                   r   prepare_metadatar      s    zz34@ HHV56biiH zz-(4 HHVM*"))< zz-.: HHV/0		B zz-.: HHV/0		B zz-(4

01=	 HH}%0E)FFbii
  zz/0< HHV12"))D zz/0<	 HH

0&9N2OPii
  zz-.: HHVZZ 3T:"))L zz/0< HHVZZ 5t<BIIN zz.)5 HHVZZ67 zz,'3 HHVZZe4BJJG{?HB ::n-LL,<,<V,D%DS%D"&&x0"(? (2CH./46HH\4JH01$&HH""((ii%H ! 02xx""((ii0H+,  &zz/7CII#NrRH^'.H#$$&HHRryy$AH ! (/H#$FFH % 0 0 2eWde& !3 v;%*****(.H$%(*rzz(JH$%,.HHXRYY,OH()##/24((OO((		3
./ ##/24((OO((		3
./ ##/68hhOO((		7
23 "*!1C!1AQ]1!1HCO Ds   
W-#W-
model_pathr   r   output_file_pathc                    t        | t              rt        |       } |j                  dd       }|rt	        d      t        d       |j                         D ci c]6  \  }}|d|v rt        ||d   |d         nd|v rt        ||d   |d         n|8 }}}|j                         D ci c]  \  }}t        |      | }}}| dz  j                         st        d	      t        j                  |       }t        ||      }|j                         D ci c]  \  }}||j                  t        j                   k(  r<|j#                  t        j$                        j#                  t        j&                        n$d
|v r|j#                  t        j$                        n| }}}|}t        j(                  |||       t        d|        y c c}}w c c}}w c c}}w )Nquantizationz4Conversion of quantized models is not yet supported.zConverting to GGUF formatzself_attn.q_proj.weightr   zself_attn.k_proj.weightr   ztokenizer.jsonzTokenizer json not foundnormzConverted GGUF model saved as: )
isinstancern   r   r   NotImplementedErrorprintr7   r   r   rh   
ValueErrorr#   rj   r   r   r   bfloat16astyper   float16	save_gguf)	r   r   r   r   r   r   r   r^   r   s	            r   convert_to_ggufr     s    *c"*%
::nd3L!B
 	
 

%&  MMO $DAq 	
 )A- 6/0&9N2O -1  v34f=R6S 	 $  $ 9@H1%a(!+GH))113344LL$E.H MMO $DAq 	
ww"++% HHRZZ ''

3)/1"**%!	<
 $   (LL!7H5	+,<+=
>?K$ Is   ;G	*GB	GrG   )rR   enumr   pathlibr   typingr   r   r   r   r	   mlx.corecorer   transformersr
   r   r    r#   r   r   r   rn   r1   r   r   r   r   <module>r      s    	   8 8  & 7 
LX LX^<(rj5@c4i 5@5@ 5@ 	5@r   