
    (HJj+                        d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Zd
 Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)AnyDictOptionalTupleUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)	SwitchGLUc                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   ded<   dZe	e
eef      ed<   dZe	eee
eef   f      ed<   dZeed<   d Zy)	ModelArgs
model_type
vocab_sizehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_headsattention_biasmoe_topknum_expertsnum_shared_expertuse_mixed_mlp_moeuse_qk_normrms_norm_eps
rope_thetause_cla   cla_share_factorNmoe_intermediate_sizerope_scalingFtie_word_embeddingsc                 l      j                   r'ddh}t         fd|D              st        d|       y y )Nfactortypec              3   :   K   | ]  }|j                   v   y wN)r$   ).0keyselfs     _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/hunyuan.py	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>)   s     I=Csd///=s   zrope_scaling must contain keys )r$   all
ValueError)r-   required_keyss   ` r.   __post_init__zModelArgs.__post_init__%   sB    %v.MI=II #B=/!RSS J     )__name__
__module____qualname__str__annotations__intboolfloatr#   r   r   listr$   r   r%   r3    r4   r.   r   r      s    OOMM8<8E#t)$45<;?L(4U5#:%6 678? %%Tr4   r   c                 0    t        | t              r| |   S | S r*   )
isinstancer=   )argidxs     r.   _int_or_listrC   -   s    #t3xJr4   c                   @     e Zd Z	 	 ddededef fdZddefdZ xZS )	DynamicNTKAlphaRoPEdimsbasescaling_alphac                     t         |           || _        ||||dz
  z  z  z  }|t        j                  d| j                  d      | j                  z  z  | _        y )Nr!   r   )super__init__rF   mxarange_freqs)r-   rF   rG   rH   	__class__s       r.   rK   zDynamicNTKAlphaRoPE.__init__4   sU     		mq(9::ryyDIIq9DIIEFr4   offsetc           	      v    t         j                  j                  || j                  dd d|| j                        S )NF      ?)traditionalrG   scalerP   freqs)rL   fastroperF   rN   )r-   xrP   s      r.   __call__zDynamicNTKAlphaRoPE.__call__?   s9    ww||II++  
 	
r4   )i'  rR   r   )r5   r6   r7   r:   r<   rK   rY   __classcell__rO   s   @r.   rE   rE   3   s<     "		G	G 	G 		G	
# 	
r4   rE   c            	            e Zd Zdedef fdZ	 	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
	Attentionkv_projargsc                 J   t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  |j                        | _        |rTt        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  ||j                        | _        |j                  | _        | j                  rJt        j                   ||j"                        | _        t        j                   ||j"                        | _        t)        ||j*                  |j,                  d         | _        y )Ng      ࿩biasalpha)rG   rH   )rJ   rK   r   r   n_headsr   
n_kv_headsrT   nnLinearr   q_projk_projv_projo_projr   RMSNormr   query_layernormkey_layernormrE   r   r$   rW   )r-   r_   r`   dimre   rf   head_dimrO   s          r.   rK   zAttention.__init__L   s`   !%!9!99w''333'+'?'??*##w.t^
iiWx%7d>Q>QR))Z(*1D1DDK ))Z(*1D1DDK ii( 2Cd>Q>QR++#%::h8I8I#JD !#Hd6G6G!HD'++G4
	r4   rX   maskcachereturnc                 \   |j                   \  }}}| j                  |      }|'| j                  |      | j                  |      }
}	|	|
f}n|\  }	}
|j	                  ||| j
                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|
j	                  ||| j                  d      j                  dddd      }
|r|j                  nd}| j                  ||      }| j                  |	|      }	| j                  r"| j                  |      }| j                  |	      }	||j                  |	|
      \  }	}
t        ||	|
|| j                  |      }|j                  dddd      j	                  ||d      }| j!                  |      |fS )Nr   r!   r	      )rP   )rs   rT   rr   )shaperi   rj   rk   reshapere   	transposerf   rP   rW   r   rn   ro   update_and_fetchr   rT   rl   )r-   rX   rr   rs   	kv_statesBLDquerieskeysvaluesrP   outputs                r.   rY   zAttention.__call__j   s    ''1a++a.;;q>4;;q>&DfI$LD& //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aP!&A))GF)3yyfy-**73G%%d+D 11$?LD&-T6djjt
 !!!Q1-55aB?{{6"I--r4   NNN)r5   r6   r7   r;   r   rK   rL   arrayr   r   rY   r[   r\   s   @r.   r^   r^   K   sb    
 
I 
B $(##.88#. rxx #. }	#. 
#.r4   r^   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFrb   )rJ   rK   rg   rh   	gate_proj	down_projup_proj)r-   rp   
hidden_dimrO   s      r.   rK   zMLP.__init__   sJ    3
?:s?yyju=r4   rt   c                 t    | j                  t        | j                  |      | j                  |                  S r*   )r   r
   r   r   r-   rX   s     r.   rY   zMLP.__call__   s)    ~~fT^^A%6QHIIr4   r5   r6   r7   rK   rL   r   rY   r[   r\   s   @r.   r   r      s    >JRXX Jr4   r   c                   >     e Zd Z fdZdej
                  fdZ xZS )Gatec                 \    t         |           t        j                  ||d      | _        y r   )rJ   rK   rg   rh   wg)r-   rp   r   rO   s      r.   rK   zGate.__init__   s"    ))C59r4   rt   c                 $    | j                  |      S r*   )r   r   s     r.   rY   zGate.__call__   s    wwqzr4   r   r\   s   @r.   r   r      s    :RXX r4   r   c                   J     e Zd Zddedef fdZdej                  fdZ xZ	S )MoeBlockr`   	layer_idxc                    t         |           |j                  }|j                  }|j                  | _        |j                  r3t        |j                  |      }t        |t        ||z              | _
        |j                  x| _        }t        |j                  |      | _        t        ||      | _        |}|j                   t        |j                   |      }t#        |||      | _        y r*   )rJ   rK   r   r   r   use_shared_mlprC   r   r   r:   
shared_mlpr   r   top_kr   gater#   r   
switch_mlp)	r-   r`   r   rp   r   
num_sharedr   expert_intermediate_sizerO   s	           r.   rK   zMoeBlock.__init__   s     22"44!!%d&<&<iHJ!#s+<z+I'JKDO)-)9)99;!$--;
k*	 $5 %%1'3**I($ $C)A;Or4   rX   c                    | j                  |      }t        j                  |dd      }| j                  }t        j                  t        j
                  | |dz
  d      dd |f         }t        j                  ||d      }| j                  ||      }||d   j                  t        j                        z  j                  d	      j                  |j                        }| j                  r| j                  |      }||z   }|S )
Nrv   T)axispreciser	   )kthr   .r   ).N)r   rL   softmaxr   stop_gradientargpartitiontake_along_axisr   astypefloat32sumdtyper   r   )r-   rX   gateskindsscoresyshared_expert_outputs           r.   rY   zMoeBlock.__call__   s     		!

5r48JJAE KCQSRSQSG TU##E4b9OOAt$	"))"**55:::CJJ177S#'??1#5 ((Ar4   rZ   )
r5   r6   r7   r   r:   rK   rL   r   rY   r[   r\   s   @r.   r   r      s*    PY P3 P088r4   r   c                        e Zd Zd
dededef fdZ	 	 	 ddej                  de	ej                     de	e
   de	eej                  ej                  f      fd	Z xZS )DecoderLayerr`   r_   r   c                    t         |           |j                  | _        t        ||      | _        |j
                  dk(  r&t        |j                  |j                        | _        nt        ||      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Nr	   eps)rJ   rK   r   r^   	self_attnr   r   r   mlpr   rg   rm   r   input_layernormpost_attention_layernormr`   )r-   r`   r_   r   rO   s       r.   rK   zDecoderLayer.__init__   s    ++"7D1q 4++T-C-CDDHi0DH!zz$*:*:@Q@QR(*

$"3"3)
% 	r4   rX   rr   rs   shared_kv_statesc                     | j                  | j                  |      |||      \  }}||z   }| j                  | j                  |            }||z   }||fS r*   )r   r   r   r   )r-   rX   rr   rs   r   rhouts           r.   rY   zDecoderLayer.__call__   sf     #nn  #T52B
 EHHT22156!e$$$r4   rZ   r   )r5   r6   r7   r   r;   r:   rK   rL   r   r   r   r   rY   r[   r\   s   @r.   r   r      s{    Y  # $ $(#@D%88% rxx % }	%
 #5288);#<=%r4   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )HunYuanModelr`   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]/  }t        ||j                   xs ||j                  z  dk(  |      1 c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r`   r_   r   r   )rJ   rK   r`   r   r   rg   	Embeddingr   embed_tokensranger   r    r"   layersrm   r   norm)r-   r`   irO   s      r.   rK   zHunYuanModel.__init__   s    	//!%!7!7"""LL$:J:JK 4112
 3 !\\)Nq43H3H/HQ.N
 3
 JJt//T5F5FG	
s   4C=inputsc                 r   | j                  |      }|d gt        | j                        z  }t        ||d         }t	        t        | j                  |            D ]J  \  }\  }}| j                  j                  r|| j                  j                  z  dk(  rd } ||||      \  }}L | j                  |      S )Nr   )
r   lenr   r   	enumeratezipr`   r    r"   r   )	r-   r   rs   r   rr   r   layercr   s	            r.   rY   zHunYuanModel.__call__  s    
 f%=FS--E$Qa1&s4;;'>?MAzqII%%!dii.H.H*HA*M#' "'44D"EA @
 yy|r4   r*   )	r5   r6   r7   r   rK   rL   r   rY   r[   r\   s   @r.   r   r      s'    HY H( r4   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr`   c                 r    t         |           || _        |j                  | _        t	        |      | _        y r*   )rJ   rK   r`   r   r   model)r-   r`   rO   s     r.   rK   zModel.__init__  s,    	//!$'
r4   r   c                 p    | j                  ||      }| j                   j                  j                  |      S r*   )r   r   	as_linear)r-   r   rs   r   s       r.   rY   zModel.__call__  s/    
 jj'zz&&0055r4   c                    d|v r:i }| j                   j                  }| j                   j                  }| j                   j                  |z  }|| j                   j                  z  }|j	                         D ]  \  }}d|v rr|j                  ||dz   |d      }|j                  ||dz   gd      }	t        g d|	      D ]1  \  }
}|j                  d|
      }t        j                  |dd      ||<   3 |d	|v rA|j                  dd      }	t        d
dg|	      D ]  \  }
}|j                  d	|
      }|||<    |||<    |}d|vr|S t        | j                   j                        D ]  }d| }dD ]|  }dD ]u  }| d| d| |v st        | j                   j                        D cg c]  }|j                  | d| d| d|         }}t        j                  |      || d| d| <   w ~  |S c c}w )Nz*model.layers.0.mlp.gate_and_up_proj.weightqkv_projr!   rv   r	   r   )ri   rj   rk   r   gate_and_up_projr   r   z+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.)r   r   r   )weightscalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.)r`   r   r   r   itemsry   splitr   replacerL   flattenr   r   r   popstack)r-   weightsnew_weightsr   rf   n_kv_groupsrq   r   vsplitsk_upv_newk_newlprefixneto_joins                     r.   sanitizezModel.sanitize&  s*   77BK		%%A66J))77:EKDII999H1?		*kAoxLAWWk;?%C!WLF'*+I6'Re !		*d ;-/ZZq!-DE* (S (1,WWQQW/F'*I{+CV'Le !		*<d C-2E* (M &'KN ( "G8GNtyy223A$QC(F:7A 1QC8GC &+499+@+@%A#%A $KK6(-s!A3as(KL%A   # GIhhwFW6(*:1#Qqc BC 8 ; 4 #s   %#G2
c                 .    | j                   j                  S r*   )r   r   )r-   s    r.   r   zModel.layersL  s    zz   r4   r*   )r5   r6   r7   r   rK   rL   r   rY   r   propertyr   r[   r\   s   @r.   r   r     s?    (Y ( 66$L ! !r4   r   ) dataclassesr   typingr   r   r   r   r   mlx.corecorerL   mlx.nnrg   activationsr
   rG   r   r   r   switch_layersr   r   rC   ModulerE   r^   r   r   r   r   r   r   r>   r4   r.   <module>r      s    " 4 4    T T $ T T T<
")) 
0B.		 B.JJ")) J299 +ryy +\%299 %@"299 "J7!BII 7!r4   