
    (HJj                         d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej,                        Z G d dej,                        Z G d dej,                        Z G d dej,                        Z G d dej,                        Z G d dej,                        Zy)    )	dataclass)AnyDictListOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)	SwitchGLUc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZ	e
eeeeef   f      ed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_expertsnum_experts_per_tokdecoder_sparse_stepmlp_only_layersmoe_intermediate_sizerms_norm_eps
vocab_sizenum_key_value_headshead_dim
rope_thetatie_word_embeddingsmax_position_embeddingsnorm_topk_probNrope_scaling)__name__
__module____qualname__str__annotations__intr   floatboolr#   r   r   r        a/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen3_moe.pyr   r      s    O#YOM  ;?L(4U5#:%6 678?r-   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
	Attentionargs	layer_idxc                    t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }t        |d|j                  |j                  z        }|dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j                  ||j                         | _        t        j                  ||j                         | _        t        j&                  |d|j(                        | _        y )Nr   g      Fbiaseps)traditionalbase)super__init__r   r   n_headsr   
n_kv_headsgetattrscalennLinearq_projk_projv_projo_projRMSNormr   q_normk_normRoPEr   rope)selfr1   r2   dimr<   r=   r   	__class__s          r.   r;   zAttention.__init__&   s/   !%!9!99w''333'+'?'??**d..$2J2JJ
 t^
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeDjjt/@/@Ajjt/@/@AGG
	r-   xmaskcachereturnc                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r	      )offset)rP   r?   rO   )shaperB   rC   rD   rG   reshaper<   	transposerH   r=   rJ   rV   update_and_fetchr   r?   rE   )rK   rN   rO   rP   BLDquerieskeysvaluesoutputs              r.   __call__zAttention.__call__A   s    ''1a $AAAv ++gooaDLL"EFPPq!Q
 {{4<<1doorBCMMq!Q
 1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r-   NNr$   r%   r&   r   r)   r;   mxarrayr   r   rb   __classcell__rM   s   @r.   r0   r0   %   s^    
Y 
3 
< $(#	#88# rxx # }	#
 
#r-   r0   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr4   )r:   r;   r@   rA   	gate_proj	down_projup_proj)rK   rL   
hidden_dimrM   s      r.   r;   zMLP.__init__d   sJ    3
?:s?yyju=r-   rQ   c                 t    | j                  t        | j                  |      | j                  |                  S N)rn   r
   rm   ro   )rK   rN   s     r.   rb   zMLP.__call__j   s)    ~~fT^^A%6QHIIr-   )r$   r%   r&   r;   re   rf   rb   rg   rh   s   @r.   rj   rj   c   s    >JRXX Jr-   rj   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Qwen3MoeSparseMoeBlockr1   c                    t         |           |j                  }|j                  }|j                  x| _        }|j
                  | _        |j                  | _        t        j                  ||d      | _
        t        |||      | _        y rl   )r:   r;   r   r   r   r   top_kr"   r@   rA   gater   
switch_mlp)rK   r1   rL   r   r   rM   s        r.   r;   zQwen3MoeSparseMoeBlock.__init__o   sx     66)-)9)99;--
"11IIc;U;	#C):KHr-   rN   rQ   c                    | j                  |      }t        j                  |dd      }| j                  }t        j                  || d      d| d f   }t        j
                  ||d      }| j                  r|t        j                  |dd      z  }| j                  ||      }||d   z  j                  d	      }|S )
NrS   T)axisprecise)kthrz   .)rz   )rz   keepdims).N)	rw   re   softmaxrv   argpartitiontake_along_axisr"   sumrx   )rK   rN   gateskindsscoresys          r.   rb   zQwen3MoeSparseMoeBlock.__call__{   s     		!

5r48JJu1"26sQBCx@##E4b9bffV"t<<FOOAt$	""''R'0r-   )	r$   r%   r&   r   r;   re   rf   rb   rg   rh   s   @r.   rt   rt   n   s0    
IY 
I88 
r-   rt   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Qwen3MoeDecoderLayerr1   r2   c                    t         |           |j                  | _        t        ||      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _	        || _
        ||j                  vr5|j                  dkD  r&|dz   |j                  z  dk(  rt        |      | _        y t!        |j                  |j"                        | _        y )Nr6   r   r	   )r:   r;   r   r0   	self_attnr@   rF   r   input_layernormpost_attention_layernormr1   r   r   r   rt   mlprj   r   )rK   r1   r2   rM   s      r.   r;   zQwen3MoeDecoderLayer.__init__   s    ++"43!zz$*:*:@Q@QR(*

$"3"3)
% 	T111q i!mt7O7O%OST%T-d3DH4++T-C-CDDHr-   rN   rO   rP   rQ   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rr   )r   r   r   r   )rK   rN   rO   rP   rhouts          r.   rb   zQwen3MoeDecoderLayer.__call__   sQ     NN4//2D%@EHHT22156!e
r-   rc   rd   rh   s   @r.   r   r      sa    EY E3 E* $(#	
88
 rxx 
 }	

 

r-   r   c                        e Zd Zdef fdZ	 	 ddej                  deej                     dej                  fdZ xZ	S )Qwen3MoeModelr1   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r1   r2   r6   )r:   r;   r1   r   r   r@   	Embeddingr   embed_tokensranger   layersrF   r   norm)rK   r1   irM   s      r.   r;   zQwen3MoeModel.__init__   s    	//!%!7!7"""LL$:J:JK 4112
2 !da82
 JJt//T5F5FG		
s   Cinputsinput_embeddingsrQ   c                     ||}n| j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r   zipr   )rK   r   rP   r   r   rO   layercs           r.   rb   zQwen3MoeModel.__call__   s     ' A!!&)A=FS--E$Qa1DKK/HE1aq!A 0 yy|r-   rc   )
r$   r%   r&   r   r;   re   rf   r   rb   rg   rh   s   @r.   r   r      sK    HY H  /3	 #288,	
 
r-   r   c                        e Zd Zdef fdZ	 	 d
dej                  deej                     dej                  fdZd Z	e
d        Ze
d	        Z xZS )Modelr1   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rl   )r:   r;   r1   r   r   modelr    r@   rA   r   r   lm_head)rK   r1   rM   s     r.   r;   zModel.__init__   sX    	//"4(
''99T%5%5tUSDL (r-   r   r   rQ   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rr   )r   r1   r    r   	as_linearr   )rK   r   rP   r   r   s        r.   rb   zModel.__call__   sZ     jj(8999((**))33C8C 
 ,,s#C
r-   c                    | j                   j                  r|j                  dd        d|vr|S t        | j                   j                        D ]{  }d| }dD ]o  }| d| d|v st        | j                   j
                        D cg c]  }|j                  | d| d| d       }}t        j                  |      || d	| d<   q } |S c c}w )
Nzlm_head.weightz+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.)ro   rn   rm   z.mlp.experts.0.z.weightz.mlp.experts..z.mlp.switch_mlp.)r1   r    popr   r   r   re   stack)rK   weightslprefixneto_joins          r.   sanitizezModel.sanitize   s    99((KK($/8GNtyy223A$QC(F:X_QCw77B "'tyy'<'<!=!=A  vhmA3as'$JK!=   FHXXgEVGvh&6qcAB ; 4 s   !Cc                     d }|S )Nc                 0    | j                  d      rdddS y)Nzmlp.gate@      )
group_sizebitsT)endswith)path_s     r.   	predicatez(Model.quant_predicate.<locals>.predicate   s    }}Z(&(!44r-   r,   )rK   r   s     r.   quant_predicatezModel.quant_predicate   s    	
 r-   c                 .    | j                   j                  S rr   )r   r   )rK   s    r.   r   zModel.layers  s    zz   r-   rc   )r$   r%   r&   r   r;   re   rf   r   rb   r   propertyr   r   rg   rh   s   @r.   r   r      sx    TY T /3	 #288,	
 
    ! !r-   r   )dataclassesr   typingr   r   r   r   r   mlx.corecorere   mlx.nnr@   activationsr
   r9   r   r   r   switch_layersr   r   Moduler0   rj   rt   r   r   r   r,   r-   r.   <module>r      s    " 3 3    T T $ @ @ @,;#		 ;#|J")) JRYY @299 @!BII !H1!BII 1!r-   