
    (HJj5                        d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	m
Z d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ e G d de             Z G d dej:                        Z G d dej:                        Zej@                  d        Z! G d dej:                        Z" G d dej:                        Z# G d dej:                        Z$ G d deej:                        Z% G d dej:                        Z&y)    N)	dataclass)partial)AnyDictOptional)shard_inplaceshard_linearsum_gradients   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)PipelineMixin)	SwitchGLUc                   *   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee	   ed<   eed<   eed<   eed<   eed<   dZ
eed<   dZeed<   y)	ModelArgs
model_type
vocab_sizehidden_sizeintermediate_sizemax_position_embeddingsmoe_intermediate_sizenorm_topk_probnum_attention_headsn_grouphead_dim
topk_groupn_shared_expertsn_routed_expertsrouted_scaling_factornum_experts_per_tokfirst_k_dense_replacenum_hidden_layersnum_key_value_headsrms_norm_eps
rope_thetarope_scalinguse_qk_normtie_word_embeddingsattention_biaspartial_rotary_factorsigmoidscoring_funcnoaux_tctopk_methodN)__name__
__module____qualname__str__annotations__intboolfloatr   r   r.   r0        `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/glm4_moe.pyr   r      s    OO  LMO  4.   !L#!!K!r:   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 :   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  }|dz  | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  |d      | _        |j                   | _        | j                   rLt        j"                  ||j$                        | _        t        j"                  ||j$                        | _        t        j*                  t-        ||j.                  z        d|j0                        | _        y )Ng      ࿩biasFeps)traditionalbase)super__init__r   r   n_headsr%   
n_kv_headsr   scalennLinearr+   q_projk_projv_projo_projr)   RMSNormr&   q_normk_normRoPEr6   r,   r'   rope)selfr>   dimrH   rI   r   	__class__s         r;   rG   zAttention.__init__2   s;   !%!9!99w'+'?'??*==t^
iiWx%7d>Q>QRiiZ(%:ATATUiiZ(%:ATATUii( 2CeD++**X43D3DEDK**X43D3DEDKGG45556
	r:   xmaskcachereturnc                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      }|j	                  ||| j                  d      }| j                  r"| j                  |      }| j                  |      }|j                  dddd      }|j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j!                  |
      S )Nr      r      )offset)r[   rJ   rZ   )shaperM   rN   rO   reshaperH   rI   r)   rR   rS   	transposerU   ra   update_and_fetchr   rJ   rP   )rV   rY   rZ   r[   BLDquerieskeysvaluesoutputs              r;   __call__zAttention.__call__L   s    ''1a $AAAv//!Qb9||Aq$//26kk'*G;;t$D##Aq!Q/~~aAq)1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r:   NNr1   r2   r3   r   rG   mxarrayr   r   rm   __classcell__rX   s   @r;   r=   r=   1   sW    
Y 
: $(#	 #88 # rxx  # }	 #
 
 #r:   r=   c                   6     e Zd Z	 ddededef fdZd Z xZS )MLPconfigr   r   c                    t         |           || _        ||j                  n|| _        ||j                  n|| _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        y NFr@   )
rF   rG   rv   r   r   rK   rL   	gate_projup_proj	down_proj)rV   rv   r   r   rX   s       r;   rG   zMLP.__init__p   s     	1<1D6--+(9(AF$$GX 	 4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr:   c                 x    | j                  t        | j                  |      | j                  |                  }|S N)r{   r   ry   rz   )rV   rY   r{   s      r;   rm   zMLP.__call__~   s.    NN6$..*;T\\!_#MN	r:   rn   )r1   r2   r3   r   r6   rG   rm   rr   rs   s   @r;   ru   ru   o   s,    SWYY.1YMPYr:   ru   c                    t        j                  | j                  t         j                              }|}||z   }|dkD  rt        j                  |d|df      }t        j
                  |dd      j                  dd      }	||z
  }
t        j                  |	|
dz
  d	      d
d |
d d f   }t        j                  |t        j                  |      t        j                  d      d      }t        j                  |dd      }|}
t        j                  | |
dz
  d	      d
d |
f   }t        j                  ||d      }|dkD  r|r|j                  dd      }||z  }||z  }||fS )Nr   r^   )axisrb   r_   r   T)r   keepdims)kthr   .g        )rp   r-   astypefloat32	unflattentopksumargpartitionput_along_axisstop_gradientrq   flattentake_along_axis)gatese_score_correction_biastop_kr   r   r!   r   scoresorig_scoresgroup_scoresk	group_idxindsdenominators                 r;   group_expert_selectr      sU    ZZRZZ01FK--F{f2gr]Cwwvqr266B6Nj OOLa!e"Ec2A2qjQ	""B$$Y/#R
 FB+A??F7AB7RaR@DT;Fqy^jjb4j8+%++F<r:   c                   *     e Zd Zdef fdZd Z xZS )MoEGaterv   c                    t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f      | _        t        j                  | j                  f      | _        |j                  dk(  sJ d       y )Nr/   zUnsupported topk method.)rF   rG   rv   r"   r   r   r    r!   r   r   rp   zerosr   weightr   r0   rV   rv   rX   s     r;   rG   zMoEGate.__init__   s    //
$33 & 7 7%+%A%A"~~ ++hh 5 5v7I7IJK')xx1F1F0H'I$!!Z/K1KK/r:   c           	          t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                        S r}   )	r   r   Tr   r   r   r   r!   r   )rV   rY   s     r;   rm   zMoEGate.__call__   sO    "((JJLLOO&&
 	
r:   r1   r2   r3   r   rG   rm   rr   rs   s   @r;   r   r      s    Ly L	
r:   r   c                   *     e Zd Zdef fdZd Z xZS )MoErv   c                 N   t         |           || _        |j                  | _        t	        |j
                  |j                  |j                        | _        t        |      | _
        |j                  +|j                  |j                  z  }t        ||      | _        d | _        y )N)rv   r   )rF   rG   rv   r"   r   r   r   r    
switch_mlpr   gater   ru   shared_expertssharding_group)rV   rv   r   rX   s      r;   rG   zMoE.__init__   s    #)#=#= #((##
 FO	"". & < <v?V?V V"%1B#D #r:   c                    | j                    t        | j                         |      }| j                  |      \  }}| j                  ||      }||d   z  j	                  d      j                  |j                        }| j                  j                  || j                  |      z   }| j                   +t        j                  j                  || j                         }|S )N).Nr   r   group)r   r
   r   r   r   r   dtyperv   r   r   rp   distributedall_sum)rV   rY   r   r   ys        r;   rm   zMoE.__call__   s    *2d11215Ayy|fOOAt$	""''R'077@;;''3D''**A*&&q0C0C&DAr:   r   rs   s   @r;   r   r      s    #y #&r:   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
DecoderLayerrv   	layer_idxc                 n   t         |           t        |      | _        |j                  ||j
                  k\  rt        |      n
t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y NrB   )rF   rG   r=   	self_attnr    r#   r   ru   mlprK   rQ   r   r&   input_layernormpost_attention_layernorm)rV   rv   r   rX   s      r;   rG   zDecoderLayer.__init__   s    "6* ''3!=!== K
 V 	  "zz&*<*<&BUBUV(*

F$7$7)
%r:   rY   rZ   r[   r\   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S r}   )r   r   r   r   )rV   rY   rZ   r[   rhs         r;   rm   zDecoderLayer.__call__   sL     NN4//2D%@EHHT221561ur:   rn   )r1   r2   r3   r   r6   rG   rp   rq   r   r   rm   rr   rs   s   @r;   r   r      s^    
y 
S 
& $(#		88	 rxx 	 }		
 
	r:   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )LanguageModelrv   c                 p   t         |           |j                  | _        t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        y c c}w r   )rF   rG   r   rK   	Embeddingr   embed_tokensranger$   r   layersrQ   r&   norm)rV   rv   idxrX   s      r;   rG   zLanguageModel.__init__  s     ++LL):):F<N<NO16v7O7O1P
1P#L%1P
 JJv11v7J7JK	
s   'B3rY   r[   r\   c                    | j                  |      }| j                  }| j                  }|d gt        | j                        z  }t        ||d         }||dz
  k  r#t        j                  j                  ||dz         }t        | j                  |      D ]  \  }} ||||      } |dk7  rVt        j                  j                  ||dz
  |z        }|d   +t        j                  |d   j                  |      |d   _        |dkD  r/t        j                  j                  |      d |j                  d    }| j                  |      S )Nr   r   )r[   r^   )r   pipeline_rankpipeline_sizelenpipeline_layersr   rp   r   	recv_likezipsenddependsrj   
all_gatherrb   r   )	rV   rY   r[   r   r   r   rZ   lcs	            r;   rm   zLanguageModel.__call__  s6   
 a ****=FS!5!566E$Qa1 =1,,((]Q->@A,,e4DAq!T#A 5 A##A(9]'JKARy$!#E"INNA!>b	 1))!,\qwwqz:Ayy|r:   r}   ro   rs   s   @r;   r   r     sC    Ly L  $88 } 
	r:   r   c                        e Zd Zdef fdZ	 ddej                  dee   fdZ	d Z
ddeej                  j                     fdZed	        Zed
        Z xZS )Modelrv   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y rx   )rF   rG   r>   r   r   modelrK   rL   r   r   lm_headr   s     r;   rG   zModel.__init__.  sN    	 ++"6*
yy!3!3V5F5FUSr:   inputsr[   c                 H    | j                  ||      }| j                  |      S r}   )r   r   )rV   r   r[   outs       r;   rm   zModel.__call__5  s#    
 jj'||C  r:   c                    | j                   j                  }t        | j                   j                        D ]  }d| }dD ]  \  }}dD ]u  }| d| d| |v st        | j                   j                        D cg c]  }|j	                  | d| d| d|         }	}t        j                  |	      || d| d| <   w   |j                         D 
ci c]  \  }}
|j                  d|       s||
 c}
}S c c}w c c}
}w )Nzmodel.layers.))w1ry   )w2r{   )w3rz   )r   scalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.)	r>   r$   r   r    poprp   stackitems
startswith)rV   weights	mpt_layerr   prefixnmr   eto_joinvs              r;   sanitizezModel.sanitize=  s1   II//	 tyy223A$QC(FU17A 1QC8GC &+499+E+E%F#%F $KK6(-s!A3as(KL%F   # GIhhwFW6(*:1#Qqc BC 8 V 4  
'1<<-	{ ;< qD'
 	
#
s   9#C<
!Dr   c                 6   |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  t               rt        |j                  j"                  d|      |j                  _        t        |j                  j$                  d|      |j                  _        t        |j                  j&                  d|      |j                  _        ||j                  _        t+        |j                  j,                  j"                  d|       t+        |j                  j,                  j$                  d|       t+        |j                  j,                  j&                  d|       t+        |j                  j.                  j"                  d|       t+        |j                  j.                  j$                  d|       t+        |j                  j.                  j&                  d|        y )Nzall-to-shardedr   zsharded-to-all)rp   r   initsizer   r   r	   r   rM   rN   rO   rP   rH   rI   
isinstancer   ru   ry   r{   rz   r   r   r   r   )rV   r   Nlayers       r;   shardzModel.shardS  sN   .,,.JJLZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO##)#OO&&1,& %))S)&2II'')9'		# '3II'')9'		# %1II%%'7u%		! ,1		(II,,668HPU II,,668HPU II,,446Fe II((224DE II((224DE II((002B%a 'r:   c                 .    | j                   j                  S r}   )r   r   )rV   s    r;   r   zModel.layers  s    zz)))r:   c                     d }|S )Nc                 
    d| vS )Nr   r9   )r   s    r;   	predicatez'Model.cast_predicate.<locals>.predicate  s    ,A55r:   r9   )rV   r   s     r;   cast_predicatezModel.cast_predicate  s    	6 r:   r}   )r1   r2   r3   r   rG   rp   rq   r   r   rm   r   r   Groupr   propertyr   r   rr   rs   s   @r;   r   r   -  s~    Ty T  $!! }!
,58BNN$8$89 5n * *  r:   r   )'mathdataclassesr   	functoolsr   typingr   r   r   mlx.corecorerp   mlx.nnrK   mlx.nn.layers.distributedr   r	   r
   activationsr   rE   r   r   r   pipeliner   switch_layersr   r   Moduler=   ru   compiler   r   r   r   r   r   r9   r:   r;   <module>r     s     !  & &   P P  T T # $ " " "<;#		 ;#|")) (  D
bii 
4!")) !H299 :)M299 )XfBII fr:   