
    (HJj]                        d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZmZmZ ddlmZmZmZ ddlmZ dd	lmZ e G d
 de             Z G d dej6                        Z eej:                  d      d        Z eej:                  d      d        Z eej:                  d      d        Z  G d dej6                        Z! G d dej6                        Z" G d dej6                        Z# G d dej6                        Z$ G d dej6                        Z% G d dej6                        Z& G d d ej6                        Z' G d! d"ej6                        Z(y)#    )	dataclass)partial)AnyDictListOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache
_BaseCache)initialize_rope)	SwitchGLUc                      e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed <   d!Zeed"<   dZee   ed#<   dZeed$<   d%Zeed&<   d'Zeed(<   dZ eed)<   d*Z!eed+<   d,Z"eed-<   dZ#eed.<   dZ$ee   ed/<   dZ%ee   ed0<   dZ&ee   ed1<   dZ'ee(e      ed2<   d,Z)eed3<   d4 Z*y)5	ModelArgsgemma4_text
model_typei   hidden_size#   num_hidden_layersi   intermediate_size   num_attention_heads   head_dimi   global_head_dim      ?global_partial_rotary_factorư>rms_norm_epsi   
vocab_sizevocab_size_per_layer_inputr	   num_key_value_headsNnum_global_key_value_heads   num_kv_shared_layersr   pad_token_idhidden_size_per_layer_inputFrope_traditional      ?partial_rotary_factorrope_parameterssliding_window   sliding_window_patterni   max_position_embeddingsattention_k_eq_vg      >@final_logit_softcappingTuse_double_wide_mlpenable_moe_blocknum_expertstop_k_expertsmoe_intermediate_sizelayer_typestie_word_embeddingsc                     | j                   ddddddddd| _         | j                  Hd	g| j                  d
z
  z  dgz   }|| j                  t	        |      z  d
z   z  d | j                   | _        y y )Nr   g    .Aproportional)r-   
rope_theta	rope_typer,        @default)full_attentionsliding_attentionrC   r	   rB   )r.   r:   r1   r   len)selfpatterns     c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gemma4_text.py__post_init__zModelArgs.__post_init__2   s    ' .2"+!/# .1")!*&$D  #*+t/J/JQ/NO S G !(4+A+AS\+QTU+U V($(( D	 $    )+__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r    floatr"   r#   r$   r%   r&   r   r(   r)   r*   r+   boolr-   r.   r   r/   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r   r;   rH    rI   rG   r   r      sm   #J#Ks!s!  HcOS*. %.L%J&,,  044 "#"L#'**"d"#&5&&*OXd^*NC"#C##)S)"d"%)U) $$"d"!%K#%#'M8C='+/8C=/'+K$s)$+ $$rI   r   c                   f     e Zd ZdZddedef fdZdej                  dej                  fdZ	 xZ
S )	RMSNormNoScalez RMSNorm without learnable scale.dimepsc                 0    t         |           || _        y N)super__init__rV   )rE   rU   rV   	__class__s      rG   rZ   zRMSNormNoScale.__init__L   s    rI   xreturnc                 X    t         j                  j                  |d | j                        S rX   )mxfastrms_normrV   rE   r\   s     rG   __call__zRMSNormNoScale.__call__P   s    ww422rI   )r!   )rJ   rK   rL   __doc__rO   rP   rZ   r_   arrayrc   __classcell__r[   s   @rG   rT   rT   I   s4    *C e 3"(( 3rxx 3rI   rT   T)	shapelessc                 8    t        j                  || z        | z  S rX   )r_   tanh)softcapr\   s     rG   logit_softcaprl   T   s    771w;'))rI   c                 @    | t        j                  |d      z   d|z  z
  S )N   )r_   expand_dims)x2y2xys      rG   _complete_squarert   Y   s!    r2&&R//rI   c                 2    t        j                  |       |z  S rX   )nngelu_approx)gater\   s     rG   geglury   ^   s    >>$!##rI   c                   b     e Zd Zddedef fdZdej                  dej                  fdZ xZ	S )MLPconfig	layer_idxc                    t         |           |j                  |j                  z
  }||cxk\  xr dkD  nc }|j                  xr |}|j
                  |rdndz  }t        j                  |j                  |d      | _	        t        j                  ||j                  d      | _
        t        j                  |j                  |d      | _        y )Nr   ro   r	   Fbias)rY   rZ   r   r(   r5   r   rv   Linearr   	gate_proj	down_projup_proj)rE   r|   r}   first_kv_shared_layer_idxis_kv_shared_layeruse_double_wider   r[   s          rG   rZ   zMLP.__init__d   s    $$v'B'BB 	" '*CGaG 44K9K"44_RST6#5#57HuU#4f6H6HuUyy!3!35FUSrI   r\   r]   c                 t    | j                  t        | j                  |      | j                  |                  S rX   )r   ry   r   r   rb   s     rG   rc   zMLP.__call__q   s)    ~~eDNN1$5t||AGHHrI   )r   )
rJ   rK   rL   r   rO   rZ   r_   re   rc   rf   rg   s   @rG   r{   r{   c   s7    Ty TS TI"(( Irxx IrI   r{   c                   H     e Zd ZdZdef fdZdej                  fdZ xZ	S )Routerz@Expert router: norm -> scale -> project -> top-k -> renormalize.r|   c                 p   t         |           || _        |j                  | _        t        j                  |j                  |j                  d      | _	        t        j                  |j                  f      | _        t        j                  |j                  f      | _        |j                  dz  | _        y )NFr         )rY   rZ   r|   r"   rV   rv   r   r   r7   projr_   onesscaleper_expert_scale
_root_sizerE   r|   r[   s     rG   rZ   zRouter.__init__x   s    &&IIf00&2D2D5Q	WWf0023
 "););(= > ,,d2rI   r\   c                    t         j                  j                  || j                  | j                  z  | j
                        }| j                  |      }t        j                  || j                  j                   d      }|d| j                  j                   d f   }t        j                  ||d      }t        j                  |d      }|| j                  |   z  }||fS )Nrn   )kthaxis.r   )r_   r`   ra   r   r   rV   r   argpartitionr|   r8   take_along_axissoftmaxr   )rE   r\   expert_scorestop_k_indicestop_k_weightss        rG   rc   zRouter.__call__   s    GGQ

T__ <dhhG		! 9 99
 &cDKK,E,E+E+G&GH**=-bQ

=r:%(=(=m(LLm++rI   
rJ   rK   rL   rd   r   rZ   r_   re   rc   rf   rg   s   @rG   r   r   u   s"    J3y 3,"(( ,rI   r   c                       e Zd ZdZd Zy)GeGLUz0GELU-gated linear unit activation for SwitchGLU.c                     t        ||      S rX   )ry   )rE   r\   rx   s      rG   rc   zGeGLU.__call__   s    T1~rI   N)rJ   rK   rL   rd   rc   rR   rI   rG   r   r      s
    :rI   r   c                        e Zd ZdZdef fdZdej                  dej                  dej                  dej                  fdZ xZ	S )	Expertsz*Sparse MoE using SwitchGLU with gather_mm.r|   c                     t         |           t        |j                  |j                  |j
                  t               d      | _        y )NF)
input_dimshidden_dimsr7   
activationr   )rY   rZ   r   r   r9   r7   r   
switch_glur   s     rG   rZ   zExperts.__init__   s=    #))44**w
rI   r\   r   r   r]   c                 z    t        j                  |d      }| j                  ||      }||z  j                  d      S )Nrn   )r_   rp   r   sum)rE   r\   r   r   wys         rG   rc   zExperts.__call__   s6     NN="-OOA}-A{{2rI   r   rg   s   @rG   r   r      sG    4	
y 	
*,((CE88	rI   r   c                        e Zd Zdedef fdZ	 	 	 	 ddej                  deej                     dee	   dee
   dee	   d	ej                  fd
Z xZS )	Attentionr|   r}   c                 ,   t         |           || _        || _        |j                  |   | _        | j
                  dk(  | _        ||j                  |j                  z
  k  | _	        | j
                  dk(  r$t        |d      r|j                  r|j                  n|j                  | _        |j                  }|j                  | _        |j                   xr | j                   | _        | j"                  r|j$                  |j$                  | _        n|j(                  | _        d| _        t-        j.                  || j                  | j                  z  d      | _        | j                  rtt-        j.                  || j&                  | j                  z  d      | _        | j"                  s4t-        j.                  || j&                  | j                  z  d      | _        t-        j.                  | j                  | j                  z  |d      | _        t-        j8                  | j                  |j:                        | _        | j                  rVt-        j8                  | j                  |j:                        | _        tA        | j                  |j:                        | _!        | j                  rdnd}|jD                  jG                  |i       }|jG                  dd	      }tI        | j                  |jJ                  |||jL                  
      | _'        y )NrC   rB   r   r,   Fr   rV   r>   r@   )dimstraditionalbasescaling_configr2   )(rY   rZ   r|   r}   r:   
layer_type
is_slidingr   r(   has_kvhasattrr   r   r   r   n_headsr3   
use_k_eq_vr&   
n_kv_headsr%   r   rv   r   q_projk_projv_projo_projRMSNormr"   q_normk_normrT   v_normr.   getr   r+   r2   rope)rE   r|   r}   rU   	layer_keyrope_paramsr>   r[   s          rG   rZ   zAttention.__init__   sT   " ,,Y7//-@@&":":V=X=X"XX "22 12&& ""  	   11 !11I$//6I??v@@L$??DO$88DO
iiT\\DMM%AN;;))C4==)HuUDK?? ii4==8u iit}} <cNjjF4G4GH;;**T]]8K8KLDK(F<O<OPDK ,0??'@P	,,00B? __\7;
#//&$*$B$B
	rI   r\   maskcache	shared_kvoffsetr]   c                 .   |j                   \  }}}| j                  |      j                  ||| j                  | j                        }	| j                  |	      }	||\  }
}n!| j                  st        d| j                   d      | j                  |      j                  ||| j                  | j                        }
|
}| j                  s7| j                  |      j                  ||| j                  | j                        }|t        j                  |j                        nd}| j!                  |
      }
|
j#                  dddd      }
| j%                  |
|      }
| j'                  |      }|j#                  dddd      }|	j#                  dddd      }	| j%                  |	|      }	||j)                  |
|      \  }
}t+        |	|
||| j,                  |      }|j#                  dddd      j                  ||d	      }| j/                  |      |
|f|fS )
NzLayer z/ is a KV-shared layer but received no shared_kvr   ro   r	      )r   )r   r   r   rn   )shaper   reshaper   r   r   r   
ValueErrorr}   r   r   r   r   r_   re   r   r   	transposer   r   update_and_fetchr   r   r   )rE   r\   r   r   r   r   BL_querieskeysvaluesoutputs                rG   rc   zAttention.__call__   s    ''1a++a.((At||T]]K++g& $LD&((WX  ;;q>))!QODF??Q//1doot}}U/4/@RXXell+aF;;t$D>>!Q1-D99T&91D[[(F%%aAq1F##Aq!Q/))GF)3 11$?LD&-T6djjt
 !!!Q1-55aB?{{6"T6NF::rI   NNNNrJ   rK   rL   r   rO   rZ   r_   re   r   r   tuplerc   rf   rg   s   @rG   r   r      s    4
y 4
S 4
r $(#%) $-;88-; rxx -; }	-;
 E?-; -; 
-;rI   r   c                        e Zd Zdedef fdZ	 	 	 	 	 ddej                  deej                     dee	   deej                     dee
   d	ee	   d
ej                  fdZ xZS )DecoderLayerr|   r}   c                    t         |           || _        || _        |j                  |   | _        t        ||      | _        t        ||      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        |j$                  | _        | j&                  rt)        |      | _        t-        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        |j6                  | _        | j6                  rt        j8                  |j                  | j6                  d      | _        t        j8                  | j6                  |j                  d      | _        t        j                  |j                  |j                        | _        nd | _        d | _        d | _        tA        jB                  d      | _"        y )Nr   Fr   )r	   )#rY   rZ   r|   r}   r:   r   r   	self_attnr{   mlprv   r   r   r"   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr6   
enable_moer   routerr   expertspost_feedforward_layernorm_1post_feedforward_layernorm_2pre_feedforward_layernorm_2r*   r   per_layer_input_gateper_layer_projectionpost_per_layer_input_normr_   r   layer_scalar)rE   r|   r}   r[   s      rG   rZ   zDecoderLayer.__init__  s   " ,,Y7"695vy)!zz&*<*<&BUBUV(*

F$7$7)
% *,F$7$7*
& +-**F$7$7+
'
 !11?? .DK"6?DL02

""(;(;1D- 13

""(;(;1D- 02zz""(;(;0D,
 ,2+M+M(++(*		""D$D$D5)D% )+		00&2D2D5)D% .0ZZ""(;(;.D* )-D%(,D%-1D* GGDMrI   r\   r   r   per_layer_inputr   r   r]   c                 n   |}| j                  |      }| j                  |||||      \  }}}| j                  |      }||z   }|}| j                  r| j	                  |      }	| j                  |	      }	| j                  |	      }	| j                  |      \  }
}| j                  |      }| j                  ||
|      }| j                  |      }|	|z   }n"| j	                  |      }| j                  |      }| j                  |      }||z   }| j                  | j                  s| j                  g|e|}| j                  |      }t        j                   |      }t#        j$                  ||      }| j                  |      }| j                  |      }||z   }| j&                  || j&                  z  }|||fS )N)r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rv   rw   r_   multiplyr   )rE   r\   r   r   r   r   r   residualhh1r   r   h2rx   s                 rG   rc   zDecoderLayer.__call__M  s      ##~~tUi  .  
9f ))!,qL??//2B"B2226B+/;;q>(M=11!4Bb-?B2226BRA..q1AA++A.qL %%1))5..:+H,,Q/D>>$'D;;t_5D,,T2D11$7D4A(D%%%A)V##rI   )NNNNNr   rg   s   @rG   r   r     s    3*y 3*S 3*p $(#.2%) $8$888$ rxx 8$ }	8$
 ""((+8$ E?8$ 8$ 
8$rI   r   c                   J    e Zd Zdef fdZ	 ddeej                     deej                     dej                  fdZ	 ddej                  deej                     dej                  fdZ	d	 Z
	 	 	 	 dd
ej                  deej                     deej                     fdZ xZS )Gemma4TextModelr|   c                    t         |           || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        |j                  dz  | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                   |j                  |j"                        | _        |j&                  | _        | j&                  rt        j                  |j(                  |j                  |j&                  z        | _        |j&                  dz  | _        d| _        |j                  dz  | _        t        j2                  |j                  |j                  |j&                  z  d      | _        t        j                   |j&                  |j"                        | _        n#d | _        d | _        d | _        d | _        d | _        t9        t        t;        | j                                    | _        |j>                  dkD  rt;        | j                        }||j>                  z
  }i }t        |      D ]  }||| j                  |   j@                  <     t        ||      D ]+  }|| j                  |   j@                     | j<                  |<   - y y c c}w )	Ng      ?)r}   r   g;f?r   Fr   r   )!rY   rZ   r|   r#   r/   window_sizer1   r   rv   	Embeddingr   embed_tokensembed_scaleranger   layersr   r"   normr*   r$   embed_tokens_per_layerembed_tokens_per_layer_scaleper_layer_input_scaleper_layer_projection_scaler   per_layer_model_projectionper_layer_projection_normlistrD   previous_kvsr(   r   )rE   r|   iNMkvs_by_typejr[   s          rG   rZ   zGemma4TextModel.__init__  su    ++!00&,&C&C#!'!9!9LL):):F<N<NO!--s27<V=U=U7V
7V!L1-7V
 JJv11v7J7JK	 ,2+M+M(++*,,,11((6+M+MM+D' 170R0RTW0WD-)2D&.4.@.@$.FD+.0ii""((6+M+MM/D+
 .0ZZ228K8K.D* +/D')-D&.2D+.2D+-1D* !s4;;'7!89&&*DKK AF///AK1X9:DKKN556 1a['24;;q>3L3L'M!!!$ ! +A
s   5K$	input_idsinput_embeddingsr]   c                    ||t        d      | j                  j                  j                         j	                  d      }|j                         j	                  d      }t        ||| j                  j                  |            }t        j                  |d      }| j                  |      }|| j                  z  }t        j                  |d| j                  j                  | j                  f      S )Nz=input_embeddings must be provided when input_ids are omitted.rn   )RuntimeErrorr   weightsquarer   rt   	as_linearr_   argminr  r  	unflattenr|   r   r*   )rE   r  r  norms_embeddingnorms_inputdistanceresults          rG   _get_per_layer_inputsz%Gemma4TextModel._get_per_layer_inputs  s    
 '"S  #//66==?CCBGO*11377;K'!!++,<=H 		(B/I,,Y7$;;;||[[**D,L,LM
 	
rI   per_layer_inputsc                     | j                  |      }|| j                  z  }t        j                  |d| j                  j
                  | j                  f      }| j                  |      }||S ||z   | j                  z  S )Nrn   )	r  r  r_   r  r|   r   r*   r  r  )rE   r  r  r   s       rG   _project_per_layer_inputsz)Gemma4TextModel._project_per_layer_inputs  s    
  $>>?OP3d6U6UU!|| [[**D,L,LM 

  $==>RS#''$'774;U;UUUrI   c                 4   i }g }t        | j                  |      D ]z  \  }}|j                  |vrI|j                  dk(  rt        ||      |d<   n*|j                  dk(  rt        ||| j                        |d<   |j                  ||j                            | |S )NrB   rC   )r   )zipr   r   r   r   append)rE   r   r   r   maskslcs          rG   _make_maskszGemma4TextModel._make_masks  s    U+DAq||4'<<#33-B1a-HD)*\\%880E1$*:*:1D,- LLall+, , rI   inputsc           
         || j                  |      }|}|| j                  z  }| j                  r&|| j                  ||      }| j	                  ||      }|5t        | j                        D cg c]  \  }}|d d d d |d d f    }}}nd gt        | j                        z  }|d gt        | j                        z  }n(|d gt        | j                        t        |      z
  z  z   }| j                  ||      }dgt        | j                        z  }	t        t        | j                  ||| j                  |            D ],  \  }
\  }}}}}|	|   \  }} |||||||      \  }}}||f|	|
<   . | j                  |      S c c}}w )N)NN)r   r   r   )r   r   r*   r  r  	enumerater   rD   r%  r   r  r   )rE   r&  r   r  r  r   r	  r   r"  intermediatesidxlayerr$  r   prev_idxr   kvsr   s                     rG   rc   zGemma4TextModel.__call__  s    ##008    ++'#'#=#=fFV#W #==aAQR'9B4;;9O 9OA Aq!,9O    !%vDKK(88 =FS--ETFc$++&6U&CDDE   E*%T[[)99@I!! A
<C<%D(O (1KC" /NAsF #&vM#)A
, yy|O s   1FrX   r   )rJ   rK   rL   r   rZ   r   r_   re   r  r  r%  rc   rf   rg   s   @rG   r   r     s    1Ny 1Nl 04&
BHH%&
 #288,&
 
	&
V 04V((V #288,V 
	V&   /3/3:: #288,	:
 #288,:rI   r   c                        e Zd Zdef fdZ	 	 	 ddej                  deej                     deej                     fdZd Z	e
d        Ze
d	        Ze
d
        Ze
d        Zd Z xZS )Modelargsc                 2   t         |           || _        |j                  | _        t	        |      | _        |j                  | _        |j                  | _        | j                  s2t        j                  |j                  |j                  d      | _        y y )NFr   )rY   rZ   r0  r   r   modelr4   r;   rv   r   r   r#   lm_head)rE   r0  r[   s     rG   rZ   zModel.__init__C  st    	//$T*
'+'C'C$#'#;#; ''99T%5%5tUSDL (rI   r&  r  r  c                     | j                  ||||      }| j                  r&| j                   j                  j                  |      }n| j	                  |      }| j
                  t        | j
                  |      }|S )N)r   r  r  )r2  r;   r   r  r3  r4   rl   )rE   r&  r   r  r  outs         rG   rc   zModel.__call__M  sy     jj--	  
 ##**))33C8C,,s#C''3 < <cBC
rI   c           	         i }|j                         D ]  \  }t        fddD              rj                  d      rUj                  d      }t	        t
        j                  t        j                  |dd            \  }}||| d<   ||| d	<   j                  d
      rj                  d      }||| d<   ||<    |S )Nc              3   &   K   | ]  }|v  
 y wrX   rR   ).0sks     rG   	<genexpr>z!Model.sanitize.<locals>.<genexpr>e  s!      	A Qs   )zself_attn.rotary_emb	input_max	input_min
output_max
output_minz.experts.gate_up_projz.gate_up_projro   r   r   z.switch_glu.gate_proj.weightz.switch_glu.up_proj.weightz.experts.down_projz
.down_projz.switch_glu.down_proj.weight)itemsanyendswithremovesuffixmapr_   
contiguoussplit)rE   weights	sanitizedvr   rx   upr:  s          @rG   sanitizezModel.sanitizeb  s    	MMODAq 		 	 zz12~~o6r}}bhhq!".EFbCG	TF">?@AC	TF"<=>zz./~~l3CD	TF">?@IaL3 $6 rI   c                     d }|S )Nc                 0    | j                  d      rdddS y)Nzrouter.proj@   r   )
group_sizebitsT)rB  )pathr   s     rG   	predicatez(Model.quant_predicate.<locals>.predicate  s    }}]+&(!44rI   rR   )rE   rR  s     rG   quant_predicatezModel.quant_predicate  s    	
 rI   c                 .    | j                   j                  S rX   )r2  r   rE   s    rG   r   zModel.layers  s    zz   rI   c                 .    | j                   j                  S rX   )r0  r   rU  s    rG   r   zModel.head_dim  s    yy!!!rI   c                 .    | j                   j                  S rX   )r0  r%   rU  s    rG   r   zModel.n_kv_heads  s    yy,,,rI   c                 P   | j                   j                  | j                   j                  z
  }g }t        |      D ]h  }| j                   j                  |   dk(  r|j                  t                      9|j                  t        | j                   j                  d             j |S )NrB   r   )max_sizekeep)	r0  r   r(   r   r:   r!  r   r   r/   )rE   first_kv_sharedcachesr	  s       rG   
make_cachezModel.make_cache  s    ))55		8V8VV'Ayy$$Q'+;;gi(#!%!9!9	 ( rI   )NNN)rJ   rK   rL   r   rZ   r_   re   r   rc   rK  propertyrS  r   r   r   r]  rf   rg   s   @rG   r/  r/  B  s    TY T /3/3 #288,	
 #288,*>   ! ! " " - -rI   r/  ))dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorer_   mlx.nnrv   r   r
   r   r   r   r   r   r   
rope_utilsr   switch_layersr   r   ModulerT   compilerl   rt   ry   r{   r   r   r   r   r   r   r/  rR   rI   rG   <module>ri     sA   "  , ,   T T 7 7 ' $ 5 5 5p3RYY 3 	t$* %* 	t$0 %0 	t$$ %$I")) I$,RYY ,:BII bii .d;		 d;Nn$299 n$bwbii wtaBII arI   