
    (HJjrS                        d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZmZ ddlmZmZmZ ddlmZmZ e G d	 d
e             Ze G d de             Z G d dej6                        Z G d dej6                        Z G d dej6                        Z eej>                  d      d        Z  G d dej6                        Z! G d dej6                        Z" G d dej6                        Z# eej>                  d      d        Z$ G d dej6                        Z% G d d ej6                        Z& G d! d"ej6                        Z'y)#    N)	dataclass)partial)AnyDictListOptional)tree_flattentree_unflatten   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCachec                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed<   ee   ed<   eed<   eed<   eed<   eed<   eed<   eed<   dZ	e
e   ed<   y)
TextConfig
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headshead_dimrms_norm_eps
vocab_sizenum_key_value_headsnum_kv_shared_layersvocab_size_per_layer_inputsliding_windowmax_position_embeddingsrope_local_base_freq
rope_thetafinal_logit_softcappinglayer_typesactivation_sparsity_patternhidden_size_per_layer_inputaltup_num_inputsaltup_coef_clipaltup_correct_scalealtup_active_idxlaurel_rankNrope_scaling)__name__
__module____qualname__str__annotations__intfloatr   boolr+   r   r        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gemma3n.pyr   r      s    OMO ##  ""c!%e,!$$#'L(4.'r5   r   c                   "    e Zd ZU eed<   eed<   y)	ModelArgsr   text_configN)r,   r-   r.   r/   r0   dictr4   r5   r6   r8   r8   -   s    Or5   r8   c                   ,     e Zd Zddef fdZd Z xZS )
RMSNoScaleepsc                 0    t         |           || _        y N)super__init__r=   )selfr=   	__class__s     r6   rA   zRMSNoScale.__init__4   s    r5   c                 X    t         j                  j                  |d | j                        S r?   )mxfastrms_normr=   )rB   xs     r6   __call__zRMSNoScale.__call__8   s    ww422r5   )gh㈵>)r,   r-   r.   r2   rA   rI   __classcell__rC   s   @r6   r<   r<   3   s    E 3r5   r<   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )Gemma3nLaurelBlockz Learned Augmented Residual Layerconfigc                    t         |           || _        t        j                  | j                  j
                  | j                  j                  d      | _        t        j                  | j                  j                  | j                  j
                  d      | _        t        j                  | j                  j
                  | j                  j                        | _        y )NFbiasdimsr=   )r@   rA   rN   nnLinearr   r*   linear_leftlinear_rightRMSNormr   post_laurel_normrB   rN   rC   s     r6   rA   zGemma3nLaurelBlock.__init__?   s    99KK##T[[%<%<5
 IIKK##T[[%<%<5
 !#

((((!
r5   rH   returnc                 r    | j                  |      }| j                  |      }| j                  |      }||z   S r?   )rV   rW   rY   )rB   rH   laurel_xnormed_laurel_xs       r6   rI   zGemma3nLaurelBlock.__call__N   s=    ##A&$$X.//9?""r5   )
r,   r-   r.   __doc__r   rA   rE   arrayrI   rJ   rK   s   @r6   rM   rM   <   s-    *
z 
#"(( #rxx #r5   rM   c            	            e Zd Zdededef fdZ	 	 d
dej                  de	ej                     de	e
   dej                  fd	Z xZS )Gemma3nAttentionrN   	layer_idxis_kv_shared_layerc                    t         |           |j                  |   dk(  | _        |j                  }|j
                  x| _        }|j                  x| _        }||z  | _	        |j                  x| _
        }|| _        d| _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j&                  |j                  |j(                        | _        t        j&                  |j                  |j(                        | _        t/        |j(                        | _        || _        t        j4                  |d| j                  r|j6                  n|j8                        | _        y )Nsliding_attention      ?FrP   rR   r=   )traditionalbase)r@   rA   r#   
is_slidingr   r   n_headsr   
n_kv_headsrepeatsr   rc   scalerT   rU   q_projk_projv_projo_projrX   r   q_normk_normr<   v_normrd   RoPEr    r!   rope)	rB   rN   rc   rd   dimrl   rm   r   rC   s	           r6   rA   zGemma3nAttention.__init__V   sb    ,,Y7;NN  !'!;!;;w'-'A'AA**,#)??2"
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeDjjfoo6;N;NOjjfoo6;N;NO V%8%89"4GG/3++FDUDU	
	r5   rH   maskcacher[   c                    |j                   \  }}}| j                  |      }|j                  ||d| j                        }| j	                  |      }d}| j
                  r||j                  \  }	}
|j                  }n||j                  }| j                  |      j                  ||d| j                        }	| j                  |	      }	|	j                  dddd      }	| j                  |	|      }	| j                  |      j                  ||d| j                        }
| j                  |
      }
|
j                  dddd      }
||j                  |	|
      \  }	}
|j                  dddd      }| j                  ||      }t        ||	|
|| j                   |      }|j                  dddd      j                  ||d      }| j#                  |      S )Nr      r      )offset)r{   ro   rz   )shaperp   reshaper   rt   rd   stater   rq   ru   	transposerx   rr   rv   update_and_fetchr   ro   rs   )rB   rH   rz   r{   BL_queriesr   keysvaluesoutputs               r6   rI   zGemma3nAttention.__call__v   s    ''1a++a.//!QDMM:++g&""u'8 ;;LD&\\F  ;;q>))!QDMMBD;;t$D>>!Q1-D99T&91D[[^++Aq"dmmDF[[(F%%aAq1F $55dFCf##Aq!Q/))GF)3-T6djjt
 !!!Q1-55aB?{{6""r5   NNr,   r-   r.   r   r1   r3   rA   rE   r`   r   r   rI   rJ   rK   s   @r6   rb   rb   U   sf    
z 
c 
t 
F $(#	*#88*# rxx *# }	*#
 
*#r5   rb   T)	shapelessc                     t        j                  | dd      }t        j                  | dd      }|||j                  |j                        z  z   }t        j                  t        j                  d| |z
              S )Nr}   Taxiskeepdimsr   )rE   meanstdastypedtyperT   gelu_approxmaximum)inputsstd_multiplierinputs_mean
inputs_stdcutoff_xs        r6   	gelu_topkr      se    ''&rD9KR$7JZ.*?*?
@P@P*QQQH>>"**Q(9:;;r5   c                   J     e Zd Zddedef fdZdej                  fdZ xZ	S )MLPrN   rc   c                    t         |           || _        |j                  | _        t	        |j
                  t              r|j
                  |   n|j
                  | _        t        j                  | j                  | j
                  d      | _	        t        j                  | j                  | j
                  d      | _
        t        j                  | j
                  | j                  d      | _        |j                  |j                  |   | _        nd| _        | j                  dkD  rAt        j                  d      t!        j"                  d| j                  z  dz
        z  | _        y y )NFrP           r   g       @r~   r   )r@   rA   rN   r   
isinstancer   listrT   rU   	gate_projup_proj	down_projr$   activation_sparsitymathsqrtrE   erfinv_std_multiplier)rB   rN   rc   rC   s      r6   rA   zMLP.__init__   s(   !-- &22D9 $$Y/)) 	
 4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWX--9'-'I'I)'TD$'*D$##a'#'99S>BIID,,,q05 $D  (r5   rH   c                     | j                  |      }| j                  dkD  rt        || j                        }nt	        j
                  |      }| j                  |      }| j                  ||z        }|S )Nr   )r   r   r   r   rT   r   r   r   )rB   rH   r   activationsr   r   s         r6   rI   zMLP.__call__   sd    NN1%	##c)#It/C/CDK..3K,,q/NN;#89	r5   )r   )
r,   r-   r.   r   r1   rA   rE   r`   rI   rJ   rK   s   @r6   r   r      s&    z c *"(( r5   r   c                        e Zd ZdZdef fdZdej                  dej                  fdZdej                  dej                  fdZ	dej                  d	ej                  fd
Z
 xZS )Gemma3nAltUpzAlternating Updates (AltUp)rN   c                    t         |           || _        t        j                  | j                  j
                  f      | _        t        j                  | j                  j                  | j                  j                  d      | _
        t        j                  | j                  j                  | j                  j                  dz  d      | _        t        j                  | j                  j
                  | j                  j                  d      | _        t        j                  | j                  j
                  | j                  j                        | _        y )NFrP   r~   rR   )r@   rA   rN   rE   zerosr   correct_output_scalerT   rU   r&   correction_coefsprediction_coefsmodality_routerrX   r   router_normrZ   s     r6   rA   zGemma3nAltUp.__init__   s    $&HHdkk.E.E-G$H! "		KK(($++*F*FU!
 !#		KK(($++*F*F*IPU!
  "yyKK##T[[%A%A 
 ::((((
r5   rH   r[   c                     | j                  |      | j                  j                  dz  z  }| j                  |      j	                  t
        j                        }t        j                  |      S )Ng      )r   rN   r   r   r   rE   float32tanh)rB   rH   router_inputsrouteds       r6   compute_router_modalitiesz&Gemma3nAltUp.compute_router_modalities   sS    ((+t{{/F/F/LM%%m4;;BJJGwwvr5   c                    | j                  || j                  j                           }| j                  j                  j                  t        j                        | j                  _        | j                  j                  ct        j                  | j                  j                  | j                  j                   | j                  j                        | j                  _         | j                  |      j                  g |j                  d d | j                  j                  | j                  j                   j                  dddd      }|j                  t        j                        }|j                  dddd      }t        j                  ||      }|j                  dddd      }||z  }|j                  |j                        S )Nr}   r   r   r   r~   )r   rN   r)   r   weightr   rE   r   r'   clipr   r   r&   r   matmulr   )rB   rH   
modalities	all_coefsx_up
x_permutedpredictionss          r6   predictzGemma3nAltUp.predict   sz   33Adkk6R6R4ST
'+'<'<'C'C'J'J2::'V$;;&&2+-77%%,,,,,++,D!!(D!!*-W!!#2& ,, ,,	 Yq!Q" 	 xx

#^^Aq!Q/
ii
I6!++Aq!Q7t!!!''**r5   r   	activatedc                 x   | j                  |      }| j                  j                  j                  t        j
                        | j                  _        | j                  j                  ct	        j                  | j                  j                  | j                  j                   | j                  j                        | j                  _        | j                  |      dz   }|| j                  j                     }||z
  }|j                  dd      }|d    |d   z  }||z  }|j                  |j                        S )Nrg   r~   r   ).N)r   r   r   r   rE   r   rN   r'   r   r)   moveaxisr   )rB   r   r   r   r   active_x
innovation	correcteds           r6   correctzGemma3nAltUp.correct  s   33I>
'+'<'<'C'C'J'J2::'V$;;&&2+-77%%,,,,,++,D!!( ))*5;	t{{;;<)
&&q!,	t$y';;	[ 		00r5   )r,   r-   r.   r_   r   rA   rE   r`   r   r   r   rJ   rK   s   @r6   r   r      sc    %
z 
&288  
+ +bhh +:1288 1 1r5   r   c            
            e Zd Zdededef fdZ	 	 	 d
dej                  de	ej                     de	e
   de	ej                     fd	Z xZS )Gemma3nDecoderLayerrN   rc   rd   c                    t         |           || _        |j                  | _        || _        t        |||      | _        t        ||      | _        t        j                  | j                  |j                        | _        t        j                  | j                  |j                        | _        t        j                  | j                  |j                        | _        t        j                  | j                  |j                        | _        | j                  j                   | _        |j"                  | _        t%        |      | _        t)        |      | _        t        j,                  | j                  | j"                  d      | _        t        j,                  | j"                  | j                  d      | _        t        j                  | j                  |j                        | _        y )N)rc   rh   FrP   )r@   rA   rN   r   rc   rb   	self_attnr   mlprT   rX   r   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormrk   r%   r   altuprM   laurelrU   per_layer_input_gateper_layer_projectionpost_per_layer_input_norm)rB   rN   rc   rd   rC   s       r6   rA   zGemma3nDecoderLayer.__init__  s   !--")&)=OPv3!zz## 

 )+

##)
% *,##*
& +-**##+
' ..33+1+M+M(!&)
(0$&IId>>U%
! %'II,,d.>.>U%
! *,##*
&r5   rH   rz   r{   per_layer_inputc                    | j                   j                  |      }|| j                  j                     }| j	                  |      }| j                  |      }| j                  |||      }	| j                  |	      }	||	z   }
|
|z   dz  }| j                  |      }| j                  |      }| j                  |      }||z   }| j                   j                  ||      }|| j                  j                     }| j                  j                  r|| j                   j                  z  }| j                  |      }t        j                   |      }t#        j$                  ||      }| j'                  |      }| j)                  |      }|dd  |z   |dd  |S )N;f?r   )r   r   rN   r)   r   r   r   r   r   r   r   r   r(   r   r   rT   r   rE   multiplyr   r   )rB   rH   rz   r{   r   r   active_predictionactive_prediction_normedlaurel_outputattn
attn_gatedattn_laurel	attn_normattn_ffwattn_ffw_normattn_ffw_laurel_gatedcorrected_predictionsfirst_predictions                     r6   rI   zGemma3nDecoderLayer.__call__E  s    jj((+'(D(DE#'#7#78I#J $<=~~$
 ,,T2&-
!M1i@22;?	88I&77A +m ; $

 2 2;@U V01M1MN;;**/$**2Q2QQ445EF>>*:;;;'7I445EF99:JK$9!"$=@P$Pab!$$r5   NNNr   rK   s   @r6   r   r     sn    '
z '
c '
t '
X $(#.2,%88,% rxx ,% }	,%
 ""((+,%r5   r   c                 @    t        j                  || z        }|| z  }|S r?   )rE   r   )softcaprH   outs      r6   logit_softcapr   t  s"    
''!g+
C
-CJr5   c                        e Zd Zdef fdZ	 	 	 ddej                  dej                  fdZdej                  dej                  fdZd	ej                  d
ej                  dej                  fdZ	d Z
 xZS )LanguageModelrN   c           	         t         	|           || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  |j                  z
  | _
        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        |||| j                  k\         c}| _        t        j                  |j                  |j                  |j                  z        | _        t        j$                  |j                  |j                  |j                  z  d      | _        t        j(                  |j                  |j*                        | _        t        d| j                  j.                        D cg c].  }t        j$                  |j                  |j                  d      0 c}| _        t        d| j                  j.                        D cg c].  }t        j$                  |j                  |j                  d      0 c}| _        t        j(                  |j                  |j*                        | _        |j6                  j9                  d      | _        |j6                  j9                  d      | _        |j>                  | _        |j6                  d | j                   }tA        |      dz
  |d d d	   j9                  d      z
  }tA        |      dz
  |d d d	   j9                  d      z
  }g | _!        tE        | j                  j6                        D ]  \  }}|| j                  k  r| jB                  jG                  |       1|dk(  r| jB                  jG                  |       R|dk(  r| jB                  jG                  |       stI        d
|        y c c}w c c}w c c}w )N)rN   rc   rd   FrP   rR   r   rh   rf   full_attentionr}   Unknown layer type: )%r@   rA   rN   r   r%   r   r   r   r"   r   first_kv_shared_layer_idxrT   	Embeddingembed_tokensranger   layersembed_tokens_per_layerrU   per_layer_model_projectionrX   r   per_layer_projection_normr&   altup_projectionsaltup_unembed_projectionsnormr#   indexfirst_sliding_idxfirst_full_idxr   lenlayer_idx_to_cache_idx	enumerateappendNotImplementedError)
rB   rN   rc   r   concrete_layersshared_full_idxshared_sliding_idxi
layer_typerC   s
            r6   rA   zLanguageModel.__init__|  s   !--+1+M+M( ++*0*K*K'!'!9!9'-'E'E$$$v'B'BB 	& LL):):F<N<NO #6#;#;<
 =	  ##,0N0N#N
 =
 ')ll--$$v'I'II'
#
 +-))$$v'I'II+
' *,33##*
& 1dkk::;"
; IIf((&*<*<5I;"
 1dkk::;*
; IIf((&*<*<5I;*
&
 JJ##
	
 "(!3!3!9!9:M!N$00667GH$33 ,,-Mt/M/MN 1$tt'<'B'BCS'TT 	  1$tt'<'B'BCV'WW 	 ')#&t{{'>'>?MAz4111++2215!11//66G#66//667IJ-0DZL.QRR @k
2"

*
s   !"O>3P43Pr   input_embeddingsc                    |"| j                  |      | j                  dz  z  }n|}| j                  |      }| j                  ||      }|d gt	        | j
                        z  }t        ||| j                           }t        ||| j                     | j                        }|}t        j                  |dz  dd      dz  }	|g}
|
j                  | j                  D cg c]
  } ||       c}       t        j                  |
d      }t        j                  |d	d  dz  dd      dz  }|d	d  |	t        j                  |t        j                   |j"                        j$                        z  z  |d	d  t'        | j
                        D ]R  \  }}|d d d d |d d f   }| j(                  j*                  |   d
k(  }|r|}n|} ||||| j,                  |      |      }T t        j                  |d   dz  dd      dz  }	t'        | j.                        D ]  \  }} |||d	z            ||d	z   <    t        j                  |d	d  dz  dd      dz  }|d	d  |	t        j                  |t        j                   |j"                        j$                        z  z  |d	d  t        j                  |d      }| j1                  |      }| j                   j3                  |      }| j4                  t7        | j4                  |      }|S c c}w )N      ?)window_sizer~   r}   Tr   r   )r   r   r   )r   r   get_per_layer_inputsproject_per_layer_inputsr  r   r   r  r  r   rE   r   extendr   stackr   finfor   minr  rN   r#   r  r  r  	as_linearr"   r   )rB   r   r{   r  hper_layer_inputsglobal_masksliding_window_maskh0target_magnitudeh_listprojmagsr  layerr   	is_globalrz   r   s                      r6   rI   zLanguageModel.__call__  s    #!!&)T-=-=s-BCA A44V<88<LM=FS--E+$%%&
 4$(()++

  772q5rDASHD,B,BC,BDtBx,BCDHHV!$wwquzT:cA!")BJJtRXXbhh=O=S=S,TTU!"!$++.HAu.q!Qz:O//26FFI"*d11!45	A /$ 771Q4192EL !?!?@GAtAa!eH~Aa!eH AwwquzT:cA!")BJJtRXXbhh=O=S=S,TTU!"GGAAiil))#.''3 < <cBC
E Ds   L	input_idsr[   c                    || j                   k  }t        j                  ||t        j                  |            }| j	                  |      | j
                  dz  z  } |j                  g |j                  | j                  | j
                   S )Nr  )	r   rE   where
zeros_liker   r%   r   r   r   )rB   r&  per_layer_inputs_masktokensresults        r6   r  z"LanguageModel.get_per_layer_inputs  s     )D,K,K K/BMM)<TU,,V4,,c1
 v~~ 
__
""
 ,,
 	
r5   inputs_embedsr  c                    | j                  |      | j                  dz  z  } |j                  g |j                  d d | j                  j
                  | j                  j                   }| j                  |      }||z   dz  S )Ng      r}   r   )r   r   r   r   rN   r   r%   r   )rB   r-  r  r   s       r6   r  z&LanguageModel.project_per_layer_inputs  s    
  $>>}Md" 
  <3;;  
  "% 
KK)) 
 KK33 

  $==>RS$'77IFFr5   c                    g }| j                   j                  d | j                   D ]d  }|dk(  r|j                  t	                      "|dk(  r1|j                  t        | j                   j                  d             Xt        d|        |S )Nr   rf   r   )max_sizekeepr   )rN   r#   r   r	  r   r   r   r
  )rB   cachesr  s      r6   
make_cachezLanguageModel.make_cache*  s    ++112RD4R4RSJ--gi(22#T[[-G-GaP *,@*MNN T r5   r   )r,   r-   r.   r   rA   rE   r`   rI   r  r  r3  rJ   rK   s   @r6   r   r   {  s    LSz LS`  %)	BB ((	BH

bhh 

288 

GxxG ((G 
	G r5   r   c                   n     e Zd Zdef fdZ	 	 ddej                  deej                     fdZd Z	 xZ
S )Gemma3nargsc                 |    t         |           t        t        j	                  |j
                              | _        y r?   )r@   rA   r   r   	from_dictr9   language_modelrB   r6  rC   s     r6   rA   zGemma3n.__init__9  s,    +J,@,@AQAQ,RSr5   r   r  c                 *    | j                  |||      S N)r{   r  )r9  rB   r   r{   r  s       r6   rI   zGemma3n.__call__=  s$     ""%2B # 
 	
r5   c                 6    | j                   j                         S r?   )r9  r3  rB   s    r6   r3  zGemma3n.make_cacheG  s    ""--//r5   r   )r,   r-   r.   r8   rA   rE   r`   r   rI   r3  rJ   rK   s   @r6   r5  r5  8  sB    TY T /3	

 #288,	
0r5   r5  c                        e Zd Zdef fdZ	 	 d	dej                  deej                     fdZd Z	e
d        Zd Z xZS )
Modelr6  c                 r    t         |           || _        t        |      | _        |j
                  | _        y r?   )r@   rA   r6  r5  modelr   r:  s     r6   rA   zModel.__init__L  s+    	T]
//r5   r   r  c                 *    | j                  |||      S r<  )rC  r=  s       r6   rI   zModel.__call__R  s     zz&@PzQQr5   c                     t        t        |j                                     }dD ]  }|d   j                  |d         t	        t        |            S )N)vision_toweraudio_towerembed_audioembed_visionrC  )r
   r   itemspopr:   r	   )rB   weightsks      r6   sanitizezModel.sanitizeZ  sG     gmmo!67OAG  D) PL)**r5   c                 B    | j                   j                  j                  S r?   )rC  r9  r   r?  s    r6   r   zModel.layers`  s    zz((///r5   c                 6    | j                   j                         S r?   )rC  r3  r?  s    r6   r3  zModel.make_cached  s    zz$$&&r5   r   )r,   r-   r.   r8   rA   rE   r`   r   rI   rN  propertyr   r3  rJ   rK   s   @r6   rA  rA  K  s\    *Y * /3	RR #288,	R+ 0 0'r5   rA  )(r   dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorerE   mlx.nnrT   	mlx.utilsr	   r
   rj   r   r   r   r{   r   r   r   r8   Moduler<   rM   rb   compiler   r   r   r   r   r   r5  rA  r4   r5   r6   <module>r[     s9    !  , ,   2 T T + ( ( (8   
3 3# #2K#ryy K#\ 	t$< %<")) BL1299 L1^V%")) V%r 	t$ %zBII zz0bii 0&'BII 'r5   