
    (HJj	H                     `   d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZ  e        G d de             Z G d dej:                        Z G d dej:                        Z G d dej:                        Z  G d dej:                        Z!ejD                  d        Z# G d dej:                        Z$ G d dej:                        Z% G d dej:                        Z& G d dej:                        Z' G d dej:                        Z(y)     )	dataclass)partial)AnyListOptionalTupleN   )swiglu)BaseModelArgscreate_attention_maskcreate_ssm_maskscaled_dot_product_attention)ArraysCacheKVCache)
ssm_update)	SwitchMLPc                   6   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZe	e
e      ed<   dZe	e
e      ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed<   dZe	e   ed <   dZe	e   ed!<   dZe	eeef      ed"<   dZe	e   ed#<   dZe	e   ed$<   d%d&d'd(d)Zd* Zy)+	ModelArgs
model_type
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersmax_position_embeddingsnum_attention_headsnum_key_value_headsattention_biasmamba_num_headsmamba_head_dimmamba_proj_biasssm_state_sizeconv_kerneln_groupsmlp_biaslayer_norm_epsilonuse_biasuse_conv_biasNhybrid_override_patternlayers_block_typehead_dimmoe_intermediate_size#moe_shared_expert_intermediate_sizemoe_latent_sizen_groupn_routed_expertsn_shared_experts
topk_groupnum_experts_per_toknorm_topk_probrouted_scaling_factortime_step_limittime_step_mintime_step_maxM*E-)mamba	attentionmoemlpc                     | j                   dt        d      f| _         | j                  8| j                  ,| j                  D cg c]  }| j                  |    c}| _        | j                  t        | j                        | _        y y c c}w )N        inf)r5   floatr(   r)   _block_type_to_charlenr   )selfts     b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/nemotron_h.py__post_init__zModelArgs.__post_init__?   s    '$'u#6D  ''/D4J4J4V595K5K,5K((+5K,D( ''3%()E)E%FD" 4,s   B)__name__
__module____qualname__str__annotations__intboolrC   r(   r   r   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r   r6   r7   rD   rI        rH   r   r      s   OO  MNN37Xd3i07-1xS	*1"Hhsm"+/8C=/9='#=%)OXc])!GXc]!&*hsm*&*hsm* $J$)-#-%)NHTN)-18E?159OXeE5L129%)M8E?)%)M8E?) %(c#cR
GrR   r   c                   ~     e Zd Zdededef fdZd	dej                  dej                  dej                  fdZ xZ	S )
MambaRMSNormGatedr   eps
group_sizec                 r    t         |           || _        t        j                  |      | _        || _        y N)super__init__rU   mxonesweightrV   )rF   r   rU   rV   	__class__s       rH   rZ   zMambaRMSNormGated.__init__M   s,    ggk*$rR   xgatereturnc                     |t        ||      }t        j                  |dd| j                  f      }t        j                  j                  |d | j                        }| j                  |j                  d      z  S )Naxisshape)r]   rU   )	r
   r[   	unflattenrV   fastrms_normrU   r]   flatten)rF   r_   r`   s      rH   __call__zMambaRMSNormGated.__call__S   sd    tQALLB+@AGGQt:{{QYYr]**rR   rX   )
rJ   rK   rL   rO   rC   rZ   r[   arrayrl   __classcell__r^   s   @rH   rT   rT   L   sC    %C %e % %+"(( +"(( +bhh +rR   rT   c                       e Zd Zdef fdZdej                  dee   deej                     dej                  fdZ	dej                  d	ej                  d
ej                  dej                  dee   deej                     dej                  fdZ
	 ddej                  deej                     dee   dej                  fdZ xZS )NemotronHMamba2Mixerargsc                 R   t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _	        |j                  | _
        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  d| j                  z  | j
                  z  z   | _        t        j                   | j                  | j                  |j                  d| j                  |j"                        | _        | j                  | j                  z   | j                  z   }t        j&                  | j                  ||j(                        | _        t-        j.                  | j                        | _        t-        j2                  t-        j4                  d| j                  dz   t,        j6                              | _        t-        j.                  | j                        | _        | j                  | j                  z  }t=        | j                  |j>                  |      | _         t        j&                  | j                  | j                  |j(                        | _!        y )N   r   )in_channelsout_channelskernel_sizepaddinggroupsbiasrz   r	   dtype)rU   rV   )"rY   rZ   r   	num_headsr   r!   r"   conv_kernel_sizer   r   r#   r*   r5   heads_per_groupconv_dimnnConv1dr'   conv1dLinearr    in_projr[   r\   dt_biaslogarangefloat32A_logDrT   r%   normout_proj)rF   rr   projection_sizerV   r^   s       rH   rZ   zNemotronHMamba2Mixer.__init__\   s   --++"11 $ 0 0!%!5!58K8K!K++#33#~~>..T]]1BTEXEX1XXii((==##
 004==@4>>QyyoD4H4H
 wwt~~.VVBIIa!);2::NO
(++t}}<
%""''!
	
 		""D$4$44;O;O
rR   
conv_inputcachemaskra   c                    |t        j                  |d   |d      }||d   It        j                  |j                  d   | j                  dz
  | j
                  f|j                        }n|d   }t        j                  ||gd      }| j                  dz
  }|j                  q|j                  d   }t        j                  |j                  d||z
        }|d d d f   t        j                  |      z   d   }	t        j                  ||	d      |d<   n;|d d | d d d f   |d<   n(t        j                  |d| j                  dz
  dfdg      }| j                  |      }
t        j                  |
      S )N.Nr   r	   r|   re   )r   r   )r[   wherezerosrf   r   r   r}   concatenatelengthsclipr   take_along_axispadr   r   silu)rF   r   r   r   
conv_statepadded_inputn_keeprG   ends	positionsconv_outputs              rH   _convzNemotronHMamba2Mixer._conv   sh    $y/:qAJQxXX%%a($*?*?!*CT]]S$**

 #1X
>>:z*BKL**Q.F}}( &&q)wwu}}aV<!!T']RYYv->>	J	--lIANa'F78Q7a66Vd&;&;a&?%CVLL kk,/ww{##rR   hidden_statesBCdtc                 <   |j                   \  }}}	|j                  ||| j                  | j                        }|j                  ||| j                  | j
                        }|j                  ||| j                  | j
                        }|r|d   }
|j                  }nd\  }
}t        || j                  ||| j                  j                  |j                        || j                  |
| j                  |
      \  }}
|r|
|d<   |j                  ||| j                        S )Nr	   NN)rf   reshaper~   r*   r#   r!   r   r   r   r   astyper}   r   r5   r   )rF   r   r   r   r   r   r   
batch_sizeseq_len_stater   ys                rH   _ssmzNemotronHMamba2Mixer._ssm   s    "/!4!4
GQ%--
 IIj'4==$:M:MNIIj'4==$:M:MN!HEmmG'NE7JJFFMM---.LL  
5 E!HyyWd.D.DEErR   c                    | j                  |      }t        j                  || j                  | j                  | j                  z   gd      \  }}}| j                  |||      }t        j                  || j                  | j                  | j                  | j                  z  z   gd      \  }	}
}| j                  |	|
||||      }|r|j                  |j                  d          | j                  ||      }| j                  |      S )Nrc   r   r	   )r   r[   splitr   r   r   r#   r!   r   advancerf   r   r   )rF   r   r   r   	projectedr`   r   r   r   hidden_states_ssmr   r   r   s                rH   rl   zNemotronHMamba2Mixer.__call__   s     LL/	!xx##T%;%;dmm%KL 
j"
 jjUD9"$((&&&&9L9L)LL #
1a II'Ar5$?MM!''!*%IIa}}QrR   rX   )rJ   rK   rL   r   rZ   r[   rm   r   r   r   r   rl   rn   ro   s   @rH   rq   rq   [   s   (
Y (
T $HH $ $ $ rxx 	 $
 
 $D%Fxx%F 88%F 88	%F
 HH%F $%F rxx %F 
%FV (,	 xx  rxx   $	 
 
 rR   rq   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	NemotronHAttentionrr   c                 \   t         |           |j                  | _        |j                  | _        |j
                  |j
                  n|j                  |j                  z  | _        |j                  | _        | j
                  dz  | _        t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j
                  z  | j                  |j                        | _        y )Ng      r{   )rY   rZ   r   r   r~   r*   r   scaler   r   r   q_projk_projv_projo_projrF   rr   r^   s     rH   rZ   zNemotronHAttention.__init__   sG   ++11 }}( MM""d&>&>> 	
 $(#;#; ]]D(
iidnnt}}<4CVCV
 ii$$t}}4$$

 ii$$t}}4$$

 iiNNT]]*D,<,<4CVCV
rR   r_   r   r   ra   c                 l   |j                   \  }}}| j                  |      j                  ||| j                  d      j	                  dddd      }| j                  |      j                  ||| j                  d      j	                  dddd      }| j                  |      j                  ||| j                  d      j	                  dddd      }	||j                  ||	      \  }}	t        |||	|| j                  |      }
|
j	                  dddd      j                  ||d      }
| j                  |
      S )Nrc   r   rt   r	      )r   r   r   )rf   r   r   r~   	transposer   r   r   update_and_fetchr   r   r   )rF   r_   r   r   r   Lr   querieskeysvaluesoutputs              rH   rl   zNemotronHAttention.__call__  s&    ''1a++a.((At~~rBLLQPQSTVWXKKNWQ433R8Yq!Q" 	 KKNWQ433R8Yq!Q" 	  11$?LD&-T6djjt
 !!!Q1-55aB?{{6""rR   r   )rJ   rK   rL   r   rZ   r[   rm   r   r   rl   rn   ro   s   @rH   r   r      sY    
Y 
@ $(#'	#88# rxx #  	#
 
#rR   r   c                   ,     e Zd Zddef fdZd Z xZS )NemotronHMLPrr   c                    t         |           |xs |j                  }t        j                  |j
                  ||j                        | _        t        j                  ||j
                  |j                        | _        y )Nr{   )	rY   rZ   r   r   r   r   r$   up_proj	down_proj)rF   rr   r   r^   s      rH   rZ   zNemotronHMLP.__init__*  sd    -G1G1Gyy/dmm
 t//dmm
rR   c                 h    | j                  t        j                  | j                  |                  S rX   )r   r   relu2r   rF   r_   s     rH   rl   zNemotronHMLP.__call__5  s"    ~~bhht||A788rR   rX   rJ   rK   rL   r   rZ   rl   rn   ro   s   @rH   r   r   )  s    	
Y 	
9rR   r   c                    t        j                  | j                  t         j                              x}}||z   }|dkD  rt        j                  |d|df      }t        j
                  |dd      j                  dd      }	||z
  }
t        j                  |	|
dz
  d	      d
d |
d d f   }t        j                  |t        j                  |      t        j                  d      d      }t        j                  |dd      }|}
t        j                  | |
dz
  d	      d
d |
f   }t        j                  ||d      }|dkD  r|r|j                  dd      }||dz   z  }||z  }||fS )Nr	   rc   rd   rt   r   T)re   keepdimsrg   )kthre   .rA   g#B;)r[   sigmoidr   r   rh   topksumargpartitionput_along_axisstop_gradientrm   rk   r   )gatese_score_correction_biastop_kr.   r1   r4   r3   orig_scoresscoresgroup_scoresk	group_idxindsdenominators                 rH   group_expert_selectr   9  sZ    ::ell2::&>??K&--F{f2gr]Cwwvqr266B6Nj OOLa!e"Ec2A2qjQ	""B$$Y/#R
 FB+A??F7AB7RaR@DT;Fqy^jjb4j8;./++F<rR   c                   *     e Zd Zdef fdZd Z xZS )MoEGateconfigc                    t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f      | _        t        j                  | j                  f      | _        y rX   )rY   rZ   r   r2   r   r3   r/   r4   r.   r1   r[   r   r   r]   r   )rF   r   r^   s     rH   rZ   zMoEGate.__init__\  s    //
$33 & 7 7%+%A%A"~~ ++hh 5 5v7I7IJK')xx1F1F0H'I$rR   c           	          t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                        S rX   )	r   r]   Tr   r   r.   r1   r4   r3   r   s     rH   rl   zMoEGate.__call__h  sO    "((JJLLOO&&
 	
rR   r   ro   s   @rH   r   r   [  s    
Jy 
J	
rR   r   c                   *     e Zd Zdef fdZd Z xZS )NemotronHMoEr   c                    t         |           || _        |j                  | _        |j                  | _        |j                  |j                  n|j
                  }t        ||j                  |j                  t        j                               | _        t        |      | _        |j                  |j                  }t!        ||      | _        |j                  wt        j$                  |j
                  |j                  |j&                        | _        t        j$                  |j                  |j
                  |j&                        | _        y y )N)
activation)r   r{   )rY   rZ   r   r2   r-   r   r   r+   r/   r   ReLU2
switch_mlpr   r`   r0   r,   r   shared_expertsr   r$   fc1_latent_projfc2_latent_proj)rF   r   expert_input_dimr   r^   s       rH   rZ   zNemotronHMoE.__init__u  s!   #)#=#= %55
 %%1 ""## 	
 $((##xxz	
 FO	"". & J J".*;#D
 !!-#%99""F$:$:$D  $&99&&(:(:$D 	 .rR   c                    |}| j                  |      \  }}| j                  | j                  |      }| j                  ||      }||d   z  j	                  d      j                  |j                        }| j                  | j                  |      }| j                  j                  || j                  |      z   }|S )Nr   rg   r   )r`   r-   r   r   r   r   r}   r   r   r0   r   )rF   r_   	residualsr   r   r   s         rH   rl   zNemotronHMoE.__call__  s    	yy|f+$$Q'AOOAt$	""''R'077@+$$Q'A;;''3D''	22ArR   r   ro   s   @rH   r   r   t  s    !y !FrR   r   c                   ^     e Zd Zdedef fdZ	 	 ddeej                     dee	   fdZ
 xZS )NemotronHBlockrr   
block_typec                    t         |           t        j                  |j                  |j
                        | _        || _        | j                  dk(  rt        |      | _	        y | j                  dk(  rt        |      | _	        y | j                  dk(  rt        |      | _	        y | j                  dk(  rt        |      | _	        y y )NrU   r8   r9   r;   r:   )rY   rZ   r   RMSNormr   r%   r   r   rq   mixerr   r   r   )rF   rr   r   r^   s      rH   rZ   zNemotronHBlock.__init__  s    JJt//T5L5LM	$??c!-d3DJ__#+D1DJ__#%d+DJ__#%d+DJ $rR   r   r   c                     | j                  |      }| j                  dk(  s| j                  dk(  r| j                  |||      }||z   S | j                  |      }||z   S )Nr8   r9   r   r   )r   r   r   )rF   r_   r   r   r   s        rH   rl   zNemotronHBlock.__call__  sd     		!??c!T__%; JJ}4uJMM =   !JJ}5M=  rR   r   )rJ   rK   rL   r   rM   rZ   r   r[   rm   r   rl   rn   ro   s   @rH   r   r     sD    ,Y ,C ,$ $(#	! rxx ! }	!rR   r   c                   :     e Zd Zdef fdZ	 ddee   fdZ xZS )NemotronHModelrr   c                 $   t         |           t        j                  |j                  |j
                        | _        |j                  D cg c]  }t        ||       c}| _	        t        j                  |j
                  |j                        | _        d| _        d| _        |j                  D ]$  }|dk(  r n|dk(  s| xj                  dz  c_        & |j                  D ]$  }|dk(  r| xj                  dz  c_        |dk(  s$ y  y c c}w )Nr   r   r9   r8   r	   )rY   rZ   r   	Embeddingr   r   
embeddingsr(   r   layersr   r%   norm_ffa_idxssm_idx)rF   rr   r   br^   s       rH   rZ   zNemotronHModel.__init__  s    ,,t8H8HI #::
:
 4,:
 jj!1!1t7N7NO--ACxcq 	 .
 --ACx!c	 .
s   Dr   c                    | j                  |      }|d gt        | j                        z  }t        ||| j                           }t        ||| j                           }d}| j                  D ]L  }|j                  dk(  s|j                  dk(  r||   }|dz  }nd }|j                  dk(  r|}	n|}	 |||	|      }N | j                  |      S )Nr   r8   r9   r	   r  )	r  rE   r  r   r	  r   r
  r   r  )
rF   inputsr   r   	attn_maskssm_maskcache_counterlayercr   s
             rH   rl   zNemotronHModel.__call__  s    
 /=FS--E)-t{{9KL	"=%2EF[[E3&%*:*:c*A-("3& !-d!DM ! {{=))rR   rX   )	rJ   rK   rL   r   rZ   r   r   rl   rn   ro   s   @rH   r  r    s'    Y 0  $* }*rR   r  c                   ~     e Zd Zdef fdZ	 d
dej                  dee   fdZ	e
d        Zd Zd Ze
d	        Z xZS )Modelrr   c                     t         |           || _        t        |      | _        t        j                  |j                  |j                  d      | _	        |j                  | _
        y )NFr{   )rY   rZ   rr   r  backboner   r   r   r   lm_headr   r   s     rH   rZ   zModel.__init__  sJ    	&t,yy!1!14??O//rR   r  r   c                 J    | j                  ||      }| j                  |      S )N)r   )r  r  )rF   r  r   outs       rH   rl   zModel.__call__  s%    
 mmF%m0||C  rR   c                 .    | j                   j                  S rX   )r  r  )rF   s    rH   r  zModel.layers  s    }}###rR   c                     g }| j                   D ]V  }|j                  dk(  r|j                  t        d             .|j                  dk(  s>|j                  t	                      X |S )Nr8   rt   )sizer9   )r  r   appendr   r   )rF   cachesls      rH   
make_cachezModel.make_cache  sS    A||s"kq12$gi(	 
 rR   c                 R   |j                         D ci c]  \  }}|j                  d      r|| }}}|j                         D ]2  \  }}d|v s|j                  d   dk7  s|j                  dd      ||<   4 t	        | j
                  j                        D ]  }d| d}dD ]r  \  }}| d	| d
|v st	        | j
                  j                        D cg c]  }|j                  | d| d| d
       }	}t        j                  |	      || d| d
<   t  |S c c}}w c c}w )Nzmtp.zconv1d.weightrc   r	   rt   zbackbone.layers.z.mixer))r   fc2)r   fc1z.experts.0.z.weightz	.experts..z.switch_mlp.)items
startswithrf   moveaxisrangerr   r   r/   popr[   stack)
rF   weightsr   vr  prefixmneto_joins
             rH   sanitizezModel.sanitize  s>   &-mmoRoFQQ\\&=Q1a4oRMMODAq!#q(8ZZ1-
 $
 tyy223A's&1FB1X[73w> "'tyy'A'A!B!BA  vhis!A3g$FG!B   BD'ARGvhl1#W=> C 4 ! Ss   DD!D$c                     d }|S )Nc                     d| vxr d| vS )Nr   r   rQ   )r   s    rH   	predicatez'Model.cast_predicate.<locals>.predicate/  s    ,A5J':JJrR   rQ   )rF   r5  s     rH   cast_predicatezModel.cast_predicate-  s    	K rR   rX   )rJ   rK   rL   r   rZ   r[   rm   r   r   rl   propertyr  r   r2  r6  rn   ro   s   @rH   r  r    se    *Y *  $!! }! $ $&  rR   r  ))dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorer[   mlx.nnr   activationsr
   baser   r   r   r   r   r   r   ssmr   switch_layersr   r   ModulerT   rq   r   r   compiler   r   r   r   r  r  rQ   rR   rH   <module>rD     s   "  - -     (  $ 2G 2G 2Gj+		 +O 299 O d9# 9#x9299 9   B
bii 
24299 4n!RYY !>0*RYY 0*f5BII 5rR   