
    (HJj                     |   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ e G d de             Z G d	 d
ej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   d
Z	eed<   dZ
eed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizeTbiasi   max_position_embeddingsNnum_key_value_heads'  
rope_thetaFrope_traditionalrope_scalingtie_word_embeddingsc                       j                    j                   _          j                  rCddh}t         fd|D              st	        d|        j                  d   dvrt	        d      y y )Nfactortypec              3   :   K   | ]  }|j                   v   y wN)r   ).0keyselfs     a/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/internlm2.py	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>$   s     I=Csd///=s   zrope_scaling must contain keys )lineardynamicz@rope_scaling 'type' currently only supports 'linear' or 'dynamic)r   r   r   all
ValueError)r%   required_keyss   ` r&   __post_init__zModelArgs.__post_init__   s    ##+'+'?'?D$%v.MI=II #B=/!RSS  (0EE V  F     )__name__
__module____qualname__str__annotations__intfloatr   boolr   r   r   r   r   r   r   r   r   r-    r.   r&   r   r      s    OOD$#(S(##J"d";?L(4U5#:%6 678? %%r.   r   c                   V     e Zd ZdZ	 	 	 	 ddededededef
 fdZd Zdd	efd
Z	 xZ
S )DynamicNTKScalingRoPEzCImplements the rotary positional encoding with Dynamic NTK scaling.dimsr   traditionalbasescalec                 h    t         |           || _        || _        || _        || _        || _        y r"   )super__init__r   original_baser:   r;   r=   )r%   r:   r   r;   r<   r=   	__class__s         r&   r@   zDynamicNTKScalingRoPE.__init__0   s6     	'>$!	&
r.   c                 l    | j                    d| j                   d| j                   d| j                   S )Nz, traditional=z, max_position_embeddings=z, scaling_factor=)r:   r;   r   scaling_factorr%   s    r&   
extra_reprz DynamicNTKScalingRoPE.extra_repr?   sV    ))N4+;+;*<<VW[WsWsVt  uF  GK  GZ  GZ  F[  \  	\r.   offsetc                    |j                   d   |z   }|| j                  kD  rW| j                  | j                  |z  | j                  z  | j                  dz
  z
  | j                  | j                  dz
  z  z  z  }n| j                  }t
        j                  j                  || j                  | j                  || j                  |      S )Nr      )r;   r<   r=   rG   )	shaper   rA   r=   r:   mxfastroper;   )r%   xrG   seq_lenr<   s        r&   __call__zDynamicNTKScalingRoPE.__call__B   s    ''!*v%T111%%g%(D(DDVWX))tyy1}-)/ /D %%Dww||II((**  
 	
r.   )i   Fr   g      ?)r   )r/   r0   r1   __doc__r4   r6   r5   r@   rF   rP   __classcell__rB   s   @r&   r9   r9   -   s[    M
 (,! "% 	
  \
# 
r.   r9   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 l   t         |           |j                  }|j                  x| _        }|j
                  x| _        }||j
                  z  | _        |j                  |z  x| _        }|dz  | _	        t        j                  ||d|z  z   |z  |j                        | _        t        j                  ||z  ||j                        | _        |j                  $|j                  d   dk(  rd|j                  d   z  nd}t!        ||j"                  |j$                  |j&                  |	      | _        y )
Ng      rI   r   r    r(   r   r   g       @)r   r;   r<   r=   )r?   r@   r   r   n_headsr   
n_kv_headsn_kv_groupshead_dimr=   nnLinearr   wqkvwor   r9   r   r   r   rM   )r%   rV   dimrY   rZ   r\   
rope_scalerB   s          r&   r@   zAttention.__init__V   s$   !%!9!99w'+'?'??*"d&>&>>#'#3#3w#>>t^
II'A
N*h6TYY
	 ))Gh.$))D   ,1B1B61Jh1V !!(++ 	 *$($@$@--
	r.   rN   maskcachereturnc                    |j                   \  }}}| j                  |      }|j                  ||dd| j                  z   | j                        }|dd | j                  d d f   }|j                  ||d| j                        }|ddd d f   }	|ddd d f   }
|j                  ||| j
                  d      j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|
j                  ||| j                  d      j                  dddd      }
|P| j                  ||j                        }| j                  |	|j                        }	|j                  |	|
      \  }	}
n"| j                  |      }| j                  |	      }	t        ||	|
|| j                  |	      }|j                  dddd      j                  ||d      }| j                  |      S )
NrI   .r   r      )rG   )rd   r=   rc   )rJ   r_   reshaper[   r\   rY   	transposerZ   rM   rG   update_and_fetchr   r=   r`   )r%   rN   rc   rd   BLD
qkv_statesquerieskeysvaluesoutputs               r&   rP   zAttention.__call__t   s    ''1aYYq\
''1b!d6F6F2FV
S"4D$4$4"4a78//!QDMM:#r1*%CQJ' //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?wwvr.   NNr/   r0   r1   r   r@   rK   arrayr   r   rP   rR   rS   s   @r&   rU   rU   U   sX    
Y 
B $(#	!88! rxx ! }	!
 
!r.   rU   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFrX   )r?   r@   r]   r^   w1w2w3)r%   ra   
hidden_dimrB   s      r&   r@   zMLP.__init__   sJ    ))C%8))J%8))C%8r.   re   c                 t    | j                  t        | j                  |      | j                  |                  S r"   )r}   r	   r|   r~   )r%   rN   s     r&   rP   zMLP.__call__   s(    wwvdggaj$''!*566r.   )r/   r0   r1   r@   rK   rw   rP   rR   rS   s   @r&   ry   ry      s    97RXX 7r.   ry   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockrV   c                 L   t         |           t        |      | _        t	        |j
                  |j                        | _        t        j                  |j
                  |j                        | _        t        j                  |j
                  |j                        | _        y )Neps)r?   r@   rU   	attentionry   r   r   feed_forwardr]   RMSNormr   attention_normffn_normr%   rV   rB   s     r&   r@   zTransformerBlock.__init__   sq    "4 0 0$2H2HI jj)9)9t?P?PQ

4#3#39J9JKr.   rN   rc   rd   re   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r"   )r   r   r   r   )r%   rN   rc   rd   rhouts          r&   rP   zTransformerBlock.__call__   sQ     NN4..q14?EdmmA./!e
r.   ru   rv   rS   s   @r&   r   r      sY    LY L $(#	
88
 rxx 
 }	

 

r.   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )InternLM2ModelrV   c                 p   t         |           |j                  dkD  sJ t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )Nr   )rV   r   )r?   r@   r   r]   	Embeddingr   tok_embeddingsranger   r   layersr   r   norm)r%   rV   _rB   s      r&   r@   zInternLM2Model.__init__   s    """ ll4??D<L<LM16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   'B3inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rd   )r   lenr   r   zipr   )r%   r   rd   r   rc   layercs          r&   rP   zInternLM2Model.__call__   su    
 '=FS--E$Qa1DKK/HE1aQ'A 0 yy|r.   r"   )	r/   r0   r1   r   r@   rK   rw   rP   rR   rS   s   @r&   r   r      s'    HY H r.   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )ModelrV   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y r{   )r?   r@   rV   r   r   modelr   r]   r^   r   r   rt   r   s     r&   r@   zModel.__init__   sX    	//#D)
''))D$4$4dooERDK (r.   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r"   )r   rV   r   r   	as_linearrt   )r%   r   rd   r   s       r&   rP   zModel.__call__   sW    
 jj'99((**++55c:C 
 ++c"C
r.   c                 `    |j                         D ci c]  \  }}d|vs|| c}}S c c}}w )Nzattention.rope.inv_freq)items)r%   weightskvs       r&   sanitizezModel.sanitize   s1    !(WA4MUV4V1WWWs   **c                 .    | j                   j                  S r"   )r   r   rE   s    r&   r   zModel.layers   s    zz   r.   r"   )r/   r0   r1   r   r@   rK   rw   rP   r   propertyr   rR   rS   s   @r&   r   r      sA    SY S 

X ! !r.   r   )dataclassesr   typingr   r   r   r   mlx.corecorerK   mlx.nnr]   activationsr	   r<   r
   r   r   r   Moduler9   rU   ry   r   r   r   r7   r.   r&   <module>r      s    " - -    T T   >%
BII %
P@		 @F7")) 7ryy *RYY 6!BII !r.   